feat(xo-lite/ObjectStatus): use ProgressCircle component (#6207)

Fix `MUI: The 'value' prop must be an array when using the 'Select' component with 'multiple'.` in case we toggle the `multiple` prop `false -> true` with a controlled input initialized with undefined value.

.eslintrc.js

@@ -1,13 +1,5 @@
 module.exports = {
-  extends: [
+  extends: ['plugin:eslint-comments/recommended', 'standard', 'standard-jsx', 'prettier'],
-    'plugin:eslint-comments/recommended',
-
-    'standard',
-    'standard-jsx',
-    'prettier',
-    'prettier/standard',
-    'prettier/react',
-  ],
   globals: {
     __DEV__: true,
     $Dict: true,
@@ -21,27 +13,26 @@ module.exports = {
 
   overrides: [
     {
-      files: ['cli.js', '*-cli.js', 'packages/*cli*/**/*.js'],
+      files: ['cli.{,c,m}js', '*-cli.{,c,m}js', '**/*cli*/**/*.{,c,m}js'],
       rules: {
         'no-console': 'off',
       },
     },
   ],
 
-  parser: 'babel-eslint',
-  parserOptions: {
-    ecmaFeatures: {
-      legacyDecorators: true,
-    },
-  },
   rules: {
     // disabled because XAPI objects are using camel case
     camelcase: ['off'],
 
+    'react/jsx-handler-names': 'off',
+
+    // disabled because not always relevant, we might reconsider in the future
+    //
+    // enabled by https://github.com/standard/eslint-config-standard/commit/319b177750899d4525eb1210686f6aca96190b2f
+    //
+    // example: https://github.com/vatesfr/xen-orchestra/blob/31ed3767c67044ca445658eb6b560718972402f2/packages/xen-api/src/index.js#L156-L157
+    'lines-between-class-members': 'off',
+
     'no-console': ['error', { allow: ['warn', 'error'] }],
-    'no-var': 'error',
-    'node/no-extraneous-import': 'error',
-    'node/no-extraneous-require': 'error',
-    'prefer-const': 'error',
   },
 }

.gitignore

@@ -4,20 +4,24 @@
 /lerna-debug.log
 /lerna-debug.log.*
 
+/@vates/*/dist/
+/@vates/*/node_modules/
 /@xen-orchestra/*/dist/
 /@xen-orchestra/*/node_modules/
 /packages/*/dist/
 /packages/*/node_modules/
 
+/@xen-orchestra/proxy/src/app/mixins/index.mjs
+
 /packages/vhd-cli/src/commands/index.js
 
 /packages/xen-api/examples/node_modules/
 /packages/xen-api/plot.dat
 
 /packages/xo-server/.xo-server.*
-/packages/xo-server/src/api/index.js
+/packages/xo-server/src/api/index.mjs
-/packages/xo-server/src/xapi/mixins/index.js
+/packages/xo-server/src/xapi/mixins/index.mjs
-/packages/xo-server/src/xo-mixins/index.js
+/packages/xo-server/src/xo-mixins/index.mjs
 
 /packages/xo-server-auth-ldap/ldap.cache.conf
 

.prettierrc.js

@@ -1,6 +1,11 @@
 module.exports = {
+  arrowParens: 'avoid',
   jsxSingleQuote: true,
   semi: false,
   singleQuote: true,
-  trailingComma: 'es5',
+
+  // 2020-11-24: Requested by nraynaud and approved by the rest of the team
+  //
+  // https://team.vates.fr/vates/pl/a1i8af1b9id7pgzm3jcg4toacy
+  printWidth: 120,
 }

.travis.yml

@@ -1,7 +1,6 @@
 language: node_js
 node_js:
-  #- stable # disable for now due to an issue of indirect dep upath with Node 9
+  - 14
-  - 8
 
 # Use containers.
 # http://docs.travis-ci.com/user/workers/container-based-infrastructure/

@vates/coalesce-calls/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/coalesce-calls/README.md

@@ -0,0 +1,46 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/coalesce-calls
+
+[![Package Version](https://badgen.net/npm/v/@vates/coalesce-calls)](https://npmjs.org/package/@vates/coalesce-calls) ![License](https://badgen.net/npm/license/@vates/coalesce-calls) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/coalesce-calls)](https://bundlephobia.com/result?p=@vates/coalesce-calls) [![Node compatibility](https://badgen.net/npm/node/@vates/coalesce-calls)](https://npmjs.org/package/@vates/coalesce-calls)
+
+> Wraps an async function so that concurrent calls will be coalesced
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/coalesce-calls):
+
+```
+> npm install --save @vates/coalesce-calls
+```
+
+## Usage
+
+```js
+import { coalesceCalls } from '@vates/coalesce-calls'
+
+const connect = coalesceCalls(async function () {
+  // async operation
+})
+
+connect()
+
+// the previous promise result will be returned if the operation is not
+// complete yet
+connect()
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/coalesce-calls/USAGE.md

@@ -0,0 +1,13 @@
+```js
+import { coalesceCalls } from '@vates/coalesce-calls'
+
+const connect = coalesceCalls(async function () {
+  // async operation
+})
+
+connect()
+
+// the previous promise result will be returned if the operation is not
+// complete yet
+connect()
+```

@vates/coalesce-calls/index.js

@@ -0,0 +1,14 @@
+exports.coalesceCalls = function (fn) {
+  let promise
+  const clean = () => {
+    promise = undefined
+  }
+  return function () {
+    if (promise !== undefined) {
+      return promise
+    }
+    promise = fn.apply(this, arguments)
+    promise.then(clean, clean)
+    return promise
+  }
+}

@vates/coalesce-calls/index.spec.js

@@ -0,0 +1,33 @@
+/* eslint-env jest */
+
+const { coalesceCalls } = require('./')
+
+const pDefer = () => {
+  const r = {}
+  r.promise = new Promise((resolve, reject) => {
+    r.reject = reject
+    r.resolve = resolve
+  })
+  return r
+}
+
+describe('coalesceCalls', () => {
+  it('decorates an async function', async () => {
+    const fn = coalesceCalls(promise => promise)
+
+    const defer1 = pDefer()
+    const promise1 = fn(defer1.promise)
+    const defer2 = pDefer()
+    const promise2 = fn(defer2.promise)
+
+    defer1.resolve('foo')
+    expect(await promise1).toBe('foo')
+    expect(await promise2).toBe('foo')
+
+    const defer3 = pDefer()
+    const promise3 = fn(defer3.promise)
+
+    defer3.resolve('bar')
+    expect(await promise3).toBe('bar')
+  })
+})

@vates/coalesce-calls/package.json

@@ -0,0 +1,35 @@
+{
+  "private": false,
+  "name": "@vates/coalesce-calls",
+  "description": "Wraps an async function so that concurrent calls will be coalesced",
+  "keywords": [
+    "async",
+    "calls",
+    "coalesce",
+    "decorate",
+    "decorator",
+    "merge",
+    "promise",
+    "wrap",
+    "wrapper"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/coalesce-calls",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/coalesce-calls",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "0.1.0",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@vates/compose/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/compose/README.md

@@ -0,0 +1,81 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/compose
+
+[![Package Version](https://badgen.net/npm/v/@vates/compose)](https://npmjs.org/package/@vates/compose) ![License](https://badgen.net/npm/license/@vates/compose) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/compose)](https://bundlephobia.com/result?p=@vates/compose) [![Node compatibility](https://badgen.net/npm/node/@vates/compose)](https://npmjs.org/package/@vates/compose)
+
+> Compose functions from left to right
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/compose):
+
+```
+> npm install --save @vates/compose
+```
+
+## Usage
+
+```js
+import { compose } from '@vates/compose'
+
+const add2 = x => x + 2
+const mul3 = x => x * 3
+
+// const f = x => mul3(add2(x))
+const f = compose(add2, mul3)
+
+console.log(f(5))
+// → 21
+```
+
+> The call context (`this`) of the composed function is forwarded to all functions.
+
+The first function is called with all arguments of the composed function:
+
+```js
+const add = (x, y) => x + y
+const mul3 = x => x * 3
+
+// const f = (x, y) => mul3(add(x, y))
+const f = compose(add, mul3)
+
+console.log(f(4, 5))
+// → 27
+```
+
+Functions may also be passed in an array:
+
+```js
+const f = compose([add2, mul3])
+```
+
+Options can be passed as first parameter:
+
+```js
+const f = compose(
+  {
+    // compose async functions
+    async: true,
+
+    // compose from right to left
+    right: true,
+  },
+  [add2, mul3]
+)
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/compose/USAGE.md

@@ -0,0 +1,48 @@
+```js
+import { compose } from '@vates/compose'
+
+const add2 = x => x + 2
+const mul3 = x => x * 3
+
+// const f = x => mul3(add2(x))
+const f = compose(add2, mul3)
+
+console.log(f(5))
+// → 21
+```
+
+> The call context (`this`) of the composed function is forwarded to all functions.
+
+The first function is called with all arguments of the composed function:
+
+```js
+const add = (x, y) => x + y
+const mul3 = x => x * 3
+
+// const f = (x, y) => mul3(add(x, y))
+const f = compose(add, mul3)
+
+console.log(f(4, 5))
+// → 27
+```
+
+Functions may also be passed in an array:
+
+```js
+const f = compose([add2, mul3])
+```
+
+Options can be passed as first parameter:
+
+```js
+const f = compose(
+  {
+    // compose async functions
+    async: true,
+
+    // compose from right to left
+    right: true,
+  },
+  [add2, mul3]
+)
+```

@vates/compose/index.js

@@ -0,0 +1,46 @@
+'use strict'
+
+const defaultOpts = { async: false, right: false }
+
+exports.compose = function compose(opts, fns) {
+  if (Array.isArray(opts)) {
+    fns = opts
+    opts = defaultOpts
+  } else if (typeof opts === 'object') {
+    opts = Object.assign({}, defaultOpts, opts)
+    if (!Array.isArray(fns)) {
+      fns = Array.prototype.slice.call(arguments, 1)
+    }
+  } else {
+    fns = Array.from(arguments)
+    opts = defaultOpts
+  }
+
+  const n = fns.length
+  if (n === 0) {
+    throw new TypeError('at least one function must be passed')
+  }
+  if (n === 1) {
+    return fns[0]
+  }
+
+  if (opts.right) {
+    fns.reverse()
+  }
+
+  return opts.async
+    ? async function () {
+        let value = await fns[0].apply(this, arguments)
+        for (let i = 1; i < n; ++i) {
+          value = await fns[i].call(this, value)
+        }
+        return value
+      }
+    : function () {
+        let value = fns[0].apply(this, arguments)
+        for (let i = 1; i < n; ++i) {
+          value = fns[i].call(this, value)
+        }
+        return value
+      }
+}

@vates/compose/index.spec.js

@@ -0,0 +1,66 @@
+/* eslint-env jest */
+
+const { compose } = require('./')
+
+const add2 = x => x + 2
+const mul3 = x => x * 3
+
+describe('compose()', () => {
+  it('throws when no functions is passed', () => {
+    expect(() => compose()).toThrow(TypeError)
+    expect(() => compose([])).toThrow(TypeError)
+  })
+
+  it('applies from left to right', () => {
+    expect(compose(add2, mul3)(5)).toBe(21)
+  })
+
+  it('accepts functions in an array', () => {
+    expect(compose([add2, mul3])(5)).toBe(21)
+  })
+
+  it('can apply from right to left', () => {
+    expect(compose({ right: true }, add2, mul3)(5)).toBe(17)
+  })
+
+  it('accepts options with functions in an array', () => {
+    expect(compose({ right: true }, [add2, mul3])(5)).toBe(17)
+  })
+
+  it('can compose async functions', async () => {
+    expect(
+      await compose(
+        { async: true },
+        async x => x + 2,
+        async x => x * 3
+      )(5)
+    ).toBe(21)
+  })
+
+  it('forwards all args to first function', () => {
+    expect.assertions(1)
+
+    const expectedArgs = [Math.random(), Math.random()]
+    compose(
+      (...args) => {
+        expect(args).toEqual(expectedArgs)
+      },
+      // add a second function to avoid the one function special case
+      Function.prototype
+    )(...expectedArgs)
+  })
+
+  it('forwards context to all functions', () => {
+    expect.assertions(2)
+
+    const expectedThis = {}
+    compose(
+      function () {
+        expect(this).toBe(expectedThis)
+      },
+      function () {
+        expect(this).toBe(expectedThis)
+      }
+    ).call(expectedThis)
+  })
+})

@vates/compose/package.json

@@ -0,0 +1,24 @@
+{
+  "private": false,
+  "name": "@vates/compose",
+  "description": "Compose functions from left to right",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/compose",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/compose",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "2.0.0",
+  "engines": {
+    "node": ">=7.6"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@vates/decorate-with/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/decorate-with/README.md

@@ -0,0 +1,75 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/decorate-with
+
+[![Package Version](https://badgen.net/npm/v/@vates/decorate-with)](https://npmjs.org/package/@vates/decorate-with) ![License](https://badgen.net/npm/license/@vates/decorate-with) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/decorate-with)](https://bundlephobia.com/result?p=@vates/decorate-with) [![Node compatibility](https://badgen.net/npm/node/@vates/decorate-with)](https://npmjs.org/package/@vates/decorate-with)
+
+> Creates a decorator from a function wrapper
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/decorate-with):
+
+```
+> npm install --save @vates/decorate-with
+```
+
+## Usage
+
+### `decorateWith(fn, ...args)`
+
+Creates a new ([legacy](https://babeljs.io/docs/en/babel-plugin-syntax-decorators#legacy)) method decorator from a function decorator, for instance, allows using Lodash's functions as decorators:
+
+```js
+import { decorateWith } from '@vates/decorate-with'
+
+class Foo {
+  @decorateWith(lodash.debounce, 150)
+  bar() {
+    // body
+  }
+}
+```
+
+### `decorateMethodsWith(class, map)`
+
+Decorates a number of methods directly, without using the decorator syntax:
+
+```js
+import { decorateMethodsWith } from '@vates/decorate-with'
+
+class Foo {
+  bar() {
+    // body
+  }
+
+  baz() {
+    // body
+  }
+}
+
+decorateMethodsWith(Foo, {
+  // without arguments
+  bar: lodash.curry,
+
+  // with arguments
+  baz: [lodash.debounce, 150],
+})
+```
+
+The decorated class is returned, so you can export it directly.
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/decorate-with/USAGE.md

@@ -0,0 +1,42 @@
+### `decorateWith(fn, ...args)`
+
+Creates a new ([legacy](https://babeljs.io/docs/en/babel-plugin-syntax-decorators#legacy)) method decorator from a function decorator, for instance, allows using Lodash's functions as decorators:
+
+```js
+import { decorateWith } from '@vates/decorate-with'
+
+class Foo {
+  @decorateWith(lodash.debounce, 150)
+  bar() {
+    // body
+  }
+}
+```
+
+### `decorateMethodsWith(class, map)`
+
+Decorates a number of methods directly, without using the decorator syntax:
+
+```js
+import { decorateMethodsWith } from '@vates/decorate-with'
+
+class Foo {
+  bar() {
+    // body
+  }
+
+  baz() {
+    // body
+  }
+}
+
+decorateMethodsWith(Foo, {
+  // without arguments
+  bar: lodash.curry,
+
+  // with arguments
+  baz: [lodash.debounce, 150],
+})
+```
+
+The decorated class is returned, so you can export it directly.

@vates/decorate-with/index.js

@@ -0,0 +1,21 @@
+exports.decorateWith = function decorateWith(fn, ...args) {
+  return (target, name, descriptor) => ({
+    ...descriptor,
+    value: fn(descriptor.value, ...args),
+  })
+}
+
+const { getOwnPropertyDescriptor, defineProperty } = Object
+
+exports.decorateMethodsWith = function decorateMethodsWith(klass, map) {
+  const { prototype } = klass
+  for (const name of Object.keys(map)) {
+    const descriptor = getOwnPropertyDescriptor(prototype, name)
+    const { value } = descriptor
+
+    const decorator = map[name]
+    descriptor.value = typeof decorator === 'function' ? decorator(value) : decorator[0](value, ...decorator.slice(1))
+    defineProperty(prototype, name, descriptor)
+  }
+  return klass
+}

@vates/decorate-with/package.json

@@ -0,0 +1,30 @@
+{
+  "private": false,
+  "name": "@vates/decorate-with",
+  "description": "Creates a decorator from a function wrapper",
+  "keywords": [
+    "apply",
+    "decorator",
+    "factory",
+    "wrapper"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/decorate-with",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/decorate-with",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "0.1.0",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@vates/disposable/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/disposable/README.md

@@ -0,0 +1,89 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/disposable
+
+[![Package Version](https://badgen.net/npm/v/@vates/disposable)](https://npmjs.org/package/@vates/disposable) ![License](https://badgen.net/npm/license/@vates/disposable) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/disposable)](https://bundlephobia.com/result?p=@vates/disposable) [![Node compatibility](https://badgen.net/npm/node/@vates/disposable)](https://npmjs.org/package/@vates/disposable)
+
+> Utilities for disposables
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/disposable):
+
+```
+> npm install --save @vates/disposable
+```
+
+## Usage
+
+This library contains utilities for disposables as defined by the [`promise-toolbox` library](https://github.com/JsCommunity/promise-toolbox#resource-management).
+
+### `deduped(fn, keyFn)`
+
+Creates a new function that wraps `fn` and instead of creating a new disposables at each call, returns copies of the same one when `keyFn` returns the same keys.
+
+Those copies contains the same value and can be disposed independently, the source disposable will only be disposed when all copies are disposed.
+
+`keyFn` is called with the same context and arguments as the wrapping function and must returns an array of keys which will be used to identify which disposables should be grouped together.
+
+```js
+import { deduped } from '@vates/disposable/deduped'
+
+// the connection with the passed host will be established once at the first call, then, it will be shared with the next calls
+const getConnection = deduped(async function (host)) {
+  const connection = new Connection(host)
+  return new Disposabe(connection, () => connection.close())
+}, host => [host])
+```
+
+### `debounceResource(disposable, delay)`
+
+Creates a new disposable with the same value and with a delayed disposer.
+
+On calling this disposer, the source disposable will be disposed when the `delay` is passed.
+
+```js
+import { createDebounceResource } from '@vates/disposable/debounceResource'
+
+const debounceResource = createDebounceResource()
+
+// it will wait for 10 seconds before calling the disposer
+Disposable.use(debounceResource(getConnection(host), 10e3), connection => {})
+```
+
+### `debounceResource.flushAll()`
+
+Disposes all delayed disposers and cancels the delaying of the disposables that are in usage.
+
+```js
+import { createDebounceResource } from '@vates/disposable/debounceResource'
+
+const debounceResource = createDebounceResource()
+
+const res1 = await debounceResource(res, 10e3)
+const res2 = await debounceResource(res, 10e3)
+const res3 = await debounceResource(res, 10e3)
+
+rest1.dispose()
+rest2.dispose()
+// res3 is in usage
+
+debounceResource.flushAll()
+// res1 and res2 are immediately disposed
+// res3 will be disposed immediately when its disposer will be called
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/disposable/USAGE.md

@@ -0,0 +1,56 @@
+This library contains utilities for disposables as defined by the [`promise-toolbox` library](https://github.com/JsCommunity/promise-toolbox#resource-management).
+
+### `deduped(fn, keyFn)`
+
+Creates a new function that wraps `fn` and instead of creating a new disposables at each call, returns copies of the same one when `keyFn` returns the same keys.
+
+Those copies contains the same value and can be disposed independently, the source disposable will only be disposed when all copies are disposed.
+
+`keyFn` is called with the same context and arguments as the wrapping function and must returns an array of keys which will be used to identify which disposables should be grouped together.
+
+```js
+import { deduped } from '@vates/disposable/deduped'
+
+// the connection with the passed host will be established once at the first call, then, it will be shared with the next calls
+const getConnection = deduped(async function (host)) {
+  const connection = new Connection(host)
+  return new Disposabe(connection, () => connection.close())
+}, host => [host])
+```
+
+### `debounceResource(disposable, delay)`
+
+Creates a new disposable with the same value and with a delayed disposer.
+
+On calling this disposer, the source disposable will be disposed when the `delay` is passed.
+
+```js
+import { createDebounceResource } from '@vates/disposable/debounceResource'
+
+const debounceResource = createDebounceResource()
+
+// it will wait for 10 seconds before calling the disposer
+Disposable.use(debounceResource(getConnection(host), 10e3), connection => {})
+```
+
+### `debounceResource.flushAll()`
+
+Disposes all delayed disposers and cancels the delaying of the disposables that are in usage.
+
+```js
+import { createDebounceResource } from '@vates/disposable/debounceResource'
+
+const debounceResource = createDebounceResource()
+
+const res1 = await debounceResource(res, 10e3)
+const res2 = await debounceResource(res, 10e3)
+const res3 = await debounceResource(res, 10e3)
+
+rest1.dispose()
+rest2.dispose()
+// res3 is in usage
+
+debounceResource.flushAll()
+// res1 and res2 are immediately disposed
+// res3 will be disposed immediately when its disposer will be called
+```

@vates/disposable/debounceResource.js

@@ -0,0 +1,56 @@
+const { asyncMap } = require('@xen-orchestra/async-map')
+const { createLogger } = require('@xen-orchestra/log')
+
+const { warn } = createLogger('vates:disposable:debounceResource')
+
+exports.createDebounceResource = () => {
+  const flushers = new Set()
+  async function debounceResource(pDisposable, delay = debounceResource.defaultDelay) {
+    if (delay === 0) {
+      return pDisposable
+    }
+
+    const disposable = await pDisposable
+
+    let timeoutId
+    const disposeWrapper = async () => {
+      if (timeoutId !== undefined) {
+        clearTimeout(timeoutId)
+        timeoutId = undefined
+        flushers.delete(flusher)
+
+        try {
+          await disposable.dispose()
+        } catch (error) {
+          warn(error)
+        }
+      }
+    }
+
+    const flusher = () => {
+      const shouldDisposeNow = timeoutId !== undefined
+      if (shouldDisposeNow) {
+        return disposeWrapper()
+      } else {
+        // will dispose ASAP
+        delay = 0
+      }
+    }
+    flushers.add(flusher)
+
+    return {
+      dispose() {
+        timeoutId = setTimeout(disposeWrapper, delay)
+      },
+      value: disposable.value,
+    }
+  }
+  debounceResource.flushAll = () => {
+    // iterate on a sync way in order to not remove a flusher added on processing flushers
+    const promise = asyncMap(flushers, flush => flush())
+    flushers.clear()
+    return promise
+  }
+
+  return debounceResource
+}

@vates/disposable/debounceResource.spec.js

@@ -0,0 +1,29 @@
+/* eslint-env jest */
+
+const { createDebounceResource } = require('./debounceResource')
+
+jest.useFakeTimers()
+
+describe('debounceResource()', () => {
+  it('calls the resource disposer after 10 seconds', async () => {
+    const debounceResource = createDebounceResource()
+    const delay = 10e3
+    const dispose = jest.fn()
+
+    const resource = await debounceResource(
+      Promise.resolve({
+        value: '',
+        dispose,
+      }),
+      delay
+    )
+
+    resource.dispose()
+
+    expect(dispose).not.toBeCalled()
+
+    jest.advanceTimersByTime(delay)
+
+    expect(dispose).toBeCalled()
+  })
+})

@vates/disposable/deduped.js

@@ -0,0 +1,52 @@
+const ensureArray = require('ensure-array')
+const { MultiKeyMap } = require('@vates/multi-key-map')
+
+function State(factory) {
+  this.factory = factory
+  this.users = 0
+}
+
+const call = fn => fn()
+
+exports.deduped = (factory, keyFn = (...args) => args) =>
+  (function () {
+    const states = new MultiKeyMap()
+    return function () {
+      const keys = ensureArray(keyFn.apply(this, arguments))
+      let state = states.get(keys)
+      if (state === undefined) {
+        const result = factory.apply(this, arguments)
+
+        const createFactory = disposable => {
+          const wrapper = {
+            dispose() {
+              if (--state.users === 0) {
+                states.delete(keys)
+                return disposable.dispose()
+              }
+            },
+            value: disposable.value,
+          }
+
+          return () => {
+            return wrapper
+          }
+        }
+
+        if (typeof result.then !== 'function') {
+          state = new State(createFactory(result))
+        } else {
+          result.catch(() => {
+            states.delete(keys)
+          })
+          const pFactory = result.then(createFactory)
+          state = new State(() => pFactory.then(call))
+        }
+
+        states.set(keys, state)
+      }
+
+      ++state.users
+      return state.factory()
+    }
+  })()

@vates/disposable/deduped.spec.js

@@ -0,0 +1,76 @@
+/* eslint-env jest */
+
+const { deduped } = require('./deduped')
+
+describe('deduped()', () => {
+  it('calls the resource function only once', async () => {
+    const value = {}
+    const getResource = jest.fn(async () => ({
+      value,
+      dispose: Function.prototype,
+    }))
+
+    const dedupedGetResource = deduped(getResource)
+
+    const { value: v1 } = await dedupedGetResource()
+    const { value: v2 } = await dedupedGetResource()
+
+    expect(getResource).toHaveBeenCalledTimes(1)
+    expect(v1).toBe(value)
+    expect(v2).toBe(value)
+  })
+
+  it('only disposes the source disposable when its all copies dispose', async () => {
+    const dispose = jest.fn()
+    const getResource = async () => ({
+      value: '',
+      dispose,
+    })
+
+    const dedupedGetResource = deduped(getResource)
+
+    const { dispose: d1 } = await dedupedGetResource()
+    const { dispose: d2 } = await dedupedGetResource()
+
+    d1()
+
+    expect(dispose).not.toHaveBeenCalled()
+
+    d2()
+
+    expect(dispose).toHaveBeenCalledTimes(1)
+  })
+
+  it('works with sync factory', () => {
+    const value = {}
+    const dispose = jest.fn()
+    const dedupedGetResource = deduped(() => ({ value, dispose }))
+
+    const d1 = dedupedGetResource()
+    expect(d1.value).toBe(value)
+
+    const d2 = dedupedGetResource()
+    expect(d2.value).toBe(value)
+
+    d1.dispose()
+
+    expect(dispose).not.toHaveBeenCalled()
+
+    d2.dispose()
+
+    expect(dispose).toHaveBeenCalledTimes(1)
+  })
+
+  it('no race condition on dispose before async acquisition', async () => {
+    const dispose = jest.fn()
+    const dedupedGetResource = deduped(async () => ({ value: 42, dispose }))
+
+    const d1 = await dedupedGetResource()
+
+    dedupedGetResource()
+
+    d1.dispose()
+
+    expect(dispose).not.toHaveBeenCalled()
+  })
+})

@vates/disposable/package.json

@@ -0,0 +1,30 @@
+{
+  "private": false,
+  "name": "@vates/disposable",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/disposable",
+  "description": "Utilities for disposables",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/disposable",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "0.1.1",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  },
+  "dependencies": {
+    "@vates/multi-key-map": "^0.1.0",
+    "@xen-orchestra/async-map": "^0.1.2",
+    "@xen-orchestra/log": "^0.3.0",
+    "ensure-array": "^1.0.0"
+  }
+}

@vates/multi-key-map/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/multi-key-map/README.md

@@ -0,0 +1,53 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/multi-key-map
+
+[![Package Version](https://badgen.net/npm/v/@vates/multi-key-map)](https://npmjs.org/package/@vates/multi-key-map) ![License](https://badgen.net/npm/license/@vates/multi-key-map) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/multi-key-map)](https://bundlephobia.com/result?p=@vates/multi-key-map) [![Node compatibility](https://badgen.net/npm/node/@vates/multi-key-map)](https://npmjs.org/package/@vates/multi-key-map)
+
+> Create map with values affected to multiple keys
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/multi-key-map):
+
+```
+> npm install --save @vates/multi-key-map
+```
+
+## Usage
+
+```js
+import { MultiKeyMap } from '@vates/multi-key-map'
+
+const map = new MultiKeyMap()
+
+const OBJ = {}
+map.set([], 0)
+map.set(['foo'], 1)
+map.set(['foo', 'bar'], 2)
+map.set(['bar', 'foo'], 3)
+map.set([OBJ], 4)
+map.set([{}], 5)
+
+map.get([]) // 0
+map.get(['foo']) // 1
+map.get(['foo', 'bar']) // 2
+map.get(['bar', 'foo']) // 3
+map.get([OBJ]) // 4
+map.get([{}]) // undefined
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/multi-key-map/USAGE.md

@@ -0,0 +1,20 @@
+```js
+import { MultiKeyMap } from '@vates/multi-key-map'
+
+const map = new MultiKeyMap()
+
+const OBJ = {}
+map.set([], 0)
+map.set(['foo'], 1)
+map.set(['foo', 'bar'], 2)
+map.set(['bar', 'foo'], 3)
+map.set([OBJ], 4)
+map.set([{}], 5)
+
+map.get([]) // 0
+map.get(['foo']) // 1
+map.get(['foo', 'bar']) // 2
+map.get(['bar', 'foo']) // 3
+map.get([OBJ]) // 4
+map.get([{}]) // undefined
+```

@vates/multi-key-map/index.js

@@ -67,7 +67,7 @@ function set(node, i, keys, value) {
   return node
 }
 
-export default class MultiKeyMap {
+exports.MultiKeyMap = class MultiKeyMap {
   constructor() {
     // each node is either a value or a Node if it contains children
     this._root = undefined

@vates/multi-key-map/index.spec.js

@@ -0,0 +1,34 @@
+/* eslint-env jest */
+
+const { MultiKeyMap } = require('./')
+
+describe('MultiKeyMap', () => {
+  it('works', () => {
+    const map = new MultiKeyMap()
+
+    const keys = [
+      // null key
+      [],
+      // simple key
+      ['foo'],
+      // composite key
+      ['foo', 'bar'],
+      // reverse composite key
+      ['bar', 'foo'],
+    ]
+    const values = keys.map(() => ({}))
+
+    // set all values first to make sure they are all stored and not only the
+    // last one
+    keys.forEach((key, i) => {
+      map.set(key, values[i])
+    })
+
+    keys.forEach((key, i) => {
+      // copy the key to make sure the array itself is not the key
+      expect(map.get(key.slice())).toBe(values[i])
+      map.delete(key.slice())
+      expect(map.get(key.slice())).toBe(undefined)
+    })
+  })
+})

@vates/multi-key-map/package.json

@@ -0,0 +1,28 @@
+{
+  "private": false,
+  "name": "@vates/multi-key-map",
+  "description": "Create map with values affected to multiple keys",
+  "keywords": [
+    "cache",
+    "map"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/multi-key-map",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/multi-key-map",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "0.1.0",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@vates/parse-duration/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/parse-duration/README.md

@@ -0,0 +1,47 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/parse-duration
+
+[![Package Version](https://badgen.net/npm/v/@vates/parse-duration)](https://npmjs.org/package/@vates/parse-duration) ![License](https://badgen.net/npm/license/@vates/parse-duration) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/parse-duration)](https://bundlephobia.com/result?p=@vates/parse-duration) [![Node compatibility](https://badgen.net/npm/node/@vates/parse-duration)](https://npmjs.org/package/@vates/parse-duration)
+
+> Small wrapper around ms to parse a duration
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/parse-duration):
+
+```
+> npm install --save @vates/parse-duration
+```
+
+## Usage
+
+`ms` without magic: always parse a duration and throws if invalid.
+
+```js
+import { parseDuration } from '@vates/parse-duration'
+
+parseDuration('2 days')
+// 172800000
+
+parseDuration(172800000)
+// 172800000
+
+parseDuration(undefined)
+// throws TypeError('not a valid duration: undefined')
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/parse-duration/USAGE.md

@@ -0,0 +1,14 @@
+`ms` without magic: always parse a duration and throws if invalid.
+
+```js
+import { parseDuration } from '@vates/parse-duration'
+
+parseDuration('2 days')
+// 172800000
+
+parseDuration(172800000)
+// 172800000
+
+parseDuration(undefined)
+// throws TypeError('not a valid duration: undefined')
+```

@vates/parse-duration/index.js

@@ -1,12 +1,12 @@
-import ms from 'ms'
+const ms = require('ms')
 
-export default value => {
+exports.parseDuration = value => {
   if (typeof value === 'number') {
     return value
   }
   const duration = ms(value)
   if (duration === undefined) {
-    throw new TypeError(`not a valid duration: ${duration}`)
+    throw new TypeError(`not a valid duration: ${value}`)
   }
   return duration
 }

@vates/parse-duration/package.json

@@ -0,0 +1,32 @@
+{
+  "private": false,
+  "name": "@vates/parse-duration",
+  "description": "Small wrapper around ms to parse a duration",
+  "keywords": [
+    "duration",
+    "ms",
+    "parse"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/parse-duration",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/parse-duration",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "0.1.1",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "dependencies": {
+    "ms": "^2.1.2"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@vates/read-chunk/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/read-chunk/README.md

@@ -0,0 +1,46 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/read-chunk
+
+[![Package Version](https://badgen.net/npm/v/@vates/read-chunk)](https://npmjs.org/package/@vates/read-chunk) ![License](https://badgen.net/npm/license/@vates/read-chunk) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/read-chunk)](https://bundlephobia.com/result?p=@vates/read-chunk) [![Node compatibility](https://badgen.net/npm/node/@vates/read-chunk)](https://npmjs.org/package/@vates/read-chunk)
+
+> Read a chunk of a Node stream
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/read-chunk):
+
+```
+> npm install --save @vates/read-chunk
+```
+
+## Usage
+
+- returns the next available chunk of data
+- like `stream.read()`, a number of bytes can be specified
+- returns `null` if the stream has ended
+
+```js
+import { readChunk } from '@vates/read-chunk'
+;(async () => {
+  let chunk
+  while ((chunk = await readChunk(stream, 1024)) !== null) {
+    // do something with chunk
+  }
+})()
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/read-chunk/USAGE.md

@@ -0,0 +1,13 @@
+- returns the next available chunk of data
+- like `stream.read()`, a number of bytes can be specified
+- returns `null` if the stream has ended
+
+```js
+import { readChunk } from '@vates/read-chunk'
+;(async () => {
+  let chunk
+  while ((chunk = await readChunk(stream, 1024)) !== null) {
+    // do something with chunk
+  }
+})()
+```

@vates/read-chunk/index.js

@@ -0,0 +1,30 @@
+const readChunk = (stream, size) =>
+  size === 0
+    ? Promise.resolve(Buffer.alloc(0))
+    : new Promise((resolve, reject) => {
+        function onEnd() {
+          resolve(null)
+          removeListeners()
+        }
+        function onError(error) {
+          reject(error)
+          removeListeners()
+        }
+        function onReadable() {
+          const data = stream.read(size)
+          if (data !== null) {
+            resolve(data)
+            removeListeners()
+          }
+        }
+        function removeListeners() {
+          stream.removeListener('end', onEnd)
+          stream.removeListener('error', onError)
+          stream.removeListener('readable', onReadable)
+        }
+        stream.on('end', onEnd)
+        stream.on('error', onError)
+        stream.on('readable', onReadable)
+        onReadable()
+      })
+exports.readChunk = readChunk

@vates/read-chunk/index.spec.js

@@ -0,0 +1,43 @@
+/* eslint-env jest */
+
+const { Readable } = require('stream')
+
+const { readChunk } = require('./')
+
+const makeStream = it => Readable.from(it, { objectMode: false })
+makeStream.obj = Readable.from
+
+describe('readChunk', () => {
+  it('returns null if stream is empty', async () => {
+    expect(await readChunk(makeStream([]))).toBe(null)
+  })
+
+  describe('with binary stream', () => {
+    it('returns the first chunk of data', async () => {
+      expect(await readChunk(makeStream(['foo', 'bar']))).toEqual(Buffer.from('foo'))
+    })
+
+    it('returns a chunk of the specified size (smaller than first)', async () => {
+      expect(await readChunk(makeStream(['foo', 'bar']), 2)).toEqual(Buffer.from('fo'))
+    })
+
+    it('returns a chunk of the specified size (larger than first)', async () => {
+      expect(await readChunk(makeStream(['foo', 'bar']), 4)).toEqual(Buffer.from('foob'))
+    })
+
+    it('returns less data if stream ends', async () => {
+      expect(await readChunk(makeStream(['foo', 'bar']), 10)).toEqual(Buffer.from('foobar'))
+    })
+
+    it('returns an empty buffer if the specified size is 0', async () => {
+      expect(await readChunk(makeStream(['foo', 'bar']), 0)).toEqual(Buffer.alloc(0))
+    })
+  })
+
+  describe('with object stream', () => {
+    it('returns the first chunk of data verbatim', async () => {
+      const chunks = [{}, {}]
+      expect(await readChunk(makeStream.obj(chunks))).toBe(chunks[0])
+    })
+  })
+})

@vates/read-chunk/package.json

@@ -0,0 +1,33 @@
+{
+  "private": false,
+  "name": "@vates/read-chunk",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/read-chunk",
+  "description": "Read a chunk of a Node stream",
+  "license": "ISC",
+  "keywords": [
+    "async",
+    "chunk",
+    "data",
+    "node",
+    "promise",
+    "read",
+    "stream"
+  ],
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/read-chunk",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "version": "0.1.2",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  }
+}

@vates/toggle-scripts/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/toggle-scripts/README.md

@@ -0,0 +1,59 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/toggle-scripts
+
+[![Package Version](https://badgen.net/npm/v/@vates/toggle-scripts)](https://npmjs.org/package/@vates/toggle-scripts) ![License](https://badgen.net/npm/license/@vates/toggle-scripts) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/toggle-scripts)](https://bundlephobia.com/result?p=@vates/toggle-scripts) [![Node compatibility](https://badgen.net/npm/node/@vates/toggle-scripts)](https://npmjs.org/package/@vates/toggle-scripts)
+
+> Easily enable/disable scripts in package.json
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/toggle-scripts):
+
+```
+> npm install --save @vates/toggle-scripts
+```
+
+## Usage
+
+```
+Usage: toggle-scripts options...
+
+  Easily enable/disable scripts in package.json
+
+  Options
+    +<script>    Enable the script <script>, ie remove the prefix `_`
+    -<script>    Disable the script <script>, ie prefix it with `_`
+
+  Examples
+    toggle-scripts +postinstall +preuninstall
+    toggle-scripts -postinstall -preuninstall
+```
+
+For example, if you want `postinstall` hook only in dev:
+
+```json
+// package.json
+{
+  "scripts": {
+    "postinstall": "<some dev only command>",
+    "prepublishOnly": "toggle-scripts -postinstall",
+    "postpublish": "toggle-scripts +postinstall"
+  }
+}
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/toggle-scripts/USAGE.md

@@ -0,0 +1,26 @@
+```
+Usage: toggle-scripts options...
+
+  Easily enable/disable scripts in package.json
+
+  Options
+    +<script>    Enable the script <script>, ie remove the prefix `_`
+    -<script>    Disable the script <script>, ie prefix it with `_`
+
+  Examples
+    toggle-scripts +postinstall +preuninstall
+    toggle-scripts -postinstall -preuninstall
+```
+
+For example, if you want `postinstall` hook only in dev:
+
+```json
+// package.json
+{
+  "scripts": {
+    "postinstall": "<some dev only command>",
+    "prepublishOnly": "toggle-scripts -postinstall",
+    "postpublish": "toggle-scripts +postinstall"
+  }
+}
+```

@vates/toggle-scripts/index.js

@@ -0,0 +1,60 @@
+#!/usr/bin/env node
+
+const fs = require('fs')
+
+const mapKeys = (object, iteratee) => {
+  const result = {}
+  for (const key of Object.keys(object)) {
+    result[iteratee(key, object)] = object[key]
+  }
+  return result
+}
+
+const args = process.argv.slice(2)
+if (args.length === 0) {
+  const { description, name, version } = require('./package.json')
+  const bin = 'toggle-scripts'
+  process.stdout.write(`Usage: ${bin} options...
+
+  ${description}
+
+  Options
+    +<script>    Enable the script <script>, ie remove the prefix \`_\`
+    -<script>    Disable the script <script>, ie prefix it with \`_\`
+
+  Examples
+    ${bin} +postinstall +preuninstall
+    ${bin} -postinstall -preuninstall
+
+${name} v${version}
+`)
+  process.exit()
+}
+
+const plan = { __proto__: null }
+for (const arg of args) {
+  const action = arg[0]
+  const script = arg.slice(1)
+
+  if (action === '+') {
+    plan['_' + script] = script
+  } else if (action === '-') {
+    plan[script] = '_' + script
+  } else {
+    throw new Error('invalid param: ' + arg)
+  }
+}
+
+const pkgPath = process.env.npm_package_json || './package.json'
+const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'))
+pkg.scripts = mapKeys(pkg.scripts, (name, scripts) => {
+  const newName = plan[name]
+  if (newName === undefined) {
+    return name
+  }
+  if (newName in scripts) {
+    throw new Error('script already defined: ' + name)
+  }
+  return newName
+})
+fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + '\n')

@vates/toggle-scripts/package.json

@@ -0,0 +1,38 @@
+{
+  "private": false,
+  "name": "@vates/toggle-scripts",
+  "description": "Easily enable/disable scripts in package.json",
+  "keywords": [
+    "dev",
+    "disable",
+    "enable",
+    "lifecycle",
+    "npm",
+    "package.json",
+    "pinst",
+    "postinstall",
+    "script",
+    "scripts",
+    "toggle"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/toggle-scripts",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/toggle-scripts",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "1.0.0",
+  "engines": {
+    "node": ">=6"
+  },
+  "bin": "./index.js",
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@xen-orchestra/async-map/.babelrc.js

@@ -1,3 +0,0 @@
-module.exports = require('../../@xen-orchestra/babel-config')(
-  require('./package.json')
-)

@xen-orchestra/async-map/.npmignore

@@ -1,24 +0,0 @@
-/benchmark/
-/benchmarks/
-*.bench.js
-*.bench.js.map
-
-/examples/
-example.js
-example.js.map
-*.example.js
-*.example.js.map
-
-/fixture/
-/fixtures/
-*.fixture.js
-*.fixture.js.map
-*.fixtures.js
-*.fixtures.js.map
-
-/test/
-/tests/
-*.spec.js
-*.spec.js.map
-
-__snapshots__/

@xen-orchestra/async-map/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@xen-orchestra/async-map/README.md

@@ -1,6 +1,10 @@
-# @xen-orchestra/async-map [![Build Status](https://travis-ci.org/vatesfr/xen-orchestra.png?branch=master)](https://travis-ci.org/vatesfr/xen-orchestra)
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
 
-> ${pkg.description}
+# @xen-orchestra/async-map
+
+[![Package Version](https://badgen.net/npm/v/@xen-orchestra/async-map)](https://npmjs.org/package/@xen-orchestra/async-map) ![License](https://badgen.net/npm/license/@xen-orchestra/async-map) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@xen-orchestra/async-map)](https://bundlephobia.com/result?p=@xen-orchestra/async-map) [![Node compatibility](https://badgen.net/npm/node/@xen-orchestra/async-map)](https://npmjs.org/package/@xen-orchestra/async-map)
+
+> Promise.all + map for all iterables
 
 ## Install
 
@@ -12,30 +16,66 @@ Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/async
 
 ## Usage
 
-**TODO**
+### `asyncMap(iterable, iteratee, thisArg = iterable)`
 
-## Development
+Similar to `Promise.all + Array#map` for all iterables: calls `iteratee` for each item in `iterable`, and returns a promise of an array containing the awaited result of each calls to `iteratee`.
 
+It rejects as soon as te first call to `iteratee` rejects.
+
+```js
+import { asyncMap } from '@xen-orchestra/async-map'
+
+const array = await asyncMap(iterable, iteratee, thisArg)
 ```
-# Install dependencies
-> yarn
 
-# Run the tests
+It can be used with any iterables (`Array`, `Map`, etc.):
-> yarn test
 
-# Continuously compile
+```js
-> yarn dev
+const map = new Map()
+map.set('foo', 42)
+map.set('bar', 3.14)
 
-# Continuously run the tests
+const array = await asyncMap(map, async function ([key, value]) {
-> yarn dev-test
+  // TODO: do async computation
+  //
+  // the map can be accessed via `this`
+})
+```
 
-# Build for production (automatically called by npm install)
+#### Use with plain objects
-> yarn build
+
+Plain objects are not iterable, but you can use `Object.keys`, `Object.values` or `Object.entries` to help:
+
+```js
+const object = {
+  foo: 42,
+  bar: 3.14,
+}
+
+const array = await asyncMap(
+  Object.entries(object),
+  async function ([key, value]) {
+    // TODO: do async computation
+    //
+    // the object can be accessed via `this` because it's been passed as third arg
+  },
+  object
+)
+```
+
+### `asyncMapSettled(iterable, iteratee, thisArg = iterable)`
+
+Similar to `asyncMap` but waits for all promises to settle before rejecting.
+
+```js
+import { asyncMapSettled } from '@xen-orchestra/async-map'
+
+const array = await asyncMapSettled(iterable, iteratee, thisArg)
 ```
 
 ## Contributions
 
-Contributions are *very* welcomed, either on the documentation or on
+Contributions are _very_ welcomed, either on the documentation or on
 the code.
 
 You may:
@@ -46,4 +86,4 @@ You may:
 
 ## License
 
-ISC © [Vates SAS](https://vates.fr)
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@xen-orchestra/async-map/USAGE.md

@@ -0,0 +1,56 @@
+### `asyncMap(iterable, iteratee, thisArg = iterable)`
+
+Similar to `Promise.all + Array#map` for all iterables: calls `iteratee` for each item in `iterable`, and returns a promise of an array containing the awaited result of each calls to `iteratee`.
+
+It rejects as soon as te first call to `iteratee` rejects.
+
+```js
+import { asyncMap } from '@xen-orchestra/async-map'
+
+const array = await asyncMap(iterable, iteratee, thisArg)
+```
+
+It can be used with any iterables (`Array`, `Map`, etc.):
+
+```js
+const map = new Map()
+map.set('foo', 42)
+map.set('bar', 3.14)
+
+const array = await asyncMap(map, async function ([key, value]) {
+  // TODO: do async computation
+  //
+  // the map can be accessed via `this`
+})
+```
+
+#### Use with plain objects
+
+Plain objects are not iterable, but you can use `Object.keys`, `Object.values` or `Object.entries` to help:
+
+```js
+const object = {
+  foo: 42,
+  bar: 3.14,
+}
+
+const array = await asyncMap(
+  Object.entries(object),
+  async function ([key, value]) {
+    // TODO: do async computation
+    //
+    // the object can be accessed via `this` because it's been passed as third arg
+  },
+  object
+)
+```
+
+### `asyncMapSettled(iterable, iteratee, thisArg = iterable)`
+
+Similar to `asyncMap` but waits for all promises to settle before rejecting.
+
+```js
+import { asyncMapSettled } from '@xen-orchestra/async-map'
+
+const array = await asyncMapSettled(iterable, iteratee, thisArg)
+```

@xen-orchestra/async-map/index.js

@@ -0,0 +1,71 @@
+const wrapCall = (fn, arg, thisArg) => {
+  try {
+    return Promise.resolve(fn.call(thisArg, arg))
+  } catch (error) {
+    return Promise.reject(error)
+  }
+}
+
+/**
+ * Similar to Promise.all + Array#map but supports all iterables and does not trigger ESLint array-callback-return
+ *
+ * WARNING: Does not handle plain objects
+ *
+ * @template Item,This
+ * @param {Iterable<Item>} iterable
+ * @param {(this: This, item: Item) => (Item | PromiseLike<Item>)} mapFn
+ * @param {This} [thisArg]
+ * @returns {Promise<Item[]>}
+ */
+exports.asyncMap = function asyncMap(iterable, mapFn, thisArg = iterable) {
+  return Promise.all(Array.from(iterable, mapFn, thisArg))
+}
+
+/**
+ * Like `asyncMap` but wait for all promises to settle before rejecting
+ *
+ * @template Item,This
+ * @param {Iterable<Item>} iterable
+ * @param {(this: This, item: Item) => (Item | PromiseLike<Item>)} mapFn
+ * @param {This} [thisArg]
+ * @returns {Promise<Item[]>}
+ */
+exports.asyncMapSettled = function asyncMapSettled(iterable, mapFn, thisArg = iterable) {
+  return new Promise((resolve, reject) => {
+    const onError = e => {
+      if (result !== undefined) {
+        error = e
+        result = undefined
+      }
+      if (--n === 0) {
+        reject(error)
+      }
+    }
+    const onValue = (i, value) => {
+      const hasError = result === undefined
+      if (!hasError) {
+        result[i] = value
+      }
+      if (--n === 0) {
+        if (hasError) {
+          reject(error)
+        } else {
+          resolve(result)
+        }
+      }
+    }
+
+    let n = 0
+    for (const item of iterable) {
+      const i = n++
+      wrapCall(mapFn, item, thisArg).then(value => onValue(i, value), onError)
+    }
+
+    if (n === 0) {
+      return resolve([])
+    }
+
+    let error
+    let result = new Array(n)
+  })
+}

@xen-orchestra/async-map/index.spec.js

@@ -0,0 +1,71 @@
+/* eslint-env jest */
+
+const { asyncMapSettled } = require('./')
+
+const noop = Function.prototype
+
+describe('asyncMapSettled', () => {
+  it('works', async () => {
+    const values = [Math.random(), Math.random()]
+    const spy = jest.fn(async v => v * 2)
+    const iterable = new Set(values)
+
+    // returns an array containing the result of each calls
+    expect(await asyncMapSettled(iterable, spy)).toEqual(values.map(value => value * 2))
+
+    for (let i = 0, n = values.length; i < n; ++i) {
+      // each call receive the current item as sole argument
+      expect(spy.mock.calls[i]).toEqual([values[i]])
+
+      // each call as this bind to the iterable
+      expect(spy.mock.instances[i]).toBe(iterable)
+    }
+  })
+
+  it('can use a specified thisArg', () => {
+    const thisArg = {}
+    const spy = jest.fn()
+    asyncMapSettled(['foo'], spy, thisArg)
+    expect(spy.mock.instances[0]).toBe(thisArg)
+  })
+
+  it('rejects only when all calls as resolved', async () => {
+    const defers = []
+    const promise = asyncMapSettled([1, 2], () => {
+      let resolve, reject
+      // eslint-disable-next-line promise/param-names
+      const promise = new Promise((_resolve, _reject) => {
+        resolve = _resolve
+        reject = _reject
+      })
+      defers.push({ promise, resolve, reject })
+      return promise
+    })
+
+    let hasSettled = false
+    promise.catch(noop).then(() => {
+      hasSettled = true
+    })
+
+    const error = new Error()
+    defers[0].reject(error)
+
+    // wait for all microtasks to settle
+    await new Promise(resolve => setImmediate(resolve))
+
+    expect(hasSettled).toBe(false)
+
+    defers[1].resolve()
+
+    // wait for all microtasks to settle
+    await new Promise(resolve => setImmediate(resolve))
+
+    expect(hasSettled).toBe(true)
+    await expect(promise).rejects.toBe(error)
+  })
+
+  it('issues when latest promise rejects', async () => {
+    const error = new Error()
+    await expect(asyncMapSettled([1], () => Promise.reject(error))).rejects.toBe(error)
+  })
+})

@xen-orchestra/async-map/legacy.js

@@ -9,14 +9,18 @@
 //   (V1, K) => MaybePromise<V2>
 // ): Promise<V2[]>
 
-import map from 'lodash/map'
+const map = require('lodash/map')
 
-// Similar to map() + Promise.all() but wait for all promises to
+/**
-// settle before rejecting (with the first error)
+ * Similar to map() + Promise.all() but wait for all promises to settle before
-const asyncMap = (collection, iteratee) => {
+ * rejecting (with the first error)
+ *
+ * @deprecated Don't support iterables, please use new implementations
+ */
+module.exports = function asyncMapLegacy(collection, iteratee) {
   let then
   if (collection != null && typeof (then = collection.then) === 'function') {
-    return then.call(collection, collection => asyncMap(collection, iteratee))
+    return then.call(collection, collection => asyncMapLegacy(collection, iteratee))
   }
 
   let errorContainer
@@ -39,5 +43,3 @@ const asyncMap = (collection, iteratee) => {
     return values
   })
 }
-
-export { asyncMap as default }

@xen-orchestra/async-map/package.json

@@ -1,9 +1,17 @@
 {
+  "private": false,
   "name": "@xen-orchestra/async-map",
-  "version": "0.0.0",
+  "version": "0.1.2",
   "license": "ISC",
-  "description": "",
+  "description": "Promise.all + map for all iterables",
-  "keywords": [],
+  "keywords": [
+    "array",
+    "async",
+    "iterable",
+    "map",
+    "settled",
+    "typescript"
+  ],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/async-map",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
@@ -12,41 +20,17 @@
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
   "author": {
-    "name": "Julien Fontanet",
+    "name": "Vates SAS",
-    "email": "julien.fontanet@isonoe.net"
+    "url": "https://vates.fr"
   },
   "preferGlobal": false,
-  "main": "dist/",
-  "bin": {},
-  "files": [
-    "dist/"
-  ],
-  "browserslist": [
-    ">2%"
-  ],
   "engines": {
     "node": ">=6"
   },
   "dependencies": {
     "lodash": "^4.17.4"
   },
-  "devDependencies": {
-    "@babel/cli": "^7.0.0",
-    "@babel/core": "^7.0.0",
-    "@babel/preset-env": "^7.0.0",
-    "@babel/preset-flow": "^7.0.0",
-    "babel-plugin-lodash": "^3.3.2",
-    "cross-env": "^5.1.3",
-    "rimraf": "^2.6.2"
-  },
   "scripts": {
-    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
-    "clean": "rimraf dist/",
-    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
-    "prebuild": "yarn run clean",
-    "predev": "yarn run prebuild",
-    "prepare": "yarn run build",
-    "prepublishOnly": "yarn run build",
     "postversion": "npm publish"
   }
 }

@xen-orchestra/audit-core/.babelrc.js

@@ -0,0 +1 @@
+module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))

@xen-orchestra/audit-core/.eslintrc.js

@@ -0,0 +1 @@
+../../scripts/babel-eslintrc.js

@xen-orchestra/audit-core/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@xen-orchestra/audit-core/README.md

@@ -0,0 +1,28 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @xen-orchestra/audit-core
+
+[![Package Version](https://badgen.net/npm/v/@xen-orchestra/audit-core)](https://npmjs.org/package/@xen-orchestra/audit-core) ![License](https://badgen.net/npm/license/@xen-orchestra/audit-core) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@xen-orchestra/audit-core)](https://bundlephobia.com/result?p=@xen-orchestra/audit-core) [![Node compatibility](https://badgen.net/npm/node/@xen-orchestra/audit-core)](https://npmjs.org/package/@xen-orchestra/audit-core)
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/audit-core):
+
+```
+> npm install --save @xen-orchestra/audit-core
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[AGPL-3.0-or-later](https://spdx.org/licenses/AGPL-3.0-or-later) © [Vates SAS](https://vates.fr)

@xen-orchestra/audit-core/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@xen-orchestra/audit-core",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/audit-core",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@xen-orchestra/audit-core",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "version": "0.2.0",
+  "engines": {
+    "node": ">=10"
+  },
+  "main": "dist/",
+  "scripts": {
+    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
+    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
+    "postversion": "npm publish --access public",
+    "prebuild": "rimraf dist/",
+    "predev": "yarn run prebuild",
+    "prepublishOnly": "yarn run build"
+  },
+  "devDependencies": {
+    "@babel/cli": "^7.7.4",
+    "@babel/core": "^7.7.4",
+    "@babel/plugin-proposal-decorators": "^7.8.0",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.8.0",
+    "@babel/preset-env": "^7.7.4",
+    "cross-env": "^7.0.2",
+    "rimraf": "^3.0.0"
+  },
+  "dependencies": {
+    "@vates/decorate-with": "^0.1.0",
+    "@xen-orchestra/log": "^0.3.0",
+    "golike-defer": "^0.5.1",
+    "object-hash": "^2.0.1"
+  },
+  "private": false,
+  "license": "AGPL-3.0-or-later",
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  }
+}

@xen-orchestra/audit-core/src/index.js

@@ -0,0 +1,191 @@
+import assert from 'assert'
+import hash from 'object-hash'
+import { createLogger } from '@xen-orchestra/log'
+import { decorateWith } from '@vates/decorate-with'
+import { defer } from 'golike-defer'
+
+const log = createLogger('xo:audit-core')
+
+export class Storage {
+  constructor() {
+    this._lock = Promise.resolve()
+  }
+
+  async acquireLock() {
+    const lock = this._lock
+    let releaseLock
+    this._lock = new Promise(resolve => {
+      releaseLock = resolve
+    })
+    await lock
+    return releaseLock
+  }
+}
+
+// Format: $<algorithm>$<salt>$<encrypted>
+//
+// http://man7.org/linux/man-pages/man3/crypt.3.html#NOTES
+const ID_TO_ALGORITHM = {
+  5: 'sha256',
+}
+
+export class AlteredRecordError extends Error {
+  constructor(id, nValid, record) {
+    super('altered record')
+
+    this.id = id
+    this.nValid = nValid
+    this.record = record
+  }
+}
+
+export class MissingRecordError extends Error {
+  constructor(id, nValid) {
+    super('missing record')
+
+    this.id = id
+    this.nValid = nValid
+  }
+}
+
+export const NULL_ID = 'nullId'
+
+const HASH_ALGORITHM_ID = '5'
+const createHash = (data, algorithmId = HASH_ALGORITHM_ID) =>
+  `$${algorithmId}$$${hash(data, {
+    algorithm: ID_TO_ALGORITHM[algorithmId],
+    excludeKeys: key => key === 'id',
+  })}`
+
+export class AuditCore {
+  constructor(storage) {
+    assert.notStrictEqual(storage, undefined)
+    this._storage = storage
+  }
+
+  @decorateWith(defer)
+  async add($defer, subject, event, data) {
+    const time = Date.now()
+    $defer(await this._storage.acquireLock())
+    return this._addUnsafe({
+      data,
+      event,
+      subject,
+      time,
+    })
+  }
+
+  async _addUnsafe({ data, event, subject, time }) {
+    const storage = this._storage
+
+    // delete "undefined" properties and normalize data with JSON.stringify
+    const record = JSON.parse(
+      JSON.stringify({
+        data,
+        event,
+        previousId: (await storage.getLastId()) ?? NULL_ID,
+        subject,
+        time,
+      })
+    )
+    record.id = createHash(record)
+    await storage.put(record)
+    await storage.setLastId(record.id)
+    return record
+  }
+
+  async checkIntegrity(oldest, newest) {
+    const storage = this._storage
+
+    // handle separated chains case
+    if (newest !== (await storage.getLastId())) {
+      let isNewestAccessible = false
+      for await (const { id } of this.getFrom()) {
+        if (id === newest) {
+          isNewestAccessible = true
+          break
+        }
+      }
+      if (!isNewestAccessible) {
+        throw new MissingRecordError(newest, 0)
+      }
+    }
+
+    let nValid = 0
+    while (newest !== oldest) {
+      const record = await storage.get(newest)
+      if (record === undefined) {
+        throw new MissingRecordError(newest, nValid)
+      }
+      if (newest !== createHash(record, newest.slice(1, newest.indexOf('$', 1)))) {
+        throw new AlteredRecordError(newest, nValid, record)
+      }
+      newest = record.previousId
+      nValid++
+    }
+    return nValid
+  }
+
+  async *getFrom(newest) {
+    const storage = this._storage
+
+    let id = newest ?? (await storage.getLastId())
+    if (id === undefined) {
+      return
+    }
+
+    let record
+    while ((record = await storage.get(id)) !== undefined) {
+      yield record
+      id = record.previousId
+    }
+  }
+
+  async deleteFrom(newest) {
+    assert.notStrictEqual(newest, undefined)
+    for await (const { id } of this.getFrom(newest)) {
+      await this._storage.del(id)
+    }
+  }
+
+  @decorateWith(defer)
+  async deleteRangeAndRewrite($defer, newest, oldest) {
+    assert.notStrictEqual(newest, undefined)
+    assert.notStrictEqual(oldest, undefined)
+
+    const storage = this._storage
+    $defer(await storage.acquireLock())
+
+    assert.notStrictEqual(await storage.get(newest), undefined)
+    const oldestRecord = await storage.get(oldest)
+    assert.notStrictEqual(oldestRecord, undefined)
+
+    const lastId = await storage.getLastId()
+    const recentRecords = []
+    for await (const record of this.getFrom(lastId)) {
+      if (record.id === newest) {
+        break
+      }
+
+      recentRecords.push(record)
+    }
+
+    for await (const record of this.getFrom(newest)) {
+      await storage.del(record.id)
+      if (record.id === oldest) {
+        break
+      }
+    }
+
+    await storage.setLastId(oldestRecord.previousId)
+
+    for (const record of recentRecords) {
+      try {
+        await this._addUnsafe(record)
+        await storage.del(record.id)
+      } catch (error) {
+        log.error(error)
+      }
+    }
+  }
+}

@xen-orchestra/audit-core/src/index.spec.js

@@ -0,0 +1,115 @@
+/* eslint-env jest */
+
+import { AlteredRecordError, AuditCore, MissingRecordError, NULL_ID, Storage } from '.'
+
+const asyncIteratorToArray = async asyncIterator => {
+  const array = []
+  for await (const entry of asyncIterator) {
+    array.push(entry)
+  }
+  return array
+}
+
+class DB extends Storage {
+  constructor() {
+    super()
+
+    this._db = new Map()
+    this._lastId = undefined
+  }
+
+  async put(record) {
+    this._db.set(record.id, record)
+  }
+
+  async setLastId(id) {
+    this._lastId = id
+  }
+
+  async getLastId() {
+    return this._lastId
+  }
+
+  async del(id) {
+    this._db.delete(id)
+  }
+
+  async get(id) {
+    return this._db.get(id)
+  }
+
+  _clear() {
+    return this._db.clear()
+  }
+}
+
+const DATA = [
+  [
+    {
+      name: 'subject0',
+    },
+    'event0',
+    {},
+  ],
+  [
+    {
+      name: 'subject1',
+    },
+    'event1',
+    {},
+  ],
+  [
+    {
+      name: 'subject2',
+    },
+    'event2',
+    {},
+  ],
+]
+
+const db = new DB()
+const auditCore = new AuditCore(db)
+const storeAuditRecords = async () => {
+  await Promise.all(DATA.map(data => auditCore.add(...data)))
+  const records = await asyncIteratorToArray(auditCore.getFrom())
+  expect(records.length).toBe(DATA.length)
+  return records
+}
+
+describe('auditCore', () => {
+  afterEach(() => db._clear())
+
+  it('detects that a record is missing', async () => {
+    const [newestRecord, deletedRecord] = await storeAuditRecords()
+
+    const nValidRecords = await auditCore.checkIntegrity(NULL_ID, newestRecord.id)
+    expect(nValidRecords).toBe(DATA.length)
+
+    await db.del(deletedRecord.id)
+    await expect(auditCore.checkIntegrity(NULL_ID, newestRecord.id)).rejects.toEqual(
+      new MissingRecordError(deletedRecord.id, 1)
+    )
+  })
+
+  it('detects that a record has been altered', async () => {
+    const [newestRecord, alteredRecord] = await storeAuditRecords()
+
+    alteredRecord.event = ''
+    await db.put(alteredRecord)
+
+    await expect(auditCore.checkIntegrity(NULL_ID, newestRecord.id)).rejects.toEqual(
+      new AlteredRecordError(alteredRecord.id, 1, alteredRecord)
+    )
+  })
+
+  it('confirms interval integrity after deletion of records outside of the interval', async () => {
+    const [thirdRecord, secondRecord, firstRecord] = await storeAuditRecords()
+
+    await auditCore.deleteFrom(secondRecord.id)
+
+    expect(await db.get(firstRecord.id)).toBe(undefined)
+    expect(await db.get(secondRecord.id)).toBe(undefined)
+
+    await auditCore.checkIntegrity(secondRecord.id, thirdRecord.id)
+  })
+})

@xen-orchestra/audit-core/src/specification.ts

@@ -0,0 +1,26 @@
+class Storage {
+  acquire: () => Promise<() => undefined>
+  del: (id: string) => Promise<void>
+  get: (id: string) => Promise<Record | void>
+  getLastId: () => Promise<string | void>
+  put: (record: Record) => Promise<void>
+  setLastId: (id: string) => Promise<void>
+}
+
+interface Record {
+  data: object
+  event: string
+  id: string
+  previousId: string
+  subject: object
+  time: number
+}
+
+export class AuditCore {
+  constructor(storage: Storage) {}
+  public add(subject: any, event: string, data: any): Promise<Record> {}
+  public checkIntegrity(oldest: string, newest: string): Promise<number> {}
+  public getFrom(newest?: string): AsyncIterator {}
+  public deleteFrom(newest: string): Promise<void> {}
+  public deleteRangeAndRewrite(newest: string, oldest: string): Promise<void> {}
+}

@xen-orchestra/babel-config/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@xen-orchestra/babel-config/README.md

@@ -0,0 +1,18 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @xen-orchestra/babel-config
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[AGPL-3.0-or-later](https://spdx.org/licenses/AGPL-3.0-or-later) © [Vates SAS](https://vates.fr)

@xen-orchestra/babel-config/index.js

@@ -14,59 +14,39 @@ const configs = {
   '@babel/plugin-proposal-pipeline-operator': {
     proposal: 'minimal',
   },
-  '@babel/preset-env'(pkg) {
+  '@babel/preset-env': {
-    return {
+    debug: !__TEST__,
-      debug: !__TEST__,
 
-      // disabled until https://github.com/babel/babel/issues/8323 is resolved
+    // disabled until https://github.com/babel/babel/issues/8323 is resolved
-      // loose: true,
+    // loose: true,
 
-      shippedProposals: true,
+    shippedProposals: true,
-      targets: (() => {
-        let node = (pkg.engines || {}).node
-        if (node !== undefined) {
-          const trimChars = '^=>~'
-          while (trimChars.includes(node[0])) {
-            node = node.slice(1)
-          }
-        }
-        return { browsers: pkg.browserslist, node }
-      })(),
-      useBuiltIns: '@babel/polyfill' in (pkg.dependencies || {}) && 'usage',
-    }
   },
 }
 
 const getConfig = (key, ...args) => {
   const config = configs[key]
-  return config === undefined
+  return config === undefined ? {} : typeof config === 'function' ? config(...args) : config
-    ? {}
-    : typeof config === 'function'
-    ? config(...args)
-    : config
 }
 
 // some plugins must be used in a specific order
-const pluginsOrder = [
+const pluginsOrder = ['@babel/plugin-proposal-decorators', '@babel/plugin-proposal-class-properties']
-  '@babel/plugin-proposal-decorators',
-  '@babel/plugin-proposal-class-properties',
-]
 
-module.exports = function(pkg, plugins, presets) {
+module.exports = function (pkg, configs = {}) {
-  plugins === undefined && (plugins = {})
+  const plugins = {}
-  presets === undefined && (presets = {})
+  const presets = {}
 
   Object.keys(pkg.devDependencies || {}).forEach(name => {
     if (!(name in presets) && PLUGINS_RE.test(name)) {
-      plugins[name] = getConfig(name, pkg)
+      plugins[name] = { ...getConfig(name, pkg), ...configs[name] }
     } else if (!(name in presets) && PRESETS_RE.test(name)) {
-      presets[name] = getConfig(name, pkg)
+      presets[name] = { ...getConfig(name, pkg), ...configs[name] }
     }
   })
 
   return {
     comments: !__PROD__,
-    ignore: __TEST__ ? undefined : [/\.spec\.js$/],
+    ignore: __PROD__ ? [/\.spec\.js$/] : undefined,
     plugins: Object.keys(plugins)
       .map(plugin => [plugin, plugins[plugin]])
       .sort(([a], [b]) => {
@@ -75,5 +55,23 @@ module.exports = function(pkg, plugins, presets) {
         return oA !== -1 && oB !== -1 ? oA - oB : a < b ? -1 : 1
       }),
     presets: Object.keys(presets).map(preset => [preset, presets[preset]]),
+    targets: (() => {
+      const targets = {}
+
+      if (pkg.browserslist !== undefined) {
+        targets.browsers = pkg.browserslist
+      }
+
+      let node = (pkg.engines || {}).node
+      if (node !== undefined) {
+        const trimChars = '^=>~'
+        while (trimChars.includes(node[0])) {
+          node = node.slice(1)
+        }
+        targets.node = node
+      }
+
+      return targets
+    })(),
   }
 }

@xen-orchestra/babel-config/package.json

@@ -8,5 +8,13 @@
     "directory": "@xen-orchestra/babel-config",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "engines": {
+    "node": ">=6"
+  },
+  "license": "AGPL-3.0-or-later",
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
   }
 }

@xen-orchestra/backups-cli/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@xen-orchestra/backups-cli/README.md

@@ -0,0 +1,48 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @xen-orchestra/backups-cli
+
+[![Package Version](https://badgen.net/npm/v/@xen-orchestra/backups-cli)](https://npmjs.org/package/@xen-orchestra/backups-cli) ![License](https://badgen.net/npm/license/@xen-orchestra/backups-cli) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@xen-orchestra/backups-cli)](https://bundlephobia.com/result?p=@xen-orchestra/backups-cli) [![Node compatibility](https://badgen.net/npm/node/@xen-orchestra/backups-cli)](https://npmjs.org/package/@xen-orchestra/backups-cli)
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/backups-cli):
+
+```
+> npm install --global @xen-orchestra/backups-cli
+```
+
+## Usage
+
+```
+> xo-backups --help
+Usage:
+
+    xo-backups clean-vms [--merge] [--remove] xo-vm-backups/*
+
+      Detects and repair issues with VM backups.
+
+      Options:
+        -m, --merge     Merge (or continue merging) VHD files that are unused
+        -r, --remove    Remove unused, incomplete, orphan, or corrupted files
+
+
+    xo-backups create-symlink-index xo-vm-backups <field path>
+
+    xo-backups info xo-vm-backups/*
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[AGPL-3.0-or-later](https://spdx.org/licenses/AGPL-3.0-or-later) © [Vates SAS](https://vates.fr)

@xen-orchestra/backups-cli/USAGE.md

@@ -0,0 +1,17 @@
+```
+> xo-backups --help
+Usage:
+
+    xo-backups clean-vms [--merge] [--remove] xo-vm-backups/*
+
+      Detects and repair issues with VM backups.
+
+      Options:
+        -m, --merge     Merge (or continue merging) VHD files that are unused
+        -r, --remove    Remove unused, incomplete, orphan, or corrupted files
+
+
+    xo-backups create-symlink-index xo-vm-backups <field path>
+
+    xo-backups info xo-vm-backups/*
+```

@xen-orchestra/backups-cli/_composeCommands.js

@@ -0,0 +1,32 @@
+const getopts = require('getopts')
+
+const { version } = require('./package.json')
+
+module.exports = commands =>
+  async function (args, prefix) {
+    const opts = getopts(args, {
+      alias: {
+        help: 'h',
+      },
+      boolean: ['help'],
+      stopEarly: true,
+    })
+
+    const commandName = opts.help || args.length === 0 ? 'help' : args[0]
+    const command = commands[commandName]
+    if (command === undefined) {
+      process.stdout.write(`Usage:
+
+${Object.keys(commands)
+  .filter(command => command !== 'help')
+  .map(command => `    ${prefix} ${command} ${commands[command].usage || ''}`)
+  .join('\n\n')}
+
+xo-backups v${version}
+`)
+      process.exitCode = commandName === 'help' ? 0 : 1
+      return
+    }
+
+    return command.main(args.slice(1), prefix + ' ' + commandName)
+  }

@xen-orchestra/backups-cli/_fs.js

@@ -0,0 +1,69 @@
+const { dirname } = require('path')
+
+const fs = require('promise-toolbox/promisifyAll')(require('fs'))
+module.exports = fs
+
+fs.getSize = path =>
+  fs.stat(path).then(
+    _ => _.size,
+    error => {
+      if (error.code === 'ENOENT') {
+        return 0
+      }
+      throw error
+    }
+  )
+
+fs.mktree = async function mkdirp(path) {
+  try {
+    await fs.mkdir(path)
+  } catch (error) {
+    const { code } = error
+    if (code === 'EEXIST') {
+      await fs.readdir(path)
+      return
+    }
+    if (code === 'ENOENT') {
+      await mkdirp(dirname(path))
+      return mkdirp(path)
+    }
+    throw error
+  }
+}
+
+// - easier:
+//   - single param for direct use in `Array#map`
+//   - files are prefixed with directory path
+// - safer: returns empty array if path is missing or not a directory
+fs.readdir2 = path =>
+  fs.readdir(path).then(
+    entries => {
+      entries.forEach((entry, i) => {
+        entries[i] = `${path}/${entry}`
+      })
+
+      return entries
+    },
+    error => {
+      const { code } = error
+      if (code === 'ENOENT') {
+        // do nothing
+      } else if (code === 'ENOTDIR') {
+        console.warn('WARN: readdir(%s)', path, error)
+      } else {
+        throw error
+      }
+      return []
+    }
+  )
+
+fs.symlink2 = async (target, path) => {
+  try {
+    await fs.symlink(target, path)
+  } catch (error) {
+    if (error.code === 'EEXIST' && (await fs.readlink(path)) === target) {
+      return
+    }
+    throw error
+  }
+}

@xen-orchestra/backups-cli/commands/clean-vms.js

@@ -0,0 +1,34 @@
+#!/usr/bin/env node
+
+// -----------------------------------------------------------------------------
+
+const asyncMap = require('lodash/curryRight')(require('@xen-orchestra/async-map').asyncMap)
+const getopts = require('getopts')
+const { RemoteAdapter } = require('@xen-orchestra/backups/RemoteAdapter')
+const { resolve } = require('path')
+
+const adapter = new RemoteAdapter(require('@xen-orchestra/fs').getHandler({ url: 'file://' }))
+
+module.exports = async function main(args) {
+  const { _, fix, remove, merge } = getopts(args, {
+    alias: {
+      fix: 'f',
+      remove: 'r',
+      merge: 'm',
+    },
+    boolean: ['fix', 'merge', 'remove'],
+    default: {
+      merge: false,
+      remove: false,
+    },
+  })
+
+  await asyncMap(_, async vmDir => {
+    vmDir = resolve(vmDir)
+    try {
+      await adapter.cleanVm(vmDir, { fixMetadata: fix, remove, merge, onLog: (...args) => console.warn(...args) })
+    } catch (error) {
+      console.error('adapter.cleanVm', vmDir, error)
+    }
+  })
+}

@xen-orchestra/backups-cli/commands/create-symlink-index.js

@@ -0,0 +1,28 @@
+const filenamify = require('filenamify')
+const get = require('lodash/get')
+const { asyncMap } = require('@xen-orchestra/async-map')
+const { dirname, join, relative } = require('path')
+
+const { mktree, readdir2, readFile, symlink2 } = require('../_fs')
+
+module.exports = async function createSymlinkIndex([backupDir, fieldPath]) {
+  const indexDir = join(backupDir, 'indexes', filenamify(fieldPath))
+  await mktree(indexDir)
+
+  await asyncMap(await readdir2(backupDir), async vmDir =>
+    asyncMap(
+      (await readdir2(vmDir)).filter(_ => _.endsWith('.json')),
+      async json => {
+        const metadata = JSON.parse(await readFile(json))
+        const value = get(metadata, fieldPath)
+        if (value !== undefined) {
+          const target = relative(indexDir, dirname(json))
+          const path = join(indexDir, filenamify(String(value)))
+          await symlink2(target, path).catch(error => {
+            console.warn('symlink(%s, %s)', target, path, error)
+          })
+        }
+      }
+    )
+  )
+}

@xen-orchestra/backups-cli/commands/info.js

@@ -0,0 +1,54 @@
+const groupBy = require('lodash/groupBy')
+const { asyncMap } = require('@xen-orchestra/async-map')
+const { createHash } = require('crypto')
+const { dirname, resolve } = require('path')
+
+const { readdir2, readFile, getSize } = require('../_fs')
+
+const sha512 = str => createHash('sha512').update(str).digest('hex')
+const sum = values => values.reduce((a, b) => a + b)
+
+module.exports = async function info(vmDirs) {
+  const jsonFiles = (
+    await asyncMap(vmDirs, async vmDir => (await readdir2(vmDir)).filter(_ => _.endsWith('.json')))
+  ).flat()
+
+  const hashes = { __proto__: null }
+
+  const info = (
+    await asyncMap(jsonFiles, async jsonFile => {
+      try {
+        const jsonDir = dirname(jsonFile)
+        const json = await readFile(jsonFile)
+
+        const hash = sha512(json)
+        if (hash in hashes) {
+          console.log(jsonFile, 'duplicate of', hashes[hash])
+          return
+        }
+        hashes[hash] = jsonFile
+
+        const metadata = JSON.parse(json)
+
+        return {
+          jsonDir,
+          jsonFile,
+          metadata,
+          size:
+            json.length +
+            (await (metadata.mode === 'delta'
+              ? asyncMap(Object.values(metadata.vhds), _ => getSize(resolve(jsonDir, _))).then(sum)
+              : getSize(resolve(jsonDir, metadata.xva)))),
+        }
+      } catch (error) {
+        console.error(jsonFile, error)
+      }
+    })
+  ).filter(_ => _ !== undefined)
+  const byJobs = groupBy(info, 'metadata.jobId')
+  Object.keys(byJobs)
+    .sort()
+    .forEach(jobId => {
+      console.log(jobId, sum(byJobs[jobId].map(_ => _.size)))
+    })
+}

@xen-orchestra/backups-cli/index.js

@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+
+require('./_composeCommands')({
+  'clean-vms': {
+    get main() {
+      return require('./commands/clean-vms')
+    },
+    usage: `[--fix] [--merge] [--remove] xo-vm-backups/*
+
+      Detects and repair issues with VM backups.
+
+      Options:
+        -f, --fix       Fix metadata issues (like size)
+        -m, --merge     Merge (or continue merging) VHD files that are unused
+        -r, --remove    Remove unused, incomplete, orphan, or corrupted files
+`,
+  },
+  'create-symlink-index': {
+    get main() {
+      return require('./commands/create-symlink-index')
+    },
+    usage: 'xo-vm-backups <field path>',
+  },
+  info: {
+    get main() {
+      return require('./commands/info')
+    },
+    usage: 'xo-vm-backups/*',
+  },
+})(process.argv.slice(2), 'xo-backups').catch(error => {
+  console.error('main', error)
+  process.exitCode = 1
+})

@xen-orchestra/backups-cli/package.json

@@ -0,0 +1,36 @@
+{
+  "private": false,
+  "bin": {
+    "xo-backups": "index.js"
+  },
+  "preferGlobal": true,
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "dependencies": {
+    "@xen-orchestra/async-map": "^0.1.2",
+    "@xen-orchestra/backups": "^0.13.0",
+    "@xen-orchestra/fs": "^0.18.0",
+    "filenamify": "^4.1.0",
+    "getopts": "^2.2.5",
+    "lodash": "^4.17.15",
+    "promise-toolbox": "^0.19.2"
+  },
+  "engines": {
+    "node": ">=7.10.1"
+  },
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/backups-cli",
+  "name": "@xen-orchestra/backups-cli",
+  "repository": {
+    "directory": "@xen-orchestra/backups-cli",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  },
+  "version": "0.6.0",
+  "license": "AGPL-3.0-or-later",
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  }
+}

@xen-orchestra/backups/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@xen-orchestra/backups/Backup.js

@@ -0,0 +1,263 @@
+const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
+const Disposable = require('promise-toolbox/Disposable.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const { compileTemplate } = require('@xen-orchestra/template')
+const { limitConcurrency } = require('limit-concurrency-decorator')
+
+const { extractIdsFromSimplePattern } = require('./_extractIdsFromSimplePattern.js')
+const { PoolMetadataBackup } = require('./_PoolMetadataBackup.js')
+const { Task } = require('./Task.js')
+const { VmBackup } = require('./_VmBackup.js')
+const { XoMetadataBackup } = require('./_XoMetadataBackup.js')
+
+const noop = Function.prototype
+
+const getAdaptersByRemote = adapters => {
+  const adaptersByRemote = {}
+  adapters.forEach(({ adapter, remoteId }) => {
+    adaptersByRemote[remoteId] = adapter
+  })
+  return adaptersByRemote
+}
+
+const runTask = (...args) => Task.run(...args).catch(noop) // errors are handled by logs
+
+exports.Backup = class Backup {
+  constructor({ config, getAdapter, getConnectedRecord, job, schedule }) {
+    this._config = config
+    this._getRecord = getConnectedRecord
+    this._job = job
+    this._schedule = schedule
+
+    this._getAdapter = Disposable.factory(function* (remoteId) {
+      return {
+        adapter: yield getAdapter(remoteId),
+        remoteId,
+      }
+    })
+
+    this._getSnapshotNameLabel = compileTemplate(config.snapshotNameLabelTpl, {
+      '{job.name}': job.name,
+      '{vm.name_label}': vm => vm.name_label,
+    })
+  }
+
+  run() {
+    const type = this._job.type
+    if (type === 'backup') {
+      return this._runVmBackup()
+    } else if (type === 'metadataBackup') {
+      return this._runMetadataBackup()
+    } else {
+      throw new Error(`No runner for the backup type ${type}`)
+    }
+  }
+
+  async _runMetadataBackup() {
+    const schedule = this._schedule
+    const job = this._job
+    const remoteIds = extractIdsFromSimplePattern(job.remotes)
+    if (remoteIds.length === 0) {
+      throw new Error('metadata backup job cannot run without remotes')
+    }
+
+    const config = this._config
+    const settings = {
+      ...config.defaultSettings,
+      ...config.metadata.defaultSettings,
+      ...job.settings[''],
+      ...job.settings[schedule.id],
+    }
+
+    const poolIds = extractIdsFromSimplePattern(job.pools)
+    const isEmptyPools = poolIds.length === 0
+    const isXoMetadata = job.xoMetadata !== undefined
+    if (!isXoMetadata && isEmptyPools) {
+      throw new Error('no metadata mode found')
+    }
+
+    const { retentionPoolMetadata, retentionXoMetadata } = settings
+
+    if (
+      (retentionPoolMetadata === 0 && retentionXoMetadata === 0) ||
+      (!isXoMetadata && retentionPoolMetadata === 0) ||
+      (isEmptyPools && retentionXoMetadata === 0)
+    ) {
+      throw new Error('no retentions corresponding to the metadata modes found')
+    }
+
+    await Disposable.use(
+      Disposable.all(
+        poolIds.map(id =>
+          this._getRecord('pool', id).catch(error => {
+            // See https://github.com/vatesfr/xen-orchestra/commit/6aa6cfba8ec939c0288f0fa740f6dfad98c43cbb
+            runTask(
+              {
+                name: 'get pool record',
+                data: { type: 'pool', id },
+              },
+              () => Promise.reject(error)
+            )
+          })
+        )
+      ),
+      Disposable.all(
+        remoteIds.map(id =>
+          this._getAdapter(id).catch(error => {
+            // See https://github.com/vatesfr/xen-orchestra/commit/6aa6cfba8ec939c0288f0fa740f6dfad98c43cbb
+            runTask(
+              {
+                name: 'get remote adapter',
+                data: { type: 'remote', id },
+              },
+              () => Promise.reject(error)
+            )
+          })
+        )
+      ),
+      async (pools, remoteAdapters) => {
+        // remove adapters that failed (already handled)
+        remoteAdapters = remoteAdapters.filter(_ => _ !== undefined)
+        if (remoteAdapters.length === 0) {
+          return
+        }
+        remoteAdapters = getAdaptersByRemote(remoteAdapters)
+
+        // remove pools that failed (already handled)
+        pools = pools.filter(_ => _ !== undefined)
+
+        const promises = []
+        if (pools.length !== 0 && settings.retentionPoolMetadata !== 0) {
+          promises.push(
+            asyncMap(pools, async pool =>
+              runTask(
+                {
+                  name: `Starting metadata backup for the pool (${pool.$id}). (${job.id})`,
+                  data: {
+                    id: pool.$id,
+                    pool,
+                    poolMaster: await ignoreErrors.call(pool.$xapi.getRecord('host', pool.master)),
+                    type: 'pool',
+                  },
+                },
+                () =>
+                  new PoolMetadataBackup({
+                    config,
+                    job,
+                    pool,
+                    remoteAdapters,
+                    schedule,
+                    settings,
+                  }).run()
+              )
+            )
+          )
+        }
+
+        if (job.xoMetadata !== undefined && settings.retentionXoMetadata !== 0) {
+          promises.push(
+            runTask(
+              {
+                name: `Starting XO metadata backup. (${job.id})`,
+                data: {
+                  type: 'xo',
+                },
+              },
+              () =>
+                new XoMetadataBackup({
+                  config,
+                  job,
+                  remoteAdapters,
+                  schedule,
+                  settings,
+                }).run()
+            )
+          )
+        }
+        await Promise.all(promises)
+      }
+    )
+  }
+
+  async _runVmBackup() {
+    const job = this._job
+
+    // FIXME: proper SimpleIdPattern handling
+    const getSnapshotNameLabel = this._getSnapshotNameLabel
+    const schedule = this._schedule
+
+    const config = this._config
+    const { settings } = job
+    const scheduleSettings = {
+      ...config.defaultSettings,
+      ...config.vm.defaultSettings,
+      ...settings[''],
+      ...settings[schedule.id],
+    }
+
+    await Disposable.use(
+      Disposable.all(
+        extractIdsFromSimplePattern(job.srs).map(id =>
+          this._getRecord('SR', id).catch(error => {
+            runTask(
+              {
+                name: 'get SR record',
+                data: { type: 'SR', id },
+              },
+              () => Promise.reject(error)
+            )
+          })
+        )
+      ),
+      Disposable.all(
+        extractIdsFromSimplePattern(job.remotes).map(id =>
+          this._getAdapter(id).catch(error => {
+            runTask(
+              {
+                name: 'get remote adapter',
+                data: { type: 'remote', id },
+              },
+              () => Promise.reject(error)
+            )
+          })
+        )
+      ),
+      async (srs, remoteAdapters) => {
+        // remove adapters that failed (already handled)
+        remoteAdapters = remoteAdapters.filter(_ => _ !== undefined)
+
+        // remove srs that failed (already handled)
+        srs = srs.filter(_ => _ !== undefined)
+
+        if (remoteAdapters.length === 0 && srs.length === 0 && scheduleSettings.snapshotRetention === 0) {
+          return
+        }
+
+        const vmIds = extractIdsFromSimplePattern(job.vms)
+
+        Task.info('vms', { vms: vmIds })
+
+        remoteAdapters = getAdaptersByRemote(remoteAdapters)
+
+        const handleVm = vmUuid =>
+          runTask({ name: 'backup VM', data: { type: 'VM', id: vmUuid } }, () =>
+            Disposable.use(this._getRecord('VM', vmUuid), vm =>
+              new VmBackup({
+                config,
+                getSnapshotNameLabel,
+                job,
+                // remotes,
+                remoteAdapters,
+                schedule,
+                settings: { ...scheduleSettings, ...settings[vmUuid] },
+                srs,
+                vm,
+              }).run()
+            )
+          )
+        const { concurrency } = scheduleSettings
+        await asyncMapSettled(vmIds, concurrency === 0 ? handleVm : limitConcurrency(concurrency)(handleVm))
+      }
+    )
+  }
+}

@xen-orchestra/backups/DurablePartition.js

@@ -0,0 +1,40 @@
+const { asyncMap } = require('@xen-orchestra/async-map')
+
+exports.DurablePartition = class DurablePartition {
+  // private resource API is used exceptionally to be able to separate resource creation and release
+  #partitionDisposers = {}
+
+  flushAll() {
+    const partitionDisposers = this.#partitionDisposers
+    return asyncMap(Object.keys(partitionDisposers), path => {
+      const disposers = partitionDisposers[path]
+      delete partitionDisposers[path]
+      return asyncMap(disposers, d => d(path).catch(noop => {}))
+    })
+  }
+
+  async mount(adapter, diskId, partitionId) {
+    const { value: path, dispose } = await adapter.getPartition(diskId, partitionId)
+
+    const partitionDisposers = this.#partitionDisposers
+    if (partitionDisposers[path] === undefined) {
+      partitionDisposers[path] = []
+    }
+    partitionDisposers[path].push(dispose)
+
+    return path
+  }
+
+  async unmount(path) {
+    const partitionDisposers = this.#partitionDisposers
+    const disposers = partitionDisposers[path]
+    if (disposers === undefined) {
+      throw new Error(`No partition corresponding to the path ${path} found`)
+    }
+
+    await disposers.pop()()
+    if (disposers.length === 0) {
+      delete partitionDisposers[path]
+    }
+  }
+}

@xen-orchestra/backups/ImportVmBackup.js

@@ -0,0 +1,66 @@
+const assert = require('assert')
+
+const { formatFilenameDate } = require('./_filenameDate.js')
+const { importDeltaVm } = require('./_deltaVm.js')
+const { Task } = require('./Task.js')
+const { watchStreamSize } = require('./_watchStreamSize.js')
+
+exports.ImportVmBackup = class ImportVmBackup {
+  constructor({ adapter, metadata, srUuid, xapi, settings: { newMacAddresses } = {} }) {
+    this._adapter = adapter
+    this._importDeltaVmSettings = { newMacAddresses }
+    this._metadata = metadata
+    this._srUuid = srUuid
+    this._xapi = xapi
+  }
+
+  async run() {
+    const adapter = this._adapter
+    const metadata = this._metadata
+    const isFull = metadata.mode === 'full'
+
+    const sizeContainer = { size: 0 }
+
+    let backup
+    if (isFull) {
+      backup = await adapter.readFullVmBackup(metadata)
+      watchStreamSize(backup, sizeContainer)
+    } else {
+      assert.strictEqual(metadata.mode, 'delta')
+
+      backup = await adapter.readDeltaVmBackup(metadata)
+      Object.values(backup.streams).forEach(stream => watchStreamSize(stream, sizeContainer))
+    }
+
+    return Task.run(
+      {
+        name: 'transfer',
+      },
+      async () => {
+        const xapi = this._xapi
+        const srRef = await xapi.call('SR.get_by_uuid', this._srUuid)
+
+        const vmRef = isFull
+          ? await xapi.VM_import(backup, srRef)
+          : await importDeltaVm(backup, await xapi.getRecord('SR', srRef), {
+              ...this._importDeltaVmSettings,
+              detectBase: false,
+            })
+
+        await Promise.all([
+          xapi.call('VM.add_tags', vmRef, 'restored from backup'),
+          xapi.call(
+            'VM.set_name_label',
+            vmRef,
+            `${metadata.vm.name_label} (${formatFilenameDate(metadata.timestamp)})`
+          ),
+        ])
+
+        return {
+          size: sizeContainer.size,
+          id: await xapi.getField('VM', vmRef, 'uuid'),
+        }
+      }
+    )
+  }
+}

@xen-orchestra/backups/README.md

@@ -0,0 +1,28 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @xen-orchestra/backups
+
+[![Package Version](https://badgen.net/npm/v/@xen-orchestra/backups)](https://npmjs.org/package/@xen-orchestra/backups) ![License](https://badgen.net/npm/license/@xen-orchestra/backups) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@xen-orchestra/backups)](https://bundlephobia.com/result?p=@xen-orchestra/backups) [![Node compatibility](https://badgen.net/npm/node/@xen-orchestra/backups)](https://npmjs.org/package/@xen-orchestra/backups)
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/backups):
+
+```
+> npm install --save @xen-orchestra/backups
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[AGPL-3.0-or-later](https://spdx.org/licenses/AGPL-3.0-or-later) © [Vates SAS](https://vates.fr)

@xen-orchestra/backups/RemoteAdapter.js

@@ -0,0 +1,559 @@
+const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
+const Disposable = require('promise-toolbox/Disposable.js')
+const fromCallback = require('promise-toolbox/fromCallback.js')
+const fromEvent = require('promise-toolbox/fromEvent.js')
+const pDefer = require('promise-toolbox/defer.js')
+const pump = require('pump')
+const { basename, dirname, join, normalize, resolve } = require('path')
+const { createLogger } = require('@xen-orchestra/log')
+const { createSyntheticStream, mergeVhd, default: Vhd } = require('vhd-lib')
+const { deduped } = require('@vates/disposable/deduped.js')
+const { execFile } = require('child_process')
+const { readdir, stat } = require('fs-extra')
+const { ZipFile } = require('yazl')
+
+const { BACKUP_DIR } = require('./_getVmBackupDir.js')
+const { cleanVm } = require('./_cleanVm.js')
+const { getTmpDir } = require('./_getTmpDir.js')
+const { isMetadataFile, isVhdFile } = require('./_backupType.js')
+const { isValidXva } = require('./_isValidXva.js')
+const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js')
+const { lvs, pvs } = require('./_lvm.js')
+
+const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
+exports.DIR_XO_CONFIG_BACKUPS = DIR_XO_CONFIG_BACKUPS
+
+const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
+exports.DIR_XO_POOL_METADATA_BACKUPS = DIR_XO_POOL_METADATA_BACKUPS
+
+const { warn } = createLogger('xo:backups:RemoteAdapter')
+
+const compareTimestamp = (a, b) => a.timestamp - b.timestamp
+
+const noop = Function.prototype
+
+const resolveRelativeFromFile = (file, path) => resolve('/', dirname(file), path).slice(1)
+
+const resolveSubpath = (root, path) => resolve(root, `.${resolve('/', path)}`)
+
+const RE_VHDI = /^vhdi(\d+)$/
+
+async function addDirectory(files, realPath, metadataPath) {
+  try {
+    const subFiles = await readdir(realPath)
+    await asyncMap(subFiles, file => addDirectory(files, realPath + '/' + file, metadataPath + '/' + file))
+  } catch (error) {
+    if (error == null || error.code !== 'ENOTDIR') {
+      throw error
+    }
+    files.push({
+      realPath,
+      metadataPath,
+    })
+  }
+}
+
+const createSafeReaddir = (handler, methodName) => (path, options) =>
+  handler.list(path, options).catch(error => {
+    if (error?.code !== 'ENOENT') {
+      warn(`${methodName} ${path}`, { error })
+    }
+    return []
+  })
+
+const debounceResourceFactory = factory =>
+  function () {
+    return this._debounceResource(factory.apply(this, arguments))
+  }
+
+class RemoteAdapter {
+  constructor(handler, { debounceResource = res => res, dirMode } = {}) {
+    this._debounceResource = debounceResource
+    this._dirMode = dirMode
+    this._handler = handler
+  }
+
+  get handler() {
+    return this._handler
+  }
+
+  async _deleteVhd(path) {
+    const handler = this._handler
+    const vhds = await asyncMapSettled(
+      await handler.list(dirname(path), {
+        filter: isVhdFile,
+        prependDir: true,
+      }),
+      async path => {
+        try {
+          const vhd = new Vhd(handler, path)
+          await vhd.readHeaderAndFooter()
+          return {
+            footer: vhd.footer,
+            header: vhd.header,
+            path,
+          }
+        } catch (error) {
+          // Do not fail on corrupted VHDs (usually uncleaned temporary files),
+          // they are probably inconsequent to the backup process and should not
+          // fail it.
+          warn(`BackupNg#_deleteVhd ${path}`, { error })
+        }
+      }
+    )
+    const base = basename(path)
+    const child = vhds.find(_ => _ !== undefined && _.header.parentUnicodeName === base)
+    if (child === undefined) {
+      await handler.unlink(path)
+      return 0
+    }
+
+    try {
+      const childPath = child.path
+      const mergedDataSize = await mergeVhd(handler, path, handler, childPath)
+      await handler.rename(path, childPath)
+      return mergedDataSize
+    } catch (error) {
+      handler.unlink(path).catch(warn)
+      throw error
+    }
+  }
+
+  async _findPartition(devicePath, partitionId) {
+    const partitions = await listPartitions(devicePath)
+    const partition = partitions.find(_ => _.id === partitionId)
+    if (partition === undefined) {
+      throw new Error(`partition ${partitionId} not found`)
+    }
+    return partition
+  }
+
+  _getLvmLogicalVolumes = Disposable.factory(this._getLvmLogicalVolumes)
+  _getLvmLogicalVolumes = deduped(this._getLvmLogicalVolumes, (devicePath, pvId, vgName) => [devicePath, pvId, vgName])
+  _getLvmLogicalVolumes = debounceResourceFactory(this._getLvmLogicalVolumes)
+  async *_getLvmLogicalVolumes(devicePath, pvId, vgName) {
+    yield this._getLvmPhysicalVolume(devicePath, pvId && (await this._findPartition(devicePath, pvId)))
+
+    await fromCallback(execFile, 'vgchange', ['-ay', vgName])
+    try {
+      yield lvs(['lv_name', 'lv_path'], vgName)
+    } finally {
+      await fromCallback(execFile, 'vgchange', ['-an', vgName])
+    }
+  }
+
+  _getLvmPhysicalVolume = Disposable.factory(this._getLvmPhysicalVolume)
+  _getLvmPhysicalVolume = deduped(this._getLvmPhysicalVolume, (devicePath, partition) => [devicePath, partition?.id])
+  _getLvmPhysicalVolume = debounceResourceFactory(this._getLvmPhysicalVolume)
+  async *_getLvmPhysicalVolume(devicePath, partition) {
+    const args = []
+    if (partition !== undefined) {
+      args.push('-o', partition.start * 512, '--sizelimit', partition.size)
+    }
+    args.push('--show', '-f', devicePath)
+    const path = (await fromCallback(execFile, 'losetup', args)).trim()
+    try {
+      await fromCallback(execFile, 'pvscan', ['--cache', path])
+      yield path
+    } finally {
+      try {
+        const vgNames = await pvs('vg_name', path)
+        await fromCallback(execFile, 'vgchange', ['-an', ...vgNames])
+      } finally {
+        await fromCallback(execFile, 'losetup', ['-d', path])
+      }
+    }
+  }
+
+  _getPartition = Disposable.factory(this._getPartition)
+  _getPartition = deduped(this._getPartition, (devicePath, partition) => [devicePath, partition?.id])
+  _getPartition = debounceResourceFactory(this._getPartition)
+  async *_getPartition(devicePath, partition) {
+    const options = ['loop', 'ro']
+
+    if (partition !== undefined) {
+      const { size, start } = partition
+      options.push(`sizelimit=${size}`)
+      if (start !== undefined) {
+        options.push(`offset=${start * 512}`)
+      }
+    }
+
+    const path = yield getTmpDir()
+    const mount = options => {
+      return fromCallback(execFile, 'mount', [
+        `--options=${options.join(',')}`,
+        `--source=${devicePath}`,
+        `--target=${path}`,
+      ])
+    }
+
+    // `norecovery` option is used for ext3/ext4/xfs, if it fails it might be
+    // another fs, try without
+    try {
+      await mount([...options, 'norecovery'])
+    } catch (error) {
+      await mount(options)
+    }
+    try {
+      yield path
+    } finally {
+      await fromCallback(execFile, 'umount', ['--lazy', path])
+    }
+  }
+
+  _listLvmLogicalVolumes(devicePath, partition, results = []) {
+    return Disposable.use(this._getLvmPhysicalVolume(devicePath, partition), async path => {
+      const lvs = await pvs(['lv_name', 'lv_path', 'lv_size', 'vg_name'], path)
+      const partitionId = partition !== undefined ? partition.id : ''
+      lvs.forEach((lv, i) => {
+        const name = lv.lv_name
+        if (name !== '') {
+          results.push({
+            id: `${partitionId}/${lv.vg_name}/${name}`,
+            name,
+            size: lv.lv_size,
+          })
+        }
+      })
+      return results
+    })
+  }
+
+  _usePartitionFiles = Disposable.factory(this._usePartitionFiles)
+  async *_usePartitionFiles(diskId, partitionId, paths) {
+    const path = yield this.getPartition(diskId, partitionId)
+
+    const files = []
+    await asyncMap(paths, file =>
+      addDirectory(files, resolveSubpath(path, file), normalize('./' + file).replace(/\/+$/, ''))
+    )
+
+    return files
+  }
+
+  fetchPartitionFiles(diskId, partitionId, paths) {
+    const { promise, reject, resolve } = pDefer()
+    Disposable.use(
+      async function* () {
+        const files = yield this._usePartitionFiles(diskId, partitionId, paths)
+        const zip = new ZipFile()
+        files.forEach(({ realPath, metadataPath }) => zip.addFile(realPath, metadataPath))
+        zip.end()
+        const { outputStream } = zip
+        resolve(outputStream)
+        await fromEvent(outputStream, 'end')
+      }.bind(this)
+    ).catch(error => {
+      warn(error)
+      reject(error)
+    })
+    return promise
+  }
+
+  async deleteDeltaVmBackups(backups) {
+    const handler = this._handler
+    let mergedDataSize = 0
+    await asyncMapSettled(backups, ({ _filename, vhds }) =>
+      Promise.all([
+        handler.unlink(_filename),
+        asyncMap(Object.values(vhds), async _ => {
+          mergedDataSize += await this._deleteVhd(resolveRelativeFromFile(_filename, _))
+        }),
+      ])
+    )
+    return mergedDataSize
+  }
+
+  async deleteMetadataBackup(backupId) {
+    const uuidReg = '\\w{8}(-\\w{4}){3}-\\w{12}'
+    const metadataDirReg = 'xo-(config|pool-metadata)-backups'
+    const timestampReg = '\\d{8}T\\d{6}Z'
+    const regexp = new RegExp(`^${metadataDirReg}/${uuidReg}(/${uuidReg})?/${timestampReg}`)
+    if (!regexp.test(backupId)) {
+      throw new Error(`The id (${backupId}) not correspond to a metadata folder`)
+    }
+
+    await this._handler.rmtree(backupId)
+  }
+
+  async deleteOldMetadataBackups(dir, retention) {
+    const handler = this.handler
+    let list = await handler.list(dir)
+    list.sort()
+    list = list.filter(timestamp => /^\d{8}T\d{6}Z$/.test(timestamp)).slice(0, -retention)
+    await asyncMapSettled(list, timestamp => handler.rmtree(`${dir}/${timestamp}`))
+  }
+
+  async deleteFullVmBackups(backups) {
+    const handler = this._handler
+    await asyncMapSettled(backups, ({ _filename, xva }) =>
+      Promise.all([handler.unlink(_filename), handler.unlink(resolveRelativeFromFile(_filename, xva))])
+    )
+  }
+
+  async deleteVmBackup(filename) {
+    const metadata = JSON.parse(String(await this._handler.readFile(filename)))
+    metadata._filename = filename
+
+    if (metadata.mode === 'delta') {
+      await this.deleteDeltaVmBackups([metadata])
+    } else if (metadata.mode === 'full') {
+      await this.deleteFullVmBackups([metadata])
+    } else {
+      throw new Error(`no deleter for backup mode ${metadata.mode}`)
+    }
+  }
+
+  getDisk = Disposable.factory(this.getDisk)
+  getDisk = deduped(this.getDisk, diskId => [diskId])
+  getDisk = debounceResourceFactory(this.getDisk)
+  async *getDisk(diskId) {
+    const handler = this._handler
+
+    const diskPath = handler._getFilePath('/' + diskId)
+    const mountDir = yield getTmpDir()
+    await fromCallback(execFile, 'vhdimount', [diskPath, mountDir])
+    try {
+      let max = 0
+      let maxEntry
+      const entries = await readdir(mountDir)
+      entries.forEach(entry => {
+        const matches = RE_VHDI.exec(entry)
+        if (matches !== null) {
+          const value = +matches[1]
+          if (value > max) {
+            max = value
+            maxEntry = entry
+          }
+        }
+      })
+      if (max === 0) {
+        throw new Error('no disks found')
+      }
+
+      yield `${mountDir}/${maxEntry}`
+    } finally {
+      await fromCallback(execFile, 'fusermount', ['-uz', mountDir])
+    }
+  }
+
+  // partitionId values:
+  //
+  // - undefined: raw disk
+  // - `<partitionId>`: partitioned disk
+  // - `<pvId>/<vgName>/<lvName>`: LVM on a partitioned disk
+  // - `/<vgName>/lvName>`: LVM on a raw disk
+  getPartition = Disposable.factory(this.getPartition)
+  async *getPartition(diskId, partitionId) {
+    const devicePath = yield this.getDisk(diskId)
+    if (partitionId === undefined) {
+      return yield this._getPartition(devicePath)
+    }
+
+    const isLvmPartition = partitionId.includes('/')
+    if (isLvmPartition) {
+      const [pvId, vgName, lvName] = partitionId.split('/')
+      const lvs = yield this._getLvmLogicalVolumes(devicePath, pvId !== '' ? pvId : undefined, vgName)
+      return yield this._getPartition(lvs.find(_ => _.lv_name === lvName).lv_path)
+    }
+
+    return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
+  }
+
+  async listAllVmBackups() {
+    const handler = this._handler
+
+    const backups = { __proto__: null }
+    await asyncMap(await handler.list(BACKUP_DIR), async vmUuid => {
+      const vmBackups = await this.listVmBackups(vmUuid)
+      backups[vmUuid] = vmBackups
+    })
+
+    return backups
+  }
+
+  listPartitionFiles(diskId, partitionId, path) {
+    return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => {
+      path = resolveSubpath(rootPath, path)
+
+      const entriesMap = {}
+      await asyncMap(await readdir(path), async name => {
+        try {
+          const stats = await stat(`${path}/${name}`)
+          entriesMap[stats.isDirectory() ? `${name}/` : name] = {}
+        } catch (error) {
+          if (error == null || error.code !== 'ENOENT') {
+            throw error
+          }
+        }
+      })
+
+      return entriesMap
+    })
+  }
+
+  listPartitions(diskId) {
+    return Disposable.use(this.getDisk(diskId), async devicePath => {
+      const partitions = await listPartitions(devicePath)
+
+      if (partitions.length === 0) {
+        try {
+          // handle potential raw LVM physical volume
+          return await this._listLvmLogicalVolumes(devicePath, undefined, partitions)
+        } catch (error) {
+          return []
+        }
+      }
+
+      const results = []
+      await asyncMapSettled(partitions, partition =>
+        partition.type === LVM_PARTITION_TYPE
+          ? this._listLvmLogicalVolumes(devicePath, partition, results)
+          : results.push(partition)
+      )
+      return results
+    })
+  }
+
+  async listPoolMetadataBackups() {
+    const handler = this._handler
+    const safeReaddir = createSafeReaddir(handler, 'listPoolMetadataBackups')
+
+    const backupsByPool = {}
+    await asyncMap(await safeReaddir(DIR_XO_POOL_METADATA_BACKUPS, { prependDir: true }), async scheduleDir =>
+      asyncMap(await safeReaddir(scheduleDir), async poolId => {
+        const backups = backupsByPool[poolId] ?? (backupsByPool[poolId] = [])
+        return asyncMap(await safeReaddir(`${scheduleDir}/${poolId}`, { prependDir: true }), async backupDir => {
+          try {
+            backups.push({
+              id: backupDir,
+              ...JSON.parse(String(await handler.readFile(`${backupDir}/metadata.json`))),
+            })
+          } catch (error) {
+            warn(`listPoolMetadataBackups ${backupDir}`, {
+              error,
+            })
+          }
+        })
+      })
+    )
+
+    // delete empty entries and sort backups
+    Object.keys(backupsByPool).forEach(poolId => {
+      const backups = backupsByPool[poolId]
+      if (backups.length === 0) {
+        delete backupsByPool[poolId]
+      } else {
+        backups.sort(compareTimestamp)
+      }
+    })
+
+    return backupsByPool
+  }
+
+  async listVmBackups(vmUuid, predicate) {
+    const handler = this._handler
+    const backups = []
+
+    try {
+      const files = await handler.list(`${BACKUP_DIR}/${vmUuid}`, {
+        filter: isMetadataFile,
+        prependDir: true,
+      })
+      await asyncMap(files, async file => {
+        try {
+          const metadata = await this.readVmBackupMetadata(file)
+          if (predicate === undefined || predicate(metadata)) {
+            // inject an id usable by importVmBackupNg()
+            metadata.id = metadata._filename
+
+            backups.push(metadata)
+          }
+        } catch (error) {
+          warn(`listVmBackups ${file}`, { error })
+        }
+      })
+    } catch (error) {
+      let code
+      if (error == null || ((code = error.code) !== 'ENOENT' && code !== 'ENOTDIR')) {
+        throw error
+      }
+    }
+
+    return backups.sort(compareTimestamp)
+  }
+
+  async listXoMetadataBackups() {
+    const handler = this._handler
+    const safeReaddir = createSafeReaddir(handler, 'listXoMetadataBackups')
+
+    const backups = []
+    await asyncMap(await safeReaddir(DIR_XO_CONFIG_BACKUPS, { prependDir: true }), async scheduleDir =>
+      asyncMap(await safeReaddir(scheduleDir, { prependDir: true }), async backupDir => {
+        try {
+          backups.push({
+            id: backupDir,
+            ...JSON.parse(String(await handler.readFile(`${backupDir}/metadata.json`))),
+          })
+        } catch (error) {
+          warn(`listXoMetadataBackups ${backupDir}`, { error })
+        }
+      })
+    )
+
+    return backups.sort(compareTimestamp)
+  }
+
+  async outputStream(path, input, { checksum = true, validator = noop } = {}) {
+    await this._handler.outputStream(path, input, {
+      checksum,
+      dirMode: this._dirMode,
+      async validator() {
+        await input.task
+        return validator.apply(this, arguments)
+      },
+    })
+  }
+
+  async readDeltaVmBackup(metadata) {
+    const handler = this._handler
+    const { vbds, vdis, vhds, vifs, vm } = metadata
+    const dir = dirname(metadata._filename)
+
+    const streams = {}
+    await asyncMapSettled(Object.keys(vdis), async id => {
+      streams[`${id}.vhd`] = await createSyntheticStream(handler, join(dir, vhds[id]))
+    })
+
+    return {
+      streams,
+      vbds,
+      vdis,
+      version: '1.0.0',
+      vifs,
+      vm,
+    }
+  }
+
+  readFullVmBackup(metadata) {
+    return this._handler.createReadStream(resolve('/', dirname(metadata._filename), metadata.xva))
+  }
+
+  async readVmBackupMetadata(path) {
+    return Object.defineProperty(JSON.parse(await this._handler.readFile(path)), '_filename', { value: path })
+  }
+}
+
+Object.assign(RemoteAdapter.prototype, {
+  cleanVm(vmDir, { lock = true } = {}) {
+    if (lock) {
+      return Disposable.use(this._handler.lock(vmDir), () => cleanVm.apply(this, arguments))
+    } else {
+      return cleanVm.apply(this, arguments)
+    }
+  },
+  isValidXva,
+})
+
+exports.RemoteAdapter = RemoteAdapter

@xen-orchestra/backups/RestoreMetadataBackup.js

@@ -0,0 +1,24 @@
+const { DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')
+const { PATH_DB_DUMP } = require('./_PoolMetadataBackup.js')
+
+exports.RestoreMetadataBackup = class RestoreMetadataBackup {
+  constructor({ backupId, handler, xapi }) {
+    this._backupId = backupId
+    this._handler = handler
+    this._xapi = xapi
+  }
+
+  async run() {
+    const backupId = this._backupId
+    const handler = this._handler
+    const xapi = this._xapi
+
+    if (backupId.split('/')[0] === DIR_XO_POOL_METADATA_BACKUPS) {
+      return xapi.putResource(await handler.createReadStream(`${backupId}/data`), PATH_DB_DUMP, {
+        task: xapi.task_create('Import pool metadata'),
+      })
+    } else {
+      return String(await handler.readFile(`${backupId}/data.json`))
+    }
+  }
+}

@xen-orchestra/backups/Task.js

@@ -0,0 +1,151 @@
+const CancelToken = require('promise-toolbox/CancelToken.js')
+const Zone = require('node-zone')
+
+const logAfterEnd = () => {
+  throw new Error('task has already ended')
+}
+
+const noop = Function.prototype
+
+// Create a serializable object from an error.
+//
+// Otherwise some fields might be non-enumerable and missing from logs.
+const serializeError = error =>
+  error instanceof Error
+    ? {
+        ...error, // Copy enumerable properties.
+        code: error.code,
+        message: error.message,
+        name: error.name,
+        stack: error.stack,
+      }
+    : error
+
+const $$task = Symbol('@xen-orchestra/backups/Task')
+
+class Task {
+  static get cancelToken() {
+    const task = Zone.current.data[$$task]
+    return task !== undefined ? task.#cancelToken : CancelToken.none
+  }
+
+  static run(opts, fn) {
+    return new this(opts).run(fn, true)
+  }
+
+  static wrapFn(opts, fn) {
+    // compatibility with @decorateWith
+    if (typeof fn !== 'function') {
+      ;[fn, opts] = [opts, fn]
+    }
+
+    return function () {
+      return Task.run(typeof opts === 'function' ? opts.apply(this, arguments) : opts, () => fn.apply(this, arguments))
+    }
+  }
+
+  #cancelToken
+  #id = Math.random().toString(36).slice(2)
+  #onLog
+  #zone
+
+  constructor({ name, data, onLog }) {
+    let parentCancelToken, parentId
+    if (onLog === undefined) {
+      const parent = Zone.current.data[$$task]
+      if (parent === undefined) {
+        onLog = noop
+      } else {
+        onLog = log => parent.#onLog(log)
+        parentCancelToken = parent.#cancelToken
+        parentId = parent.#id
+      }
+    }
+
+    const zone = Zone.current.fork('@xen-orchestra/backups/Task')
+    zone.data[$$task] = this
+    this.#zone = zone
+
+    const { cancel, token } = CancelToken.source(parentCancelToken && [parentCancelToken])
+    this.#cancelToken = token
+    this.cancel = cancel
+
+    this.#onLog = onLog
+
+    this.#log('start', {
+      data,
+      message: name,
+      parentId,
+    })
+  }
+
+  failure(error) {
+    this.#end('failure', serializeError(error))
+  }
+
+  info(message, data) {
+    this.#log('info', { data, message })
+  }
+
+  /**
+   * Run a function in the context of this task
+   *
+   * In case of error, the task will be failed.
+   *
+   * @typedef Result
+   * @param {() => Result)} fn
+   * @param {boolean} last - Whether the task should succeed if there is no error
+   * @returns Result
+   */
+  run(fn, last = false) {
+    return this.#zone.run(() => {
+      try {
+        const result = fn()
+        let then
+        if (result != null && typeof (then = result.then) === 'function') {
+          then.call(result, last && (value => this.success(value)), error => this.failure(error))
+        } else if (last) {
+          this.success(result)
+        }
+        return result
+      } catch (error) {
+        this.failure(error)
+        throw error
+      }
+    })
+  }
+
+  success(value) {
+    this.#end('success', value)
+  }
+
+  warning(message, data) {
+    this.#log('warning', { data, message })
+  }
+
+  wrapFn(fn, last) {
+    const task = this
+    return function () {
+      return task.run(() => fn.apply(this, arguments), last)
+    }
+  }
+
+  #end(status, result) {
+    this.#log('end', { result, status })
+    this.#onLog = logAfterEnd
+  }
+
+  #log(event, props) {
+    this.#onLog({
+      ...props,
+      event,
+      taskId: this.#id,
+      timestamp: Date.now(),
+    })
+  }
+}
+exports.Task = Task
+
+for (const method of ['info', 'warning']) {
+  Task[method] = (...args) => Zone.current.data[$$task]?.[method](...args)
+}

@xen-orchestra/backups/_PoolMetadataBackup.js

@@ -0,0 +1,75 @@
+const { asyncMap } = require('@xen-orchestra/async-map')
+
+const { DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')
+const { forkStreamUnpipe } = require('./_forkStreamUnpipe.js')
+const { formatFilenameDate } = require('./_filenameDate.js')
+const { Task } = require('./Task.js')
+
+const PATH_DB_DUMP = '/pool/xmldbdump'
+exports.PATH_DB_DUMP = PATH_DB_DUMP
+
+exports.PoolMetadataBackup = class PoolMetadataBackup {
+  constructor({ config, job, pool, remoteAdapters, schedule, settings }) {
+    this._config = config
+    this._job = job
+    this._pool = pool
+    this._remoteAdapters = remoteAdapters
+    this._schedule = schedule
+    this._settings = settings
+  }
+
+  _exportPoolMetadata() {
+    const xapi = this._pool.$xapi
+    return xapi.getResource(PATH_DB_DUMP, {
+      task: xapi.task_create('Export pool metadata'),
+    })
+  }
+
+  async run() {
+    const timestamp = Date.now()
+
+    const { _job: job, _schedule: schedule, _pool: pool } = this
+    const poolDir = `${DIR_XO_POOL_METADATA_BACKUPS}/${schedule.id}/${pool.$id}`
+    const dir = `${poolDir}/${formatFilenameDate(timestamp)}`
+
+    const stream = await this._exportPoolMetadata()
+    const fileName = `${dir}/data`
+
+    const metadata = JSON.stringify(
+      {
+        jobId: job.id,
+        jobName: job.name,
+        pool,
+        poolMaster: pool.$master,
+        scheduleId: schedule.id,
+        scheduleName: schedule.name,
+        timestamp,
+      },
+      null,
+      2
+    )
+    const metaDataFileName = `${dir}/metadata.json`
+
+    await asyncMap(
+      Object.entries(this._remoteAdapters),
+      ([remoteId, adapter]) =>
+        Task.run(
+          {
+            name: `Starting metadata backup for the pool (${pool.$id}) for the remote (${remoteId}). (${job.id})`,
+            data: {
+              id: remoteId,
+              type: 'remote',
+            },
+          },
+          async () => {
+            // forkStreamUnpipe should be used in a sync way, do not wait for a promise before using it
+            await adapter.outputStream(fileName, forkStreamUnpipe(stream), { checksum: false })
+            await adapter.handler.outputFile(metaDataFileName, metadata, {
+              dirMode: this._config.dirMode,
+            })
+            await adapter.deleteOldMetadataBackups(poolDir, this._settings.retentionPoolMetadata)
+          }
+        ).catch(() => {}) // errors are handled by logs
+    )
+  }
+}

@xen-orchestra/backups/_VmBackup.js

@@ -0,0 +1,394 @@
+const assert = require('assert')
+const findLast = require('lodash/findLast.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const keyBy = require('lodash/keyBy.js')
+const mapValues = require('lodash/mapValues.js')
+const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
+const { createLogger } = require('@xen-orchestra/log')
+const { defer } = require('golike-defer')
+const { formatDateTime } = require('@xen-orchestra/xapi')
+
+const { DeltaBackupWriter } = require('./writers/DeltaBackupWriter.js')
+const { DeltaReplicationWriter } = require('./writers/DeltaReplicationWriter.js')
+const { exportDeltaVm } = require('./_deltaVm.js')
+const { forkStreamUnpipe } = require('./_forkStreamUnpipe.js')
+const { FullBackupWriter } = require('./writers/FullBackupWriter.js')
+const { FullReplicationWriter } = require('./writers/FullReplicationWriter.js')
+const { getOldEntries } = require('./_getOldEntries.js')
+const { Task } = require('./Task.js')
+const { watchStreamSize } = require('./_watchStreamSize.js')
+
+const { debug, warn } = createLogger('xo:backups:VmBackup')
+
+const asyncEach = async (iterable, fn, thisArg = iterable) => {
+  for (const item of iterable) {
+    await fn.call(thisArg, item)
+  }
+}
+
+const forkDeltaExport = deltaExport =>
+  Object.create(deltaExport, {
+    streams: {
+      value: mapValues(deltaExport.streams, forkStreamUnpipe),
+    },
+  })
+
+exports.VmBackup = class VmBackup {
+  constructor({ config, getSnapshotNameLabel, job, remoteAdapters, remotes, schedule, settings, srs, vm }) {
+    this.config = config
+    this.job = job
+    this.remoteAdapters = remoteAdapters
+    this.remotes = remotes
+    this.scheduleId = schedule.id
+    this.timestamp = undefined
+
+    // VM currently backed up
+    this.vm = vm
+    const { tags } = this.vm
+
+    // VM (snapshot) that is really exported
+    this.exportedVm = undefined
+
+    this._fullVdisRequired = undefined
+    this._getSnapshotNameLabel = getSnapshotNameLabel
+    this._isDelta = job.mode === 'delta'
+    this._jobId = job.id
+    this._jobSnapshots = undefined
+    this._xapi = vm.$xapi
+
+    // Base VM for the export
+    this._baseVm = undefined
+
+    // Settings for this specific run (job, schedule, VM)
+    if (tags.includes('xo-memory-backup')) {
+      settings.checkpointSnapshot = true
+    }
+    if (tags.includes('xo-offline-backup')) {
+      settings.offlineSnapshot = true
+    }
+    this._settings = settings
+
+    // Create writers
+    {
+      const writers = new Set()
+      this._writers = writers
+
+      const [BackupWriter, ReplicationWriter] = this._isDelta
+        ? [DeltaBackupWriter, DeltaReplicationWriter]
+        : [FullBackupWriter, FullReplicationWriter]
+
+      const allSettings = job.settings
+
+      Object.keys(remoteAdapters).forEach(remoteId => {
+        const targetSettings = {
+          ...settings,
+          ...allSettings[remoteId],
+        }
+        if (targetSettings.exportRetention !== 0) {
+          writers.add(new BackupWriter({ backup: this, remoteId, settings: targetSettings }))
+        }
+      })
+      srs.forEach(sr => {
+        const targetSettings = {
+          ...settings,
+          ...allSettings[sr.uuid],
+        }
+        if (targetSettings.copyRetention !== 0) {
+          writers.add(new ReplicationWriter({ backup: this, sr, settings: targetSettings }))
+        }
+      })
+    }
+  }
+
+  // calls fn for each function, warns of any errors, and throws only if there are no writers left
+  async _callWriters(fn, warnMessage, parallel = true) {
+    const writers = this._writers
+    const n = writers.size
+    if (n === 0) {
+      return
+    }
+    if (n === 1) {
+      const [writer] = writers
+      try {
+        await fn(writer)
+      } catch (error) {
+        writers.delete(writer)
+        throw error
+      }
+      return
+    }
+
+    await (parallel ? asyncMap : asyncEach)(writers, async function (writer) {
+      try {
+        await fn(writer)
+      } catch (error) {
+        this.delete(writer)
+        warn(warnMessage, { error, writer: writer.constructor.name })
+      }
+    })
+    if (writers.size === 0) {
+      throw new Error('all targets have failed, step: ' + warnMessage)
+    }
+  }
+
+  // ensure the VM itself does not have any backup metadata which would be
+  // copied on manual snapshots and interfere with the backup jobs
+  async _cleanMetadata() {
+    const { vm } = this
+    if ('xo:backup:job' in vm.other_config) {
+      await vm.update_other_config({
+        'xo:backup:datetime': null,
+        'xo:backup:deltaChainLength': null,
+        'xo:backup:exported': null,
+        'xo:backup:job': null,
+        'xo:backup:schedule': null,
+        'xo:backup:vm': null,
+      })
+    }
+  }
+
+  async _snapshot() {
+    const { vm } = this
+    const xapi = this._xapi
+
+    const settings = this._settings
+
+    const doSnapshot =
+      this._isDelta || (!settings.offlineBackup && vm.power_state === 'Running') || settings.snapshotRetention !== 0
+    if (doSnapshot) {
+      await Task.run({ name: 'snapshot' }, async () => {
+        if (!settings.bypassVdiChainsCheck) {
+          await vm.$assertHealthyVdiChains()
+        }
+
+        const snapshotRef = await vm[settings.checkpointSnapshot ? '$checkpoint' : '$snapshot']({
+          name_label: this._getSnapshotNameLabel(vm),
+        })
+        this.timestamp = Date.now()
+
+        await xapi.setFieldEntries('VM', snapshotRef, 'other_config', {
+          'xo:backup:datetime': formatDateTime(this.timestamp),
+          'xo:backup:job': this._jobId,
+          'xo:backup:schedule': this.scheduleId,
+          'xo:backup:vm': vm.uuid,
+        })
+
+        this.exportedVm = await xapi.getRecord('VM', snapshotRef)
+
+        return this.exportedVm.uuid
+      })
+    } else {
+      this.exportedVm = vm
+      this.timestamp = Date.now()
+    }
+  }
+
+  async _copyDelta() {
+    const { exportedVm } = this
+    const baseVm = this._baseVm
+    const fullVdisRequired = this._fullVdisRequired
+
+    const isFull = fullVdisRequired === undefined || fullVdisRequired.size !== 0
+
+    await this._callWriters(writer => writer.prepare({ isFull }), 'writer.prepare()')
+
+    const deltaExport = await exportDeltaVm(exportedVm, baseVm, {
+      fullVdisRequired,
+    })
+    const sizeContainers = mapValues(deltaExport.streams, stream => watchStreamSize(stream))
+
+    const timestamp = Date.now()
+
+    await this._callWriters(
+      writer =>
+        writer.transfer({
+          deltaExport: forkDeltaExport(deltaExport),
+          sizeContainers,
+          timestamp,
+        }),
+      'writer.transfer()'
+    )
+
+    this._baseVm = exportedVm
+
+    if (baseVm !== undefined) {
+      await exportedVm.update_other_config(
+        'xo:backup:deltaChainLength',
+        String(+(baseVm.other_config['xo:backup:deltaChainLength'] ?? 0) + 1)
+      )
+    }
+
+    // not the case if offlineBackup
+    if (exportedVm.is_a_snapshot) {
+      await exportedVm.update_other_config('xo:backup:exported', 'true')
+    }
+
+    const size = Object.values(sizeContainers).reduce((sum, { size }) => sum + size, 0)
+    const end = Date.now()
+    const duration = end - timestamp
+    debug('transfer complete', {
+      duration,
+      speed: duration !== 0 ? (size * 1e3) / 1024 / 1024 / duration : 0,
+      size,
+    })
+
+    await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
+  }
+
+  async _copyFull() {
+    const { compression } = this.job
+    const stream = await this._xapi.VM_export(this.exportedVm.$ref, {
+      compress: Boolean(compression) && (compression === 'native' ? 'gzip' : 'zstd'),
+      useSnapshot: false,
+    })
+    const sizeContainer = watchStreamSize(stream)
+
+    const timestamp = Date.now()
+
+    await this._callWriters(
+      writer =>
+        writer.run({
+          sizeContainer,
+          stream: forkStreamUnpipe(stream),
+          timestamp,
+        }),
+      'writer.run()'
+    )
+
+    const { size } = sizeContainer
+    const end = Date.now()
+    const duration = end - timestamp
+    debug('transfer complete', {
+      duration,
+      speed: duration !== 0 ? (size * 1e3) / 1024 / 1024 / duration : 0,
+      size,
+    })
+  }
+
+  async _fetchJobSnapshots() {
+    const jobId = this._jobId
+    const vmRef = this.vm.$ref
+    const xapi = this._xapi
+
+    const snapshotsRef = await xapi.getField('VM', vmRef, 'snapshots')
+    const snapshotsOtherConfig = await asyncMap(snapshotsRef, ref => xapi.getField('VM', ref, 'other_config'))
+
+    const snapshots = []
+    snapshotsOtherConfig.forEach((other_config, i) => {
+      if (other_config['xo:backup:job'] === jobId) {
+        snapshots.push({ other_config, $ref: snapshotsRef[i] })
+      }
+    })
+    snapshots.sort((a, b) => (a.other_config['xo:backup:datetime'] < b.other_config['xo:backup:datetime'] ? -1 : 1))
+    this._jobSnapshots = snapshots
+  }
+
+  async _removeUnusedSnapshots() {
+    // TODO: handle all schedules (no longer existing schedules default to 0 retention)
+
+    const { scheduleId } = this
+    const scheduleSnapshots = this._jobSnapshots.filter(_ => _.other_config['xo:backup:schedule'] === scheduleId)
+
+    const baseVmRef = this._baseVm?.$ref
+    const xapi = this._xapi
+    await asyncMap(getOldEntries(this._settings.snapshotRetention, scheduleSnapshots), ({ $ref }) => {
+      if ($ref !== baseVmRef) {
+        return xapi.VM_destroy($ref)
+      }
+    })
+  }
+
+  async _selectBaseVm() {
+    const xapi = this._xapi
+
+    let baseVm = findLast(this._jobSnapshots, _ => 'xo:backup:exported' in _.other_config)
+    if (baseVm === undefined) {
+      return
+    }
+
+    const fullInterval = this._settings.fullInterval
+    const deltaChainLength = +(baseVm.other_config['xo:backup:deltaChainLength'] ?? 0) + 1
+    if (!(fullInterval === 0 || fullInterval > deltaChainLength)) {
+      return
+    }
+
+    const srcVdis = keyBy(await xapi.getRecords('VDI', await this.vm.$getDisks()), '$ref')
+
+    // resolve full record
+    baseVm = await xapi.getRecord('VM', baseVm.$ref)
+
+    const baseUuidToSrcVdi = new Map()
+    await asyncMap(await baseVm.$getDisks(), async baseRef => {
+      const snapshotOf = await xapi.getField('VDI', baseRef, 'snapshot_of')
+      const srcVdi = srcVdis[snapshotOf]
+      if (srcVdi !== undefined) {
+        baseUuidToSrcVdi.set(await xapi.getField('VDI', baseRef, 'uuid'), srcVdi)
+      }
+    })
+
+    const presentBaseVdis = new Map(baseUuidToSrcVdi)
+    await this._callWriters(
+      writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis, baseVm),
+      'writer.checkBaseVdis()',
+      false
+    )
+
+    const fullVdisRequired = new Set()
+    baseUuidToSrcVdi.forEach((srcVdi, baseUuid) => {
+      if (!presentBaseVdis.has(baseUuid)) {
+        fullVdisRequired.add(srcVdi.uuid)
+      }
+    })
+
+    this._baseVm = baseVm
+    this._fullVdisRequired = fullVdisRequired
+  }
+
+  run = defer(this.run)
+  async run($defer) {
+    const settings = this._settings
+    assert(
+      !settings.offlineBackup || settings.snapshotRetention === 0,
+      'offlineBackup is not compatible with snapshotRetention'
+    )
+
+    await this._callWriters(async writer => {
+      await writer.beforeBackup()
+      $defer(() => writer.afterBackup())
+    }, 'writer.beforeBackup()')
+
+    await this._fetchJobSnapshots()
+
+    if (this._isDelta) {
+      await this._selectBaseVm()
+    }
+
+    await this._cleanMetadata()
+    await this._removeUnusedSnapshots()
+
+    const { vm } = this
+    const isRunning = vm.power_state === 'Running'
+    const startAfter = isRunning && (settings.offlineBackup ? 'backup' : settings.offlineSnapshot && 'snapshot')
+    if (startAfter) {
+      await vm.$callAsync('clean_shutdown')
+    }
+
+    try {
+      await this._snapshot()
+      if (startAfter === 'snapshot') {
+        ignoreErrors.call(vm.$callAsync('start', false, false))
+      }
+
+      if (this._writers.size !== 0) {
+        await (this._isDelta ? this._copyDelta() : this._copyFull())
+      }
+    } finally {
+      if (startAfter) {
+        ignoreErrors.call(vm.$callAsync('start', false, false))
+      }
+
+      await this._fetchJobSnapshots()
+      await this._removeUnusedSnapshots()
+    }
+  }
+}

@xen-orchestra/backups/_XoMetadataBackup.js

@@ -0,0 +1,62 @@
+const { asyncMap } = require('@xen-orchestra/async-map')
+
+const { DIR_XO_CONFIG_BACKUPS } = require('./RemoteAdapter.js')
+const { formatFilenameDate } = require('./_filenameDate.js')
+const { Task } = require('./Task.js')
+
+exports.XoMetadataBackup = class XoMetadataBackup {
+  constructor({ config, job, remoteAdapters, schedule, settings }) {
+    this._config = config
+    this._job = job
+    this._remoteAdapters = remoteAdapters
+    this._schedule = schedule
+    this._settings = settings
+  }
+
+  async run() {
+    const timestamp = Date.now()
+
+    const { _job: job, _schedule: schedule } = this
+    const scheduleDir = `${DIR_XO_CONFIG_BACKUPS}/${schedule.id}`
+    const dir = `${scheduleDir}/${formatFilenameDate(timestamp)}`
+
+    const data = job.xoMetadata
+    const fileName = `${dir}/data.json`
+
+    const metadata = JSON.stringify(
+      {
+        jobId: job.id,
+        jobName: job.name,
+        scheduleId: schedule.id,
+        scheduleName: schedule.name,
+        timestamp,
+      },
+      null,
+      2
+    )
+    const metaDataFileName = `${dir}/metadata.json`
+
+    await asyncMap(
+      Object.entries(this._remoteAdapters),
+      ([remoteId, adapter]) =>
+        Task.run(
+          {
+            name: `Starting XO metadata backup for the remote (${remoteId}). (${job.id})`,
+            data: {
+              id: remoteId,
+              type: 'remote',
+            },
+          },
+          async () => {
+            const handler = adapter.handler
+            const dirMode = this._config.dirMode
+            await handler.outputFile(fileName, data, { dirMode })
+            await handler.outputFile(metaDataFileName, metadata, {
+              dirMode,
+            })
+            await adapter.deleteOldMetadataBackups(scheduleDir, this._settings.retentionXoMetadata)
+          }
+        ).catch(() => {}) // errors are handled by logs
+    )
+  }
+}

@xen-orchestra/backups/_backupType.js

@@ -0,0 +1,4 @@
+exports.isMetadataFile = filename => filename.endsWith('.json')
+exports.isVhdFile = filename => filename.endsWith('.vhd')
+exports.isXvaFile = filename => filename.endsWith('.xva')
+exports.isXvaSumFile = filename => filename.endsWith('.xva.cheksum')

@xen-orchestra/backups/_backupWorker.js

@@ -0,0 +1,155 @@
+require('@xen-orchestra/log/configure.js').catchGlobalErrors(
+  require('@xen-orchestra/log').createLogger('xo:backups:worker')
+)
+
+const Disposable = require('promise-toolbox/Disposable.js')
+const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
+const { compose } = require('@vates/compose')
+const { createDebounceResource } = require('@vates/disposable/debounceResource.js')
+const { deduped } = require('@vates/disposable/deduped.js')
+const { getHandler } = require('@xen-orchestra/fs')
+const { parseDuration } = require('@vates/parse-duration')
+const { Xapi } = require('@xen-orchestra/xapi')
+
+const { Backup } = require('./Backup.js')
+const { RemoteAdapter } = require('./RemoteAdapter.js')
+const { Task } = require('./Task.js')
+
+class BackupWorker {
+  #config
+  #job
+  #recordToXapi
+  #remoteOptions
+  #remotes
+  #schedule
+  #xapiOptions
+  #xapis
+
+  constructor({ config, job, recordToXapi, remoteOptions, remotes, resourceCacheDelay, schedule, xapiOptions, xapis }) {
+    this.#config = config
+    this.#job = job
+    this.#recordToXapi = recordToXapi
+    this.#remoteOptions = remoteOptions
+    this.#remotes = remotes
+    this.#schedule = schedule
+    this.#xapiOptions = xapiOptions
+    this.#xapis = xapis
+
+    const debounceResource = createDebounceResource()
+    debounceResource.defaultDelay = parseDuration(resourceCacheDelay)
+    this.debounceResource = debounceResource
+  }
+
+  run() {
+    return new Backup({
+      config: this.#config,
+      getAdapter: remoteId => this.getAdapter(this.#remotes[remoteId]),
+      getConnectedRecord: Disposable.factory(async function* getConnectedRecord(type, uuid) {
+        const xapiId = this.#recordToXapi[uuid]
+        if (xapiId === undefined) {
+          throw new Error('no XAPI associated to ' + uuid)
+        }
+
+        const xapi = yield this.getXapi(this.#xapis[xapiId])
+        return xapi.getRecordByUuid(type, uuid)
+      }).bind(this),
+      job: this.#job,
+      schedule: this.#schedule,
+    }).run()
+  }
+
+  getAdapter = Disposable.factory(this.getAdapter)
+  getAdapter = deduped(this.getAdapter, remote => [remote.url])
+  getAdapter = compose(this.getAdapter, function (resource) {
+    return this.debounceResource(resource)
+  })
+  async *getAdapter(remote) {
+    const handler = getHandler(remote, this.#remoteOptions)
+    await handler.sync()
+    try {
+      yield new RemoteAdapter(handler, {
+        debounceResource: this.debounceResource,
+        dirMode: this.#config.dirMode,
+      })
+    } finally {
+      await handler.forget()
+    }
+  }
+
+  getXapi = Disposable.factory(this.getXapi)
+  getXapi = deduped(this.getXapi, ({ url }) => [url])
+  getXapi = compose(this.getXapi, function (resource) {
+    return this.debounceResource(resource)
+  })
+  async *getXapi({ credentials: { username: user, password }, ...opts }) {
+    const xapi = new Xapi({
+      ...this.#xapiOptions,
+      ...opts,
+      auth: {
+        user,
+        password,
+      },
+    })
+
+    await xapi.connect()
+    try {
+      await xapi.objectsFetched
+
+      yield xapi
+    } finally {
+      await xapi.disconnect()
+    }
+  }
+}
+
+// Received message:
+//
+// Message {
+//   action: 'run'
+//   data: object
+//   runWithLogs: boolean
+// }
+//
+// Sent message:
+//
+// Message {
+//   type: 'log' | 'result'
+//   data?: object
+//   status?: 'success' | 'failure'
+//   result?: any
+// }
+process.on('message', async message => {
+  if (message.action === 'run') {
+    const backupWorker = new BackupWorker(message.data)
+    try {
+      const result = message.runWithLogs
+        ? await Task.run(
+            {
+              name: 'backup run',
+              onLog: data =>
+                process.send({
+                  data,
+                  type: 'log',
+                }),
+            },
+            () => backupWorker.run()
+          )
+        : await backupWorker.run()
+
+      process.send({
+        type: 'result',
+        result,
+        status: 'success',
+      })
+    } catch (error) {
+      process.send({
+        type: 'result',
+        result: error,
+        status: 'failure',
+      })
+    } finally {
+      await ignoreErrors.call(backupWorker.debounceResource.flushAll())
+      process.disconnect()
+    }
+  }
+})

@xen-orchestra/backups/_cancelableMap.js

@@ -0,0 +1,20 @@
+const cancelable = require('promise-toolbox/cancelable.js')
+const CancelToken = require('promise-toolbox/CancelToken.js')
+
+// Similar to `Promise.all` + `map` but pass a cancel token to the callback
+//
+// If any of the executions fails, the cancel token will be triggered and the
+// first reason will be rejected.
+exports.cancelableMap = cancelable(async function cancelableMap($cancelToken, iterable, callback) {
+  const { cancel, token } = CancelToken.source([$cancelToken])
+  try {
+    return await Promise.all(
+      Array.from(iterable, function (item) {
+        return callback.call(this, token, item)
+      })
+    )
+  } catch (error) {
+    await cancel()
+    throw error
+  }
+})

@xen-orchestra/backups/_cleanVm.js

@@ -0,0 +1,358 @@
+const assert = require('assert')
+const sum = require('lodash/sum')
+const { asyncMap } = require('@xen-orchestra/async-map')
+const { default: Vhd, mergeVhd } = require('vhd-lib')
+const { dirname, resolve } = require('path')
+const { DISK_TYPE_DIFFERENCING } = require('vhd-lib/dist/_constants.js')
+const { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } = require('./_backupType.js')
+const { limitConcurrency } = require('limit-concurrency-decorator')
+
+// chain is an array of VHDs from child to parent
+//
+// the whole chain will be merged into parent, parent will be renamed to child
+// and all the others will deleted
+const mergeVhdChain = limitConcurrency(1)(async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
+  assert(chain.length >= 2)
+
+  let child = chain[0]
+  const parent = chain[chain.length - 1]
+  const children = chain.slice(0, -1).reverse()
+
+  chain
+    .slice(1)
+    .reverse()
+    .forEach(parent => {
+      onLog(`the parent ${parent} of the child ${child} is unused`)
+    })
+
+  if (merge) {
+    // `mergeVhd` does not work with a stream, either
+    // - make it accept a stream
+    // - or create synthetic VHD which is not a stream
+    if (children.length !== 1) {
+      // TODO: implement merging multiple children
+      children.length = 1
+      child = children[0]
+    }
+
+    onLog(`merging ${child} into ${parent}`)
+
+    let done, total
+    const handle = setInterval(() => {
+      if (done !== undefined) {
+        onLog(`merging ${child}: ${done}/${total}`)
+      }
+    }, 10e3)
+
+    await mergeVhd(
+      handler,
+      parent,
+      handler,
+      child,
+      // children.length === 1
+      //   ? child
+      //   : await createSyntheticStream(handler, children),
+      {
+        onProgress({ done: d, total: t }) {
+          done = d
+          total = t
+        },
+      }
+    )
+
+    clearInterval(handle)
+
+    await Promise.all([
+      handler.rename(parent, child),
+      asyncMap(children.slice(0, -1), child => {
+        onLog(`the VHD ${child} is unused`)
+        if (remove) {
+          onLog(`deleting unused VHD ${child}`)
+          return handler.unlink(child)
+        }
+      }),
+    ])
+  }
+})
+
+const noop = Function.prototype
+
+const INTERRUPTED_VHDS_REG = /^(?:(.+)\/)?\.(.+)\.merge.json$/
+const listVhds = async (handler, vmDir) => {
+  const vhds = []
+  const interruptedVhds = new Set()
+
+  await asyncMap(
+    await handler.list(`${vmDir}/vdis`, {
+      ignoreMissing: true,
+      prependDir: true,
+    }),
+    async jobDir =>
+      asyncMap(
+        await handler.list(jobDir, {
+          prependDir: true,
+        }),
+        async vdiDir => {
+          const list = await handler.list(vdiDir, {
+            filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file),
+            prependDir: true,
+          })
+
+          list.forEach(file => {
+            const res = INTERRUPTED_VHDS_REG.exec(file)
+            if (res === null) {
+              vhds.push(file)
+            } else {
+              const [, dir, file] = res
+              interruptedVhds.add(`${dir}/${file}`)
+            }
+          })
+        }
+      )
+  )
+
+  return { vhds, interruptedVhds }
+}
+
+exports.cleanVm = async function cleanVm(vmDir, { fixMetadata, remove, merge, onLog = noop }) {
+  const handler = this._handler
+
+  const vhds = new Set()
+  const vhdParents = { __proto__: null }
+  const vhdChildren = { __proto__: null }
+
+  const vhdsList = await listVhds(handler, vmDir)
+
+  // remove broken VHDs
+  await asyncMap(vhdsList.vhds, async path => {
+    try {
+      const vhd = new Vhd(handler, path)
+      await vhd.readHeaderAndFooter(!vhdsList.interruptedVhds.has(path))
+      vhds.add(path)
+      if (vhd.footer.diskType === DISK_TYPE_DIFFERENCING) {
+        const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
+        vhdParents[path] = parent
+        if (parent in vhdChildren) {
+          const error = new Error('this script does not support multiple VHD children')
+          error.parent = parent
+          error.child1 = vhdChildren[parent]
+          error.child2 = path
+          throw error // should we throw?
+        }
+        vhdChildren[parent] = path
+      }
+    } catch (error) {
+      onLog(`error while checking the VHD with path ${path}`, { error })
+      if (error?.code === 'ERR_ASSERTION' && remove) {
+        onLog(`deleting broken ${path}`)
+        await handler.unlink(path)
+      }
+    }
+  })
+
+  // remove VHDs with missing ancestors
+  {
+    const deletions = []
+
+    // return true if the VHD has been deleted or is missing
+    const deleteIfOrphan = vhd => {
+      const parent = vhdParents[vhd]
+      if (parent === undefined) {
+        return
+      }
+
+      // no longer needs to be checked
+      delete vhdParents[vhd]
+
+      deleteIfOrphan(parent)
+
+      if (!vhds.has(parent)) {
+        vhds.delete(vhd)
+
+        onLog(`the parent ${parent} of the VHD ${vhd} is missing`)
+        if (remove) {
+          onLog(`deleting orphan VHD ${vhd}`)
+          deletions.push(handler.unlink(vhd))
+        }
+      }
+    }
+
+    // > A property that is deleted before it has been visited will not be
+    // > visited later.
+    // >
+    // > -- https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for...in#Deleted_added_or_modified_properties
+    for (const child in vhdParents) {
+      deleteIfOrphan(child)
+    }
+
+    await Promise.all(deletions)
+  }
+
+  const jsons = []
+  const xvas = new Set()
+  const xvaSums = []
+  const entries = await handler.list(vmDir, {
+    prependDir: true,
+  })
+  entries.forEach(path => {
+    if (isMetadataFile(path)) {
+      jsons.push(path)
+    } else if (isXvaFile(path)) {
+      xvas.add(path)
+    } else if (isXvaSumFile(path)) {
+      xvaSums.push(path)
+    }
+  })
+
+  await asyncMap(xvas, async path => {
+    // check is not good enough to delete the file, the best we can do is report
+    // it
+    if (!(await this.isValidXva(path))) {
+      onLog(`the XVA with path ${path} is potentially broken`)
+    }
+  })
+
+  const unusedVhds = new Set(vhds)
+  const unusedXvas = new Set(xvas)
+
+  // compile the list of unused XVAs and VHDs, and remove backup metadata which
+  // reference a missing XVA/VHD
+  await asyncMap(jsons, async json => {
+    const metadata = JSON.parse(await handler.readFile(json))
+    const { mode } = metadata
+    let size
+    if (mode === 'full') {
+      const linkedXva = resolve('/', vmDir, metadata.xva)
+
+      if (xvas.has(linkedXva)) {
+        unusedXvas.delete(linkedXva)
+
+        size = await handler.getSize(linkedXva).catch(error => {
+          onLog(`failed to get size of ${json}`, { error })
+        })
+      } else {
+        onLog(`the XVA linked to the metadata ${json} is missing`)
+        if (remove) {
+          onLog(`deleting incomplete backup ${json}`)
+          await handler.unlink(json)
+        }
+      }
+    } else if (mode === 'delta') {
+      const linkedVhds = (() => {
+        const { vhds } = metadata
+        return Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
+      })()
+
+      // FIXME: find better approach by keeping as much of the backup as
+      // possible (existing disks) even if one disk is missing
+      if (linkedVhds.every(_ => vhds.has(_))) {
+        linkedVhds.forEach(_ => unusedVhds.delete(_))
+
+        size = await asyncMap(linkedVhds, vhd => handler.getSize(vhd)).then(sum, error => {
+          onLog(`failed to get size of ${json}`, { error })
+        })
+      } else {
+        onLog(`Some VHDs linked to the metadata ${json} are missing`)
+        if (remove) {
+          onLog(`deleting incomplete backup ${json}`)
+          await handler.unlink(json)
+        }
+      }
+    }
+
+    const metadataSize = metadata.size
+    if (size !== undefined && metadataSize !== size) {
+      onLog(`incorrect size in metadata: ${metadataSize ?? 'none'} instead of ${size}`)
+
+      // don't update if the the stored size is greater than found files,
+      // it can indicates a problem
+      if (fixMetadata && (metadataSize === undefined || metadataSize < size)) {
+        try {
+          metadata.size = size
+          await handler.writeFile(json, JSON.stringify(metadata), { flags: 'w' })
+        } catch (error) {
+          onLog(`failed to update size in backup metadata ${json}`, { error })
+        }
+      }
+    }
+  })
+
+  // TODO: parallelize by vm/job/vdi
+  const unusedVhdsDeletion = []
+  {
+    // VHD chains (as list from child to ancestor) to merge indexed by last
+    // ancestor
+    const vhdChainsToMerge = { __proto__: null }
+
+    const toCheck = new Set(unusedVhds)
+
+    const getUsedChildChainOrDelete = vhd => {
+      if (vhd in vhdChainsToMerge) {
+        const chain = vhdChainsToMerge[vhd]
+        delete vhdChainsToMerge[vhd]
+        return chain
+      }
+
+      if (!unusedVhds.has(vhd)) {
+        return [vhd]
+      }
+
+      // no longer needs to be checked
+      toCheck.delete(vhd)
+
+      const child = vhdChildren[vhd]
+      if (child !== undefined) {
+        const chain = getUsedChildChainOrDelete(child)
+        if (chain !== undefined) {
+          chain.push(vhd)
+          return chain
+        }
+      }
+
+      onLog(`the VHD ${vhd} is unused`)
+      if (remove) {
+        onLog(`deleting unused VHD ${vhd}`)
+        unusedVhdsDeletion.push(handler.unlink(vhd))
+      }
+    }
+
+    toCheck.forEach(vhd => {
+      vhdChainsToMerge[vhd] = getUsedChildChainOrDelete(vhd)
+    })
+
+    // merge interrupted VHDs
+    if (merge) {
+      vhdsList.interruptedVhds.forEach(parent => {
+        vhdChainsToMerge[parent] = [vhdChildren[parent], parent]
+      })
+    }
+
+    Object.keys(vhdChainsToMerge).forEach(key => {
+      const chain = vhdChainsToMerge[key]
+      if (chain !== undefined) {
+        unusedVhdsDeletion.push(mergeVhdChain(chain, { handler, onLog, remove, merge }))
+      }
+    })
+  }
+
+  await Promise.all([
+    ...unusedVhdsDeletion,
+    asyncMap(unusedXvas, path => {
+      onLog(`the XVA ${path} is unused`)
+      if (remove) {
+        onLog(`deleting unused XVA ${path}`)
+        return handler.unlink(path)
+      }
+    }),
+    asyncMap(xvaSums, path => {
+      // no need to handle checksums for XVAs deleted by the script, they will be handled by `unlink()`
+      if (!xvas.has(path.slice(0, -'.checksum'.length))) {
+        onLog(`the XVA checksum ${path} is unused`)
+        if (remove) {
+          onLog(`deleting unused XVA checksum ${path}`)
+          return handler.unlink(path)
+        }
+      }
+    }),
+  ])
+}