show debug once per cleanVm

Introduced by 3f1c41a4f

Fixes zammad#6819 zammad#6781 zammad#6827

In #6186 the behavior was changed to always pass hosts, which broke the pool wide detection.

.eslintrc.js

@@ -1,5 +1,7 @@
+'use strict'
+
 module.exports = {
-  extends: ['plugin:eslint-comments/recommended', 'standard', 'standard-jsx', 'prettier'],
+  extends: ['plugin:eslint-comments/recommended', 'plugin:n/recommended', 'standard', 'standard-jsx', 'prettier'],
   globals: {
     __DEV__: true,
     $Dict: true,
@@ -13,19 +15,42 @@ module.exports = {
 
   overrides: [
     {
-      files: ['cli.js', '*-cli.js', '**/*cli*/**/*.js'],
+      files: ['cli.{,c,m}js', '*-cli.{,c,m}js', '**/*cli*/**/*.{,c,m}js'],
       rules: {
+        'n/no-process-exit': 'off',
         'no-console': 'off',
       },
     },
+    {
+      files: ['*.mjs'],
+      parserOptions: {
+        sourceType: 'module',
+      },
+    },
+    {
+      files: ['*.spec.{,c,m}js'],
+      rules: {
+        'n/no-unsupported-features/node-builtins': [
+          'error',
+          {
+            version: '>=16',
+          },
+        ],
+        'n/no-unsupported-features/es-syntax': [
+          'error',
+          {
+            version: '>=16',
+          },
+        ],
+      },
+    },
   ],
 
-  parser: 'babel-eslint',
   parserOptions: {
-    ecmaFeatures: {
-      legacyDecorators: true,
-    },
+    ecmaVersion: 13,
+    sourceType: 'script',
   },
+
   rules: {
     // disabled because XAPI objects are using camel case
     camelcase: ['off'],
@@ -40,5 +65,7 @@ module.exports = {
     'lines-between-class-members': 'off',
 
     'no-console': ['error', { allow: ['warn', 'error'] }],
+
+    strict: 'error',
   },
 }

.flowconfig

@@ -1,16 +0,0 @@
-[ignore]
-<PROJECT_ROOT>/node_modules/.*
-
-[include]
-
-[libs]
-
-[lints]
-
-[options]
-esproposal.decorators=ignore
-esproposal.optional_chaining=enable
-include_warnings=true
-module.use_strict=true
-
-[strict]

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,46 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: 'status: triaging :triangular_flag_on_post:, type: bug :bug:'
+assignees: ''
+---
+
+**XOA or XO from the sources?**
+
+If XOA:
+
+- which release channel? (`stable` vs `latest`)
+- please consider creating a support ticket in [your dedicated support area](https://xen-orchestra.com/#!/member/support)
+
+If XO from the sources:
+
+- Don't forget to [read this first](https://xen-orchestra.com/docs/community.html)
+- As well as follow [this guide](https://xen-orchestra.com/docs/community.html#report-a-bug)
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Environment (please provide the following information):**
+
+- Node: [e.g. 16.12.1]
+- xo-server: [e.g. 5.82.3]
+- xo-web: [e.g. 5.87.0]
+- hypervisor: [e.g. XCP-ng 8.2.0]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/push.yml

@@ -0,0 +1,13 @@
+name: CI
+on: [push]
+jobs:
+  build:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: satackey/action-docker-layer-caching@v0.0.11
+      # Ignore the failure of a step and avoid terminating the job.
+        continue-on-error: true
+      - run: docker-compose  -f docker/docker-compose.dev.yml build
+      - run: docker-compose  -f docker/docker-compose.dev.yml up

.gitignore

@@ -1,5 +1,4 @@
 /_book/
-/coverage/
 /node_modules/
 /lerna-debug.log
 /lerna-debug.log.*
@@ -11,17 +10,15 @@
 /packages/*/dist/
 /packages/*/node_modules/
 
-/@xen-orchestra/proxy/src/app/mixins/index.js
-
 /packages/vhd-cli/src/commands/index.js
 
 /packages/xen-api/examples/node_modules/
 /packages/xen-api/plot.dat
 
 /packages/xo-server/.xo-server.*
-/packages/xo-server/src/api/index.js
-/packages/xo-server/src/xapi/mixins/index.js
-/packages/xo-server/src/xo-mixins/index.js
+/packages/xo-server/src/api/index.mjs
+/packages/xo-server/src/xapi/mixins/index.mjs
+/packages/xo-server/src/xo-mixins/index.mjs
 
 /packages/xo-server-auth-ldap/ldap.cache.conf
 
@@ -35,3 +32,7 @@ pnpm-debug.log.*
 yarn-error.log
 yarn-error.log.*
 .env
+
+# code coverage
+.nyc_output/
+coverage/

.travis.yml

@@ -1,23 +0,0 @@
-language: node_js
-node_js:
-  - 14
-
-# Use containers.
-# http://docs.travis-ci.com/user/workers/container-based-infrastructure/
-sudo: false
-addons:
-  apt:
-    packages:
-      - qemu-utils
-      - blktap-utils
-      - vmdk-stream-converter
-
-before_install:
-  - curl -o- -L https://yarnpkg.com/install.sh | bash
-  - export PATH="$HOME/.yarn/bin:$PATH"
-
-cache:
-  yarn: true
-
-script:
-  - yarn run travis-tests

@vates/async-each/.USAGE.md

@@ -0,0 +1,35 @@
+### `asyncEach(iterable, iteratee, [opts])`
+
+Executes `iteratee` in order for each value yielded by `iterable`.
+
+Returns a promise wich rejects as soon as a call to `iteratee` throws or a promise returned by it rejects, and which resolves when all promises returned by `iteratee` have resolved.
+
+`iterable` must be an iterable or async iterable.
+
+`iteratee` is called with the same `this` value as `asyncEach`, and with the following arguments:
+
+- `value`: the value yielded by `iterable`
+- `index`: the 0-based index for this value
+- `iterable`: the iterable itself
+
+`opts` is an object that can contains the following options:
+
+- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
+- `signal`: an abort signal to stop the iteration
+- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
+
+```js
+import { asyncEach } from '@vates/async-each'
+
+const contents = []
+await asyncEach(
+  ['foo.txt', 'bar.txt', 'baz.txt'],
+  async function (filename, i) {
+    contents[i] = await readFile(filename)
+  },
+  {
+    // reads two files at a time
+    concurrency: 2,
+  }
+)
+```

@vates/async-each/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/async-each/README.md

@@ -0,0 +1,68 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/async-each
+
+[![Package Version](https://badgen.net/npm/v/@vates/async-each)](https://npmjs.org/package/@vates/async-each) ![License](https://badgen.net/npm/license/@vates/async-each) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/async-each)](https://bundlephobia.com/result?p=@vates/async-each) [![Node compatibility](https://badgen.net/npm/node/@vates/async-each)](https://npmjs.org/package/@vates/async-each)
+
+> Run async fn for each item in (async) iterable
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/async-each):
+
+```
+> npm install --save @vates/async-each
+```
+
+## Usage
+
+### `asyncEach(iterable, iteratee, [opts])`
+
+Executes `iteratee` in order for each value yielded by `iterable`.
+
+Returns a promise wich rejects as soon as a call to `iteratee` throws or a promise returned by it rejects, and which resolves when all promises returned by `iteratee` have resolved.
+
+`iterable` must be an iterable or async iterable.
+
+`iteratee` is called with the same `this` value as `asyncEach`, and with the following arguments:
+
+- `value`: the value yielded by `iterable`
+- `index`: the 0-based index for this value
+- `iterable`: the iterable itself
+
+`opts` is an object that can contains the following options:
+
+- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
+- `signal`: an abort signal to stop the iteration
+- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
+
+```js
+import { asyncEach } from '@vates/async-each'
+
+const contents = []
+await asyncEach(
+  ['foo.txt', 'bar.txt', 'baz.txt'],
+  async function (filename, i) {
+    contents[i] = await readFile(filename)
+  },
+  {
+    // reads two files at a time
+    concurrency: 2,
+  }
+)
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/async-each/index.js

@@ -0,0 +1,99 @@
+'use strict'
+
+const noop = Function.prototype
+
+class AggregateError extends Error {
+  constructor(errors, message) {
+    super(message)
+    this.errors = errors
+  }
+}
+
+exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 1, signal, stopOnError = true } = {}) {
+  return new Promise((resolve, reject) => {
+    const it = (iterable[Symbol.iterator] || iterable[Symbol.asyncIterator]).call(iterable)
+    const errors = []
+    let running = 0
+    let index = 0
+
+    let onAbort
+    if (signal !== undefined) {
+      onAbort = () => {
+        onRejectedWrapper(new Error('asyncEach aborted'))
+      }
+      signal.addEventListener('abort', onAbort)
+    }
+
+    const clean = () => {
+      onFulfilled = onRejected = noop
+      if (onAbort !== undefined) {
+        signal.removeEventListener('abort', onAbort)
+      }
+    }
+
+    resolve = (resolve =>
+      function resolveAndClean(value) {
+        resolve(value)
+        clean()
+      })(resolve)
+    reject = (reject =>
+      function rejectAndClean(reason) {
+        reject(reason)
+        clean()
+      })(reject)
+
+    let onFulfilled = value => {
+      --running
+      next()
+    }
+    const onFulfilledWrapper = value => onFulfilled(value)
+
+    let onRejected = stopOnError
+      ? reject
+      : error => {
+          --running
+          errors.push(error)
+          next()
+        }
+    const onRejectedWrapper = reason => onRejected(reason)
+
+    let nextIsRunning = false
+    let next = async () => {
+      if (nextIsRunning) {
+        return
+      }
+      nextIsRunning = true
+      if (running < concurrency) {
+        const cursor = await it.next()
+        if (cursor.done) {
+          next = () => {
+            if (running === 0) {
+              if (errors.length === 0) {
+                resolve()
+              } else {
+                reject(new AggregateError(errors))
+              }
+            }
+          }
+        } else {
+          ++running
+          try {
+            const result = iteratee.call(this, cursor.value, index++, iterable)
+            let then
+            if (result != null && typeof result === 'object' && typeof (then = result.then) === 'function') {
+              then.call(result, onFulfilledWrapper, onRejectedWrapper)
+            } else {
+              onFulfilled(result)
+            }
+          } catch (error) {
+            onRejected(error)
+          }
+        }
+        nextIsRunning = false
+        return next()
+      }
+      nextIsRunning = false
+    }
+    next()
+  })
+}

@vates/async-each/index.spec.js

@@ -0,0 +1,99 @@
+'use strict'
+
+/* eslint-env jest */
+
+const { asyncEach } = require('./')
+
+const randomDelay = (max = 10) =>
+  new Promise(resolve => {
+    setTimeout(resolve, Math.floor(Math.random() * max + 1))
+  })
+
+const rejectionOf = p =>
+  new Promise((resolve, reject) => {
+    p.then(reject, resolve)
+  })
+
+describe('asyncEach', () => {
+  const thisArg = 'qux'
+  const values = ['foo', 'bar', 'baz']
+
+  Object.entries({
+    'sync iterable': () => values,
+    'async iterable': async function* () {
+      for (const value of values) {
+        await randomDelay()
+        yield value
+      }
+    },
+  }).forEach(([what, getIterable]) =>
+    describe('with ' + what, () => {
+      let iterable
+      beforeEach(() => {
+        iterable = getIterable()
+      })
+
+      it('works', async () => {
+        const iteratee = jest.fn(async () => {})
+
+        await asyncEach.call(thisArg, iterable, iteratee)
+
+        expect(iteratee.mock.instances).toEqual(Array.from(values, () => thisArg))
+        expect(iteratee.mock.calls).toEqual(Array.from(values, (value, index) => [value, index, iterable]))
+      })
+      ;[1, 2, 4].forEach(concurrency => {
+        it('respects a concurrency of ' + concurrency, async () => {
+          let running = 0
+
+          await asyncEach(
+            values,
+            async () => {
+              ++running
+              expect(running).toBeLessThanOrEqual(concurrency)
+              await randomDelay()
+              --running
+            },
+            { concurrency }
+          )
+        })
+      })
+
+      it('stops on first error when stopOnError is true', async () => {
+        const error = new Error()
+        const iteratee = jest.fn((_, i) => {
+          if (i === 1) {
+            throw error
+          }
+        })
+
+        expect(await rejectionOf(asyncEach(iterable, iteratee, { stopOnError: true }))).toBe(error)
+        expect(iteratee).toHaveBeenCalledTimes(2)
+      })
+
+      it('rejects AggregateError when stopOnError is false', async () => {
+        const errors = []
+        const iteratee = jest.fn(() => {
+          const error = new Error()
+          errors.push(error)
+          throw error
+        })
+
+        const error = await rejectionOf(asyncEach(iterable, iteratee, { stopOnError: false }))
+        expect(error.errors).toEqual(errors)
+        expect(iteratee.mock.calls).toEqual(Array.from(values, (value, index) => [value, index, iterable]))
+      })
+
+      it('can be interrupted with an AbortSignal', async () => {
+        const ac = new AbortController()
+        const iteratee = jest.fn((_, i) => {
+          if (i === 1) {
+            ac.abort()
+          }
+        })
+
+        await expect(asyncEach(iterable, iteratee, { signal: ac.signal })).rejects.toThrow('asyncEach aborted')
+        expect(iteratee).toHaveBeenCalledTimes(2)
+      })
+    })
+  )
+})

@vates/async-each/package.json

@@ -0,0 +1,34 @@
+{
+  "private": false,
+  "name": "@vates/async-each",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/async-each",
+  "description": "Run async fn for each item in (async) iterable",
+  "keywords": [
+    "array",
+    "async",
+    "collection",
+    "each",
+    "for",
+    "foreach",
+    "iterable",
+    "iterator"
+  ],
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/async-each",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "0.1.0",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@vates/cached-dns.lookup/.USAGE.md

@@ -0,0 +1,30 @@
+Node does not cache queries to `dns.lookup`, which can lead application doing a lot of connections to have perf issues and to saturate Node threads pool.
+
+This library attempts to mitigate these problems by providing a version of this function with a version short cache, applied on both errors and results.
+
+> Limitation: `verbatim: false` option is not supported.
+
+It has exactly the same API as the native method and can be used directly:
+
+```js
+import { createCachedLookup } from '@vates/cached-dns.lookup'
+
+const lookup = createCachedLookup()
+
+lookup('example.net', { all: true, family: 0 }, (error, result) => {
+  if (error != null) {
+    return console.warn(error)
+  }
+  console.log(result)
+})
+```
+
+Or it can be used to replace the native implementation and speed up the whole app:
+
+```js
+// assign our cached implementation to dns.lookup
+const restore = createCachedLookup().patchGlobal()
+
+// to restore the previous implementation
+restore()
+```

@vates/cached-dns.lookup/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/cached-dns.lookup/README.md

@@ -0,0 +1,63 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/cached-dns.lookup
+
+[![Package Version](https://badgen.net/npm/v/@vates/cached-dns.lookup)](https://npmjs.org/package/@vates/cached-dns.lookup) ![License](https://badgen.net/npm/license/@vates/cached-dns.lookup) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/cached-dns.lookup)](https://bundlephobia.com/result?p=@vates/cached-dns.lookup) [![Node compatibility](https://badgen.net/npm/node/@vates/cached-dns.lookup)](https://npmjs.org/package/@vates/cached-dns.lookup)
+
+> Cached implementation of dns.lookup
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/cached-dns.lookup):
+
+```
+> npm install --save @vates/cached-dns.lookup
+```
+
+## Usage
+
+Node does not cache queries to `dns.lookup`, which can lead application doing a lot of connections to have perf issues and to saturate Node threads pool.
+
+This library attempts to mitigate these problems by providing a version of this function with a version short cache, applied on both errors and results.
+
+> Limitation: `verbatim: false` option is not supported.
+
+It has exactly the same API as the native method and can be used directly:
+
+```js
+import { createCachedLookup } from '@vates/cached-dns.lookup'
+
+const lookup = createCachedLookup()
+
+lookup('example.net', { all: true, family: 0 }, (error, result) => {
+  if (error != null) {
+    return console.warn(error)
+  }
+  console.log(result)
+})
+```
+
+Or it can be used to replace the native implementation and speed up the whole app:
+
+```js
+// assign our cached implementation to dns.lookup
+const restore = createCachedLookup().patchGlobal()
+
+// to restore the previous implementation
+restore()
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/cached-dns.lookup/index.js

@@ -0,0 +1,72 @@
+'use strict'
+
+const assert = require('assert')
+const dns = require('dns')
+const LRU = require('lru-cache')
+
+function reportResults(all, results, callback) {
+  if (all) {
+    callback(null, results)
+  } else {
+    const first = results[0]
+    callback(null, first.address, first.family)
+  }
+}
+
+exports.createCachedLookup = function createCachedLookup({ lookup = dns.lookup } = {}) {
+  const cache = new LRU({
+    max: 500,
+
+    // 1 minute: long enough to be effective, short enough so there is no need to bother with DNS TTLs
+    ttl: 60e3,
+  })
+
+  function cachedLookup(hostname, options, callback) {
+    let all = false
+    let family = 0
+    if (typeof options === 'function') {
+      callback = options
+    } else if (typeof options === 'number') {
+      family = options
+    } else if (options != null) {
+      assert.notStrictEqual(options.verbatim, false, 'not supported by this implementation')
+      ;({ all = all, family = family } = options)
+    }
+
+    // cache by family option because there will be an error if there is no
+    // entries for the requestion family so we cannot easily cache all families
+    // and filter on reporting back
+    const key = hostname + '/' + family
+
+    const result = cache.get(key)
+    if (result !== undefined) {
+      setImmediate(reportResults, all, result, callback)
+    } else {
+      lookup(hostname, { all: true, family, verbatim: true }, function onLookup(error, results) {
+        // errors are not cached because this will delay recovery after DNS/network issues
+        //
+        // there are no reliable way to detect if the error is real or simply
+        // that there are no results for the requested hostname
+        //
+        // there should be much fewer errors than success, therefore it should
+        // not be a big deal to not cache them
+        if (error != null) {
+          return callback(error)
+        }
+
+        cache.set(key, results)
+        reportResults(all, results, callback)
+      })
+    }
+  }
+  cachedLookup.patchGlobal = function patchGlobal() {
+    const previous = dns.lookup
+    dns.lookup = cachedLookup
+    return function restoreGlobal() {
+      assert.strictEqual(dns.lookup, cachedLookup)
+      dns.lookup = previous
+    }
+  }
+
+  return cachedLookup
+}

@vates/cached-dns.lookup/package.json

@@ -0,0 +1,32 @@
+{
+  "engines": {
+    "node": ">=8"
+  },
+  "dependencies": {
+    "lru-cache": "^7.0.4"
+  },
+  "private": false,
+  "name": "@vates/cached-dns.lookup",
+  "description": "Cached implementation of dns.lookup",
+  "keywords": [
+    "cache",
+    "dns",
+    "lookup"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/cached-dns.lookup",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/cached-dns.lookup",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "1.0.0",
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@vates/coalesce-calls/index.js

@@ -1,3 +1,5 @@
+'use strict'
+
 exports.coalesceCalls = function (fn) {
   let promise
   const clean = () => {

@vates/coalesce-calls/index.spec.js

@@ -1,3 +1,5 @@
+'use strict'
+
 /* eslint-env jest */
 
 const { coalesceCalls } = require('./')

@vates/coalesce-calls/package.json

@@ -20,9 +20,6 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "files": [
-    "index.js"
-  ],
   "author": {
     "name": "Vates SAS",
     "url": "https://vates.fr"

@vates/compose/.USAGE.md

@@ -46,3 +46,20 @@ const f = compose(
   [add2, mul3]
 )
 ```
+
+Functions can receive extra parameters:
+
+```js
+const isIn = (value, min, max) => min <= value && value <= max
+
+// Only compatible when `fns` is passed as an array!
+const f = compose([
+  [add, 2],
+  [isIn, 3, 10],
+])
+
+console.log(f(1))
+// → true
+```
+
+> Note: if the first function is defined with extra parameters, it will only receive the first value passed to the composed function, instead of all the parameters.

@vates/compose/README.md

@@ -65,6 +65,23 @@ const f = compose(
 )
 ```
 
+Functions can receive extra parameters:
+
+```js
+const isIn = (value, min, max) => min <= value && value <= max
+
+// Only compatible when `fns` is passed as an array!
+const f = compose([
+  [add, 2],
+  [isIn, 3, 10],
+])
+
+console.log(f(1))
+// → true
+```
+
+> Note: if the first function is defined with extra parameters, it will only receive the first value passed to the composed function, instead of all the parameters.
+
 ## Contributions
 
 Contributions are _very_ welcomed, either on the documentation or on

@vates/compose/index.js

@@ -4,11 +4,13 @@ const defaultOpts = { async: false, right: false }
 
 exports.compose = function compose(opts, fns) {
   if (Array.isArray(opts)) {
-    fns = opts
+    fns = opts.slice() // don't mutate passed array
     opts = defaultOpts
   } else if (typeof opts === 'object') {
     opts = Object.assign({}, defaultOpts, opts)
-    if (!Array.isArray(fns)) {
+    if (Array.isArray(fns)) {
+      fns = fns.slice() // don't mutate passed array
+    } else {
       fns = Array.prototype.slice.call(arguments, 1)
     }
   } else {
@@ -20,6 +22,24 @@ exports.compose = function compose(opts, fns) {
   if (n === 0) {
     throw new TypeError('at least one function must be passed')
   }
+
+  for (let i = 0; i < n; ++i) {
+    const entry = fns[i]
+    if (Array.isArray(entry)) {
+      const fn = entry[0]
+      const args = entry.slice()
+      args[0] = undefined
+      fns[i] = function composeWithArgs(value) {
+        args[0] = value
+        try {
+          return fn.apply(this, args)
+        } finally {
+          args[0] = undefined
+        }
+      }
+    }
+  }
+
   if (n === 1) {
     return fns[0]
   }

@vates/compose/index.spec.js

@@ -1,3 +1,5 @@
+'use strict'
+
 /* eslint-env jest */
 
 const { compose } = require('./')

@vates/compose/package.json

@@ -14,7 +14,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "engines": {
     "node": ">=7.6"
   },

@vates/decorate-with/.USAGE.md

@@ -0,0 +1,86 @@
+### `decorateWith(fn, ...args)`
+
+Creates a new ([legacy](https://babeljs.io/docs/en/babel-plugin-syntax-decorators#legacy)) method decorator from a function decorator, for instance, allows using Lodash's functions as decorators:
+
+```js
+import { decorateWith } from '@vates/decorate-with'
+
+class Foo {
+  @decorateWith(lodash.debounce, 150)
+  bar() {
+    // body
+  }
+}
+```
+
+### `decorateClass(class, map)`
+
+Decorates a number of accessors and methods directly, without using the decorator syntax:
+
+```js
+import { decorateClass } from '@vates/decorate-with'
+
+class Foo {
+  get bar() {
+    // body
+  }
+
+  set bar(value) {
+    // body
+  }
+
+  baz() {
+    // body
+  }
+}
+
+decorateClass(Foo, {
+  // getter and/or setter
+  bar: {
+    // without arguments
+    get: lodash.memoize,
+
+    // with arguments
+    set: [lodash.debounce, 150],
+  },
+
+  // method (with or without arguments)
+  baz: lodash.curry,
+})
+```
+
+The decorated class is returned, so you can export it directly.
+
+To apply multiple transforms to an accessor/method, you can either call `decorateClass` multiple times or use [`@vates/compose`](https://www.npmjs.com/package/@vates/compose):
+
+```js
+decorateClass(Foo, {
+  baz: compose([
+    [lodash.debounce, 150]
+    lodash.curry,
+  ])
+})
+```
+
+### `perInstance(fn, ...args)`
+
+Helper to decorate the method by instance instead of for the whole class.
+
+This is often necessary for caching or deduplicating calls.
+
+```js
+import { perInstance } from '@vates/decorateWith'
+
+class Foo {
+  @decorateWith(perInstance, lodash.memoize)
+  bar() {
+    // body
+  }
+}
+```
+
+Because it's a normal function, it can also be used with `decorateClass`, with `compose` or even by itself.
+
+### `decorateMethodsWith(class, map)`
+
+> Deprecated alias for [`decorateClass(class, map)`](#decorateclassclass-map).

@vates/decorate-with/README.md

@@ -16,7 +16,9 @@ Installation of the [npm package](https://npmjs.org/package/@vates/decorate-with
 
 ## Usage
 
-For instance, allows using Lodash's functions as decorators:
+### `decorateWith(fn, ...args)`
+
+Creates a new ([legacy](https://babeljs.io/docs/en/babel-plugin-syntax-decorators#legacy)) method decorator from a function decorator, for instance, allows using Lodash's functions as decorators:
 
 ```js
 import { decorateWith } from '@vates/decorate-with'
@@ -29,6 +31,78 @@ class Foo {
 }
 ```
 
+### `decorateClass(class, map)`
+
+Decorates a number of accessors and methods directly, without using the decorator syntax:
+
+```js
+import { decorateClass } from '@vates/decorate-with'
+
+class Foo {
+  get bar() {
+    // body
+  }
+
+  set bar(value) {
+    // body
+  }
+
+  baz() {
+    // body
+  }
+}
+
+decorateClass(Foo, {
+  // getter and/or setter
+  bar: {
+    // without arguments
+    get: lodash.memoize,
+
+    // with arguments
+    set: [lodash.debounce, 150],
+  },
+
+  // method (with or without arguments)
+  baz: lodash.curry,
+})
+```
+
+The decorated class is returned, so you can export it directly.
+
+To apply multiple transforms to an accessor/method, you can either call `decorateClass` multiple times or use [`@vates/compose`](https://www.npmjs.com/package/@vates/compose):
+
+```js
+decorateClass(Foo, {
+  baz: compose([
+    [lodash.debounce, 150]
+    lodash.curry,
+  ])
+})
+```
+
+### `perInstance(fn, ...args)`
+
+Helper to decorate the method by instance instead of for the whole class.
+
+This is often necessary for caching or deduplicating calls.
+
+```js
+import { perInstance } from '@vates/decorateWith'
+
+class Foo {
+  @decorateWith(perInstance, lodash.memoize)
+  bar() {
+    // body
+  }
+}
+```
+
+Because it's a normal function, it can also be used with `decorateClass`, with `compose` or even by itself.
+
+### `decorateMethodsWith(class, map)`
+
+> Deprecated alias for [`decorateClass(class, map)`](#decorateclassclass-map).
+
 ## Contributions
 
 Contributions are _very_ welcomed, either on the documentation or on

@vates/decorate-with/USAGE.md

@@ -1,12 +0,0 @@
-For instance, allows using Lodash's functions as decorators:
-
-```js
-import { decorateWith } from '@vates/decorate-with'
-
-class Foo {
-  @decorateWith(lodash.debounce, 150)
-  bar() {
-    // body
-  }
-}
-```

@vates/decorate-with/index.js

@@ -1,4 +1,48 @@
-exports.decorateWith = (fn, ...args) => (target, name, descriptor) => ({
-  ...descriptor,
-  value: fn(descriptor.value, ...args),
-})
+'use strict'
+
+exports.decorateWith = function decorateWith(fn, ...args) {
+  return (target, name, descriptor) => ({
+    ...descriptor,
+    value: fn(descriptor.value, ...args),
+  })
+}
+
+const { getOwnPropertyDescriptor, defineProperty } = Object
+
+function applyDecorator(decorator, value) {
+  return typeof decorator === 'function' ? decorator(value) : decorator[0](value, ...decorator.slice(1))
+}
+
+exports.decorateClass = exports.decorateMethodsWith = function decorateClass(klass, map) {
+  const { prototype } = klass
+  for (const name of Object.keys(map)) {
+    const decorator = map[name]
+    const descriptor = getOwnPropertyDescriptor(prototype, name)
+    if (typeof decorator === 'function' || Array.isArray(decorator)) {
+      descriptor.value = applyDecorator(decorator, descriptor.value)
+    } else {
+      const { get, set } = decorator
+      if (get !== undefined) {
+        descriptor.get = applyDecorator(get, descriptor.get)
+      }
+      if (set !== undefined) {
+        descriptor.set = applyDecorator(set, descriptor.set)
+      }
+    }
+
+    defineProperty(prototype, name, descriptor)
+  }
+  return klass
+}
+
+exports.perInstance = function perInstance(fn, decorator, ...args) {
+  const map = new WeakMap()
+  return function () {
+    let decorated = map.get(this)
+    if (decorated === undefined) {
+      decorated = decorator(fn, ...args)
+      map.set(this, decorated)
+    }
+    return decorated.apply(this, arguments)
+  }
+}

@vates/decorate-with/index.spec.js

@@ -0,0 +1,152 @@
+'use strict'
+
+const assert = require('assert')
+const { describe, it } = require('tap').mocha
+
+const { decorateClass, decorateWith, decorateMethodsWith, perInstance } = require('./')
+
+const identity = _ => _
+
+describe('decorateWith', () => {
+  it('works', () => {
+    const expectedArgs = [Math.random(), Math.random()]
+    const expectedFn = Function.prototype
+    const newFn = () => {}
+
+    const decorator = decorateWith(function wrapper(fn, ...args) {
+      assert.deepStrictEqual(fn, expectedFn)
+      assert.deepStrictEqual(args, expectedArgs)
+
+      return newFn
+    }, ...expectedArgs)
+
+    const descriptor = {
+      configurable: true,
+      enumerable: false,
+      value: expectedFn,
+      writable: true,
+    }
+    assert.deepStrictEqual(decorator({}, 'foo', descriptor), {
+      ...descriptor,
+      value: newFn,
+    })
+  })
+})
+
+describe('decorateClass', () => {
+  it('works', () => {
+    class C {
+      foo() {}
+      bar() {}
+      get baz() {}
+      // eslint-disable-next-line accessor-pairs
+      set qux(_) {}
+    }
+
+    const expectedArgs = [Math.random(), Math.random()]
+    const P = C.prototype
+
+    const descriptors = Object.getOwnPropertyDescriptors(P)
+
+    const newFoo = () => {}
+    const newBar = () => {}
+    const newGetBaz = () => {}
+    const newSetQux = _ => {}
+
+    decorateClass(C, {
+      foo(fn) {
+        assert.strictEqual(arguments.length, 1)
+        assert.strictEqual(fn, P.foo)
+        return newFoo
+      },
+      bar: [
+        function (fn, ...args) {
+          assert.strictEqual(fn, P.bar)
+          assert.deepStrictEqual(args, expectedArgs)
+          return newBar
+        },
+        ...expectedArgs,
+      ],
+      baz: {
+        get(fn) {
+          assert.strictEqual(arguments.length, 1)
+          assert.strictEqual(fn, descriptors.baz.get)
+          return newGetBaz
+        },
+      },
+      qux: {
+        set: [
+          function (fn, ...args) {
+            assert.strictEqual(fn, descriptors.qux.set)
+            assert.deepStrictEqual(args, expectedArgs)
+            return newSetQux
+          },
+          ...expectedArgs,
+        ],
+      },
+    })
+
+    const newDescriptors = Object.getOwnPropertyDescriptors(P)
+    assert.deepStrictEqual(newDescriptors.foo, { ...descriptors.foo, value: newFoo })
+    assert.deepStrictEqual(newDescriptors.bar, { ...descriptors.bar, value: newBar })
+    assert.deepStrictEqual(newDescriptors.baz, { ...descriptors.baz, get: newGetBaz })
+    assert.deepStrictEqual(newDescriptors.qux, { ...descriptors.qux, set: newSetQux })
+  })
+
+  it('throws if using an accessor decorator for a method', function () {
+    assert.throws(() =>
+      decorateClass(
+        class {
+          foo() {}
+        },
+        { foo: { get: identity, set: identity } }
+      )
+    )
+  })
+
+  it('throws if using a method decorator for an accessor', function () {
+    assert.throws(() =>
+      decorateClass(
+        class {
+          get foo() {}
+        },
+        { foo: identity }
+      )
+    )
+  })
+})
+
+it('decorateMethodsWith is an alias of decorateClass', function () {
+  assert.strictEqual(decorateMethodsWith, decorateClass)
+})
+
+describe('perInstance', () => {
+  it('works', () => {
+    let calls = 0
+
+    const expectedArgs = [Math.random(), Math.random()]
+    const expectedFn = Function.prototype
+    function wrapper(fn, ...args) {
+      assert.strictEqual(fn, expectedFn)
+      assert.deepStrictEqual(args, expectedArgs)
+      const i = ++calls
+      return () => i
+    }
+
+    const wrapped = perInstance(expectedFn, wrapper, ...expectedArgs)
+
+    // decorator is not called before decorated called
+    assert.strictEqual(calls, 0)
+
+    const o1 = {}
+    const o2 = {}
+
+    assert.strictEqual(wrapped.call(o1), 1)
+
+    // the same decorated function is returned for the same instance
+    assert.strictEqual(wrapped.call(o1), 1)
+
+    // a new decorated function is returned for another instance
+    assert.strictEqual(wrapped.call(o2), 2)
+  })
+})

@vates/decorate-with/package.json

@@ -20,11 +20,15 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "0.0.1",
+  "version": "2.0.0",
   "engines": {
     "node": ">=8.10"
   },
   "scripts": {
-    "postversion": "npm publish --access public"
+    "postversion": "npm publish --access public",
+    "test": "tap"
+  },
+  "devDependencies": {
+    "tap": "^16.0.1"
   }
 }

@vates/disposable/.USAGE.md

@@ -30,7 +30,7 @@ import { createDebounceResource } from '@vates/disposable/debounceResource'
 const debounceResource = createDebounceResource()
 
 // it will wait for 10 seconds before calling the disposer
-using(debounceResource(getConnection(host), 10e3), connection => {})
+Disposable.use(debounceResource(getConnection(host), 10e3), connection => {})
 ```
 
 ### `debounceResource.flushAll()`

@vates/disposable/README.md

@@ -48,7 +48,7 @@ import { createDebounceResource } from '@vates/disposable/debounceResource'
 const debounceResource = createDebounceResource()
 
 // it will wait for 10 seconds before calling the disposer
-using(debounceResource(getConnection(host), 10e3), connection => {})
+Disposable.use(debounceResource(getConnection(host), 10e3), connection => {})
 ```
 
 ### `debounceResource.flushAll()`

@vates/disposable/debounceResource.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { createLogger } = require('@xen-orchestra/log')
 

@vates/disposable/debounceResource.spec.js

@@ -1,3 +1,5 @@
+'use strict'
+
 /* eslint-env jest */
 
 const { createDebounceResource } = require('./debounceResource')

@vates/disposable/deduped.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const ensureArray = require('ensure-array')
 const { MultiKeyMap } = require('@vates/multi-key-map')
 

@vates/disposable/deduped.spec.js

@@ -1,3 +1,5 @@
+'use strict'
+
 /* eslint-env jest */
 
 const { deduped } = require('./deduped')

@vates/disposable/package.json

@@ -23,7 +23,8 @@
   },
   "dependencies": {
     "@vates/multi-key-map": "^0.1.0",
-    "@xen-orchestra/log": "^0.2.0",
+    "@xen-orchestra/async-map": "^0.1.2",
+    "@xen-orchestra/log": "^0.3.0",
     "ensure-array": "^1.0.0"
   }
 }

@vates/event-listeners-manager/.USAGE.md

@@ -0,0 +1,50 @@
+> This library is compatible with Node's `EventEmitter` and web browsers' `EventTarget` APIs.
+
+### API
+
+```js
+import { EventListenersManager } from '@vates/event-listeners-manager'
+
+const events = new EventListenersManager(emitter)
+
+// adding listeners
+events.add('foo', onFoo).add('bar', onBar).on('baz', onBaz)
+
+// removing a specific listener
+events.remove('foo', onFoo)
+
+// removing all listeners for a specific event
+events.removeAll('foo')
+
+// removing all listeners
+events.removeAll()
+```
+
+### Typical use case
+
+> Removing all listeners when no longer necessary.
+
+Manually:
+
+```js
+const onFoo = () => {}
+const onBar = () => {}
+const onBaz = () => {}
+emitter.on('foo', onFoo).on('bar', onBar).on('baz', onBaz)
+
+// CODE LOGIC
+
+emitter.off('foo', onFoo).off('bar', onBar).off('baz', onBaz)
+```
+
+With this library:
+
+```js
+const events = new EventListenersManager(emitter)
+
+events.add('foo', () => {})).add('bar', () => {})).add('baz', () => {}))
+
+// CODE LOGIC
+
+events.removeAll()
+```

@vates/event-listeners-manager/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/event-listeners-manager/README.md

@@ -0,0 +1,81 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/event-listeners-manager
+
+[![Package Version](https://badgen.net/npm/v/@vates/event-listeners-manager)](https://npmjs.org/package/@vates/event-listeners-manager) ![License](https://badgen.net/npm/license/@vates/event-listeners-manager) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/event-listeners-manager)](https://bundlephobia.com/result?p=@vates/event-listeners-manager) [![Node compatibility](https://badgen.net/npm/node/@vates/event-listeners-manager)](https://npmjs.org/package/@vates/event-listeners-manager)
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/event-listeners-manager):
+
+```
+> npm install --save @vates/event-listeners-manager
+```
+
+## Usage
+
+> This library is compatible with Node's `EventEmitter` and web browsers' `EventTarget` APIs.
+
+### API
+
+```js
+import { EventListenersManager } from '@vates/event-listeners-manager'
+
+const events = new EventListenersManager(emitter)
+
+// adding listeners
+events.add('foo', onFoo).add('bar', onBar).on('baz', onBaz)
+
+// removing a specific listener
+events.remove('foo', onFoo)
+
+// removing all listeners for a specific event
+events.removeAll('foo')
+
+// removing all listeners
+events.removeAll()
+```
+
+### Typical use case
+
+> Removing all listeners when no longer necessary.
+
+Manually:
+
+```js
+const onFoo = () => {}
+const onBar = () => {}
+const onBaz = () => {}
+emitter.on('foo', onFoo).on('bar', onBar).on('baz', onBaz)
+
+// CODE LOGIC
+
+emitter.off('foo', onFoo).off('bar', onBar).off('baz', onBaz)
+```
+
+With this library:
+
+```js
+const events = new EventListenersManager(emitter)
+
+events.add('foo', () => {})).add('bar', () => {})).add('baz', () => {}))
+
+// CODE LOGIC
+
+events.removeAll()
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/event-listeners-manager/index.js

@@ -0,0 +1,56 @@
+'use strict'
+
+exports.EventListenersManager = class EventListenersManager {
+  constructor(emitter) {
+    this._listeners = new Map()
+
+    this._add = (emitter.addListener || emitter.addEventListener).bind(emitter)
+    this._remove = (emitter.removeListener || emitter.removeEventListener).bind(emitter)
+  }
+
+  add(type, listener) {
+    let listeners = this._listeners[type]
+    if (listeners === undefined) {
+      listeners = new Set()
+      this._listeners.set(type, listeners)
+    }
+
+    // don't add the same listener multiple times (allowed on Node.js)
+    if (!listeners.has(listener)) {
+      listeners.add(listener)
+      this._add(type, listener)
+    }
+
+    return this
+  }
+
+  remove(type, listener) {
+    const allListeners = this._listeners
+    const listeners = allListeners.get(type)
+    if (listeners !== undefined && listeners.delete(listener)) {
+      this._remove(type, listener)
+      if (listeners.size === 0) {
+        allListeners.delete(type)
+      }
+    }
+
+    return this
+  }
+
+  removeAll(type) {
+    const allListeners = this._listeners
+    const remove = this._remove
+    const types = type !== undefined ? [type] : allListeners.keys()
+    for (const type of types) {
+      const listeners = allListeners.get(type)
+      if (listeners !== undefined) {
+        allListeners.delete(type)
+        for (const listener of listeners) {
+          remove(type, listener)
+        }
+      }
+    }
+
+    return this
+  }
+}

@vates/event-listeners-manager/package.json

@@ -0,0 +1,42 @@
+{
+  "engines": {
+    "node": ">=6"
+  },
+  "private": false,
+  "name": "@vates/event-listeners-manager",
+  "descriptions": "Easy way to clean up event listeners",
+  "keywords": [
+    "add",
+    "addEventListener",
+    "addListener",
+    "browser",
+    "clear",
+    "DOM",
+    "emitter",
+    "event",
+    "EventEmitter",
+    "EventTarget",
+    "management",
+    "manager",
+    "node",
+    "remove",
+    "removeEventListener",
+    "removeListener"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/event-listeners-manager",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/event-listeners-manager",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "1.0.0",
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@vates/multi-key-map/index.js

@@ -1,3 +1,5 @@
+'use strict'
+
 class Node {
   constructor(value) {
     this.children = new Map()

@vates/multi-key-map/index.spec.js

@@ -1,3 +1,5 @@
+'use strict'
+
 /* eslint-env jest */
 
 const { MultiKeyMap } = require('./')

@vates/parse-duration/README.md

@@ -44,4 +44,4 @@ You may:
 
 ## License
 
-[AGPL-3.0-or-later](https://spdx.org/licenses/AGPL-3.0-or-later) © [Vates SAS](https://vates.fr)
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/parse-duration/index.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const ms = require('ms')
 
 exports.parseDuration = value => {
@@ -6,7 +8,7 @@ exports.parseDuration = value => {
   }
   const duration = ms(value)
   if (duration === undefined) {
-    throw new TypeError(`not a valid duration: ${duration}`)
+    throw new TypeError(`not a valid duration: ${value}`)
   }
   return duration
 }

@vates/parse-duration/package.json

@@ -18,8 +18,8 @@
     "name": "Vates SAS",
     "url": "https://vates.fr"
   },
-  "license": "AGPL-3.0-or-later",
-  "version": "0.1.0",
+  "license": "ISC",
+  "version": "0.1.1",
   "engines": {
     "node": ">=8.10"
   },

@vates/predicates/.USAGE.md

@@ -0,0 +1,57 @@
+`undefined` predicates are ignored and `undefined` is returned if all predicates are `undefined`, this permits the most efficient composition:
+
+```js
+const compositePredicate = every(undefined, some(predicate2, undefined))
+
+// ends up as
+
+const compositePredicate = predicate2
+```
+
+Predicates can also be passed wrapped in an array:
+
+```js
+const compositePredicate = every([predicate1, some([predicate2, predicate3])])
+```
+
+`this` and all arguments are passed to the nested predicates.
+
+### `every(predicates)`
+
+> Returns a predicate that returns `true` iff every predicate returns `true`.
+
+```js
+const isBetween3And7 = every(
+  n => n >= 3,
+  n => n <= 7
+)
+
+isBetween3And10(0)
+// → false
+
+isBetween3And10(5)
+// → true
+
+isBetween3And10(10)
+// → false
+```
+
+### `some(predicates)`
+
+> Returns a predicate that returns `true` iff some predicate returns `true`.
+
+```js
+const isAliceOrBob = some(
+  name => name === 'Alice',
+  name => name === 'Bob'
+)
+
+isAliceOrBob('Alice')
+// → true
+
+isAliceOrBob('Bob')
+// → true
+
+isAliceOrBob('Oscar')
+// → false
+```

@vates/predicates/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/predicates/README.md

@@ -0,0 +1,90 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/predicates
+
+[![Package Version](https://badgen.net/npm/v/@vates/predicates)](https://npmjs.org/package/@vates/predicates) ![License](https://badgen.net/npm/license/@vates/predicates) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/predicates)](https://bundlephobia.com/result?p=@vates/predicates) [![Node compatibility](https://badgen.net/npm/node/@vates/predicates)](https://npmjs.org/package/@vates/predicates)
+
+> Utilities to compose predicates
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/predicates):
+
+```
+> npm install --save @vates/predicates
+```
+
+## Usage
+
+`undefined` predicates are ignored and `undefined` is returned if all predicates are `undefined`, this permits the most efficient composition:
+
+```js
+const compositePredicate = every(undefined, some(predicate2, undefined))
+
+// ends up as
+
+const compositePredicate = predicate2
+```
+
+Predicates can also be passed wrapped in an array:
+
+```js
+const compositePredicate = every([predicate1, some([predicate2, predicate3])])
+```
+
+`this` and all arguments are passed to the nested predicates.
+
+### `every(predicates)`
+
+> Returns a predicate that returns `true` iff every predicate returns `true`.
+
+```js
+const isBetween3And7 = every(
+  n => n >= 3,
+  n => n <= 7
+)
+
+isBetween3And10(0)
+// → false
+
+isBetween3And10(5)
+// → true
+
+isBetween3And10(10)
+// → false
+```
+
+### `some(predicates)`
+
+> Returns a predicate that returns `true` iff some predicate returns `true`.
+
+```js
+const isAliceOrBob = some(
+  name => name === 'Alice',
+  name => name === 'Bob'
+)
+
+isAliceOrBob('Alice')
+// → true
+
+isAliceOrBob('Bob')
+// → true
+
+isAliceOrBob('Oscar')
+// → false
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/predicates/index.js

@@ -0,0 +1,71 @@
+'use strict'
+
+const {
+  isArray,
+  prototype: { filter },
+} = Array
+
+class InvalidPredicate extends TypeError {
+  constructor(value) {
+    super('not a valid predicate')
+    this.value = value
+  }
+}
+
+function isDefinedPredicate(value) {
+  if (value === undefined) {
+    return false
+  }
+
+  if (typeof value !== 'function') {
+    throw new InvalidPredicate(value)
+  }
+
+  return true
+}
+
+function handleArgs() {
+  let predicates
+  if (!(arguments.length === 1 && isArray((predicates = arguments[0])))) {
+    predicates = arguments
+  }
+  return filter.call(predicates, isDefinedPredicate)
+}
+
+exports.every = function every() {
+  const predicates = handleArgs.apply(this, arguments)
+  const n = predicates.length
+  if (n === 0) {
+    return
+  }
+  if (n === 1) {
+    return predicates[0]
+  }
+  return function everyPredicate() {
+    for (let i = 0; i < n; ++i) {
+      if (!predicates[i].apply(this, arguments)) {
+        return false
+      }
+    }
+    return true
+  }
+}
+
+exports.some = function some() {
+  const predicates = handleArgs.apply(this, arguments)
+  const n = predicates.length
+  if (n === 0) {
+    return
+  }
+  if (n === 1) {
+    return predicates[0]
+  }
+  return function somePredicate() {
+    for (let i = 0; i < n; ++i) {
+      if (predicates[i].apply(this, arguments)) {
+        return true
+      }
+    }
+    return false
+  }
+}

@vates/predicates/index.spec.js

@@ -0,0 +1,65 @@
+'use strict'
+
+const assert = require('assert/strict')
+const { describe, it } = require('tap').mocha
+
+const { every, some } = require('./')
+
+const T = () => true
+const F = () => false
+
+const testArgsHandling = fn => {
+  it('returns undefined if all predicates are undefined', () => {
+    assert.equal(fn(undefined), undefined)
+    assert.equal(fn([undefined]), undefined)
+  })
+
+  it('returns the predicate if only a single one is passed', () => {
+    assert.equal(fn(undefined, T), T)
+    assert.equal(fn([undefined, T]), T)
+  })
+
+  it('throws if it receives a non-predicate', () => {
+    const error = new TypeError('not a valid predicate')
+    error.value = 3
+    assert.throws(() => fn(3), error)
+  })
+
+  it('forwards this and arguments to predicates', () => {
+    const thisArg = 'qux'
+    const args = ['foo', 'bar', 'baz']
+    const predicate = function () {
+      assert.equal(this, thisArg)
+      assert.deepEqual(Array.from(arguments), args)
+    }
+    fn(predicate, predicate).apply(thisArg, args)
+  })
+}
+
+const runTests = (fn, truthTable) =>
+  it('works', () => {
+    truthTable.forEach(([result, ...predicates]) => {
+      assert.equal(fn(...predicates)(), result)
+      assert.equal(fn(predicates)(), result)
+    })
+  })
+
+describe('every', () => {
+  testArgsHandling(every)
+  runTests(every, [
+    [true, T, T],
+    [false, T, F],
+    [false, F, T],
+    [false, F, F],
+  ])
+})
+
+describe('some', () => {
+  testArgsHandling(some)
+  runTests(some, [
+    [true, T, T],
+    [true, T, F],
+    [true, F, T],
+    [false, F, F],
+  ])
+})

@vates/predicates/package.json

@@ -0,0 +1,40 @@
+{
+  "private": false,
+  "name": "@vates/predicates",
+  "description": "Utilities to compose predicates",
+  "keywords": [
+    "and",
+    "combine",
+    "compose",
+    "every",
+    "function",
+    "functions",
+    "or",
+    "predicate",
+    "predicates",
+    "some"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/predicates",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/predicates",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "1.0.0",
+  "engines": {
+    "node": ">=6"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public",
+    "test": "tap"
+  },
+  "devDependencies": {
+    "tap": "^16.0.1"
+  }
+}

@vates/read-chunk/index.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const readChunk = (stream, size) =>
   size === 0
     ? Promise.resolve(Buffer.alloc(0))

@vates/read-chunk/index.spec.js

@@ -1,3 +1,5 @@
+'use strict'
+
 /* eslint-env jest */
 
 const { Readable } = require('stream')

@vates/read-chunk/package.json

@@ -23,9 +23,6 @@
   "engines": {
     "node": ">=8.10"
   },
-  "files": [
-    "index.js"
-  ],
   "scripts": {
     "postversion": "npm publish --access public"
   },

@vates/toggle-scripts/index.js

@@ -1,5 +1,7 @@
 #!/usr/bin/env node
 
+'use strict'
+
 const fs = require('fs')
 
 const mapKeys = (object, iteratee) => {

@vates/toggle-scripts/package.json

@@ -31,9 +31,6 @@
   "engines": {
     "node": ">=6"
   },
-  "files": [
-    "index.js"
-  ],
   "bin": "./index.js",
   "scripts": {
     "postversion": "npm publish --access public"

@xen-orchestra/async-map/index.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const wrapCall = (fn, arg, thisArg) => {
   try {
     return Promise.resolve(fn.call(thisArg, arg))

@xen-orchestra/async-map/index.spec.js

@@ -1,3 +1,5 @@
+'use strict'
+
 /* eslint-env jest */
 
 const { asyncMapSettled } = require('./')

@xen-orchestra/async-map/legacy.js

@@ -1,3 +1,5 @@
+'use strict'
+
 // type MaybePromise<T> = Promise<T> | T
 //
 // declare export function asyncMap<T1, T2>(

@xen-orchestra/async-map/package.json

@@ -24,10 +24,6 @@
     "url": "https://vates.fr"
   },
   "preferGlobal": false,
-  "files": [
-    "index.js",
-    "legacy.js"
-  ],
   "engines": {
     "node": ">=6"
   },

@xen-orchestra/audit-core/docs/specification.ts

@@ -17,10 +17,10 @@ interface Record {
 }
 
 export class AuditCore {
-  constructor(storage: Storage) { }
-  public add(subject: any, event: string, data: any): Promise<Record> { }
-  public checkIntegrity(oldest: string, newest: string): Promise<number> { }
-  public getFrom(newest?: string): AsyncIterator { }
-  public deleteFrom(newest: string): Promise<void> { }
-  public deleteRangeAndRewrite(newest: string, oldest: string): Promise<void> { }
+  constructor(storage: Storage) {}
+  public add(subject: any, event: string, data: any): Promise<Record> {}
+  public checkIntegrity(oldest: string, newest: string): Promise<number> {}
+  public getFrom(newest?: string): AsyncIterator {}
+  public deleteFrom(newest: string): Promise<void> {}
+  public deleteRangeAndRewrite(newest: string, oldest: string): Promise<void> {}
 }

@xen-orchestra/audit-core/index.js

@@ -1,14 +1,14 @@
-// see https://github.com/babel/babel/issues/8450
-import 'core-js/features/symbol/async-iterator'
+'use strict'
 
-import assert from 'assert'
-import defer from 'golike-defer'
-import hash from 'object-hash'
-import { createLogger } from '@xen-orchestra/log'
+const assert = require('assert')
+const hash = require('object-hash')
+const { createLogger } = require('@xen-orchestra/log')
+const { decorateClass } = require('@vates/decorate-with')
+const { defer } = require('golike-defer')
 
 const log = createLogger('xo:audit-core')
 
-export class Storage {
+exports.Storage = class Storage {
   constructor() {
     this._lock = Promise.resolve()
   }
@@ -31,7 +31,7 @@ const ID_TO_ALGORITHM = {
   5: 'sha256',
 }
 
-export class AlteredRecordError extends Error {
+class AlteredRecordError extends Error {
   constructor(id, nValid, record) {
     super('altered record')
 
@@ -40,8 +40,9 @@ export class AlteredRecordError extends Error {
     this.record = record
   }
 }
+exports.AlteredRecordError = AlteredRecordError
 
-export class MissingRecordError extends Error {
+class MissingRecordError extends Error {
   constructor(id, nValid) {
     super('missing record')
 
@@ -49,8 +50,10 @@ export class MissingRecordError extends Error {
     this.nValid = nValid
   }
 }
+exports.MissingRecordError = MissingRecordError
 
-export const NULL_ID = 'nullId'
+const NULL_ID = 'nullId'
+exports.NULL_ID = NULL_ID
 
 const HASH_ALGORITHM_ID = '5'
 const createHash = (data, algorithmId = HASH_ALGORITHM_ID) =>
@@ -59,13 +62,12 @@ const createHash = (data, algorithmId = HASH_ALGORITHM_ID) =>
     excludeKeys: key => key === 'id',
   })}`
 
-export class AuditCore {
+class AuditCore {
   constructor(storage) {
     assert.notStrictEqual(storage, undefined)
     this._storage = storage
   }
 
-  @defer
   async add($defer, subject, event, data) {
     const time = Date.now()
     $defer(await this._storage.acquireLock())
@@ -150,7 +152,6 @@ export class AuditCore {
     }
   }
 
-  @defer
   async deleteRangeAndRewrite($defer, newest, oldest) {
     assert.notStrictEqual(newest, undefined)
     assert.notStrictEqual(oldest, undefined)
@@ -191,3 +192,9 @@ export class AuditCore {
     }
   }
 }
+exports.AuditCore = AuditCore
+
+decorateClass(AuditCore, {
+  add: defer,
+  deleteRangeAndRewrite: defer,
+})

@xen-orchestra/audit-core/index.spec.js

@@ -1,6 +1,9 @@
-/* eslint-env jest */
+'use strict'
 
-import { AlteredRecordError, AuditCore, MissingRecordError, NULL_ID, Storage } from '.'
+const assert = require('assert/strict')
+const { afterEach, describe, it } = require('tap').mocha
+
+const { AlteredRecordError, AuditCore, MissingRecordError, NULL_ID, Storage } = require('.')
 
 const asyncIteratorToArray = async asyncIterator => {
   const array = []
@@ -72,7 +75,7 @@ const auditCore = new AuditCore(db)
 const storeAuditRecords = async () => {
   await Promise.all(DATA.map(data => auditCore.add(...data)))
   const records = await asyncIteratorToArray(auditCore.getFrom())
-  expect(records.length).toBe(DATA.length)
+  assert.equal(records.length, DATA.length)
   return records
 }
 
@@ -83,10 +86,11 @@ describe('auditCore', () => {
     const [newestRecord, deletedRecord] = await storeAuditRecords()
 
     const nValidRecords = await auditCore.checkIntegrity(NULL_ID, newestRecord.id)
-    expect(nValidRecords).toBe(DATA.length)
+    assert.equal(nValidRecords, DATA.length)
 
     await db.del(deletedRecord.id)
-    await expect(auditCore.checkIntegrity(NULL_ID, newestRecord.id)).rejects.toEqual(
+    await assert.rejects(
+      auditCore.checkIntegrity(NULL_ID, newestRecord.id),
       new MissingRecordError(deletedRecord.id, 1)
     )
   })
@@ -97,7 +101,8 @@ describe('auditCore', () => {
     alteredRecord.event = ''
     await db.put(alteredRecord)
 
-    await expect(auditCore.checkIntegrity(NULL_ID, newestRecord.id)).rejects.toEqual(
+    await assert.rejects(
+      auditCore.checkIntegrity(NULL_ID, newestRecord.id),
       new AlteredRecordError(alteredRecord.id, 1, alteredRecord)
     )
   })
@@ -107,8 +112,8 @@ describe('auditCore', () => {
 
     await auditCore.deleteFrom(secondRecord.id)
 
-    expect(await db.get(firstRecord.id)).toBe(undefined)
-    expect(await db.get(secondRecord.id)).toBe(undefined)
+    assert.equal(await db.get(firstRecord.id), undefined)
+    assert.equal(await db.get(secondRecord.id), undefined)
 
     await auditCore.checkIntegrity(secondRecord.id, thirdRecord.id)
   })

@xen-orchestra/audit-core/package.json

@@ -9,31 +9,16 @@
   },
   "version": "0.2.0",
   "engines": {
-    "node": ">=8.10"
+    "node": ">=14"
   },
-  "main": "dist/",
   "scripts": {
-    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
-    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
     "postversion": "npm publish --access public",
-    "prebuild": "rimraf dist/",
-    "predev": "yarn run prebuild",
-    "prepublishOnly": "yarn run build"
-  },
-  "devDependencies": {
-    "@babel/cli": "^7.7.4",
-    "@babel/core": "^7.7.4",
-    "@babel/plugin-proposal-decorators": "^7.8.0",
-    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.8.0",
-    "@babel/preset-env": "^7.7.4",
-    "cross": "^1.0.0",
-    "rimraf": "^3.0.0"
+    "test": "tap --lines 67 --functions 92 --branches 52 --statements 67"
   },
   "dependencies": {
-    "@xen-orchestra/log": "^0.2.0",
-    "core-js": "^3.6.4",
+    "@vates/decorate-with": "^2.0.0",
+    "@xen-orchestra/log": "^0.3.0",
     "golike-defer": "^0.5.1",
-    "lodash": "^4.17.15",
     "object-hash": "^2.0.1"
   },
   "private": false,
@@ -41,5 +26,8 @@
   "author": {
     "name": "Vates SAS",
     "url": "https://vates.fr"
+  },
+  "devDependencies": {
+    "tap": "^16.0.1"
   }
 }

@xen-orchestra/babel-config/index.js

@@ -14,25 +14,13 @@ const configs = {
   '@babel/plugin-proposal-pipeline-operator': {
     proposal: 'minimal',
   },
-  '@babel/preset-env'(pkg) {
-    return {
-      debug: !__TEST__,
+  '@babel/preset-env': {
+    debug: !__TEST__,
 
-      // disabled until https://github.com/babel/babel/issues/8323 is resolved
-      // loose: true,
+    // disabled until https://github.com/babel/babel/issues/8323 is resolved
+    // loose: true,
 
-      shippedProposals: true,
-      targets: (() => {
-        let node = (pkg.engines || {}).node
-        if (node !== undefined) {
-          const trimChars = '^=>~'
-          while (trimChars.includes(node[0])) {
-            node = node.slice(1)
-          }
-        }
-        return { browsers: pkg.browserslist, node }
-      })(),
-    }
+    shippedProposals: true,
   },
 }
 
@@ -44,21 +32,21 @@ const getConfig = (key, ...args) => {
 // some plugins must be used in a specific order
 const pluginsOrder = ['@babel/plugin-proposal-decorators', '@babel/plugin-proposal-class-properties']
 
-module.exports = function (pkg, plugins, presets) {
-  plugins === undefined && (plugins = {})
-  presets === undefined && (presets = {})
+module.exports = function (pkg, configs = {}) {
+  const plugins = {}
+  const presets = {}
 
   Object.keys(pkg.devDependencies || {}).forEach(name => {
     if (!(name in presets) && PLUGINS_RE.test(name)) {
-      plugins[name] = getConfig(name, pkg)
+      plugins[name] = { ...getConfig(name, pkg), ...configs[name] }
     } else if (!(name in presets) && PRESETS_RE.test(name)) {
-      presets[name] = getConfig(name, pkg)
+      presets[name] = { ...getConfig(name, pkg), ...configs[name] }
     }
   })
 
   return {
     comments: !__PROD__,
-    ignore: __TEST__ ? undefined : [/\.spec\.js$/],
+    ignore: __PROD__ ? [/\btests?\//, /\.spec\.js$/] : undefined,
     plugins: Object.keys(plugins)
       .map(plugin => [plugin, plugins[plugin]])
       .sort(([a], [b]) => {
@@ -67,5 +55,15 @@ module.exports = function (pkg, plugins, presets) {
         return oA !== -1 && oB !== -1 ? oA - oB : a < b ? -1 : 1
       }),
     presets: Object.keys(presets).map(preset => [preset, presets[preset]]),
+    targets: (() => {
+      let node = (pkg.engines || {}).node
+      if (node !== undefined) {
+        const trimChars = '^=>~'
+        while (trimChars.includes(node[0])) {
+          node = node.slice(1)
+        }
+      }
+      return { browsers: pkg.browserslist, node }
+    })(),
   }
 }

@xen-orchestra/babel-config/package.json

@@ -10,7 +10,7 @@
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
   "engines": {
-    "node": ">=6"
+    "node": ">=8.3"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

@xen-orchestra/backups-cli/_composeCommands.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const getopts = require('getopts')
 
 const { version } = require('./package.json')

@xen-orchestra/backups-cli/_fs.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { dirname } = require('path')
 
 const fs = require('promise-toolbox/promisifyAll')(require('fs'))

@xen-orchestra/backups-cli/commands/clean-vms.js

@@ -1,4 +1,4 @@
-#!/usr/bin/env node
+'use strict'
 
 // -----------------------------------------------------------------------------
 
@@ -10,12 +10,13 @@ const { resolve } = require('path')
 const adapter = new RemoteAdapter(require('@xen-orchestra/fs').getHandler({ url: 'file://' }))
 
 module.exports = async function main(args) {
-  const { _, remove, merge } = getopts(args, {
+  const { _, fix, remove, merge } = getopts(args, {
     alias: {
+      fix: 'f',
       remove: 'r',
       merge: 'm',
     },
-    boolean: ['merge', 'remove'],
+    boolean: ['fix', 'merge', 'remove'],
     default: {
       merge: false,
       remove: false,
@@ -25,7 +26,7 @@ module.exports = async function main(args) {
   await asyncMap(_, async vmDir => {
     vmDir = resolve(vmDir)
     try {
-      await adapter.cleanVm(vmDir, { remove, merge, onLog: log => console.warn(log) })
+      await adapter.cleanVm(vmDir, { fixMetadata: fix, remove, merge, onLog: (...args) => console.warn(...args) })
     } catch (error) {
       console.error('adapter.cleanVm', vmDir, error)
     }

@xen-orchestra/backups-cli/commands/create-symlink-index.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const filenamify = require('filenamify')
 const get = require('lodash/get')
 const { asyncMap } = require('@xen-orchestra/async-map')

@xen-orchestra/backups-cli/commands/info.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const groupBy = require('lodash/groupBy')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { createHash } = require('crypto')

@xen-orchestra/backups-cli/index.js

@@ -1,15 +1,18 @@
 #!/usr/bin/env node
 
+'use strict'
+
 require('./_composeCommands')({
   'clean-vms': {
     get main() {
       return require('./commands/clean-vms')
     },
-    usage: `[--merge] [--remove] xo-vm-backups/*
+    usage: `[--fix] [--merge] [--remove] xo-vm-backups/*
 
       Detects and repair issues with VM backups.
 
       Options:
+        -f, --fix       Fix metadata issues (like size)
         -m, --merge     Merge (or continue merging) VHD files that are unused
         -r, --remove    Remove unused, incomplete, orphan, or corrupted files
 `,

@xen-orchestra/backups-cli/package.json

@@ -7,21 +7,16 @@
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.9.1",
-    "@xen-orchestra/fs": "^0.14.0",
+    "@xen-orchestra/backups": "^0.23.0",
+    "@xen-orchestra/fs": "^1.0.1",
     "filenamify": "^4.1.0",
     "getopts": "^2.2.5",
     "lodash": "^4.17.15",
-    "promise-toolbox": "^0.18.0",
-    "vhd-lib": "^1.0.0"
+    "promise-toolbox": "^0.21.0"
   },
   "engines": {
     "node": ">=7.10.1"
   },
-  "files": [
-    "commands",
-    "*.js"
-  ],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/backups-cli",
   "name": "@xen-orchestra/backups-cli",
   "repository": {
@@ -32,7 +27,7 @@
   "scripts": {
     "postversion": "npm publish --access public"
   },
-  "version": "0.5.0",
+  "version": "0.7.1",
   "license": "AGPL-3.0-or-later",
   "author": {
     "name": "Vates SAS",

@xen-orchestra/backups/Backup.js

@@ -1,14 +1,16 @@
+'use strict'
+
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
 const Disposable = require('promise-toolbox/Disposable')
 const ignoreErrors = require('promise-toolbox/ignoreErrors')
-const limitConcurrency = require('limit-concurrency-decorator').default
 const { compileTemplate } = require('@xen-orchestra/template')
+const { limitConcurrency } = require('limit-concurrency-decorator')
 
-const { extractIdsFromSimplePattern } = require('./_extractIdsFromSimplePattern')
-const { PoolMetadataBackup } = require('./_PoolMetadataBackup')
-const { Task } = require('./Task')
-const { VmBackup } = require('./_VmBackup')
-const { XoMetadataBackup } = require('./_XoMetadataBackup')
+const { extractIdsFromSimplePattern } = require('./_extractIdsFromSimplePattern.js')
+const { PoolMetadataBackup } = require('./_PoolMetadataBackup.js')
+const { Task } = require('./Task.js')
+const { VmBackup } = require('./_VmBackup.js')
+const { XoMetadataBackup } = require('./_XoMetadataBackup.js')
 
 const noop = Function.prototype
 

@xen-orchestra/backups/DurablePartition.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const { asyncMap } = require('@xen-orchestra/async-map')
 
 exports.DurablePartition = class DurablePartition {

@xen-orchestra/backups/ImportVmBackup.js

@@ -1,13 +1,16 @@
+'use strict'
+
 const assert = require('assert')
 
-const { formatFilenameDate } = require('./_filenameDate')
-const { importDeltaVm } = require('./_deltaVm')
-const { Task } = require('./Task')
+const { formatFilenameDate } = require('./_filenameDate.js')
+const { importDeltaVm } = require('./_deltaVm.js')
+const { Task } = require('./Task.js')
+const { watchStreamSize } = require('./_watchStreamSize.js')
 
 exports.ImportVmBackup = class ImportVmBackup {
-  constructor({ adapter, metadata, srUuid, xapi, settings: { newMacAddresses } = {} }) {
+  constructor({ adapter, metadata, srUuid, xapi, settings: { newMacAddresses, mapVdisSrs = {} } = {} }) {
     this._adapter = adapter
-    this._importDeltaVmSettings = { newMacAddresses }
+    this._importDeltaVmSettings = { newMacAddresses, mapVdisSrs }
     this._metadata = metadata
     this._srUuid = srUuid
     this._xapi = xapi
@@ -18,13 +21,22 @@ exports.ImportVmBackup = class ImportVmBackup {
     const metadata = this._metadata
     const isFull = metadata.mode === 'full'
 
+    const sizeContainer = { size: 0 }
+
     let backup
     if (isFull) {
       backup = await adapter.readFullVmBackup(metadata)
+      watchStreamSize(backup, sizeContainer)
     } else {
       assert.strictEqual(metadata.mode, 'delta')
 
-      backup = await adapter.readDeltaVmBackup(metadata)
+      const ignoredVdis = new Set(
+        Object.entries(this._importDeltaVmSettings.mapVdisSrs)
+          .filter(([_, srUuid]) => srUuid === null)
+          .map(([vdiUuid]) => vdiUuid)
+      )
+      backup = await adapter.readDeltaVmBackup(metadata, ignoredVdis)
+      Object.values(backup.streams).forEach(stream => watchStreamSize(stream, sizeContainer))
     }
 
     return Task.run(
@@ -52,7 +64,7 @@ exports.ImportVmBackup = class ImportVmBackup {
         ])
 
         return {
-          size: metadata.size,
+          size: sizeContainer.size,
           id: await xapi.getField('VM', vmRef, 'uuid'),
         }
       }

@xen-orchestra/backups/RemoteAdapter.js

@@ -1,23 +1,30 @@
+'use strict'
+
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
 const Disposable = require('promise-toolbox/Disposable')
 const fromCallback = require('promise-toolbox/fromCallback')
 const fromEvent = require('promise-toolbox/fromEvent')
 const pDefer = require('promise-toolbox/defer')
-const pump = require('pump')
-const { basename, dirname, join, normalize, resolve } = require('path')
+const groupBy = require('lodash/groupBy.js')
+const pickBy = require('lodash/pickBy.js')
+const { dirname, join, normalize, resolve } = require('path')
 const { createLogger } = require('@xen-orchestra/log')
-const { createSyntheticStream, mergeVhd, default: Vhd } = require('vhd-lib')
-const { deduped } = require('@vates/disposable/deduped')
+const { Constants, createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib')
+const { deduped } = require('@vates/disposable/deduped.js')
+const { decorateMethodsWith } = require('@vates/decorate-with')
+const { compose } = require('@vates/compose')
 const { execFile } = require('child_process')
 const { readdir, stat } = require('fs-extra')
+const { v4: uuidv4 } = require('uuid')
 const { ZipFile } = require('yazl')
 
-const { BACKUP_DIR } = require('./_getVmBackupDir')
-const { cleanVm } = require('./_cleanVm')
-const { getTmpDir } = require('./_getTmpDir')
-const { isMetadataFile, isVhdFile } = require('./_backupType')
-const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions')
-const { lvs, pvs } = require('./_lvm')
+const { BACKUP_DIR } = require('./_getVmBackupDir.js')
+const { cleanVm } = require('./_cleanVm.js')
+const { getTmpDir } = require('./_getTmpDir.js')
+const { isMetadataFile } = require('./_backupType.js')
+const { isValidXva } = require('./_isValidXva.js')
+const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js')
+const { lvs, pvs } = require('./_lvm.js')
 
 const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
 exports.DIR_XO_CONFIG_BACKUPS = DIR_XO_CONFIG_BACKUPS
@@ -66,58 +73,17 @@ const debounceResourceFactory = factory =>
   }
 
 class RemoteAdapter {
-  constructor(handler, { debounceResource = res => res, dirMode } = {}) {
+  constructor(handler, { debounceResource = res => res, dirMode, vhdDirectoryCompression } = {}) {
     this._debounceResource = debounceResource
     this._dirMode = dirMode
     this._handler = handler
+    this._vhdDirectoryCompression = vhdDirectoryCompression
   }
 
   get handler() {
     return this._handler
   }
 
-  async _deleteVhd(path) {
-    const handler = this._handler
-    const vhds = await asyncMapSettled(
-      await handler.list(dirname(path), {
-        filter: isVhdFile,
-        prependDir: true,
-      }),
-      async path => {
-        try {
-          const vhd = new Vhd(handler, path)
-          await vhd.readHeaderAndFooter()
-          return {
-            footer: vhd.footer,
-            header: vhd.header,
-            path,
-          }
-        } catch (error) {
-          // Do not fail on corrupted VHDs (usually uncleaned temporary files),
-          // they are probably inconsequent to the backup process and should not
-          // fail it.
-          warn(`BackupNg#_deleteVhd ${path}`, { error })
-        }
-      }
-    )
-    const base = basename(path)
-    const child = vhds.find(_ => _ !== undefined && _.header.parentUnicodeName === base)
-    if (child === undefined) {
-      await handler.unlink(path)
-      return 0
-    }
-
-    try {
-      const childPath = child.path
-      const mergedDataSize = await mergeVhd(handler, path, handler, childPath)
-      await handler.rename(path, childPath)
-      return mergedDataSize
-    } catch (error) {
-      handler.unlink(path).catch(warn)
-      throw error
-    }
-  }
-
   async _findPartition(devicePath, partitionId) {
     const partitions = await listPartitions(devicePath)
     const partition = partitions.find(_ => _.id === partitionId)
@@ -127,9 +93,6 @@ class RemoteAdapter {
     return partition
   }
 
-  _getLvmLogicalVolumes = Disposable.factory(this._getLvmLogicalVolumes)
-  _getLvmLogicalVolumes = deduped(this._getLvmLogicalVolumes, (devicePath, pvId, vgName) => [devicePath, pvId, vgName])
-  _getLvmLogicalVolumes = debounceResourceFactory(this._getLvmLogicalVolumes)
   async *_getLvmLogicalVolumes(devicePath, pvId, vgName) {
     yield this._getLvmPhysicalVolume(devicePath, pvId && (await this._findPartition(devicePath, pvId)))
 
@@ -141,9 +104,6 @@ class RemoteAdapter {
     }
   }
 
-  _getLvmPhysicalVolume = Disposable.factory(this._getLvmPhysicalVolume)
-  _getLvmPhysicalVolume = deduped(this._getLvmPhysicalVolume, (devicePath, partition) => [devicePath, partition?.id])
-  _getLvmPhysicalVolume = debounceResourceFactory(this._getLvmPhysicalVolume)
   async *_getLvmPhysicalVolume(devicePath, partition) {
     const args = []
     if (partition !== undefined) {
@@ -164,9 +124,6 @@ class RemoteAdapter {
     }
   }
 
-  _getPartition = Disposable.factory(this._getPartition)
-  _getPartition = deduped(this._getPartition, (devicePath, partition) => [devicePath, partition?.id])
-  _getPartition = debounceResourceFactory(this._getPartition)
   async *_getPartition(devicePath, partition) {
     const options = ['loop', 'ro']
 
@@ -219,7 +176,6 @@ class RemoteAdapter {
     })
   }
 
-  _usePartitionFiles = Disposable.factory(this._usePartitionFiles)
   async *_usePartitionFiles(diskId, partitionId, paths) {
     const path = yield this.getPartition(diskId, partitionId)
 
@@ -231,6 +187,22 @@ class RemoteAdapter {
     return files
   }
 
+  // check if we will be allowed to merge a a vhd created in this adapter
+  // with the vhd at path `path`
+  async isMergeableParent(packedParentUid, path) {
+    return await Disposable.use(openVhd(this.handler, path), vhd => {
+      // this baseUuid is not linked with this vhd
+      if (!vhd.footer.uuid.equals(packedParentUid)) {
+        return false
+      }
+
+      const isVhdDirectory = vhd instanceof VhdDirectory
+      return isVhdDirectory
+        ? this.#useVhdDirectory() && this.#getCompressionType() === vhd.compressionType
+        : !this.#useVhdDirectory()
+    })
+  }
+
   fetchPartitionFiles(diskId, partitionId, paths) {
     const { promise, reject, resolve } = pDefer()
     Disposable.use(
@@ -252,16 +224,9 @@ class RemoteAdapter {
 
   async deleteDeltaVmBackups(backups) {
     const handler = this._handler
-    let mergedDataSize = 0
-    await asyncMapSettled(backups, ({ _filename, vhds }) =>
-      Promise.all([
-        handler.unlink(_filename),
-        asyncMap(Object.values(vhds), async _ => {
-          mergedDataSize += await this._deleteVhd(resolveRelativeFromFile(_filename, _))
-        }),
-      ])
-    )
-    return mergedDataSize
+
+    // this will delete the json, unused VHDs will be detected by `cleanVm`
+    await asyncMapSettled(backups, ({ _filename }) => handler.unlink(_filename))
   }
 
   async deleteMetadataBackup(backupId) {
@@ -291,22 +256,42 @@ class RemoteAdapter {
     )
   }
 
-  async deleteVmBackup(filename) {
-    const metadata = JSON.parse(String(await this._handler.readFile(filename)))
-    metadata._filename = filename
+  deleteVmBackup(file) {
+    return this.deleteVmBackups([file])
+  }
 
-    if (metadata.mode === 'delta') {
-      await this.deleteDeltaVmBackups([metadata])
-    } else if (metadata.mode === 'full') {
-      await this.deleteFullVmBackups([metadata])
-    } else {
-      throw new Error(`no deleter for backup mode ${metadata.mode}`)
+  async deleteVmBackups(files) {
+    const { delta, full, ...others } = groupBy(await asyncMap(files, file => this.readVmBackupMetadata(file)), 'mode')
+
+    const unsupportedModes = Object.keys(others)
+    if (unsupportedModes.length !== 0) {
+      throw new Error('no deleter for backup modes: ' + unsupportedModes.join(', '))
+    }
+
+    await Promise.all([
+      delta !== undefined && this.deleteDeltaVmBackups(delta),
+      full !== undefined && this.deleteFullVmBackups(full),
+    ])
+
+    const dirs = new Set(files.map(file => dirname(file)))
+    for (const dir of dirs) {
+      // don't merge in main process, unused VHDs will be merged in the next backup run
+      await this.cleanVm(dir, { remove: true, onLog: warn })
     }
   }
 
-  getDisk = Disposable.factory(this.getDisk)
-  getDisk = deduped(this.getDisk, diskId => [diskId])
-  getDisk = debounceResourceFactory(this.getDisk)
+  #getCompressionType() {
+    return this._vhdDirectoryCompression
+  }
+
+  #useVhdDirectory() {
+    return this.handler.type === 's3'
+  }
+
+  #useAlias() {
+    return this.#useVhdDirectory()
+  }
+
   async *getDisk(diskId) {
     const handler = this._handler
 
@@ -343,7 +328,6 @@ class RemoteAdapter {
   // - `<partitionId>`: partitioned disk
   // - `<pvId>/<vgName>/<lvName>`: LVM on a partitioned disk
   // - `/<vgName>/lvName>`: LVM on a raw disk
-  getPartition = Disposable.factory(this.getPartition)
   async *getPartition(diskId, partitionId) {
     const devicePath = yield this.getDisk(diskId)
     if (partitionId === undefined) {
@@ -360,13 +344,26 @@ class RemoteAdapter {
     return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
   }
 
+  // if we use alias on this remote, we have to name the file alias.vhd
+  getVhdFileName(baseName) {
+    if (this.#useAlias()) {
+      return `${baseName}.alias.vhd`
+    }
+    return `${baseName}.vhd`
+  }
+
   async listAllVmBackups() {
     const handler = this._handler
 
     const backups = { __proto__: null }
-    await asyncMap(await handler.list(BACKUP_DIR), async vmUuid => {
-      const vmBackups = await this.listVmBackups(vmUuid)
-      backups[vmUuid] = vmBackups
+    await asyncMap(await handler.list(BACKUP_DIR), async entry => {
+      // ignore hidden and lock files
+      if (entry[0] !== '.' && !entry.endsWith('.lock')) {
+        const vmBackups = await this.listVmBackups(entry)
+        if (vmBackups.length !== 0) {
+          backups[entry] = vmBackups
+        }
+      }
     })
 
     return backups
@@ -504,32 +501,91 @@ class RemoteAdapter {
     return backups.sort(compareTimestamp)
   }
 
-  async outputStream(path, input, { checksum = true, validator = noop } = {}) {
+  async writeVhd(path, input, { checksum = true, validator = noop } = {}) {
     const handler = this._handler
-    input = await input
-    const tmpPath = `${dirname(path)}/.${basename(path)}`
-    const output = await handler.createOutputStream(tmpPath, {
-      checksum,
-      dirMode: this._dirMode,
-    })
-    try {
-      await Promise.all([fromCallback(pump, input, output), output.checksumWritten, input.task])
-      await validator(tmpPath)
-      await handler.rename(tmpPath, path, { checksum })
-    } catch (error) {
-      await handler.unlink(tmpPath, { checksum })
-      throw error
+
+    if (this.#useVhdDirectory()) {
+      const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
+      await createVhdDirectoryFromStream(handler, dataPath, input, {
+        concurrency: 16,
+        compression: this.#getCompressionType(),
+        async validator() {
+          await input.task
+          return validator.apply(this, arguments)
+        },
+      })
+      await VhdAbstract.createAlias(handler, path, dataPath)
+    } else {
+      await this.outputStream(path, input, { checksum, validator })
     }
   }
 
-  async readDeltaVmBackup(metadata) {
+  async outputStream(path, input, { checksum = true, validator = noop } = {}) {
+    await this._handler.outputStream(path, input, {
+      checksum,
+      dirMode: this._dirMode,
+      async validator() {
+        await input.task
+        return validator.apply(this, arguments)
+      },
+    })
+  }
+
+  async _createSyntheticStream(handler, paths) {
+    let disposableVhds = []
+
+    // if it's a path : open all hierarchy of parent
+    if (typeof paths === 'string') {
+      let vhd
+      let vhdPath = paths
+      do {
+        const disposable = await openVhd(handler, vhdPath)
+        vhd = disposable.value
+        disposableVhds.push(disposable)
+        vhdPath = resolveRelativeFromFile(vhdPath, vhd.header.parentUnicodeName)
+      } while (vhd.footer.diskType !== Constants.DISK_TYPES.DYNAMIC)
+    } else {
+      // only open the list of path given
+      disposableVhds = paths.map(path => openVhd(handler, path))
+    }
+
+    // I don't want the vhds to be disposed on return
+    // but only when the stream is done ( or failed )
+    const disposables = await Disposable.all(disposableVhds)
+    const vhds = disposables.value
+
+    let disposed = false
+    const disposeOnce = async () => {
+      if (!disposed) {
+        disposed = true
+
+        try {
+          await disposables.dispose()
+        } catch (error) {
+          warn('_createSyntheticStream: failed to dispose VHDs', { error })
+        }
+      }
+    }
+
+    const synthetic = new VhdSynthetic(vhds)
+    await synthetic.readHeaderAndFooter()
+    await synthetic.readBlockAllocationTable()
+    const stream = await synthetic.stream()
+    stream.on('end', disposeOnce)
+    stream.on('close', disposeOnce)
+    stream.on('error', disposeOnce)
+    return stream
+  }
+
+  async readDeltaVmBackup(metadata, ignoredVdis) {
     const handler = this._handler
-    const { vbds, vdis, vhds, vifs, vm } = metadata
+    const { vbds, vhds, vifs, vm } = metadata
     const dir = dirname(metadata._filename)
+    const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
 
     const streams = {}
-    await asyncMapSettled(Object.entries(vdis), async ([id, vdi]) => {
-      streams[`${id}.vhd`] = await createSyntheticStream(handler, join(dir, vhds[id]))
+    await asyncMapSettled(Object.keys(vdis), async ref => {
+      streams[`${ref}.vhd`] = await this._createSyntheticStream(handler, join(dir, vhds[ref]))
     })
 
     return {
@@ -551,8 +607,41 @@ class RemoteAdapter {
   }
 }
 
-RemoteAdapter.prototype.cleanVm = function (vmDir) {
-  return Disposable.use(this._handler.lock(vmDir), () => cleanVm.apply(this, arguments))
-}
+Object.assign(RemoteAdapter.prototype, {
+  cleanVm(vmDir, { lock = true } = {}) {
+    if (lock) {
+      return Disposable.use(this._handler.lock(vmDir), () => cleanVm.apply(this, arguments))
+    } else {
+      return cleanVm.apply(this, arguments)
+    }
+  },
+  isValidXva,
+})
+
+decorateMethodsWith(RemoteAdapter, {
+  _getLvmLogicalVolumes: compose([
+    Disposable.factory,
+    [deduped, (devicePath, pvId, vgName) => [devicePath, pvId, vgName]],
+    debounceResourceFactory,
+  ]),
+
+  _getLvmPhysicalVolume: compose([
+    Disposable.factory,
+    [deduped, (devicePath, partition) => [devicePath, partition?.id]],
+    debounceResourceFactory,
+  ]),
+
+  _getPartition: compose([
+    Disposable.factory,
+    [deduped, (devicePath, partition) => [devicePath, partition?.id]],
+    debounceResourceFactory,
+  ]),
+
+  _usePartitionFiles: Disposable.factory,
+
+  getDisk: compose([Disposable.factory, [deduped, diskId => [diskId]], debounceResourceFactory]),
+
+  getPartition: Disposable.factory,
+})
 
 exports.RemoteAdapter = RemoteAdapter

@xen-orchestra/backups/RestoreMetadataBackup.js

@@ -1,5 +1,7 @@
-const { DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter')
-const { PATH_DB_DUMP } = require('./_PoolMetadataBackup')
+'use strict'
+
+const { DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')
+const { PATH_DB_DUMP } = require('./_PoolMetadataBackup.js')
 
 exports.RestoreMetadataBackup = class RestoreMetadataBackup {
   constructor({ backupId, handler, xapi }) {

@xen-orchestra/backups/Task.js

@@ -1,11 +1,16 @@
-const Zone = require('node-zone')
+'use strict'
 
-const { SyncThenable } = require('./_syncThenable')
+const CancelToken = require('promise-toolbox/CancelToken')
+const Zone = require('node-zone')
 
 const logAfterEnd = () => {
   throw new Error('task has already ended')
 }
 
+const noop = Function.prototype
+
+const serializeErrors = errors => (Array.isArray(errors) ? errors.map(serializeError) : errors)
+
 // Create a serializable object from an error.
 //
 // Otherwise some fields might be non-enumerable and missing from logs.
@@ -14,168 +19,138 @@ const serializeError = error =>
     ? {
         ...error, // Copy enumerable properties.
         code: error.code,
+        errors: serializeErrors(error.errors), // supports AggregateError
         message: error.message,
         name: error.name,
         stack: error.stack,
       }
     : error
-exports.serializeError = serializeError
 
-class TaskLogger {
-  constructor(logFn, parentId) {
-    this._log = logFn
-    this._parentId = parentId
-    this._taskId = undefined
+const $$task = Symbol('@xen-orchestra/backups/Task')
+
+class Task {
+  static get cancelToken() {
+    const task = Zone.current.data[$$task]
+    return task !== undefined ? task.#cancelToken : CancelToken.none
   }
 
-  get taskId() {
-    const taskId = this._taskId
-    if (taskId === undefined) {
-      throw new Error('start the task first')
-    }
-    return taskId
+  static run(opts, fn) {
+    return new this(opts).run(fn, true)
   }
 
-  // create a subtask
-  fork() {
-    return new TaskLogger(this._log, this.taskId)
-  }
-
-  info(message, data) {
-    return this._log({
-      data,
-      event: 'info',
-      message,
-      taskId: this.taskId,
-      timestamp: Date.now(),
-    })
-  }
-
-  run(message, data, fn) {
-    if (arguments.length === 2) {
-      fn = data
-      data = undefined
-    }
-
-    return SyncThenable.tryUnwrap(
-      SyncThenable.fromFunction(() => {
-        if (this._taskId !== undefined) {
-          throw new Error('task has already started')
-        }
-
-        this._taskId = Math.random().toString(36).slice(2)
-
-        return this._log({
-          data,
-          event: 'start',
-          message,
-          parentId: this._parentId,
-          taskId: this.taskId,
-          timestamp: Date.now(),
-        })
-      })
-        .then(fn)
-        .then(
-          result => {
-            const log = this._log
-            this._log = logAfterEnd
-            return SyncThenable.resolve(
-              log({
-                event: 'end',
-                result,
-                status: 'success',
-                taskId: this.taskId,
-                timestamp: Date.now(),
-              })
-            ).then(() => result)
-          },
-          error => {
-            const log = this._log
-            this._log = logAfterEnd
-            return SyncThenable.resolve(
-              log({
-                event: 'end',
-                result: serializeError(error),
-                status: 'failure',
-                taskId: this.taskId,
-                timestamp: Date.now(),
-              })
-            ).then(() => {
-              throw error
-            })
-          }
-        )
-    )
-  }
-
-  warning(message, data) {
-    return this._log({
-      data,
-      event: 'warning',
-      message,
-      taskId: this.taskId,
-      timestamp: Date.now(),
-    })
-  }
-
-  wrapFn(fn, message, data) {
-    const logger = this
-    return function () {
-      const evaluate = v => (typeof v === 'function' ? v.apply(this, arguments) : v)
-
-      return logger.run(evaluate(message), evaluate(data), () => fn.apply(this, arguments))
-    }
-  }
-}
-
-const $$task = Symbol('current task logger')
-
-const getCurrent = () => Zone.current.data[$$task]
-
-const Task = {
-  info(message, data) {
-    const task = getCurrent()
-    if (task !== undefined) {
-      return task.info(message, data)
-    }
-  },
-
-  run({ name, data, onLog }, fn) {
-    let parentId
-    if (onLog === undefined) {
-      const parent = getCurrent()
-      if (parent === undefined) {
-        return fn()
-      }
-      onLog = parent._log
-      parentId = parent.taskId
-    }
-
-    const task = new TaskLogger(onLog, parentId)
-    const zone = Zone.current.fork('task')
-    zone.data[$$task] = task
-    return task.run(name, data, zone.wrap(fn))
-  },
-
-  warning(message, data) {
-    const task = getCurrent()
-    if (task !== undefined) {
-      return task.warning(message, data)
-    }
-  },
-
-  wrapFn(opts, fn) {
+  static wrapFn(opts, fn) {
     // compatibility with @decorateWith
     if (typeof fn !== 'function') {
       ;[fn, opts] = [opts, fn]
     }
 
-    const { name, data, onLog } = opts
-
     return function () {
-      const evaluate = v => (typeof v === 'function' ? v.apply(this, arguments) : v)
-      return Task.run({ name: evaluate(name), data: evaluate(data), onLog }, () => fn.apply(this, arguments))
+      return Task.run(typeof opts === 'function' ? opts.apply(this, arguments) : opts, () => fn.apply(this, arguments))
     }
-  },
+  }
+
+  #cancelToken
+  #id = Math.random().toString(36).slice(2)
+  #onLog
+  #zone
+
+  constructor({ name, data, onLog }) {
+    let parentCancelToken, parentId
+    if (onLog === undefined) {
+      const parent = Zone.current.data[$$task]
+      if (parent === undefined) {
+        onLog = noop
+      } else {
+        onLog = log => parent.#onLog(log)
+        parentCancelToken = parent.#cancelToken
+        parentId = parent.#id
+      }
+    }
+
+    const zone = Zone.current.fork('@xen-orchestra/backups/Task')
+    zone.data[$$task] = this
+    this.#zone = zone
+
+    const { cancel, token } = CancelToken.source(parentCancelToken && [parentCancelToken])
+    this.#cancelToken = token
+    this.cancel = cancel
+
+    this.#onLog = onLog
+
+    this.#log('start', {
+      data,
+      message: name,
+      parentId,
+    })
+  }
+
+  failure(error) {
+    this.#end('failure', serializeError(error))
+  }
+
+  info(message, data) {
+    this.#log('info', { data, message })
+  }
+
+  /**
+   * Run a function in the context of this task
+   *
+   * In case of error, the task will be failed.
+   *
+   * @typedef Result
+   * @param {() => Result)} fn
+   * @param {boolean} last - Whether the task should succeed if there is no error
+   * @returns Result
+   */
+  run(fn, last = false) {
+    return this.#zone.run(() => {
+      try {
+        const result = fn()
+        let then
+        if (result != null && typeof (then = result.then) === 'function') {
+          then.call(result, last && (value => this.success(value)), error => this.failure(error))
+        } else if (last) {
+          this.success(result)
+        }
+        return result
+      } catch (error) {
+        this.failure(error)
+        throw error
+      }
+    })
+  }
+
+  success(value) {
+    this.#end('success', value)
+  }
+
+  warning(message, data) {
+    this.#log('warning', { data, message })
+  }
+
+  wrapFn(fn, last) {
+    const task = this
+    return function () {
+      return task.run(() => fn.apply(this, arguments), last)
+    }
+  }
+
+  #end(status, result) {
+    this.#log('end', { result, status })
+    this.#onLog = logAfterEnd
+  }
+
+  #log(event, props) {
+    this.#onLog({
+      ...props,
+      event,
+      taskId: this.#id,
+      timestamp: Date.now(),
+    })
+  }
 }
 exports.Task = Task
+
+for (const method of ['info', 'warning']) {
+  Task[method] = (...args) => Zone.current.data[$$task]?.[method](...args)
+}

@xen-orchestra/backups/_PoolMetadataBackup.js

@@ -1,9 +1,11 @@
+'use strict'
+
 const { asyncMap } = require('@xen-orchestra/async-map')
 
-const { DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter')
-const { forkStreamUnpipe } = require('./_forkStreamUnpipe')
-const { formatFilenameDate } = require('./_filenameDate')
-const { Task } = require('./Task')
+const { DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')
+const { forkStreamUnpipe } = require('./_forkStreamUnpipe.js')
+const { formatFilenameDate } = require('./_filenameDate.js')
+const { Task } = require('./Task.js')
 
 const PATH_DB_DUMP = '/pool/xmldbdump'
 exports.PATH_DB_DUMP = PATH_DB_DUMP

@xen-orchestra/backups/_VmBackup.js

@@ -1,23 +1,42 @@
-const findLast = require('lodash/findLast')
+'use strict'
+
+const assert = require('assert')
+const findLast = require('lodash/findLast.js')
+const groupBy = require('lodash/groupBy.js')
 const ignoreErrors = require('promise-toolbox/ignoreErrors')
-const keyBy = require('lodash/keyBy')
-const mapValues = require('lodash/mapValues')
+const keyBy = require('lodash/keyBy.js')
+const mapValues = require('lodash/mapValues.js')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { createLogger } = require('@xen-orchestra/log')
+const { decorateMethodsWith } = require('@vates/decorate-with')
+const { defer } = require('golike-defer')
 const { formatDateTime } = require('@xen-orchestra/xapi')
 
-const { ContinuousReplicationWriter } = require('./_ContinuousReplicationWriter')
-const { DeltaBackupWriter } = require('./_DeltaBackupWriter')
-const { DisasterRecoveryWriter } = require('./_DisasterRecoveryWriter')
-const { exportDeltaVm } = require('./_deltaVm')
-const { forkStreamUnpipe } = require('./_forkStreamUnpipe')
-const { FullBackupWriter } = require('./_FullBackupWriter')
-const { getOldEntries } = require('./_getOldEntries')
-const { Task } = require('./Task')
-const { watchStreamSize } = require('./_watchStreamSize')
+const { DeltaBackupWriter } = require('./writers/DeltaBackupWriter.js')
+const { DeltaReplicationWriter } = require('./writers/DeltaReplicationWriter.js')
+const { exportDeltaVm } = require('./_deltaVm.js')
+const { forkStreamUnpipe } = require('./_forkStreamUnpipe.js')
+const { FullBackupWriter } = require('./writers/FullBackupWriter.js')
+const { FullReplicationWriter } = require('./writers/FullReplicationWriter.js')
+const { getOldEntries } = require('./_getOldEntries.js')
+const { Task } = require('./Task.js')
+const { watchStreamSize } = require('./_watchStreamSize.js')
 
 const { debug, warn } = createLogger('xo:backups:VmBackup')
 
+class AggregateError extends Error {
+  constructor(errors, message) {
+    super(message)
+    this.errors = errors
+  }
+}
+
+const asyncEach = async (iterable, fn, thisArg = iterable) => {
+  for (const item of iterable) {
+    await fn.call(thisArg, item)
+  }
+}
+
 const forkDeltaExport = deltaExport =>
   Object.create(deltaExport, {
     streams: {
@@ -25,8 +44,14 @@ const forkDeltaExport = deltaExport =>
     },
   })
 
-exports.VmBackup = class VmBackup {
+class VmBackup {
   constructor({ config, getSnapshotNameLabel, job, remoteAdapters, remotes, schedule, settings, srs, vm }) {
+    if (vm.other_config['xo:backup:job'] === job.id && 'start' in vm.blocked_operations) {
+      // don't match replicated VMs created by this very job otherwise they
+      // will be replicated again and again
+      throw new Error('cannot backup a VM created by this very job')
+    }
+
     this.config = config
     this.job = job
     this.remoteAdapters = remoteAdapters
@@ -62,12 +87,12 @@ exports.VmBackup = class VmBackup {
 
     // Create writers
     {
-      const writers = []
+      const writers = new Set()
       this._writers = writers
 
       const [BackupWriter, ReplicationWriter] = this._isDelta
-        ? [DeltaBackupWriter, ContinuousReplicationWriter]
-        : [FullBackupWriter, DisasterRecoveryWriter]
+        ? [DeltaBackupWriter, DeltaReplicationWriter]
+        : [FullBackupWriter, FullReplicationWriter]
 
       const allSettings = job.settings
 
@@ -77,7 +102,7 @@ exports.VmBackup = class VmBackup {
           ...allSettings[remoteId],
         }
         if (targetSettings.exportRetention !== 0) {
-          writers.push(new BackupWriter(this, remoteId, targetSettings))
+          writers.add(new BackupWriter({ backup: this, remoteId, settings: targetSettings }))
         }
       })
       srs.forEach(sr => {
@@ -86,12 +111,45 @@ exports.VmBackup = class VmBackup {
           ...allSettings[sr.uuid],
         }
         if (targetSettings.copyRetention !== 0) {
-          writers.push(new ReplicationWriter(this, sr, targetSettings))
+          writers.add(new ReplicationWriter({ backup: this, sr, settings: targetSettings }))
         }
       })
     }
   }
 
+  // calls fn for each function, warns of any errors, and throws only if there are no writers left
+  async _callWriters(fn, warnMessage, parallel = true) {
+    const writers = this._writers
+    const n = writers.size
+    if (n === 0) {
+      return
+    }
+    if (n === 1) {
+      const [writer] = writers
+      try {
+        await fn(writer)
+      } catch (error) {
+        writers.delete(writer)
+        throw error
+      }
+      return
+    }
+
+    const errors = []
+    await (parallel ? asyncMap : asyncEach)(writers, async function (writer) {
+      try {
+        await fn(writer)
+      } catch (error) {
+        errors.push(error)
+        this.delete(writer)
+        warn(warnMessage, { error, writer: writer.constructor.name })
+      }
+    })
+    if (writers.size === 0) {
+      throw new AggregateError(errors, 'all targets have failed, step: ' + warnMessage)
+    }
+  }
+
   // ensure the VM itself does not have any backup metadata which would be
   // copied on manual snapshots and interfere with the backup jobs
   async _cleanMetadata() {
@@ -114,7 +172,8 @@ exports.VmBackup = class VmBackup {
 
     const settings = this._settings
 
-    const doSnapshot = this._isDelta || vm.power_state === 'Running' || settings.snapshotRetention !== 0
+    const doSnapshot =
+      this._isDelta || (!settings.offlineBackup && vm.power_state === 'Running') || settings.snapshotRetention !== 0
     if (doSnapshot) {
       await Task.run({ name: 'snapshot' }, async () => {
         if (!settings.bypassVdiChainsCheck) {
@@ -122,6 +181,7 @@ exports.VmBackup = class VmBackup {
         }
 
         const snapshotRef = await vm[settings.checkpointSnapshot ? '$checkpoint' : '$snapshot']({
+          ignoreNobakVdis: true,
           name_label: this._getSnapshotNameLabel(vm),
         })
         this.timestamp = Date.now()
@@ -146,31 +206,28 @@ exports.VmBackup = class VmBackup {
   async _copyDelta() {
     const { exportedVm } = this
     const baseVm = this._baseVm
+    const fullVdisRequired = this._fullVdisRequired
 
-    await asyncMap(this._writers, writer => writer.prepare && writer.prepare())
+    const isFull = fullVdisRequired === undefined || fullVdisRequired.size !== 0
+
+    await this._callWriters(writer => writer.prepare({ isFull }), 'writer.prepare()')
 
     const deltaExport = await exportDeltaVm(exportedVm, baseVm, {
-      fullVdisRequired: this._fullVdisRequired,
+      fullVdisRequired,
     })
-    const sizeContainers = mapValues(deltaExport.streams, watchStreamSize)
+    const sizeContainers = mapValues(deltaExport.streams, stream => watchStreamSize(stream))
 
     const timestamp = Date.now()
 
-    await asyncMap(this._writers, async writer => {
-      try {
-        await writer.transfer({
+    await this._callWriters(
+      writer =>
+        writer.transfer({
           deltaExport: forkDeltaExport(deltaExport),
           sizeContainers,
           timestamp,
-        })
-      } catch (error) {
-        warn('copy failure', {
-          error,
-          target: writer.target,
-          vm: this.vm,
-        })
-      }
-    })
+        }),
+      'writer.transfer()'
+    )
 
     this._baseVm = exportedVm
 
@@ -195,7 +252,7 @@ exports.VmBackup = class VmBackup {
       size,
     })
 
-    await asyncMap(this._writers, writer => writer.cleanup && writer.cleanup())
+    await this._callWriters(writer => writer.cleanup(), 'writer.cleanup()')
   }
 
   async _copyFull() {
@@ -208,21 +265,15 @@ exports.VmBackup = class VmBackup {
 
     const timestamp = Date.now()
 
-    await asyncMap(this._writers, async writer => {
-      try {
-        await writer.run({
+    await this._callWriters(
+      writer =>
+        writer.run({
           sizeContainer,
           stream: forkStreamUnpipe(stream),
           timestamp,
-        })
-      } catch (error) {
-        warn('copy failure', {
-          error,
-          target: writer.target,
-          vm: this.vm,
-        })
-      }
-    })
+        }),
+      'writer.run()'
+    )
 
     const { size } = sizeContainer
     const end = Date.now()
@@ -253,17 +304,28 @@ exports.VmBackup = class VmBackup {
   }
 
   async _removeUnusedSnapshots() {
-    // TODO: handle all schedules (no longer existing schedules default to 0 retention)
-
-    const { scheduleId } = this
-    const scheduleSnapshots = this._jobSnapshots.filter(_ => _.other_config['xo:backup:schedule'] === scheduleId)
-
+    const jobSettings = this.job.settings
     const baseVmRef = this._baseVm?.$ref
+    const { config } = this
+    const baseSettings = {
+      ...config.defaultSettings,
+      ...config.metadata.defaultSettings,
+      ...jobSettings[''],
+    }
+
+    const snapshotsPerSchedule = groupBy(this._jobSnapshots, _ => _.other_config['xo:backup:schedule'])
     const xapi = this._xapi
-    await asyncMap(getOldEntries(this._settings.snapshotRetention, scheduleSnapshots), ({ $ref }) => {
-      if ($ref !== baseVmRef) {
-        return xapi.VM_destroy($ref)
+    await asyncMap(Object.entries(snapshotsPerSchedule), ([scheduleId, snapshots]) => {
+      const settings = {
+        ...baseSettings,
+        ...jobSettings[scheduleId],
+        ...jobSettings[this.vm.uuid],
       }
+      return asyncMap(getOldEntries(settings.snapshotRetention, snapshots), ({ $ref }) => {
+        if ($ref !== baseVmRef) {
+          return xapi.VM_destroy($ref)
+        }
+      })
     })
   }
 
@@ -272,12 +334,14 @@ exports.VmBackup = class VmBackup {
 
     let baseVm = findLast(this._jobSnapshots, _ => 'xo:backup:exported' in _.other_config)
     if (baseVm === undefined) {
+      debug('no base VM found')
       return
     }
 
     const fullInterval = this._settings.fullInterval
     const deltaChainLength = +(baseVm.other_config['xo:backup:deltaChainLength'] ?? 0) + 1
     if (!(fullInterval === 0 || fullInterval > deltaChainLength)) {
+      debug('not using base VM becaust fullInterval reached')
       return
     }
 
@@ -288,26 +352,44 @@ exports.VmBackup = class VmBackup {
 
     const baseUuidToSrcVdi = new Map()
     await asyncMap(await baseVm.$getDisks(), async baseRef => {
-      const snapshotOf = await xapi.getField('VDI', baseRef, 'snapshot_of')
+      const [baseUuid, snapshotOf] = await Promise.all([
+        xapi.getField('VDI', baseRef, 'uuid'),
+        xapi.getField('VDI', baseRef, 'snapshot_of'),
+      ])
       const srcVdi = srcVdis[snapshotOf]
       if (srcVdi !== undefined) {
-        baseUuidToSrcVdi.set(await xapi.getField('VDI', baseRef, 'uuid'), srcVdi)
+        baseUuidToSrcVdi.set(baseUuid, srcVdi)
+      } else {
+        debug('ignore snapshot VDI because no longer present on VM', {
+          vdi: baseUuid,
+        })
       }
     })
 
     const presentBaseVdis = new Map(baseUuidToSrcVdi)
-    const writers = this._writers
-    for (let i = 0, n = writers.length; presentBaseVdis.size !== 0 && i < n; ++i) {
-      await writers[i].checkBaseVdis(presentBaseVdis, baseVm)
-    }
+    await this._callWriters(
+      writer => presentBaseVdis.size !== 0 && writer.checkBaseVdis(presentBaseVdis, baseVm),
+      'writer.checkBaseVdis()',
+      false
+    )
 
     if (presentBaseVdis.size === 0) {
+      debug('no base VM found')
       return
     }
 
     const fullVdisRequired = new Set()
     baseUuidToSrcVdi.forEach((srcVdi, baseUuid) => {
-      if (!presentBaseVdis.has(baseUuid)) {
+      if (presentBaseVdis.has(baseUuid)) {
+        debug('found base VDI', {
+          base: baseUuid,
+          vdi: srcVdi.uuid,
+        })
+      } else {
+        debug('missing base VDI', {
+          base: baseUuid,
+          vdi: srcVdi.uuid,
+        })
         fullVdisRequired.add(srcVdi.uuid)
       }
     })
@@ -316,7 +398,18 @@ exports.VmBackup = class VmBackup {
     this._fullVdisRequired = fullVdisRequired
   }
 
-  async run() {
+  async run($defer) {
+    const settings = this._settings
+    assert(
+      !settings.offlineBackup || settings.snapshotRetention === 0,
+      'offlineBackup is not compatible with snapshotRetention'
+    )
+
+    await this._callWriters(async writer => {
+      await writer.beforeBackup()
+      $defer(() => writer.afterBackup())
+    }, 'writer.beforeBackup()')
+
     await this._fetchJobSnapshots()
 
     if (this._isDelta) {
@@ -326,7 +419,7 @@ exports.VmBackup = class VmBackup {
     await this._cleanMetadata()
     await this._removeUnusedSnapshots()
 
-    const { _settings: settings, vm } = this
+    const { vm } = this
     const isRunning = vm.power_state === 'Running'
     const startAfter = isRunning && (settings.offlineBackup ? 'backup' : settings.offlineSnapshot && 'snapshot')
     if (startAfter) {
@@ -339,7 +432,7 @@ exports.VmBackup = class VmBackup {
         ignoreErrors.call(vm.$callAsync('start', false, false))
       }
 
-      if (this._writers.length !== 0) {
+      if (this._writers.size !== 0) {
         await (this._isDelta ? this._copyDelta() : this._copyFull())
       }
     } finally {
@@ -352,3 +445,8 @@ exports.VmBackup = class VmBackup {
     }
   }
 }
+exports.VmBackup = VmBackup
+
+decorateMethodsWith(VmBackup, {
+  run: defer,
+})

@xen-orchestra/backups/_XoMetadataBackup.js

@@ -1,8 +1,10 @@
+'use strict'
+
 const { asyncMap } = require('@xen-orchestra/async-map')
 
-const { DIR_XO_CONFIG_BACKUPS } = require('./RemoteAdapter')
-const { formatFilenameDate } = require('./_filenameDate')
-const { Task } = require('./Task')
+const { DIR_XO_CONFIG_BACKUPS } = require('./RemoteAdapter.js')
+const { formatFilenameDate } = require('./_filenameDate.js')
+const { Task } = require('./Task.js')
 
 exports.XoMetadataBackup = class XoMetadataBackup {
   constructor({ config, job, remoteAdapters, schedule, settings }) {

@xen-orchestra/backups/_backupType.js

@@ -1,4 +1,6 @@
+'use strict'
+
 exports.isMetadataFile = filename => filename.endsWith('.json')
 exports.isVhdFile = filename => filename.endsWith('.vhd')
 exports.isXvaFile = filename => filename.endsWith('.xva')
-exports.isXvaSumFile = filename => filename.endsWith('.xva.cheksum')
+exports.isXvaSumFile = filename => filename.endsWith('.xva.checksum')

@xen-orchestra/backups/_backupWorker.js

@@ -1,15 +1,24 @@
+'use strict'
+
+require('@xen-orchestra/log/configure.js').catchGlobalErrors(
+  require('@xen-orchestra/log').createLogger('xo:backups:worker')
+)
+
+require('@vates/cached-dns.lookup').createCachedLookup().patchGlobal()
+
 const Disposable = require('promise-toolbox/Disposable')
 const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { compose } = require('@vates/compose')
-const { createDebounceResource } = require('@vates/disposable/debounceResource')
-const { deduped } = require('@vates/disposable/deduped')
+const { createDebounceResource } = require('@vates/disposable/debounceResource.js')
+const { decorateMethodsWith } = require('@vates/decorate-with')
+const { deduped } = require('@vates/disposable/deduped.js')
 const { getHandler } = require('@xen-orchestra/fs')
 const { parseDuration } = require('@vates/parse-duration')
 const { Xapi } = require('@xen-orchestra/xapi')
 
-const { Backup } = require('./Backup')
-const { RemoteAdapter } = require('./RemoteAdapter')
-const { Task } = require('./Task')
+const { Backup } = require('./Backup.js')
+const { RemoteAdapter } = require('./RemoteAdapter.js')
+const { Task } = require('./Task.js')
 
 class BackupWorker {
   #config
@@ -54,11 +63,6 @@ class BackupWorker {
     }).run()
   }
 
-  getAdapter = Disposable.factory(this.getAdapter)
-  getAdapter = deduped(this.getAdapter, remote => [remote.url])
-  getAdapter = compose(this.getAdapter, function (resource) {
-    return this.debounceResource(resource)
-  })
   async *getAdapter(remote) {
     const handler = getHandler(remote, this.#remoteOptions)
     await handler.sync()
@@ -66,17 +70,13 @@ class BackupWorker {
       yield new RemoteAdapter(handler, {
         debounceResource: this.debounceResource,
         dirMode: this.#config.dirMode,
+        vhdDirectoryCompression: this.#config.vhdDirectoryCompression,
       })
     } finally {
       await handler.forget()
     }
   }
 
-  getXapi = Disposable.factory(this.getXapi)
-  getXapi = deduped(this.getXapi, ({ url }) => [url])
-  getXapi = compose(this.getXapi, function (resource) {
-    return this.debounceResource(resource)
-  })
   async *getXapi({ credentials: { username: user, password }, ...opts }) {
     const xapi = new Xapi({
       ...this.#xapiOptions,
@@ -98,6 +98,30 @@ class BackupWorker {
   }
 }
 
+decorateMethodsWith(BackupWorker, {
+  getAdapter: compose([
+    Disposable.factory,
+    [deduped, remote => [remote.url]],
+    [
+      compose,
+      function (resource) {
+        return this.debounceResource(resource)
+      },
+    ],
+  ]),
+
+  getXapi: compose([
+    Disposable.factory,
+    [deduped, xapi => [xapi.url]],
+    [
+      compose,
+      function (resource) {
+        return this.debounceResource(resource)
+      },
+    ],
+  ]),
+})
+
 // Received message:
 //
 // Message {

@xen-orchestra/backups/_cancelableMap.js

@@ -1,3 +1,5 @@
+'use strict'
+
 const cancelable = require('promise-toolbox/cancelable')
 const CancelToken = require('promise-toolbox/CancelToken')
 

@xen-orchestra/backups/_checkVhd.js

@@ -1,5 +0,0 @@
-const Vhd = require('vhd-lib').default
-
-exports.checkVhd = async function checkVhd(handler, path) {
-  await new Vhd(handler, path).readHeaderAndFooter()
-}