fix: identify and reconnect to master

.gitignore

@@ -10,6 +10,8 @@
 /packages/*/dist/
 /packages/*/node_modules/
 
+/packages/vhd-cli/src/commands/index.js
+
 /packages/xen-api/examples/node_modules/
 /packages/xen-api/plot.dat
 

@vates/async-each/.USAGE.md

@@ -1,4 +1,4 @@
-### `pEach(iterable, iteratee, [opts])`
+### `asyncEach(iterable, iteratee, [opts])`
 
 Executes `iteratee` in order for each value yielded by `iterable`.
 
@@ -6,7 +6,7 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
 
 `iterable` must be an iterable or async iterable.
 
-`iteratee` is called with the same `this` value as `pEach`, and with the following arguments:
+`iteratee` is called with the same `this` value as `asyncEach`, and with the following arguments:
 
 - `value`: the value yielded by `iterable`
 - `index`: the 0-based index for this value
@@ -14,15 +14,15 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
 
 `opts` is an object that can contains the following options:
 
-- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`. The value `0` means no concurrency limit.
+- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
 - `signal`: an abort signal to stop the iteration
 - `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
 
 ```js
-import { pEach } from '@vates/async-each'
+import { asyncEach } from '@vates/async-each'
 
 const contents = []
-await pEach(
+await asyncEach(
   ['foo.txt', 'bar.txt', 'baz.txt'],
   async function (filename, i) {
     contents[i] = await readFile(filename)

@vates/async-each/README.md

@@ -16,7 +16,7 @@ Installation of the [npm package](https://npmjs.org/package/@vates/async-each):
 
 ## Usage
 
-### `pEach(iterable, iteratee, [opts])`
+### `asyncEach(iterable, iteratee, [opts])`
 
 Executes `iteratee` in order for each value yielded by `iterable`.
 
@@ -24,7 +24,7 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
 
 `iterable` must be an iterable or async iterable.
 
-`iteratee` is called with the same `this` value as `pEach`, and with the following arguments:
+`iteratee` is called with the same `this` value as `asyncEach`, and with the following arguments:
 
 - `value`: the value yielded by `iterable`
 - `index`: the 0-based index for this value
@@ -32,15 +32,15 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
 
 `opts` is an object that can contains the following options:
 
-- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`. The value `0` means no concurrency limit.
+- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
 - `signal`: an abort signal to stop the iteration
 - `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
 
 ```js
-import { pEach } from '@vates/async-each'
+import { asyncEach } from '@vates/async-each'
 
 const contents = []
-await pEach(
+await asyncEach(
   ['foo.txt', 'bar.txt', 'baz.txt'],
   async function (filename, i) {
     contents[i] = await readFile(filename)

@vates/async-each/index.js

@@ -9,16 +9,7 @@ class AggregateError extends Error {
   }
 }
 
-/**
- * @template Item
- * @param {Iterable<Item>} iterable
- * @param {(item: Item, index: number, iterable: Iterable<Item>) => Promise<void>} iteratee
- * @returns {Promise<void>}
- */
-exports.pEach = function pEach(iterable, iteratee, { concurrency = 10, signal, stopOnError = true } = {}) {
-  if (concurrency === 0) {
-    concurrency = Infinity
-  }
+exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 1, signal, stopOnError = true } = {}) {
   return new Promise((resolve, reject) => {
     const it = (iterable[Symbol.iterator] || iterable[Symbol.asyncIterator]).call(iterable)
     const errors = []
@@ -28,7 +19,7 @@ exports.pEach = function pEach(iterable, iteratee, { concurrency = 10, signal, s
     let onAbort
     if (signal !== undefined) {
       onAbort = () => {
-        onRejectedWrapper(new Error('pEach aborted'))
+        onRejectedWrapper(new Error('asyncEach aborted'))
       }
       signal.addEventListener('abort', onAbort)
     }
@@ -51,10 +42,11 @@ exports.pEach = function pEach(iterable, iteratee, { concurrency = 10, signal, s
         clean()
       })(reject)
 
-    let onFulfilled = () => {
+    let onFulfilled = value => {
       --running
       next()
     }
+    const onFulfilledWrapper = value => onFulfilled(value)
 
     let onRejected = stopOnError
       ? reject
@@ -65,14 +57,6 @@ exports.pEach = function pEach(iterable, iteratee, { concurrency = 10, signal, s
         }
     const onRejectedWrapper = reason => onRejected(reason)
 
-    const run = async (value, index) => {
-      try {
-        onFulfilled(await iteratee.call(this, value, index++))
-      } catch (error) {
-        onRejected(error)
-      }
-    }
-
     let nextIsRunning = false
     let next = async () => {
       if (nextIsRunning) {
@@ -93,7 +77,17 @@ exports.pEach = function pEach(iterable, iteratee, { concurrency = 10, signal, s
           }
         } else {
           ++running
-          run(cursor.value, index++, iterable)
+          try {
+            const result = iteratee.call(this, cursor.value, index++, iterable)
+            let then
+            if (result != null && typeof result === 'object' && typeof (then = result.then) === 'function') {
+              then.call(result, onFulfilledWrapper, onRejectedWrapper)
+            } else {
+              onFulfilled(result)
+            }
+          } catch (error) {
+            onRejected(error)
+          }
         }
         nextIsRunning = false
         return next()

@vates/async-each/index.spec.js

@@ -2,7 +2,7 @@
 
 /* eslint-env jest */
 
-const { pEach } = require('./')
+const { asyncEach } = require('./')
 
 const randomDelay = (max = 10) =>
   new Promise(resolve => {
@@ -14,7 +14,7 @@ const rejectionOf = p =>
     p.then(reject, resolve)
   })
 
-describe('pEach', () => {
+describe('asyncEach', () => {
   const thisArg = 'qux'
   const values = ['foo', 'bar', 'baz']
 
@@ -36,7 +36,7 @@ describe('pEach', () => {
       it('works', async () => {
         const iteratee = jest.fn(async () => {})
 
-        await pEach.call(thisArg, iterable, iteratee)
+        await asyncEach.call(thisArg, iterable, iteratee)
 
         expect(iteratee.mock.instances).toEqual(Array.from(values, () => thisArg))
         expect(iteratee.mock.calls).toEqual(Array.from(values, (value, index) => [value, index, iterable]))
@@ -45,7 +45,7 @@ describe('pEach', () => {
         it('respects a concurrency of ' + concurrency, async () => {
           let running = 0
 
-          await pEach(
+          await asyncEach(
             values,
             async () => {
               ++running
@@ -66,7 +66,7 @@ describe('pEach', () => {
           }
         })
 
-        expect(await rejectionOf(pEach(iterable, iteratee, { stopOnError: true }))).toBe(error)
+        expect(await rejectionOf(asyncEach(iterable, iteratee, { stopOnError: true }))).toBe(error)
         expect(iteratee).toHaveBeenCalledTimes(2)
       })
 
@@ -78,7 +78,7 @@ describe('pEach', () => {
           throw error
         })
 
-        const error = await rejectionOf(pEach(iterable, iteratee, { stopOnError: false }))
+        const error = await rejectionOf(asyncEach(iterable, iteratee, { stopOnError: false }))
         expect(error.errors).toEqual(errors)
         expect(iteratee.mock.calls).toEqual(Array.from(values, (value, index) => [value, index, iterable]))
       })
@@ -91,7 +91,7 @@ describe('pEach', () => {
           }
         })
 
-        await expect(pEach(iterable, iteratee, { signal: ac.signal })).rejects.toThrow('pEach aborted')
+        await expect(asyncEach(iterable, iteratee, { signal: ac.signal })).rejects.toThrow('asyncEach aborted')
         expect(iteratee).toHaveBeenCalledTimes(2)
       })
     })

@vates/cached-dns.lookup/.USAGE.md

@@ -1,30 +0,0 @@
-Node does not cache queries to `dns.lookup`, which can lead application doing a lot of connections to have perf issues and to saturate Node threads pool.
-
-This library attempts to mitigate these problems by providing a version of this function with a version short cache, applied on both errors and results.
-
-> Limitation: `verbatim: false` option is not supported.
-
-It has exactly the same API as the native method and can be used directly:
-
-```js
-import { createCachedLookup } from '@vates/cached-dns.lookup'
-
-const lookup = createCachedLookup()
-
-lookup('example.net', { all: true, family: 0 }, (error, result) => {
-  if (error != null) {
-    return console.warn(error)
-  }
-  console.log(result)
-})
-```
-
-Or it can be used to replace the native implementation and speed up the whole app:
-
-```js
-// assign our cached implementation to dns.lookup
-const restore = createCachedLookup().patchGlobal()
-
-// to restore the previous implementation
-restore()
-```

@vates/cached-dns.lookup/.npmignore

@@ -1 +0,0 @@
-../../scripts/npmignore

@vates/cached-dns.lookup/README.md

@@ -1,63 +0,0 @@
-<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
-
-# @vates/cached-dns.lookup
-
-[![Package Version](https://badgen.net/npm/v/@vates/cached-dns.lookup)](https://npmjs.org/package/@vates/cached-dns.lookup) ![License](https://badgen.net/npm/license/@vates/cached-dns.lookup) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/cached-dns.lookup)](https://bundlephobia.com/result?p=@vates/cached-dns.lookup) [![Node compatibility](https://badgen.net/npm/node/@vates/cached-dns.lookup)](https://npmjs.org/package/@vates/cached-dns.lookup)
-
-> Cached implementation of dns.lookup
-
-## Install
-
-Installation of the [npm package](https://npmjs.org/package/@vates/cached-dns.lookup):
-
-```
-> npm install --save @vates/cached-dns.lookup
-```
-
-## Usage
-
-Node does not cache queries to `dns.lookup`, which can lead application doing a lot of connections to have perf issues and to saturate Node threads pool.
-
-This library attempts to mitigate these problems by providing a version of this function with a version short cache, applied on both errors and results.
-
-> Limitation: `verbatim: false` option is not supported.
-
-It has exactly the same API as the native method and can be used directly:
-
-```js
-import { createCachedLookup } from '@vates/cached-dns.lookup'
-
-const lookup = createCachedLookup()
-
-lookup('example.net', { all: true, family: 0 }, (error, result) => {
-  if (error != null) {
-    return console.warn(error)
-  }
-  console.log(result)
-})
-```
-
-Or it can be used to replace the native implementation and speed up the whole app:
-
-```js
-// assign our cached implementation to dns.lookup
-const restore = createCachedLookup().patchGlobal()
-
-// to restore the previous implementation
-restore()
-```
-
-## Contributions
-
-Contributions are _very_ welcomed, either on the documentation or on
-the code.
-
-You may:
-
-- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
-  you've encountered;
-- fork and create a pull request.
-
-## License
-
-[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/cached-dns.lookup/index.js

@@ -1,72 +0,0 @@
-'use strict'
-
-const assert = require('assert')
-const dns = require('dns')
-const LRU = require('lru-cache')
-
-function reportResults(all, results, callback) {
-  if (all) {
-    callback(null, results)
-  } else {
-    const first = results[0]
-    callback(null, first.address, first.family)
-  }
-}
-
-exports.createCachedLookup = function createCachedLookup({ lookup = dns.lookup } = {}) {
-  const cache = new LRU({
-    max: 500,
-
-    // 1 minute: long enough to be effective, short enough so there is no need to bother with DNS TTLs
-    ttl: 60e3,
-  })
-
-  function cachedLookup(hostname, options, callback) {
-    let all = false
-    let family = 0
-    if (typeof options === 'function') {
-      callback = options
-    } else if (typeof options === 'number') {
-      family = options
-    } else if (options != null) {
-      assert.notStrictEqual(options.verbatim, false, 'not supported by this implementation')
-      ;({ all = all, family = family } = options)
-    }
-
-    // cache by family option because there will be an error if there is no
-    // entries for the requestion family so we cannot easily cache all families
-    // and filter on reporting back
-    const key = hostname + '/' + family
-
-    const result = cache.get(key)
-    if (result !== undefined) {
-      setImmediate(reportResults, all, result, callback)
-    } else {
-      lookup(hostname, { all: true, family, verbatim: true }, function onLookup(error, results) {
-        // errors are not cached because this will delay recovery after DNS/network issues
-        //
-        // there are no reliable way to detect if the error is real or simply
-        // that there are no results for the requested hostname
-        //
-        // there should be much fewer errors than success, therefore it should
-        // not be a big deal to not cache them
-        if (error != null) {
-          return callback(error)
-        }
-
-        cache.set(key, results)
-        reportResults(all, results, callback)
-      })
-    }
-  }
-  cachedLookup.patchGlobal = function patchGlobal() {
-    const previous = dns.lookup
-    dns.lookup = cachedLookup
-    return function restoreGlobal() {
-      assert.strictEqual(dns.lookup, cachedLookup)
-      dns.lookup = previous
-    }
-  }
-
-  return cachedLookup
-}

@vates/cached-dns.lookup/package.json

@@ -1,32 +0,0 @@
-{
-  "engines": {
-    "node": ">=8"
-  },
-  "dependencies": {
-    "lru-cache": "^7.0.4"
-  },
-  "private": false,
-  "name": "@vates/cached-dns.lookup",
-  "description": "Cached implementation of dns.lookup",
-  "keywords": [
-    "cache",
-    "dns",
-    "lookup"
-  ],
-  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/cached-dns.lookup",
-  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
-  "repository": {
-    "directory": "@vates/cached-dns.lookup",
-    "type": "git",
-    "url": "https://github.com/vatesfr/xen-orchestra.git"
-  },
-  "author": {
-    "name": "Vates SAS",
-    "url": "https://vates.fr"
-  },
-  "license": "ISC",
-  "version": "1.0.0",
-  "scripts": {
-    "postversion": "npm publish --access public"
-  }
-}

@vates/event-listeners-manager/.USAGE.md

@@ -1,50 +0,0 @@
-> This library is compatible with Node's `EventEmitter` and web browsers' `EventTarget` APIs.
-
-### API
-
-```js
-import { EventListenersManager } from '@vates/event-listeners-manager'
-
-const events = new EventListenersManager(emitter)
-
-// adding listeners
-events.add('foo', onFoo).add('bar', onBar).on('baz', onBaz)
-
-// removing a specific listener
-events.remove('foo', onFoo)
-
-// removing all listeners for a specific event
-events.removeAll('foo')
-
-// removing all listeners
-events.removeAll()
-```
-
-### Typical use case
-
-> Removing all listeners when no longer necessary.
-
-Manually:
-
-```js
-const onFoo = () => {}
-const onBar = () => {}
-const onBaz = () => {}
-emitter.on('foo', onFoo).on('bar', onBar).on('baz', onBaz)
-
-// CODE LOGIC
-
-emitter.off('foo', onFoo).off('bar', onBar).off('baz', onBaz)
-```
-
-With this library:
-
-```js
-const events = new EventListenersManager(emitter)
-
-events.add('foo', () => {})).add('bar', () => {})).add('baz', () => {}))
-
-// CODE LOGIC
-
-events.removeAll()
-```

@vates/event-listeners-manager/.npmignore

@@ -1 +0,0 @@
-../../scripts/npmignore

@vates/event-listeners-manager/README.md

@@ -1,81 +0,0 @@
-<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
-
-# @vates/event-listeners-manager
-
-[![Package Version](https://badgen.net/npm/v/@vates/event-listeners-manager)](https://npmjs.org/package/@vates/event-listeners-manager) ![License](https://badgen.net/npm/license/@vates/event-listeners-manager) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/event-listeners-manager)](https://bundlephobia.com/result?p=@vates/event-listeners-manager) [![Node compatibility](https://badgen.net/npm/node/@vates/event-listeners-manager)](https://npmjs.org/package/@vates/event-listeners-manager)
-
-## Install
-
-Installation of the [npm package](https://npmjs.org/package/@vates/event-listeners-manager):
-
-```
-> npm install --save @vates/event-listeners-manager
-```
-
-## Usage
-
-> This library is compatible with Node's `EventEmitter` and web browsers' `EventTarget` APIs.
-
-### API
-
-```js
-import { EventListenersManager } from '@vates/event-listeners-manager'
-
-const events = new EventListenersManager(emitter)
-
-// adding listeners
-events.add('foo', onFoo).add('bar', onBar).on('baz', onBaz)
-
-// removing a specific listener
-events.remove('foo', onFoo)
-
-// removing all listeners for a specific event
-events.removeAll('foo')
-
-// removing all listeners
-events.removeAll()
-```
-
-### Typical use case
-
-> Removing all listeners when no longer necessary.
-
-Manually:
-
-```js
-const onFoo = () => {}
-const onBar = () => {}
-const onBaz = () => {}
-emitter.on('foo', onFoo).on('bar', onBar).on('baz', onBaz)
-
-// CODE LOGIC
-
-emitter.off('foo', onFoo).off('bar', onBar).off('baz', onBaz)
-```
-
-With this library:
-
-```js
-const events = new EventListenersManager(emitter)
-
-events.add('foo', () => {})).add('bar', () => {})).add('baz', () => {}))
-
-// CODE LOGIC
-
-events.removeAll()
-```
-
-## Contributions
-
-Contributions are _very_ welcomed, either on the documentation or on
-the code.
-
-You may:
-
-- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
-  you've encountered;
-- fork and create a pull request.
-
-## License
-
-[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/event-listeners-manager/index.js

@@ -1,56 +0,0 @@
-'use strict'
-
-exports.EventListenersManager = class EventListenersManager {
-  constructor(emitter) {
-    this._listeners = new Map()
-
-    this._add = (emitter.addListener || emitter.addEventListener).bind(emitter)
-    this._remove = (emitter.removeListener || emitter.removeEventListener).bind(emitter)
-  }
-
-  add(type, listener) {
-    let listeners = this._listeners.get(type)
-    if (listeners === undefined) {
-      listeners = new Set()
-      this._listeners.set(type, listeners)
-    }
-
-    // don't add the same listener multiple times (allowed on Node.js)
-    if (!listeners.has(listener)) {
-      listeners.add(listener)
-      this._add(type, listener)
-    }
-
-    return this
-  }
-
-  remove(type, listener) {
-    const allListeners = this._listeners
-    const listeners = allListeners.get(type)
-    if (listeners !== undefined && listeners.delete(listener)) {
-      this._remove(type, listener)
-      if (listeners.size === 0) {
-        allListeners.delete(type)
-      }
-    }
-
-    return this
-  }
-
-  removeAll(type) {
-    const allListeners = this._listeners
-    const remove = this._remove
-    const types = type !== undefined ? [type] : allListeners.keys()
-    for (const type of types) {
-      const listeners = allListeners.get(type)
-      if (listeners !== undefined) {
-        allListeners.delete(type)
-        for (const listener of listeners) {
-          remove(type, listener)
-        }
-      }
-    }
-
-    return this
-  }
-}

@vates/event-listeners-manager/index.spec.js

@@ -1,67 +0,0 @@
-'use strict'
-
-const t = require('tap')
-const { EventEmitter } = require('events')
-
-const { EventListenersManager } = require('./')
-
-const noop = Function.prototype
-
-// function spy (impl = Function.prototype) {
-//   function spy() {
-//     spy.calls.push([Array.from(arguments), this])
-//   }
-//   spy.calls = []
-//   return spy
-// }
-
-function assertListeners(t, event, listeners) {
-  t.strictSame(t.context.ee.listeners(event), listeners)
-}
-
-t.beforeEach(function (t) {
-  t.context.ee = new EventEmitter()
-  t.context.em = new EventListenersManager(t.context.ee)
-})
-
-t.test('.add adds a listener', function (t) {
-  t.context.em.add('foo', noop)
-
-  assertListeners(t, 'foo', [noop])
-
-  t.end()
-})
-
-t.test('.add does not add a duplicate listener', function (t) {
-  t.context.em.add('foo', noop).add('foo', noop)
-
-  assertListeners(t, 'foo', [noop])
-
-  t.end()
-})
-
-t.test('.remove removes a listener', function (t) {
-  t.context.em.add('foo', noop).remove('foo', noop)
-
-  assertListeners(t, 'foo', [])
-
-  t.end()
-})
-
-t.test('.removeAll removes all listeners of a given type', function (t) {
-  t.context.em.add('foo', noop).add('bar', noop).removeAll('foo')
-
-  assertListeners(t, 'foo', [])
-  assertListeners(t, 'bar', [noop])
-
-  t.end()
-})
-
-t.test('.removeAll removes all listeners', function (t) {
-  t.context.em.add('foo', noop).add('bar', noop).removeAll()
-
-  assertListeners(t, 'foo', [])
-  assertListeners(t, 'bar', [])
-
-  t.end()
-})

@vates/event-listeners-manager/package.json

@@ -1,46 +0,0 @@
-{
-  "engines": {
-    "node": ">=6"
-  },
-  "private": false,
-  "name": "@vates/event-listeners-manager",
-  "descriptions": "Easy way to clean up event listeners",
-  "keywords": [
-    "add",
-    "addEventListener",
-    "addListener",
-    "browser",
-    "clear",
-    "DOM",
-    "emitter",
-    "event",
-    "EventEmitter",
-    "EventTarget",
-    "management",
-    "manager",
-    "node",
-    "remove",
-    "removeEventListener",
-    "removeListener"
-  ],
-  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/event-listeners-manager",
-  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
-  "repository": {
-    "directory": "@vates/event-listeners-manager",
-    "type": "git",
-    "url": "https://github.com/vatesfr/xen-orchestra.git"
-  },
-  "author": {
-    "name": "Vates SAS",
-    "url": "https://vates.fr"
-  },
-  "license": "ISC",
-  "version": "1.0.1",
-  "scripts": {
-    "postversion": "npm publish --access public",
-    "test": "tap --branches=72"
-  },
-  "devDependencies": {
-    "tap": "^16.2.0"
-  }
-}

@vates/read-chunk/.USAGE.md

@@ -1,9 +1,6 @@
-### `readChunk(stream, [size])`
-
 - returns the next available chunk of data
 - like `stream.read()`, a number of bytes can be specified
-- returns with less data than expected if stream has ended
-- returns `null` if the stream has ended and no data has been read
+- returns `null` if the stream has ended
 
 ```js
 import { readChunk } from '@vates/read-chunk'
@@ -14,13 +11,3 @@ import { readChunk } from '@vates/read-chunk'
   }
 })()
 ```
-
-### `readChunkStrict(stream, [size])`
-
-Similar behavior to `readChunk` but throws if the stream ended before the requested data could be read.
-
-```js
-import { readChunkStrict } from '@vates/read-chunk'
-
-const chunk = await readChunkStrict(stream, 1024)
-```

@vates/read-chunk/README.md

@@ -16,12 +16,9 @@ Installation of the [npm package](https://npmjs.org/package/@vates/read-chunk):
 
 ## Usage
 
-### `readChunk(stream, [size])`
-
 - returns the next available chunk of data
 - like `stream.read()`, a number of bytes can be specified
-- returns with less data than expected if stream has ended
-- returns `null` if the stream has ended and no data has been read
+- returns `null` if the stream has ended
 
 ```js
 import { readChunk } from '@vates/read-chunk'
@@ -33,16 +30,6 @@ import { readChunk } from '@vates/read-chunk'
 })()
 ```
 
-### `readChunkStrict(stream, [size])`
-
-Similar behavior to `readChunk` but throws if the stream ended before the requested data could be read.
-
-```js
-import { readChunkStrict } from '@vates/read-chunk'
-
-const chunk = await readChunkStrict(stream, 1024)
-```
-
 ## Contributions
 
 Contributions are _very_ welcomed, either on the documentation or on

@vates/read-chunk/index.js

@@ -30,22 +30,3 @@ const readChunk = (stream, size) =>
         onReadable()
       })
 exports.readChunk = readChunk
-
-exports.readChunkStrict = async function readChunkStrict(stream, size) {
-  const chunk = await readChunk(stream, size)
-  if (chunk === null) {
-    throw new Error('stream has ended without data')
-  }
-
-  if (size !== undefined && chunk.length !== size) {
-    const error = new Error('stream has ended with not enough data')
-    Object.defineProperties(error, {
-      chunk: {
-        value: chunk,
-      },
-    })
-    throw error
-  }
-
-  return chunk
-}

@vates/read-chunk/index.spec.js

@@ -4,7 +4,7 @@
 
 const { Readable } = require('stream')
 
-const { readChunk, readChunkStrict } = require('./')
+const { readChunk } = require('./')
 
 const makeStream = it => Readable.from(it, { objectMode: false })
 makeStream.obj = Readable.from
@@ -43,27 +43,3 @@ describe('readChunk', () => {
     })
   })
 })
-
-const rejectionOf = promise =>
-  promise.then(
-    value => {
-      throw value
-    },
-    error => error
-  )
-
-describe('readChunkStrict', function () {
-  it('throws if stream is empty', async () => {
-    const error = await rejectionOf(readChunkStrict(makeStream([])))
-    expect(error).toBeInstanceOf(Error)
-    expect(error.message).toBe('stream has ended without data')
-    expect(error.chunk).toEqual(undefined)
-  })
-
-  it('throws if stream ends with not enough data', async () => {
-    const error = await rejectionOf(readChunkStrict(makeStream(['foo', 'bar']), 10))
-    expect(error).toBeInstanceOf(Error)
-    expect(error.message).toBe('stream has ended with not enough data')
-    expect(error.chunk).toEqual(Buffer.from('foobar'))
-  })
-})

@vates/read-chunk/package.json

@@ -19,7 +19,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "1.0.0",
+  "version": "0.1.2",
   "engines": {
     "node": ">=8.10"
   },

@xen-orchestra/backups-cli/commands/clean-vms.js

@@ -23,7 +23,7 @@ module.exports = async function main(args) {
     },
   })
 
-  await pEach(_, async vmDir => {
+  await asyncMap(_, async vmDir => {
     vmDir = resolve(vmDir)
     try {
       await adapter.cleanVm(vmDir, { fixMetadata: fix, remove, merge, onLog: (...args) => console.warn(...args) })

@xen-orchestra/backups-cli/commands/create-symlink-index.js

@@ -11,8 +11,8 @@ module.exports = async function createSymlinkIndex([backupDir, fieldPath]) {
   const indexDir = join(backupDir, 'indexes', filenamify(fieldPath))
   await mktree(indexDir)
 
-  await pEach(await readdir2(backupDir), async vmDir =>
-    pEach(
+  await asyncMap(await readdir2(backupDir), async vmDir =>
+    asyncMap(
       (await readdir2(vmDir)).filter(_ => _.endsWith('.json')),
       async json => {
         const metadata = JSON.parse(await readFile(json))

@xen-orchestra/backups-cli/package.json

@@ -7,8 +7,8 @@
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.26.0",
-    "@xen-orchestra/fs": "^1.1.0",
+    "@xen-orchestra/backups": "^0.21.0",
+    "@xen-orchestra/fs": "^1.0.0",
     "filenamify": "^4.1.0",
     "getopts": "^2.2.5",
     "lodash": "^4.17.15",
@@ -27,7 +27,7 @@
   "scripts": {
     "postversion": "npm publish --access public"
   },
-  "version": "0.7.4",
+  "version": "0.7.0",
   "license": "AGPL-3.0-or-later",
   "author": {
     "name": "Vates SAS",

@xen-orchestra/backups/Backup.js

@@ -6,7 +6,7 @@ const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { compileTemplate } = require('@xen-orchestra/template')
 const { limitConcurrency } = require('limit-concurrency-decorator')
 
-const { extractIdsFromSimplePattern } = require('./extractIdsFromSimplePattern.js')
+const { extractIdsFromSimplePattern } = require('./_extractIdsFromSimplePattern.js')
 const { PoolMetadataBackup } = require('./_PoolMetadataBackup.js')
 const { Task } = require('./Task.js')
 const { VmBackup } = require('./_VmBackup.js')
@@ -24,34 +24,6 @@ const getAdaptersByRemote = adapters => {
 
 const runTask = (...args) => Task.run(...args).catch(noop) // errors are handled by logs
 
-const DEFAULT_SETTINGS = {
-  reportWhen: 'failure',
-}
-
-const DEFAULT_VM_SETTINGS = {
-  bypassVdiChainsCheck: false,
-  checkpointSnapshot: false,
-  concurrency: 2,
-  copyRetention: 0,
-  deleteFirst: false,
-  exportRetention: 0,
-  fullInterval: 0,
-  healthCheckSr: undefined,
-  healthCheckVmsWithTags: [],
-  maxMergedDeltasPerRun: 2,
-  offlineBackup: false,
-  offlineSnapshot: false,
-  snapshotRetention: 0,
-  timeout: 0,
-  unconditionalSnapshot: false,
-  vmTimeout: 0,
-}
-
-const DEFAULT_METADATA_SETTINGS = {
-  retentionPoolMetadata: 0,
-  retentionXoMetadata: 0,
-}
-
 exports.Backup = class Backup {
   constructor({ config, getAdapter, getConnectedRecord, job, schedule }) {
     this._config = config
@@ -70,22 +42,17 @@ exports.Backup = class Backup {
       '{job.name}': job.name,
       '{vm.name_label}': vm => vm.name_label,
     })
+  }
 
-    const { type } = job
-    const baseSettings = { ...DEFAULT_SETTINGS }
+  run() {
+    const type = this._job.type
     if (type === 'backup') {
-      Object.assign(baseSettings, DEFAULT_VM_SETTINGS, config.defaultSettings, config.vm?.defaultSettings)
-      this.run = this._runVmBackup
+      return this._runVmBackup()
     } else if (type === 'metadataBackup') {
-      Object.assign(baseSettings, DEFAULT_METADATA_SETTINGS, config.defaultSettings, config.metadata?.defaultSettings)
-      this.run = this._runMetadataBackup
+      return this._runMetadataBackup()
     } else {
       throw new Error(`No runner for the backup type ${type}`)
     }
-    Object.assign(baseSettings, job.settings[''])
-
-    this._baseSettings = baseSettings
-    this._settings = { ...baseSettings, ...job.settings[schedule.id] }
   }
 
   async _runMetadataBackup() {
@@ -97,6 +64,13 @@ exports.Backup = class Backup {
     }
 
     const config = this._config
+    const settings = {
+      ...config.defaultSettings,
+      ...config.metadata.defaultSettings,
+      ...job.settings[''],
+      ...job.settings[schedule.id],
+    }
+
     const poolIds = extractIdsFromSimplePattern(job.pools)
     const isEmptyPools = poolIds.length === 0
     const isXoMetadata = job.xoMetadata !== undefined
@@ -104,8 +78,6 @@ exports.Backup = class Backup {
       throw new Error('no metadata mode found')
     }
 
-    const settings = this._settings
-
     const { retentionPoolMetadata, retentionXoMetadata } = settings
 
     if (
@@ -159,7 +131,7 @@ exports.Backup = class Backup {
         const promises = []
         if (pools.length !== 0 && settings.retentionPoolMetadata !== 0) {
           promises.push(
-            pEach(pools, async pool =>
+            asyncMap(pools, async pool =>
               runTask(
                 {
                   name: `Starting metadata backup for the pool (${pool.$id}). (${job.id})`,
@@ -217,7 +189,14 @@ exports.Backup = class Backup {
     const schedule = this._schedule
 
     const config = this._config
-    const settings = this._settings
+    const { settings } = job
+    const scheduleSettings = {
+      ...config.defaultSettings,
+      ...config.vm.defaultSettings,
+      ...settings[''],
+      ...settings[schedule.id],
+    }
+
     await Disposable.use(
       Disposable.all(
         extractIdsFromSimplePattern(job.srs).map(id =>
@@ -245,15 +224,14 @@ exports.Backup = class Backup {
           })
         )
       ),
-      () => (settings.healthCheckSr !== undefined ? this._getRecord('SR', settings.healthCheckSr) : undefined),
-      async (srs, remoteAdapters, healthCheckSr) => {
+      async (srs, remoteAdapters) => {
         // remove adapters that failed (already handled)
         remoteAdapters = remoteAdapters.filter(_ => _ !== undefined)
 
         // remove srs that failed (already handled)
         srs = srs.filter(_ => _ !== undefined)
 
-        if (remoteAdapters.length === 0 && srs.length === 0 && settings.snapshotRetention === 0) {
+        if (remoteAdapters.length === 0 && srs.length === 0 && scheduleSettings.snapshotRetention === 0) {
           return
         }
 
@@ -263,31 +241,24 @@ exports.Backup = class Backup {
 
         remoteAdapters = getAdaptersByRemote(remoteAdapters)
 
-        const allSettings = this._job.settings
-        const baseSettings = this._baseSettings
-
         const handleVm = vmUuid =>
           runTask({ name: 'backup VM', data: { type: 'VM', id: vmUuid } }, () =>
             Disposable.use(this._getRecord('VM', vmUuid), vm =>
               new VmBackup({
-                baseSettings,
                 config,
                 getSnapshotNameLabel,
-                healthCheckSr,
                 job,
+                // remotes,
                 remoteAdapters,
                 schedule,
-                settings: { ...settings, ...allSettings[vm.uuid] },
+                settings: { ...scheduleSettings, ...settings[vmUuid] },
                 srs,
                 vm,
               }).run()
             )
           )
-        const { concurrency } = settings
-        await pEach(vmIds, handleVm, {
-          concurrency,
-          stopOnError: false,
-        })
+        const { concurrency } = scheduleSettings
+        await asyncMapSettled(vmIds, concurrency === 0 ? handleVm : limitConcurrency(concurrency)(handleVm))
       }
     )
   }

@xen-orchestra/backups/DurablePartition.js

@@ -1,8 +1,6 @@
 'use strict'
 
-const { pEach } = require('@xen-orchestra/async-map')
-
-const noop = Function.protoype
+const { asyncMap } = require('@xen-orchestra/async-map')
 
 exports.DurablePartition = class DurablePartition {
   // private resource API is used exceptionally to be able to separate resource creation and release
@@ -10,10 +8,10 @@ exports.DurablePartition = class DurablePartition {
 
   flushAll() {
     const partitionDisposers = this.#partitionDisposers
-    return pEach(Object.keys(partitionDisposers), path => {
+    return asyncMap(Object.keys(partitionDisposers), path => {
       const disposers = partitionDisposers[path]
       delete partitionDisposers[path]
-      return pEach(disposers, d => d(path).catch(noop))
+      return asyncMap(disposers, d => d(path).catch(noop => {}))
     })
   }
 

@xen-orchestra/backups/HealthCheckVmBackup.js

@@ -1,64 +0,0 @@
-'use strict'
-
-const { Task } = require('./Task')
-
-exports.HealthCheckVmBackup = class HealthCheckVmBackup {
-  #xapi
-  #restoredVm
-
-  constructor({ restoredVm, xapi }) {
-    this.#restoredVm = restoredVm
-    this.#xapi = xapi
-  }
-
-  async run() {
-    return Task.run(
-      {
-        name: 'vmstart',
-      },
-      async () => {
-        let restoredVm = this.#restoredVm
-        const xapi = this.#xapi
-        const restoredId = restoredVm.uuid
-
-        // remove vifs
-        await Promise.all(restoredVm.$VIFs.map(vif => xapi.callAsync('VIF.destroy', vif.$ref)))
-
-        const start = new Date()
-        // start Vm
-
-        await xapi.callAsync(
-          'VM.start',
-          restoredVm.$ref,
-          false, // Start paused?
-          false // Skip pre-boot checks?
-        )
-        const started = new Date()
-        const timeout = 10 * 60 * 1000
-        const startDuration = started - start
-
-        let remainingTimeout = timeout - startDuration
-
-        if (remainingTimeout < 0) {
-          throw new Error(`VM ${restoredId} not started after ${timeout / 1000} second`)
-        }
-
-        // wait for the 'Running' event to be really stored in local xapi object cache
-        restoredVm = await xapi.waitObjectState(restoredVm.$ref, vm => vm.power_state === 'Running', {
-          timeout: remainingTimeout,
-        })
-
-        const running = new Date()
-        remainingTimeout -= running - started
-
-        if (remainingTimeout < 0) {
-          throw new Error(`local xapi  did not get Runnig state for VM ${restoredId} after ${timeout / 1000} second`)
-        }
-        // wait for the guest tool version to be defined
-        await xapi.waitObjectState(restoredVm.guest_metrics, gm => gm?.PV_drivers_version?.major !== undefined, {
-          timeout: remainingTimeout,
-        })
-      }
-    )
-  }
-}

@xen-orchestra/backups/RemoteAdapter.js

@@ -1,7 +1,6 @@
 'use strict'
 
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
-const { synchronized } = require('decorator-synchronized')
 const Disposable = require('promise-toolbox/Disposable')
 const fromCallback = require('promise-toolbox/fromCallback')
 const fromEvent = require('promise-toolbox/fromEvent')
@@ -10,15 +9,14 @@ const groupBy = require('lodash/groupBy.js')
 const pickBy = require('lodash/pickBy.js')
 const { dirname, join, normalize, resolve } = require('path')
 const { createLogger } = require('@xen-orchestra/log')
-const { createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib')
+const { Constants, createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib')
 const { deduped } = require('@vates/disposable/deduped.js')
 const { decorateMethodsWith } = require('@vates/decorate-with')
 const { compose } = require('@vates/compose')
 const { execFile } = require('child_process')
-const { readdir, lstat } = require('fs-extra')
+const { readdir, stat } = require('fs-extra')
 const { v4: uuidv4 } = require('uuid')
 const { ZipFile } = require('yazl')
-const zlib = require('zlib')
 
 const { BACKUP_DIR } = require('./_getVmBackupDir.js')
 const { cleanVm } = require('./_cleanVm.js')
@@ -47,10 +45,13 @@ const resolveSubpath = (root, path) => resolve(root, `.${resolve('/', path)}`)
 const RE_VHDI = /^vhdi(\d+)$/
 
 async function addDirectory(files, realPath, metadataPath) {
-  const stats = await lstat(realPath)
-  if (stats.isDirectory()) {
-    await pEach(await readdir(realPath), file => addDirectory(files, realPath + '/' + file, metadataPath + '/' + file))
-  } else if (stats.isFile()) {
+  try {
+    const subFiles = await readdir(realPath)
+    await asyncMap(subFiles, file => addDirectory(files, realPath + '/' + file, metadataPath + '/' + file))
+  } catch (error) {
+    if (error == null || error.code !== 'ENOTDIR') {
+      throw error
+    }
     files.push({
       realPath,
       metadataPath,
@@ -77,7 +78,6 @@ class RemoteAdapter {
     this._dirMode = dirMode
     this._handler = handler
     this._vhdDirectoryCompression = vhdDirectoryCompression
-    this._readCacheListVmBackups = synchronized.withKey()(this._readCacheListVmBackups)
   }
 
   get handler() {
@@ -180,7 +180,7 @@ class RemoteAdapter {
     const path = yield this.getPartition(diskId, partitionId)
 
     const files = []
-    await pEach(paths, file =>
+    await asyncMap(paths, file =>
       addDirectory(files, resolveSubpath(path, file), normalize('./' + file).replace(/\/+$/, ''))
     )
 
@@ -226,9 +226,7 @@ class RemoteAdapter {
     const handler = this._handler
 
     // this will delete the json, unused VHDs will be detected by `cleanVm`
-    await pEach(backups, ({ _filename }) => handler.unlink(_filename), {
-      stopOnError: false,
-    })
+    await asyncMapSettled(backups, ({ _filename }) => handler.unlink(_filename))
   }
 
   async deleteMetadataBackup(backupId) {
@@ -248,16 +246,13 @@ class RemoteAdapter {
     let list = await handler.list(dir)
     list.sort()
     list = list.filter(timestamp => /^\d{8}T\d{6}Z$/.test(timestamp)).slice(0, -retention)
-    await pEach(list, timestamp => handler.rmtree(`${dir}/${timestamp}`), { stopOnError: false })
+    await asyncMapSettled(list, timestamp => handler.rmtree(`${dir}/${timestamp}`))
   }
 
   async deleteFullVmBackups(backups) {
     const handler = this._handler
-    await pEach(
-      backups,
-      ({ _filename, xva }) =>
-        Promise.all([handler.unlink(_filename), handler.unlink(resolveRelativeFromFile(_filename, xva))]),
-      { stopOnError: false }
+    await asyncMapSettled(backups, ({ _filename, xva }) =>
+      Promise.all([handler.unlink(_filename), handler.unlink(resolveRelativeFromFile(_filename, xva))])
     )
   }
 
@@ -266,8 +261,7 @@ class RemoteAdapter {
   }
 
   async deleteVmBackups(files) {
-    const metadatas = await pEach(files, file => this.readVmBackupMetadata(file))
-    const { delta, full, ...others } = groupBy(metadatas, 'mode')
+    const { delta, full, ...others } = groupBy(await asyncMap(files, file => this.readVmBackupMetadata(file)), 'mode')
 
     const unsupportedModes = Object.keys(others)
     if (unsupportedModes.length !== 0) {
@@ -284,9 +278,6 @@ class RemoteAdapter {
       // don't merge in main process, unused VHDs will be merged in the next backup run
       await this.cleanVm(dir, { remove: true, onLog: warn })
     }
-
-    const dedupedVmUuid = new Set(metadatas.map(_ => _.vm.uuid))
-    await pEach(dedupedVmUuid, vmUuid => this.invalidateVmBackupListCache(vmUuid))
   }
 
   #getCompressionType() {
@@ -294,7 +285,7 @@ class RemoteAdapter {
   }
 
   #useVhdDirectory() {
-    return this.handler.useVhdDirectory()
+    return this.handler.type === 's3'
   }
 
   #useAlias() {
@@ -365,7 +356,7 @@ class RemoteAdapter {
     const handler = this._handler
 
     const backups = { __proto__: null }
-    await pEach(await handler.list(BACKUP_DIR), async entry => {
+    await asyncMap(await handler.list(BACKUP_DIR), async entry => {
       // ignore hidden and lock files
       if (entry[0] !== '.' && !entry.endsWith('.lock')) {
         const vmBackups = await this.listVmBackups(entry)
@@ -383,14 +374,10 @@ class RemoteAdapter {
       path = resolveSubpath(rootPath, path)
 
       const entriesMap = {}
-      await pEach(await readdir(path), async name => {
+      await asyncMap(await readdir(path), async name => {
         try {
-          const stats = await lstat(`${path}/${name}`)
-          if (stats.isDirectory()) {
-            entriesMap[name + '/'] = {}
-          } else if (stats.isFile()) {
-            entriesMap[name] = {}
-          }
+          const stats = await stat(`${path}/${name}`)
+          entriesMap[stats.isDirectory() ? `${name}/` : name] = {}
         } catch (error) {
           if (error == null || error.code !== 'ENOENT') {
             throw error
@@ -416,13 +403,10 @@ class RemoteAdapter {
       }
 
       const results = []
-      await pEach(
-        partitions,
-        partition =>
-          partition.type === LVM_PARTITION_TYPE
-            ? this._listLvmLogicalVolumes(devicePath, partition, results)
-            : results.push(partition),
-        { stopOnError: false }
+      await asyncMapSettled(partitions, partition =>
+        partition.type === LVM_PARTITION_TYPE
+          ? this._listLvmLogicalVolumes(devicePath, partition, results)
+          : results.push(partition)
       )
       return results
     })
@@ -433,10 +417,10 @@ class RemoteAdapter {
     const safeReaddir = createSafeReaddir(handler, 'listPoolMetadataBackups')
 
     const backupsByPool = {}
-    await pEach(await safeReaddir(DIR_XO_POOL_METADATA_BACKUPS, { prependDir: true }), async scheduleDir =>
-      pEach(await safeReaddir(scheduleDir), async poolId => {
+    await asyncMap(await safeReaddir(DIR_XO_POOL_METADATA_BACKUPS, { prependDir: true }), async scheduleDir =>
+      asyncMap(await safeReaddir(scheduleDir), async poolId => {
         const backups = backupsByPool[poolId] ?? (backupsByPool[poolId] = [])
-        return pEach(await safeReaddir(`${scheduleDir}/${poolId}`, { prependDir: true }), async backupDir => {
+        return asyncMap(await safeReaddir(`${scheduleDir}/${poolId}`, { prependDir: true }), async backupDir => {
           try {
             backups.push({
               id: backupDir,
@@ -464,94 +448,34 @@ class RemoteAdapter {
     return backupsByPool
   }
 
-  async invalidateVmBackupListCache(vmUuid) {
-    await this.handler.unlink(`${BACKUP_DIR}/${vmUuid}/cache.json.gz`)
-  }
-
-  async #getCachabledDataListVmBackups(dir) {
+  async listVmBackups(vmUuid, predicate) {
     const handler = this._handler
-    const backups = {}
+    const backups = []
 
     try {
-      const files = await handler.list(dir, {
+      const files = await handler.list(`${BACKUP_DIR}/${vmUuid}`, {
         filter: isMetadataFile,
         prependDir: true,
       })
-      await pEach(files, async file => {
+      await asyncMap(files, async file => {
         try {
           const metadata = await this.readVmBackupMetadata(file)
-          // inject an id usable by importVmBackupNg()
-          metadata.id = metadata._filename
-          backups[file] = metadata
+          if (predicate === undefined || predicate(metadata)) {
+            // inject an id usable by importVmBackupNg()
+            metadata.id = metadata._filename
+
+            backups.push(metadata)
+          }
         } catch (error) {
-          warn(`can't read vm backup metadata`, { error, file, dir })
+          warn(`listVmBackups ${file}`, { error })
         }
       })
-      return backups
     } catch (error) {
       let code
       if (error == null || ((code = error.code) !== 'ENOENT' && code !== 'ENOTDIR')) {
         throw error
       }
     }
-  }
-
-  // use _ to mark this method as private by convention
-  // since we decorate it with synchronized.withKey in the constructor
-  // and # function are not writeable.
-  //
-  // read the list of backup of a Vm from cache
-  // if cache is missing  or broken  => regenerate it and return
-
-  async _readCacheListVmBackups(vmUuid) {
-    const dir = `${BACKUP_DIR}/${vmUuid}`
-    const path = `${dir}/cache.json.gz`
-
-    try {
-      const gzipped = await this.handler.readFile(path)
-      const text = await fromCallback(zlib.gunzip, gzipped)
-      return JSON.parse(text)
-    } catch (error) {
-      if (error.code !== 'ENOENT') {
-        warn('Cache file was unreadable', { vmUuid, error })
-      }
-    }
-
-    // nothing cached, or cache unreadable => regenerate it
-    const backups = await this.#getCachabledDataListVmBackups(dir)
-    if (backups === undefined) {
-      return
-    }
-
-    // detached async action, will not reject
-    this.#writeVmBackupsCache(path, backups)
-
-    return backups
-  }
-
-  async #writeVmBackupsCache(cacheFile, backups) {
-    try {
-      const text = JSON.stringify(backups)
-      const zipped = await fromCallback(zlib.gzip, text)
-      await this.handler.writeFile(cacheFile, zipped, { flags: 'w' })
-    } catch (error) {
-      warn('writeVmBackupsCache', { cacheFile, error })
-    }
-  }
-
-  async listVmBackups(vmUuid, predicate) {
-    const backups = []
-    const cached = await this._readCacheListVmBackups(vmUuid)
-
-    if (cached === undefined) {
-      return []
-    }
-
-    Object.values(cached).forEach(metadata => {
-      if (predicate === undefined || predicate(metadata)) {
-        backups.push(metadata)
-      }
-    })
 
     return backups.sort(compareTimestamp)
   }
@@ -561,8 +485,8 @@ class RemoteAdapter {
     const safeReaddir = createSafeReaddir(handler, 'listXoMetadataBackups')
 
     const backups = []
-    await pEach(await safeReaddir(DIR_XO_CONFIG_BACKUPS, { prependDir: true }), async scheduleDir =>
-      pEach(await safeReaddir(scheduleDir, { prependDir: true }), async backupDir => {
+    await asyncMap(await safeReaddir(DIR_XO_CONFIG_BACKUPS, { prependDir: true }), async scheduleDir =>
+      asyncMap(await safeReaddir(scheduleDir, { prependDir: true }), async backupDir => {
         try {
           backups.push({
             id: backupDir,
@@ -607,27 +531,46 @@ class RemoteAdapter {
     })
   }
 
-  // open the  hierarchy of ancestors until we find a full one
-  async _createSyntheticStream(handler, path) {
-    const disposableSynthetic = await VhdSynthetic.fromVhdChain(handler, path)
+  async _createSyntheticStream(handler, paths) {
+    let disposableVhds = []
+
+    // if it's a path : open all hierarchy of parent
+    if (typeof paths === 'string') {
+      let vhd
+      let vhdPath = paths
+      do {
+        const disposable = await openVhd(handler, vhdPath)
+        vhd = disposable.value
+        disposableVhds.push(disposable)
+        vhdPath = resolveRelativeFromFile(vhdPath, vhd.header.parentUnicodeName)
+      } while (vhd.footer.diskType !== Constants.DISK_TYPES.DYNAMIC)
+    } else {
+      // only open the list of path given
+      disposableVhds = paths.map(path => openVhd(handler, path))
+    }
+
     // I don't want the vhds to be disposed on return
     // but only when the stream is done ( or failed )
+    const disposables = await Disposable.all(disposableVhds)
+    const vhds = disposables.value
 
     let disposed = false
     const disposeOnce = async () => {
       if (!disposed) {
         disposed = true
+
         try {
-          await disposableSynthetic.dispose()
+          await disposables.dispose()
         } catch (error) {
-          warn('openVhd: failed to dispose VHDs', { error })
+          warn('_createSyntheticStream: failed to dispose VHDs', { error })
         }
       }
     }
-    const synthetic = disposableSynthetic.value
+
+    const synthetic = new VhdSynthetic(vhds)
+    await synthetic.readHeaderAndFooter()
     await synthetic.readBlockAllocationTable()
     const stream = await synthetic.stream()
-
     stream.on('end', disposeOnce)
     stream.on('close', disposeOnce)
     stream.on('error', disposeOnce)
@@ -641,13 +584,9 @@ class RemoteAdapter {
     const vdis = ignoredVdis === undefined ? metadata.vdis : pickBy(metadata.vdis, vdi => !ignoredVdis.has(vdi.uuid))
 
     const streams = {}
-    await pEach(
-      Object.keys(vdis),
-      async ref => {
-        streams[`${ref}.vhd`] = await this._createSyntheticStream(handler, join(dir, vhds[ref]))
-      },
-      { stopOnError: false }
-    )
+    await asyncMapSettled(Object.keys(vdis), async ref => {
+      streams[`${ref}.vhd`] = await this._createSyntheticStream(handler, join(dir, vhds[ref]))
+    })
 
     return {
       streams,
@@ -664,10 +603,7 @@ class RemoteAdapter {
   }
 
   async readVmBackupMetadata(path) {
-    // _filename is a private field used to compute the backup id
-    //
-    // it's enumerable to make it cacheable
-    return { ...JSON.parse(await this._handler.readFile(path)), _filename: path }
+    return Object.defineProperty(JSON.parse(await this._handler.readFile(path)), '_filename', { value: path })
   }
 }
 

@xen-orchestra/backups/_PoolMetadataBackup.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { pEach } = require('@vates/async-each')
+const { asyncMap } = require('@xen-orchestra/async-map')
 
 const { DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')
 const { forkStreamUnpipe } = require('./_forkStreamUnpipe.js')
@@ -52,7 +52,7 @@ exports.PoolMetadataBackup = class PoolMetadataBackup {
     )
     const metaDataFileName = `${dir}/metadata.json`
 
-    await pEach(
+    await asyncMap(
       Object.entries(this._remoteAdapters),
       ([remoteId, adapter]) =>
         Task.run(

@xen-orchestra/backups/_VmBackup.js

@@ -45,18 +45,7 @@ const forkDeltaExport = deltaExport =>
   })
 
 class VmBackup {
-  constructor({
-    config,
-    getSnapshotNameLabel,
-    healthCheckSr,
-    job,
-    remoteAdapters,
-    remotes,
-    schedule,
-    settings,
-    srs,
-    vm,
-  }) {
+  constructor({ config, getSnapshotNameLabel, job, remoteAdapters, remotes, schedule, settings, srs, vm }) {
     if (vm.other_config['xo:backup:job'] === job.id && 'start' in vm.blocked_operations) {
       // don't match replicated VMs created by this very job otherwise they
       // will be replicated again and again
@@ -66,6 +55,7 @@ class VmBackup {
     this.config = config
     this.job = job
     this.remoteAdapters = remoteAdapters
+    this.remotes = remotes
     this.scheduleId = schedule.id
     this.timestamp = undefined
 
@@ -79,7 +69,6 @@ class VmBackup {
     this._fullVdisRequired = undefined
     this._getSnapshotNameLabel = getSnapshotNameLabel
     this._isDelta = job.mode === 'delta'
-    this._healthCheckSr = healthCheckSr
     this._jobId = job.id
     this._jobSnapshots = undefined
     this._xapi = vm.$xapi
@@ -106,6 +95,7 @@ class VmBackup {
         : [FullBackupWriter, FullReplicationWriter]
 
       const allSettings = job.settings
+
       Object.keys(remoteAdapters).forEach(remoteId => {
         const targetSettings = {
           ...settings,
@@ -146,20 +136,13 @@ class VmBackup {
     }
 
     const errors = []
-    await (parallel ? pEach : asyncEach)(writers, async function (writer) {
+    await (parallel ? asyncMap : asyncEach)(writers, async function (writer) {
       try {
         await fn(writer)
       } catch (error) {
         errors.push(error)
         this.delete(writer)
         warn(warnMessage, { error, writer: writer.constructor.name })
-
-        // these two steps are the only one that are not already in their own sub tasks
-        if (warnMessage === 'writer.checkBaseVdis()' || warnMessage === 'writer.beforeBackup()') {
-          Task.warning(
-            `the writer ${writer.constructor.name} has failed the step ${warnMessage} with error ${error.message}. It won't be used anymore in this job execution.`
-          )
-        }
       }
     })
     if (writers.size === 0) {
@@ -190,10 +173,7 @@ class VmBackup {
     const settings = this._settings
 
     const doSnapshot =
-      settings.unconditionalSnapshot ||
-      this._isDelta ||
-      (!settings.offlineBackup && vm.power_state === 'Running') ||
-      settings.snapshotRetention !== 0
+      this._isDelta || (!settings.offlineBackup && vm.power_state === 'Running') || settings.snapshotRetention !== 0
     if (doSnapshot) {
       await Task.run({ name: 'snapshot' }, async () => {
         if (!settings.bypassVdiChainsCheck) {
@@ -201,9 +181,7 @@ class VmBackup {
         }
 
         const snapshotRef = await vm[settings.checkpointSnapshot ? '$checkpoint' : '$snapshot']({
-          ignoreNobakVdis: true,
           name_label: this._getSnapshotNameLabel(vm),
-          unplugVusbs: true,
         })
         this.timestamp = Date.now()
 
@@ -325,19 +303,24 @@ class VmBackup {
   }
 
   async _removeUnusedSnapshots() {
-    const allSettings = this.job.settings
-    const baseSettings = this._baseSettings
+    const jobSettings = this.job.settings
     const baseVmRef = this._baseVm?.$ref
+    const { config } = this
+    const baseSettings = {
+      ...config.defaultSettings,
+      ...config.metadata.defaultSettings,
+      ...jobSettings[''],
+    }
 
     const snapshotsPerSchedule = groupBy(this._jobSnapshots, _ => _.other_config['xo:backup:schedule'])
     const xapi = this._xapi
-    await pEach(Object.entries(snapshotsPerSchedule), ([scheduleId, snapshots]) => {
+    await asyncMap(Object.entries(snapshotsPerSchedule), ([scheduleId, snapshots]) => {
       const settings = {
         ...baseSettings,
-        ...allSettings[scheduleId],
-        ...allSettings[this.vm.uuid],
+        ...jobSettings[scheduleId],
+        ...jobSettings[this.vm.uuid],
       }
-      return pEach(getOldEntries(settings.snapshotRetention, snapshots), ({ $ref }) => {
+      return asyncMap(getOldEntries(settings.snapshotRetention, snapshots), ({ $ref }) => {
         if ($ref !== baseVmRef) {
           return xapi.VM_destroy($ref)
         }
@@ -367,7 +350,7 @@ class VmBackup {
     baseVm = await xapi.getRecord('VM', baseVm.$ref)
 
     const baseUuidToSrcVdi = new Map()
-    await pEach(await baseVm.$getDisks(), async baseRef => {
+    await asyncMap(await baseVm.$getDisks(), async baseRef => {
       const [baseUuid, snapshotOf] = await Promise.all([
         xapi.getField('VDI', baseRef, 'uuid'),
         xapi.getField('VDI', baseRef, 'snapshot_of'),
@@ -414,24 +397,6 @@ class VmBackup {
     this._fullVdisRequired = fullVdisRequired
   }
 
-  async _healthCheck() {
-    const settings = this._settings
-
-    if (this._healthCheckSr === undefined) {
-      return
-    }
-
-    // check if current VM has tags
-    const { tags } = this.vm
-    const intersect = settings.healthCheckVmsWithTags.some(t => tags.includes(t))
-
-    if (settings.healthCheckVmsWithTags.length !== 0 && !intersect) {
-      return
-    }
-
-    await this._callWriters(writer => writer.healthCheck(this._healthCheckSr), 'writer.healthCheck()')
-  }
-
   async run($defer) {
     const settings = this._settings
     assert(
@@ -441,9 +406,7 @@ class VmBackup {
 
     await this._callWriters(async writer => {
       await writer.beforeBackup()
-      $defer(async () => {
-        await writer.afterBackup()
-      })
+      $defer(() => writer.afterBackup())
     }, 'writer.beforeBackup()')
 
     await this._fetchJobSnapshots()
@@ -479,7 +442,6 @@ class VmBackup {
       await this._fetchJobSnapshots()
       await this._removeUnusedSnapshots()
     }
-    await this._healthCheck()
   }
 }
 exports.VmBackup = VmBackup

@xen-orchestra/backups/_XoMetadataBackup.js

@@ -38,7 +38,7 @@ exports.XoMetadataBackup = class XoMetadataBackup {
     )
     const metaDataFileName = `${dir}/metadata.json`
 
-    await pEach(
+    await asyncMap(
       Object.entries(this._remoteAdapters),
       ([remoteId, adapter]) =>
         Task.run(

@xen-orchestra/backups/_backupWorker.js

@@ -4,8 +4,6 @@ require('@xen-orchestra/log/configure.js').catchGlobalErrors(
   require('@xen-orchestra/log').createLogger('xo:backups:worker')
 )
 
-require('@vates/cached-dns.lookup').createCachedLookup().patchGlobal()
-
 const Disposable = require('promise-toolbox/Disposable')
 const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { compose } = require('@vates/compose')

@xen-orchestra/backups/_cleanVm.integ.spec.js

@@ -5,9 +5,9 @@
 const rimraf = require('rimraf')
 const tmp = require('tmp')
 const fs = require('fs-extra')
-const uuid = require('uuid')
 const { getHandler } = require('@xen-orchestra/fs')
 const { pFromCallback } = require('promise-toolbox')
+const crypto = require('crypto')
 const { RemoteAdapter } = require('./RemoteAdapter')
 const { VHDFOOTER, VHDHEADER } = require('./tests.fixtures.js')
 const { VhdFile, Constants, VhdDirectory, VhdAbstract } = require('vhd-lib')
@@ -34,8 +34,7 @@ afterEach(async () => {
   await handler.forget()
 })
 
-const uniqueId = () => uuid.v1()
-const uniqueIdBuffer = () => Buffer.from(uniqueId(), 'utf-8')
+const uniqueId = () => crypto.randomBytes(16).toString('hex')
 
 async function generateVhd(path, opts = {}) {
   let vhd
@@ -54,9 +53,10 @@ async function generateVhd(path, opts = {}) {
   }
 
   vhd.header = { ...VHDHEADER, ...opts.header }
-  vhd.footer = { ...VHDFOOTER, ...opts.footer, uuid: uniqueIdBuffer() }
+  vhd.footer = { ...VHDFOOTER, ...opts.footer }
+  vhd.footer.uuid = Buffer.from(crypto.randomBytes(16))
 
-  if (vhd.header.parentUuid) {
+  if (vhd.header.parentUnicodeName) {
     vhd.footer.diskType = Constants.DISK_TYPES.DIFFERENCING
   } else {
     vhd.footer.diskType = Constants.DISK_TYPES.DYNAMIC
@@ -91,31 +91,24 @@ test('It remove broken vhd', async () => {
 })
 
 test('it remove vhd with missing or multiple ancestors', async () => {
-  // one with a broken parent, should be deleted
+  // one with a broken parent
   await generateVhd(`${basePath}/abandonned.vhd`, {
     header: {
       parentUnicodeName: 'gone.vhd',
-      parentUuid: uniqueIdBuffer(),
+      parentUid: Buffer.from(crypto.randomBytes(16)),
     },
   })
 
-  // one orphan, which is a full vhd, no parent : should stay
+  // one orphan, which is a full vhd, no parent
   const orphan = await generateVhd(`${basePath}/orphan.vhd`)
-  // a child to the orphan in the metadata : should stay
+  // a child to the orphan
   await generateVhd(`${basePath}/child.vhd`, {
     header: {
       parentUnicodeName: 'orphan.vhd',
-      parentUuid: orphan.footer.uuid,
+      parentUid: orphan.footer.uuid,
     },
   })
-  await handler.writeFile(
-    `metadata.json`,
-    JSON.stringify({
-      mode: 'delta',
-      vhds: [`${basePath}/child.vhd`, `${basePath}/abandonned.vhd`],
-    }),
-    { flags: 'w' }
-  )
+
   // clean
   let loggued = ''
   const onLog = message => {
@@ -154,7 +147,7 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy
   await generateVhd(`${basePath}/child.vhd`, {
     header: {
       parentUnicodeName: 'orphan.vhd',
-      parentUuid: orphan.footer.uuid,
+      parentUid: orphan.footer.uuid,
     },
   })
 
@@ -208,14 +201,14 @@ test('it merges delta of non destroyed chain', async () => {
   const child = await generateVhd(`${basePath}/child.vhd`, {
     header: {
       parentUnicodeName: 'orphan.vhd',
-      parentUuid: orphan.footer.uuid,
+      parentUid: orphan.footer.uuid,
     },
   })
   // a grand child
   await generateVhd(`${basePath}/grandchild.vhd`, {
     header: {
       parentUnicodeName: 'child.vhd',
-      parentUuid: child.footer.uuid,
+      parentUid: child.footer.uuid,
     },
   })
 
@@ -224,12 +217,14 @@ test('it merges delta of non destroyed chain', async () => {
     loggued.push(message)
   }
   await adapter.cleanVm('/', { remove: true, onLog })
-  expect(loggued[0]).toEqual(`incorrect size in metadata: 12000 instead of 209920`)
+  expect(loggued[0]).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused`)
+  expect(loggued[1]).toEqual(`incorrect size in metadata: 12000 instead of 209920`)
 
   loggued = []
   await adapter.cleanVm('/', { remove: true, merge: true, onLog })
-  const [merging] = loggued
-  expect(merging).toEqual(`merging 1 children into /${basePath}/orphan.vhd`)
+  const [unused, merging] = loggued
+  expect(unused).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused`)
+  expect(merging).toEqual(`merging /${basePath}/child.vhd into /${basePath}/orphan.vhd`)
 
   const metadata = JSON.parse(await handler.readFile(`metadata.json`))
   // size should be the size of children + grand children after the merge
@@ -259,7 +254,7 @@ test('it finish unterminated merge ', async () => {
   const child = await generateVhd(`${basePath}/child.vhd`, {
     header: {
       parentUnicodeName: 'orphan.vhd',
-      parentUuid: orphan.footer.uuid,
+      parentUid: orphan.footer.uuid,
     },
   })
   // a merge in progress file
@@ -315,7 +310,7 @@ describe('tests multiple combination ', () => {
           mode: vhdMode,
           header: {
             parentUnicodeName: 'gone.vhd',
-            parentUuid: uniqueIdBuffer(),
+            parentUid: crypto.randomBytes(16),
           },
         })
 
@@ -329,7 +324,7 @@ describe('tests multiple combination ', () => {
           mode: vhdMode,
           header: {
             parentUnicodeName: 'ancestor.vhd' + (useAlias ? '.alias.vhd' : ''),
-            parentUuid: ancestor.footer.uuid,
+            parentUid: ancestor.footer.uuid,
           },
         })
         // a grand child  vhd in metadata
@@ -338,7 +333,7 @@ describe('tests multiple combination ', () => {
           mode: vhdMode,
           header: {
             parentUnicodeName: 'child.vhd' + (useAlias ? '.alias.vhd' : ''),
-            parentUuid: child.footer.uuid,
+            parentUid: child.footer.uuid,
           },
         })
 
@@ -353,7 +348,7 @@ describe('tests multiple combination ', () => {
           mode: vhdMode,
           header: {
             parentUnicodeName: 'cleanAncestor.vhd' + (useAlias ? '.alias.vhd' : ''),
-            parentUuid: cleanAncestor.footer.uuid,
+            parentUid: cleanAncestor.footer.uuid,
           },
         })
 

@xen-orchestra/backups/_cleanVm.js

@@ -4,12 +4,11 @@ const assert = require('assert')
 const sum = require('lodash/sum')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { Constants, mergeVhd, openVhd, VhdAbstract, VhdFile } = require('vhd-lib')
+const { isVhdAlias, resolveVhdAlias } = require('vhd-lib/aliases')
 const { dirname, resolve } = require('path')
 const { DISK_TYPES } = Constants
 const { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } = require('./_backupType.js')
-const { isVhdAlias, resolveVhdAlias } = require('vhd-lib/aliases')
 const { limitConcurrency } = require('limit-concurrency-decorator')
-const { pEach } = require('@vates/async-each')
 
 const { Task } = require('./Task.js')
 const { Disposable } = require('promise-toolbox')
@@ -32,48 +31,71 @@ const computeVhdsSize = (handler, vhdPaths) =>
     }
   )
 
-// chain is [ ancestor, child1, ..., childn]
-// 1. Create a VhdSynthetic from all children
-// 2. Merge the VhdSynthetic into the ancestor
-// 3. Delete all (now) unused VHDs
-// 4. Rename the ancestor with the merged data to the latest child
+// chain is an array of VHDs from child to parent
 //
-//                  VhdSynthetic
-//                       |
-//              /‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾\
-//  [ ancestor, child1, ...,child n-1,  childn ]
-//         |    \___________________/     ^
-//         |             |                |
-//         |       unused VHDs            |
-//         |                              |
-//         \___________rename_____________/
-
-async function mergeVhdChain(chain, { handler, logInfo, remove, merge }) {
+// the whole chain will be merged into parent, parent will be renamed to child
+// and all the others will deleted
+async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
   assert(chain.length >= 2)
-  const chainCopy = [...chain]
-  const parent = chainCopy.pop()
-  const children = chainCopy
+
+  let child = chain[0]
+  const parent = chain[chain.length - 1]
+  const children = chain.slice(0, -1).reverse()
+
+  chain
+    .slice(1)
+    .reverse()
+    .forEach(parent => {
+      onLog(`the parent ${parent} of the child ${child} is unused`)
+    })
 
   if (merge) {
-    logInfo(`merging children into parent`, { childrenCount: children.length, parent })
+    // `mergeVhd` does not work with a stream, either
+    // - make it accept a stream
+    // - or create synthetic VHD which is not a stream
+    if (children.length !== 1) {
+      // TODO: implement merging multiple children
+      children.length = 1
+      child = children[0]
+    }
+
+    onLog(`merging ${child} into ${parent}`)
 
     let done, total
     const handle = setInterval(() => {
       if (done !== undefined) {
-        logInfo(`merging children in progress`, { children, parent, doneCount: done, totalCount: total })
+        onLog(`merging ${child}: ${done}/${total}`)
       }
     }, 10e3)
 
-    const mergedSize = await mergeVhd(handler, parent, handler, children, {
-      logInfo,
-      onProgress({ done: d, total: t }) {
-        done = d
-        total = t
-      },
-      remove,
-    })
+    const mergedSize = await mergeVhd(
+      handler,
+      parent,
+      handler,
+      child,
+      // children.length === 1
+      //   ? child
+      //   : await createSyntheticStream(handler, children),
+      {
+        onProgress({ done: d, total: t }) {
+          done = d
+          total = t
+        },
+      }
+    )
 
     clearInterval(handle)
+    await Promise.all([
+      VhdAbstract.rename(handler, parent, child),
+      asyncMap(children.slice(0, -1), child => {
+        onLog(`the VHD ${child} is unused`)
+        if (remove) {
+          onLog(`deleting unused VHD ${child}`)
+          return VhdAbstract.unlink(handler, child)
+        }
+      }),
+    ])
+
     return mergedSize
   }
 }
@@ -86,13 +108,13 @@ const listVhds = async (handler, vmDir) => {
   const aliases = {}
   const interruptedVhds = new Map()
 
-  await pEach(
+  await asyncMap(
     await handler.list(`${vmDir}/vdis`, {
       ignoreMissing: true,
       prependDir: true,
     }),
     async jobDir =>
-      pEach(
+      asyncMap(
         await handler.list(jobDir, {
           prependDir: true,
         }),
@@ -116,19 +138,14 @@ const listVhds = async (handler, vmDir) => {
   return { vhds, interruptedVhds, aliases }
 }
 
-async function checkAliases(
-  aliasPaths,
-  targetDataRepository,
-  { handler, logInfo = noop, logWarn = console.warn, remove = false }
-) {
+async function checkAliases(aliasPaths, targetDataRepository, { handler, onLog = noop, remove = false }) {
   const aliasFound = []
   for (const path of aliasPaths) {
     const target = await resolveVhdAlias(handler, path)
 
     if (!isVhdFile(target)) {
-      logWarn('alias references non VHD target', { path, target })
+      onLog(`Alias ${path} references a non vhd target:  ${target}`)
       if (remove) {
-        logInfo('removing alias and non VHD target', { path, target })
         await handler.unlink(target)
         await handler.unlink(path)
       }
@@ -143,13 +160,13 @@ async function checkAliases(
         // error during dispose should not trigger a deletion
       }
     } catch (error) {
-      logWarn('missing or broken alias target', { target, path, error })
+      onLog(`target ${target} of alias ${path} is missing or broken`, { error })
       if (remove) {
         try {
           await VhdAbstract.unlink(handler, path)
-        } catch (error) {
-          if (error.code !== 'ENOENT') {
-            logWarn('error deleting alias target', { target, path, error })
+        } catch (e) {
+          if (e.code !== 'ENOENT') {
+            onLog(`Error while deleting target ${target} of alias ${path}`, { error: e })
           }
         }
       }
@@ -166,22 +183,20 @@ async function checkAliases(
 
   entries.forEach(async entry => {
     if (!aliasFound.includes(entry)) {
-      logWarn('no alias references VHD', { entry })
+      onLog(`the Vhd  ${entry} is not referenced by a an alias`)
       if (remove) {
-        logInfo('deleting unaliased VHD')
         await VhdAbstract.unlink(handler, entry)
       }
     }
   })
 }
-
 exports.checkAliases = checkAliases
 
 const defaultMergeLimiter = limitConcurrency(1)
 
 exports.cleanVm = async function cleanVm(
   vmDir,
-  { fixMetadata, remove, merge, mergeLimiter = defaultMergeLimiter, logInfo = noop, logWarn = console.warn }
+  { fixMetadata, remove, merge, mergeLimiter = defaultMergeLimiter, onLog = noop }
 ) {
   const limitedMergeVhdChain = mergeLimiter(mergeVhdChain)
 
@@ -194,7 +209,7 @@ exports.cleanVm = async function cleanVm(
   const { vhds, interruptedVhds, aliases } = await listVhds(handler, vmDir)
 
   // remove broken VHDs
-  await pEach(vhds, async path => {
+  await asyncMap(vhds, async path => {
     try {
       await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), vhd => {
         if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {
@@ -212,9 +227,9 @@ exports.cleanVm = async function cleanVm(
       })
     } catch (error) {
       vhds.delete(path)
-      logWarn('VHD check error', { path, error })
+      onLog(`error while checking the VHD with path ${path}`, { error })
       if (error?.code === 'ERR_ASSERTION' && remove) {
-        logInfo('deleting broken path', { path })
+        onLog(`deleting broken ${path}`)
         return VhdAbstract.unlink(handler, path)
       }
     }
@@ -226,12 +241,12 @@ exports.cleanVm = async function cleanVm(
       const statePath = interruptedVhds.get(interruptedVhd)
       interruptedVhds.delete(interruptedVhd)
 
-      logWarn('orphan merge state', {
+      onLog('orphan merge state', {
         mergeStatePath: statePath,
         missingVhdPath: interruptedVhd,
       })
       if (remove) {
-        logInfo('deleting orphan merge state', { statePath })
+        onLog(`deleting orphan merge state ${statePath}`)
         await handler.unlink(statePath)
       }
     }
@@ -239,8 +254,8 @@ exports.cleanVm = async function cleanVm(
 
   // check if alias are correct
   // check if all vhd in data subfolder have a corresponding alias
-  await pEach(Object.keys(aliases), async dir => {
-    await checkAliases(aliases[dir], `${dir}/data`, { handler, logInfo, logWarn, remove })
+  await asyncMap(Object.keys(aliases), async dir => {
+    await checkAliases(aliases[dir], `${dir}/data`, { handler, onLog, remove })
   })
 
   // remove VHDs with missing ancestors
@@ -262,9 +277,9 @@ exports.cleanVm = async function cleanVm(
       if (!vhds.has(parent)) {
         vhds.delete(vhdPath)
 
-        logWarn('parent VHD is missing', { parent, vhdPath })
+        onLog(`the parent ${parent} of the VHD ${vhdPath} is missing`)
         if (remove) {
-          logInfo('deleting orphan VHD', { vhdPath })
+          onLog(`deleting orphan VHD ${vhdPath}`)
           deletions.push(VhdAbstract.unlink(handler, vhdPath))
         }
       }
@@ -297,11 +312,11 @@ exports.cleanVm = async function cleanVm(
     }
   })
 
-  await pEach(xvas, async path => {
+  await asyncMap(xvas, async path => {
     // check is not good enough to delete the file, the best we can do is report
     // it
     if (!(await this.isValidXva(path))) {
-      logWarn('XVA might be broken', { path })
+      onLog(`the XVA with path ${path} is potentially broken`)
     }
   })
 
@@ -310,12 +325,12 @@ exports.cleanVm = async function cleanVm(
 
   // compile the list of unused XVAs and VHDs, and remove backup metadata which
   // reference a missing XVA/VHD
-  await pEach(jsons, async json => {
+  await asyncMap(jsons, async json => {
     let metadata
     try {
       metadata = JSON.parse(await handler.readFile(json))
     } catch (error) {
-      logWarn('failed to read metadata file', { json, error })
+      onLog(`failed to read metadata file ${json}`, { error })
       jsons.delete(json)
       return
     }
@@ -326,9 +341,9 @@ exports.cleanVm = async function cleanVm(
       if (xvas.has(linkedXva)) {
         unusedXvas.delete(linkedXva)
       } else {
-        logWarn('metadata XVA is missing', { json })
+        onLog(`the XVA linked to the metadata ${json} is missing`)
         if (remove) {
-          logInfo('deleting incomplete backup', { json })
+          onLog(`deleting incomplete backup ${json}`)
           jsons.delete(json)
           await handler.unlink(json)
         }
@@ -349,9 +364,9 @@ exports.cleanVm = async function cleanVm(
           vhdsToJSons[path] = json
         })
       } else {
-        logWarn('some metadata VHDs are missing', { json, missingVhds })
+        onLog(`Some VHDs linked to the metadata ${json} are missing`, { missingVhds })
         if (remove) {
-          logInfo('deleting incomplete backup', { json })
+          onLog(`deleting incomplete backup ${json}`)
           jsons.delete(json)
           await handler.unlink(json)
         }
@@ -392,9 +407,9 @@ exports.cleanVm = async function cleanVm(
         }
       }
 
-      logWarn('unused VHD', { vhd })
+      onLog(`the VHD ${vhd} is unused`)
       if (remove) {
-        logInfo('deleting unused VHD', { vhd })
+        onLog(`deleting unused VHD ${vhd}`)
         unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
       }
     }
@@ -417,8 +432,8 @@ exports.cleanVm = async function cleanVm(
 
   const metadataWithMergedVhd = {}
   const doMerge = async () => {
-    await pEach(toMerge, async chain => {
-      const merged = await limitedMergeVhdChain(chain, { handler, logInfo, logWarn, remove, merge })
+    await asyncMap(toMerge, async chain => {
+      const merged = await limitedMergeVhdChain(chain, { handler, onLog, remove, merge })
       if (merged !== undefined) {
         const metadataPath = vhdsToJSons[chain[0]] // all the chain should have the same metada file
         metadataWithMergedVhd[metadataPath] = true
@@ -429,19 +444,19 @@ exports.cleanVm = async function cleanVm(
   await Promise.all([
     ...unusedVhdsDeletion,
     toMerge.length !== 0 && (merge ? Task.run({ name: 'merge' }, doMerge) : doMerge()),
-    pEach(unusedXvas, path => {
-      logWarn('unused XVA', { path })
+    asyncMap(unusedXvas, path => {
+      onLog(`the XVA ${path} is unused`)
       if (remove) {
-        logInfo('deleting unused XVA', { path })
+        onLog(`deleting unused XVA ${path}`)
         return handler.unlink(path)
       }
     }),
-    pEach(xvaSums, path => {
+    asyncMap(xvaSums, path => {
       // no need to handle checksums for XVAs deleted by the script, they will be handled by `unlink()`
       if (!xvas.has(path.slice(0, -'.checksum'.length))) {
-        logInfo('unused XVA checksum', { path })
+        onLog(`the XVA checksum ${path} is unused`)
         if (remove) {
-          logInfo('deleting unused XVA checksum', { path })
+          onLog(`deleting unused XVA checksum ${path}`)
           return handler.unlink(path)
         }
       }
@@ -451,7 +466,7 @@ exports.cleanVm = async function cleanVm(
   // update size for delta metadata with merged VHD
   // check for the other that the size is the same as the real file size
 
-  await pEach(jsons, async metadataPath => {
+  await asyncMap(jsons, async metadataPath => {
     const metadata = JSON.parse(await handler.readFile(metadataPath))
 
     let fileSystemSize
@@ -475,11 +490,11 @@ exports.cleanVm = async function cleanVm(
 
         // don't warn if the size has changed after a merge
         if (!merged && fileSystemSize !== size) {
-          logWarn('incorrect size in metadata', { size: size ?? 'none', fileSystemSize })
+          onLog(`incorrect size in metadata: ${size ?? 'none'} instead of ${fileSystemSize}`)
         }
       }
     } catch (error) {
-      logWarn('failed to get metadata size', { metadataPath, error })
+      onLog(`failed to get size of ${metadataPath}`, { error })
       return
     }
 
@@ -489,7 +504,7 @@ exports.cleanVm = async function cleanVm(
       try {
         await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })
       } catch (error) {
-        logWarn('metadata size update failed', { metadataPath, error })
+        onLog(`failed to update size in backup metadata ${metadataPath} after merge`, { error })
       }
     }
   })

@xen-orchestra/backups/_deltaVm.js

@@ -5,10 +5,10 @@ const find = require('lodash/find.js')
 const groupBy = require('lodash/groupBy.js')
 const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const omit = require('lodash/omit.js')
+const { asyncMap } = require('@xen-orchestra/async-map')
 const { CancelToken } = require('promise-toolbox')
 const { createVhdStreamWithLength } = require('vhd-lib')
 const { defer } = require('golike-defer')
-const { pEach } = require('@vates/async-each')
 
 const { cancelableMap } = require('./_cancelableMap.js')
 const { Task } = require('./Task.js')
@@ -65,6 +65,17 @@ exports.exportDeltaVm = async function exportDeltaVm(
       return
     }
 
+    // If the VDI name start with `[NOBAK]`, do not export it.
+    if (vdi.name_label.startsWith('[NOBAK]')) {
+      // FIXME: find a way to not create the VDI snapshot in the
+      // first time.
+      //
+      // The snapshot must not exist otherwise it could break the
+      // next export.
+      ignoreErrors.call(vdi.$destroy())
+      return
+    }
+
     vbds[vbd.$ref] = vbd
 
     const vdiRef = vdi.$ref
@@ -237,13 +248,13 @@ exports.importDeltaVm = defer(async function importDeltaVm(
   $defer.onFailure.call(xapi, 'VM_destroy', vmRef)
 
   // 2. Delete all VBDs which may have been created by the import.
-  await pEach(await xapi.getField('VM', vmRef, 'VBDs'), ref => ignoreErrors.call(xapi.call('VBD.destroy', ref)))
+  await asyncMap(await xapi.getField('VM', vmRef, 'VBDs'), ref => ignoreErrors.call(xapi.call('VBD.destroy', ref)))
 
   // 3. Create VDIs & VBDs.
   const vbdRecords = deltaVm.vbds
   const vbds = groupBy(vbdRecords, 'VDI')
   const newVdis = {}
-  await pEach(Object.keys(vdiRecords), async vdiRef => {
+  await asyncMap(Object.keys(vdiRecords), async vdiRef => {
     const vdi = vdiRecords[vdiRef]
     let newVdi
 
@@ -279,7 +290,7 @@ exports.importDeltaVm = defer(async function importDeltaVm(
 
     const vdiVbds = vbds[vdiRef]
     if (vdiVbds !== undefined) {
-      await pEach(Object.values(vdiVbds), vbd =>
+      await asyncMap(Object.values(vdiVbds), vbd =>
         xapi.VBD_create({
           ...vbd,
           VDI: newVdi.$ref,
@@ -323,7 +334,7 @@ exports.importDeltaVm = defer(async function importDeltaVm(
     }),
 
     // Create VIFs.
-    pEach(Object.values(deltaVm.vifs), vif => {
+    asyncMap(Object.values(deltaVm.vifs), vif => {
       let network = vif.$network$uuid && xapi.getObjectByUuid(vif.$network$uuid, undefined)
 
       if (network === undefined) {

@xen-orchestra/backups/docs/VM backups/README.md

@@ -6,11 +6,6 @@
 - [Task logs](#task-logs)
   - [During backup](#during-backup)
   - [During restoration](#during-restoration)
-- [API](#api)
-  - [Run description object](#run-description-object)
-  - [`IdPattern`](#idpattern)
-  - [Settings](#settings)
-- [Writer API](#writer-api)
 
 ## File structure on remote
 
@@ -69,30 +64,24 @@ job.start(data: { mode: Mode, reportWhen: ReportWhen })
 ├─ task.warning(message: string)
 ├─ task.start(data: { type: 'VM', id: string })
 │  ├─ task.warning(message: string)
-|  ├─ task.start(message: 'clean-vm')
-│  │  └─ task.end
 │  ├─ task.start(message: 'snapshot')
 │  │  └─ task.end
-│  ├─ task.start(message: 'export', data: { type: 'SR' | 'remote', id: string, isFull: boolean })
+│  ├─ task.start(message: 'export', data: { type: 'SR' | 'remote', id: string })
 │  │  ├─ task.warning(message: string)
 │  │  ├─ task.start(message: 'transfer')
 │  │  │  ├─ task.warning(message: string)
 │  │  │  └─ task.end(result: { size: number })
 │  │  │
-│  │  │  // in case there is a healthcheck scheduled for this vm in this job
-│  │  ├─ task.start(message: 'health check')
-│  │  │  ├─ task.start(message: 'transfer')
-│  │  │  │  └─ task.end(result: { size: number })
-│  │  │  ├─ task.start(message: 'vmstart')
-│  │  │  │  └─ task.end
-│  │  │  └─ task.end
-│  │  │
 │  │  │  // in case of full backup, DR and CR
 │  │  ├─ task.start(message: 'clean')
 │  │  │  ├─ task.warning(message: string)
 │  │  │  └─ task.end
-│  │  └─ task.end
-|  ├─ task.start(message: 'clean-vm')
+│  │  │
+│  │  │ // in case of delta backup
+│  │  ├─ task.start(message: 'merge')
+│  │  │  ├─ task.warning(message: string)
+│  │  │  └─ task.end(result: { size: number })
+│  │  │
 │  │  └─ task.end
 │  └─ task.end
 └─ job.end
@@ -106,102 +95,3 @@ task.start(message: 'restore', data: { jobId: string, srId: string, time: number
 │  └─ task.end(result: { id: string, size: number })
 └─ task.end
 ```
-
-## API
-
-### Run description object
-
-This is a JavaScript object containing all the information necessary to run a backup job.
-
-```coffee
-# Information about the job itself
-job:
-
-  # Unique identifier
-  id: string
-
-  # Human readable identifier
-  name: string
-
-  # Whether this job is doing Full Backup / Disaster Recovery or
-  # Delta Backup / Continuous Replication
-  mode: 'full' | 'delta'
-
-  # For backup jobs, indicates which remotes to use
-  remotes: IdPattern
-
-  settings:
-
-    # Used for the whole job
-    '': Settings
-
-    # Used for a specific schedule
-    [ScheduleId]: Settings
-
-    # Used for a specific VM
-    [VmId]: Settings
-
-  # For replication jobs, indicates which SRs to use
-  srs: IdPattern
-
-  # Here for historical reasons
-  type: 'backup'
-
-  # Indicates which VMs to backup/replicate
-  vms: IdPattern
-
-# Indicates which XAPI to use to connect to a specific VM or SR
-recordToXapi:
-  [ObjectId]: XapiId
-
-# Information necessary to connect to each remote
-remotes:
-  [RemoteId]:
-    url: string
-
-# Indicates which schedule is used for this run
-schedule:
-  id: ScheduleId
-
-# Information necessary to connect to each XAPI
-xapis:
-  [XapiId]:
-    allowUnauthorized: boolean
-    credentials:
-      password: string
-      username: string
-    url: string
-```
-
-### `IdPattern`
-
-For a single object:
-
-```
-{ id: string }
-```
-
-For multiple objects:
-
-```
-{ id: { __or: string[] } }
-```
-
-> This syntax is compatible with [`value-matcher`](https://github.com/vatesfr/xen-orchestra/tree/master/packages/value-matcher).
-
-### Settings
-
-Settings are described in [`@xen-orchestra/backups/Backup.js](https://github.com/vatesfr/xen-orchestra/blob/master/%40xen-orchestra/backups/Backup.js).
-
-## Writer API
-
-- `beforeBackup()`
-  - **Delta**
-    - `checkBaseVdis(baseUuidToSrcVdi, baseVm)`
-    - `prepare({ isFull })`
-    - `transfer({ timestamp, deltaExport, sizeContainers })`
-    - `cleanup()`
-    - `healthCheck(sr)`
-  - **Full**
-    - `run({ timestamp, sizeContainer, stream })`
-- `afterBackup()`

@xen-orchestra/backups/merge-worker/cli.js

@@ -1,6 +1,4 @@
 #!/usr/bin/env node
-// eslint-disable-next-line eslint-comments/disable-enable-pair
-/* eslint-disable n/shebang */
 
 'use strict'
 

@xen-orchestra/backups/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.26.0",
+  "version": "0.21.0",
   "engines": {
     "node": ">=14.6"
   },
@@ -16,18 +16,16 @@
     "postversion": "npm publish --access public"
   },
   "dependencies": {
-    "@vates/cached-dns.lookup": "^1.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.0.0",
     "@vates/disposable": "^0.1.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^1.1.0",
+    "@xen-orchestra/fs": "^1.0.0",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/template": "^0.1.0",
     "compare-versions": "^4.0.1",
     "d3-time-format": "^3.0.0",
-    "decorator-synchronized": "^0.6.0",
     "end-of-stream": "^1.4.4",
     "fs-extra": "^10.0.0",
     "golike-defer": "^0.5.1",
@@ -38,7 +36,7 @@
     "promise-toolbox": "^0.21.0",
     "proper-lockfile": "^4.1.2",
     "uuid": "^8.3.2",
-    "vhd-lib": "^3.3.1",
+    "vhd-lib": "^3.1.0",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
@@ -46,7 +44,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^1.4.0"
+    "@xen-orchestra/xapi": "^0.10.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

@xen-orchestra/backups/writers/DeltaBackupWriter.js

@@ -19,8 +19,6 @@ const { AbstractDeltaWriter } = require('./_AbstractDeltaWriter.js')
 const { checkVhd } = require('./_checkVhd.js')
 const { packUuid } = require('./_packUuid.js')
 const { Disposable } = require('promise-toolbox')
-const { HealthCheckVmBackup } = require('../HealthCheckVmBackup.js')
-const { ImportVmBackup } = require('../ImportVmBackup.js')
 
 const { warn } = createLogger('xo:backups:DeltaBackupWriter')
 
@@ -33,7 +31,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
     const backupDir = getVmBackupDir(backup.vm.uuid)
     const vdisDir = `${backupDir}/vdis/${backup.job.id}`
 
-    await pEach(baseUuidToSrcVdi, async ([baseUuid, srcVdi]) => {
+    await asyncMap(baseUuidToSrcVdi, async ([baseUuid, srcVdi]) => {
       let found = false
       try {
         const vhds = await handler.list(`${vdisDir}/${srcVdi.uuid}`, {
@@ -41,7 +39,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           prependDir: true,
         })
         const packedBaseUuid = packUuid(baseUuid)
-        await pEach(vhds, async path => {
+        await asyncMap(vhds, async path => {
           try {
             await checkVhdChain(handler, path)
             // Warning, this should not be written as found = found || await adapter.isMergeableParent(packedBaseUuid, path)
@@ -71,35 +69,6 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
     return this._cleanVm({ merge: true })
   }
 
-  healthCheck(sr) {
-    return Task.run(
-      {
-        name: 'health check',
-      },
-      async () => {
-        const xapi = sr.$xapi
-        const srUuid = sr.uuid
-        const adapter = this._adapter
-        const metadata = await adapter.readVmBackupMetadata(this._metadataFileName)
-        const { id: restoredId } = await new ImportVmBackup({
-          adapter,
-          metadata,
-          srUuid,
-          xapi,
-        }).run()
-        const restoredVm = xapi.getObject(restoredId)
-        try {
-          await new HealthCheckVmBackup({
-            restoredVm,
-            xapi,
-          }).run()
-        } finally {
-          await xapi.VM_destroy(restoredVm.$ref)
-        }
-      }
-    )
-  }
-
   prepare({ isFull }) {
     // create the task related to this export and ensure all methods are called in this context
     const task = new Task({
@@ -111,9 +80,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
       },
     })
     this.transfer = task.wrapFn(this.transfer)
-    this.healthCheck = task.wrapFn(this.healthCheck)
-    this.cleanup = task.wrapFn(this.cleanup)
-    this.afterBackup = task.wrapFn(this.afterBackup, true)
+    this.cleanup = task.wrapFn(this.cleanup, true)
 
     return task.run(() => this._prepare())
   }
@@ -189,7 +156,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
         }/${adapter.getVhdFileName(basename)}`
     )
 
-    const metadataFilename = (this._metadataFileName = `${backupDir}/${basename}.json`)
+    const metadataFilename = `${backupDir}/${basename}.json`
     const metadataContent = {
       jobId,
       mode: job.mode,

@xen-orchestra/backups/writers/DeltaReplicationWriter.js

@@ -25,7 +25,7 @@ exports.DeltaReplicationWriter = class DeltaReplicationWriter extends MixinRepli
 
     const xapi = replicatedVm.$xapi
     const replicatedVdis = new Set(
-      await pEach(await replicatedVm.$getDisks(), async vdiRef => {
+      await asyncMap(await replicatedVm.$getDisks(), async vdiRef => {
         const otherConfig = await xapi.getField('VDI', vdiRef, 'other_config')
         return otherConfig[TAG_COPY_SRC]
       })
@@ -60,10 +60,7 @@ exports.DeltaReplicationWriter = class DeltaReplicationWriter extends MixinRepli
     const { scheduleId, vm } = this._backup
 
     // delete previous interrupted copies
-    ignoreErrors.call(
-      pEach(listReplicatedVms(xapi, scheduleId, undefined, vm.uuid), vm => vm.$destroy),
-      { stopOnError: false }
-    )
+    ignoreErrors.call(asyncMapSettled(listReplicatedVms(xapi, scheduleId, undefined, vm.uuid), vm => vm.$destroy))
 
     this._oldEntries = getOldEntries(settings.copyRetention - 1, listReplicatedVms(xapi, scheduleId, srUuid, vm.uuid))
 
@@ -79,7 +76,7 @@ exports.DeltaReplicationWriter = class DeltaReplicationWriter extends MixinRepli
   }
 
   async _deleteOldEntries() {
-    return pEach(this._oldEntries, vm => vm.$destroy(), { stopOnError: false })
+    return asyncMapSettled(this._oldEntries, vm => vm.$destroy())
   }
 
   async _transfer({ timestamp, deltaExport, sizeContainers }) {
@@ -111,7 +108,7 @@ exports.DeltaReplicationWriter = class DeltaReplicationWriter extends MixinRepli
       targetVm.ha_restart_priority !== '' &&
         Promise.all([targetVm.set_ha_restart_priority(''), targetVm.add_tags('HA disabled')]),
       targetVm.set_name_label(`${vm.name_label} - ${job.name} - (${formatFilenameDate(timestamp)})`),
-      pEach(['start', 'start_on'], op =>
+      asyncMap(['start', 'start_on'], op =>
         targetVm.update_blocked_operations(
           op,
           'Start operation for this vm is blocked, clone it if you want to use it.'

@xen-orchestra/backups/writers/FullReplicationWriter.js

@@ -40,14 +40,12 @@ exports.FullReplicationWriter = class FullReplicationWriter extends MixinReplica
 
     // delete previous interrupted copies
     ignoreErrors.call(
-      pEach(listReplicatedVms(xapi, scheduleId, undefined, vm.uuid), vm => xapi.VM_destroy(vm.$ref), {
-        stopOnError: false,
-      })
+      asyncMapSettled(listReplicatedVms(xapi, scheduleId, undefined, vm.uuid), vm => xapi.VM_destroy(vm.$ref))
     )
 
     const oldVms = getOldEntries(settings.copyRetention - 1, listReplicatedVms(xapi, scheduleId, srUuid, vm.uuid))
 
-    const deleteOldBackups = () => pEach(oldVms, vm => xapi.VM_destroy(vm.$ref), { stopOnError: false })
+    const deleteOldBackups = () => asyncMapSettled(oldVms, vm => xapi.VM_destroy(vm.$ref))
     const { deleteFirst } = settings
     if (deleteFirst) {
       await deleteOldBackups()
@@ -68,7 +66,7 @@ exports.FullReplicationWriter = class FullReplicationWriter extends MixinReplica
     const targetVm = await xapi.getRecord('VM', targetVmRef)
 
     await Promise.all([
-      pEach(['start', 'start_on'], op =>
+      asyncMap(['start', 'start_on'], op =>
         targetVm.update_blocked_operations(
           op,
           'Start operation for this vm is blocked, clone it if you want to use it.'

@xen-orchestra/backups/writers/_AbstractWriter.js

@@ -9,6 +9,4 @@ exports.AbstractWriter = class AbstractWriter {
   beforeBackup() {}
 
   afterBackup() {}
-
-  healthCheck(sr) {}
 }

@xen-orchestra/backups/writers/_MixinBackupWriter.js

@@ -6,9 +6,8 @@ const { join } = require('path')
 const { getVmBackupDir } = require('../_getVmBackupDir.js')
 const MergeWorker = require('../merge-worker/index.js')
 const { formatFilenameDate } = require('../_filenameDate.js')
-const { Task } = require('../Task.js')
 
-const { info, warn } = createLogger('xo:backups:MixinBackupWriter')
+const { warn } = createLogger('xo:backups:MixinBackupWriter')
 
 exports.MixinBackupWriter = (BaseClass = Object) =>
   class MixinBackupWriter extends BaseClass {
@@ -26,17 +25,11 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
 
     async _cleanVm(options) {
       try {
-        return await Task.run({ name: 'clean-vm' }, () => {
-          return this._adapter.cleanVm(this.#vmBackupDir, {
-            ...options,
-            fixMetadata: true,
-            logInfo: info,
-            logWarn: (message, data) => {
-              warn(message, data)
-              Task.warning(message, data)
-            },
-            lock: false,
-          })
+        return await this._adapter.cleanVm(this.#vmBackupDir, {
+          ...options,
+          fixMetadata: true,
+          onLog: warn,
+          lock: false,
         })
       } catch (error) {
         warn(error)
@@ -71,6 +64,5 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
         const remotePath = handler._getRealPath()
         await MergeWorker.run(remotePath)
       }
-      await this._adapter.invalidateVmBackupListCache(this._backup.vm.uuid)
     }
   }

@xen-orchestra/cr-seed-cli/package.json

@@ -18,7 +18,7 @@
   "preferGlobal": true,
   "dependencies": {
     "golike-defer": "^0.5.1",
-    "xen-api": "^1.2.1"
+    "xen-api": "^1.1.0"
   },
   "scripts": {
     "postversion": "npm publish"

@xen-orchestra/emit-async/.USAGE.md

@@ -22,7 +22,7 @@ await ee.emitAsync('start')
 // error handling though:
 await ee.emitAsync(
   {
-    onError(error, event, listener) {
+    onError(error) {
       console.warn(error)
     },
   },

@xen-orchestra/emit-async/README.md

@@ -40,7 +40,7 @@ await ee.emitAsync('start')
 // error handling though:
 await ee.emitAsync(
   {
-    onError(error, event, listener) {
+    onError(error) {
       console.warn(error)
     },
   },

@xen-orchestra/emit-async/index.js

@@ -1,7 +1,5 @@
 'use strict'
 
-const identity = v => v
-
 module.exports = function emitAsync(event) {
   let opts
   let i = 1
@@ -19,18 +17,12 @@ module.exports = function emitAsync(event) {
   }
 
   const onError = opts != null && opts.onError
-  const addErrorHandler = onError
-    ? (promise, listener) => promise.catch(error => onError(error, event, listener))
-    : identity
 
   return Promise.all(
     this.listeners(event).map(listener =>
-      addErrorHandler(
-        new Promise(resolve => {
-          resolve(listener.apply(this, args))
-        }),
-        listener
-      )
+      new Promise(resolve => {
+        resolve(listener.apply(this, args))
+      }).catch(onError)
     )
   )
 }

@xen-orchestra/emit-async/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/emit-async",
-  "version": "1.0.0",
+  "version": "0.1.0",
   "license": "ISC",
   "description": "Emit an event for async listeners to settle",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/emit-async",

@xen-orchestra/fs/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/fs",
-  "version": "1.1.0",
+  "version": "1.0.0",
   "license": "AGPL-3.0-or-later",
   "description": "The File System for Xen Orchestra backups.",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
@@ -20,10 +20,6 @@
     "node": ">=14"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.54.0",
-    "@aws-sdk/lib-storage": "^3.54.0",
-    "@aws-sdk/middleware-apply-body-checksum": "^3.58.0",
-    "@aws-sdk/node-http-handler": "^3.54.0",
     "@marsaud/smb2": "^0.18.0",
     "@sindresorhus/df": "^3.1.1",
     "@vates/async-each": "^0.1.0",
@@ -31,6 +27,9 @@
     "@vates/decorate-with": "^2.0.0",
     "@xen-orchestra/async-map": "^0.1.2",
     "@xen-orchestra/log": "^0.3.0",
+    "@aws-sdk/client-s3": "^3.54.0",
+    "@aws-sdk/lib-storage": "^3.54.0",
+    "@aws-sdk/node-http-handler": "^3.54.0",
     "bind-property-descriptor": "^2.0.0",
     "decorator-synchronized": "^0.6.0",
     "execa": "^5.0.0",
@@ -42,7 +41,7 @@
     "proper-lockfile": "^4.1.2",
     "readable-stream": "^3.0.6",
     "through2": "^4.0.2",
-    "xo-remote-parser": "^0.9.1"
+    "xo-remote-parser": "^0.8.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",
@@ -53,7 +52,7 @@
     "async-iterator-to-stream": "^1.1.0",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^7.0.2",
-    "dotenv": "^16.0.0",
+    "dotenv": "^15.0.0",
     "rimraf": "^3.0.0"
   },
   "scripts": {

@xen-orchestra/fs/src/abstract.js

@@ -1,7 +1,6 @@
 import asyncMapSettled from '@xen-orchestra/async-map/legacy'
 import getStream from 'get-stream'
 import { coalesceCalls } from '@vates/coalesce-calls'
-import { createLogger } from '@xen-orchestra/log'
 import { fromCallback, fromEvent, ignoreErrors, timeout } from 'promise-toolbox'
 import { limitConcurrency } from 'limit-concurrency-decorator'
 import { parse } from 'xo-remote-parser'
@@ -12,8 +11,6 @@ import { synchronized } from 'decorator-synchronized'
 import { basename, dirname, normalize as normalizePath } from './_path'
 import { createChecksumStream, validChecksumOfReadStream } from './checksum'
 
-const { warn } = createLogger('@xen-orchestra:fs')
-
 const checksumFile = file => file + '.checksum'
 const computeRate = (hrtime, size) => {
   const seconds = hrtime[0] + hrtime[1] / 1e9
@@ -360,12 +357,11 @@ export default class RemoteHandlerAbstract {
         readRate: computeRate(readDuration, SIZE),
       }
     } catch (error) {
-      warn(`error while testing the remote at step ${step}`, { error })
       return {
         success: false,
         step,
         file: testFileName,
-        error,
+        error: error.message || String(error),
       }
     } finally {
       ignoreErrors.call(this._unlink(testFileName))
@@ -424,10 +420,6 @@ export default class RemoteHandlerAbstract {
 
   // Methods that can be implemented by inheriting classes
 
-  useVhdDirectory() {
-    return this._remote.useVhdDirectory ?? false
-  }
-
   async _closeFile(fd) {
     throw new Error('Not implemented')
   }
@@ -554,18 +546,15 @@ export default class RemoteHandlerAbstract {
     }
 
     const files = await this._list(dir)
-    await pEach(files, file =>
-      this._unlink(`${dir}/${file}`).catch(
-        error => {
-          // Unlink dir behavior is not consistent across platforms
-          // https://github.com/nodejs/node-v0.x-archive/issues/5791
-          if (error.code === 'EISDIR' || error.code === 'EPERM') {
-            return this._rmtree(`${dir}/${file}`)
-          }
-          throw error
-        },
-        { stopOnError: false }
-      )
+    await asyncMapSettled(files, file =>
+      this._unlink(`${dir}/${file}`).catch(error => {
+        // Unlink dir behavior is not consistent across platforms
+        // https://github.com/nodejs/node-v0.x-archive/issues/5791
+        if (error.code === 'EISDIR' || error.code === 'EPERM') {
+          return this._rmtree(`${dir}/${file}`)
+        }
+        throw error
+      })
     )
     return this._rmtree(dir)
   }

@xen-orchestra/fs/src/s3.js

@@ -14,7 +14,6 @@ import {
 } from '@aws-sdk/client-s3'
 import { Upload } from '@aws-sdk/lib-storage'
 import { NodeHttpHandler } from '@aws-sdk/node-http-handler'
-import { getApplyMd5BodyChecksumPlugin } from '@aws-sdk/middleware-apply-body-checksum'
 import assert from 'assert'
 import { Agent as HttpAgent } from 'http'
 import { Agent as HttpsAgent } from 'https'
@@ -28,7 +27,7 @@ import createBufferFromStream from './_createBufferFromStream.js'
 import guessAwsRegion from './_guessAwsRegion.js'
 import RemoteHandlerAbstract from './abstract'
 import { basename, join, split } from './_path'
-import { pEach } from '@vates/async-each'
+import { asyncEach } from '@vates/async-each'
 
 // endpoints https://docs.aws.amazon.com/general/latest/gr/s3.html
 
@@ -76,9 +75,6 @@ export default class S3Handler extends RemoteHandlerAbstract {
       }),
     })
 
-    // Workaround for https://github.com/aws/aws-sdk-js-v3/issues/2673
-    this._s3.middlewareStack.use(getApplyMd5BodyChecksumPlugin(this._s3.config))
-
     const parts = split(path)
     this._bucket = parts.shift()
     this._dir = join(...parts)
@@ -97,12 +93,7 @@ export default class S3Handler extends RemoteHandlerAbstract {
   }
 
   _makePrefix(dir) {
-    const prefix = join(this._dir, dir, '/')
-
-    // no prefix for root
-    if (prefix !== './') {
-      return prefix
-    }
+    return join(this._dir, dir, '/')
   }
 
   _createParams(file) {
@@ -235,17 +226,14 @@ export default class S3Handler extends RemoteHandlerAbstract {
   }
 
   async _createReadStream(path, options) {
-    try {
-      return (await this._s3.send(new GetObjectCommand(this._createParams(path)))).Body
-    } catch (e) {
-      if (e.name === 'NoSuchKey') {
-        const error = new Error(`ENOENT: no such file '${path}'`)
-        error.code = 'ENOENT'
-        error.path = path
-        throw error
-      }
-      throw e
+    if (!(await this._isFile(path))) {
+      const error = new Error(`ENOENT: no such file '${path}'`)
+      error.code = 'ENOENT'
+      error.path = path
+      throw error
     }
+
+    return (await this._s3.send(new GetObjectCommand(this._createParams(path)))).Body
   }
 
   async _unlink(path) {
@@ -369,7 +357,7 @@ export default class S3Handler extends RemoteHandlerAbstract {
       )
 
       NextContinuationToken = result.IsTruncated ? result.NextContinuationToken : undefined
-      await pEach(
+      await asyncEach(
         result.Contents ?? [],
         async ({ Key }) => {
           // _unlink will add the prefix, but Key contains everything
@@ -525,8 +513,4 @@ export default class S3Handler extends RemoteHandlerAbstract {
   }
 
   async _closeFile(fd) {}
-
-  useVhdDirectory() {
-    return true
-  }
 }

@xen-orchestra/mixins/Config.js

@@ -1,16 +1,15 @@
-import get from 'lodash/get.js'
-import identity from 'lodash/identity.js'
-import isEqual from 'lodash/isEqual.js'
-import { createLogger } from '@xen-orchestra/log'
-import { parseDuration } from '@vates/parse-duration'
-import { watch } from 'app-conf'
+'use strict'
+
+const get = require('lodash/get')
+const identity = require('lodash/identity')
+const isEqual = require('lodash/isEqual')
+const { createLogger } = require('@xen-orchestra/log')
+const { parseDuration } = require('@vates/parse-duration')
+const { watch } = require('app-conf')
 
 const { warn } = createLogger('xo:mixins:config')
 
-// if path is undefined, an empty string or an empty array, returns the root value
-const niceGet = (value, path) => (path === undefined || path.length === 0 ? value : get(value, path))
-
-export default class Config {
+module.exports = class Config {
   constructor(app, { appDir, appName, config }) {
     this._config = config
     const watchers = (this._watchers = new Set())
@@ -33,7 +32,7 @@ export default class Config {
   }
 
   get(path) {
-    const value = niceGet(this._config, path)
+    const value = get(this._config, path)
     if (value === undefined) {
       throw new TypeError('missing config entry: ' + path)
     }
@@ -45,27 +44,20 @@ export default class Config {
   }
 
   getOptional(path) {
-    return niceGet(this._config, path)
+    return get(this._config, path)
   }
 
   watch(path, cb) {
-    // short syntax for the whole config: watch(cb)
-    if (typeof path === 'function') {
-      cb = path
-      path = undefined
-    }
-
     // internal arg
     const processor = arguments.length > 2 ? arguments[2] : identity
 
     let prev
     const watcher = config => {
       try {
-        const value = processor(niceGet(config, path))
+        const value = processor(get(config, path))
         if (!isEqual(value, prev)) {
-          const previous = prev
           prev = value
-          cb(value, previous, path)
+          cb(value)
         }
       } catch (error) {
         warn('watch', { error, path })

@xen-orchestra/mixins/Hooks.js

@@ -1,7 +1,9 @@
-import assert from 'assert'
-import emitAsync from '@xen-orchestra/emit-async'
-import EventEmitter from 'events'
-import { createLogger } from '@xen-orchestra/log'
+'use strict'
+
+const assert = require('assert')
+const emitAsync = require('@xen-orchestra/emit-async')
+const EventEmitter = require('events')
+const { createLogger } = require('@xen-orchestra/log')
 
 const { debug, warn } = createLogger('xo:mixins:hooks')
 
@@ -17,7 +19,7 @@ const runHook = async (emitter, hook) => {
   debug(`${hook} finished`)
 }
 
-export default class Hooks extends EventEmitter {
+module.exports = class Hooks extends EventEmitter {
   // Run *clean* async listeners.
   //
   // They normalize existing data, clear invalid entries, etc.

@xen-orchestra/mixins/HttpProxy.mjs

@@ -1,144 +0,0 @@
-import { createLogger } from '@xen-orchestra/log'
-import { EventListenersManager } from '@vates/event-listeners-manager'
-import { pipeline } from 'stream'
-import { ServerResponse, request } from 'http'
-import assert from 'assert'
-import fromCallback from 'promise-toolbox/fromCallback'
-import fromEvent from 'promise-toolbox/fromEvent'
-import net from 'net'
-
-import { parseBasicAuth } from './_parseBasicAuth.mjs'
-
-const { debug, warn } = createLogger('xo:mixins:HttpProxy')
-
-const IGNORED_HEADERS = new Set([
-  // https://datatracker.ietf.org/doc/html/rfc2616#section-13.5.1
-  'connection',
-  'keep-alive',
-  'proxy-authenticate',
-  'proxy-authorization',
-  'te',
-  'trailers',
-  'transfer-encoding',
-  'upgrade',
-
-  // don't forward original host
-  'host',
-])
-
-export default class HttpProxy {
-  #app
-
-  constructor(app, { httpServer }) {
-    // don't setup the proxy if httpServer is not present
-    //
-    // that can happen when the app is instanciated in another context like xo-server-recover-account
-    if (httpServer === undefined) {
-      return
-    }
-
-    this.#app = app
-
-    const events = new EventListenersManager(httpServer)
-    app.config.watch('http.proxy.enabled', (enabled = false) => {
-      events.removeAll()
-      if (enabled) {
-        events.add('connect', this.#handleConnect.bind(this)).add('request', this.#handleRequest.bind(this))
-      }
-    })
-  }
-
-  async #handleAuthentication(req, res, next) {
-    const auth = parseBasicAuth(req.headers['proxy-authorization'])
-
-    let authenticated = false
-
-    if (auth !== undefined) {
-      const app = this.#app
-
-      if (app.authenticateUser !== undefined) {
-        // xo-server
-        try {
-          const { user } = await app.authenticateUser(auth)
-          authenticated = user.permission === 'admin'
-        } catch (error) {}
-      } else {
-        // xo-proxy
-        authenticated = (await app.authentication.findProfile(auth)) !== undefined
-      }
-    }
-
-    if (authenticated) {
-      return next()
-    }
-
-    // https://datatracker.ietf.org/doc/html/rfc7235#section-3.2
-    res.statusCode = '407'
-    res.setHeader('proxy-authenticate', 'Basic realm="proxy"')
-    return res.end('Proxy Authentication Required')
-  }
-
-  // https://nodejs.org/api/http.html#event-connect
-  async #handleConnect(req, clientSocket, head) {
-    const { url } = req
-
-    debug('CONNECT proxy', { url })
-
-    // https://github.com/TooTallNate/proxy/blob/d677ef31fd4ca9f7e868b34c18b9cb22b0ff69da/proxy.js#L391-L398
-    const res = new ServerResponse(req)
-    res.assignSocket(clientSocket)
-
-    try {
-      await this.#handleAuthentication(req, res, async () => {
-        const { port, hostname } = new URL('http://' + req.url)
-        const serverSocket = net.connect(port || 80, hostname)
-
-        await fromEvent(serverSocket, 'connect')
-
-        clientSocket.write('HTTP/1.1 200 Connection Established\r\n\r\n')
-        serverSocket.write(head)
-        fromCallback(pipeline, clientSocket, serverSocket).catch(warn)
-        fromCallback(pipeline, serverSocket, clientSocket).catch(warn)
-      })
-    } catch (error) {
-      warn(error)
-      clientSocket.end()
-    }
-  }
-
-  async #handleRequest(req, res) {
-    const { url } = req
-
-    if (url.startsWith('/')) {
-      // not a proxy request
-      return
-    }
-
-    debug('HTTP proxy', { url })
-
-    try {
-      assert(url.startsWith('http:'), 'HTTPS should use connect')
-
-      await this.#handleAuthentication(req, res, async () => {
-        const { headers } = req
-        const pHeaders = {}
-        for (const key of Object.keys(headers)) {
-          if (!IGNORED_HEADERS.has(key)) {
-            pHeaders[key] = headers[key]
-          }
-        }
-
-        const pReq = request(url, { headers: pHeaders, method: req.method })
-        fromCallback(pipeline, req, pReq).catch(warn)
-
-        const pRes = await fromEvent(pReq, 'response')
-        res.writeHead(pRes.statusCode, pRes.statusMessage, pRes.headers)
-        await fromCallback(pipeline, pRes, res)
-      })
-    } catch (error) {
-      res.statusCode = 500
-      res.end('Internal Server Error')
-      warn(error)
-    }
-  }
-}

@xen-orchestra/mixins/_parseBasicAuth.mjs

@@ -1,27 +0,0 @@
-const RE = /^\s*basic\s+(.+?)\s*$/i
-
-export function parseBasicAuth(header) {
-  if (header === undefined) {
-    return
-  }
-
-  const matches = RE.exec(header)
-  if (matches === null) {
-    return
-  }
-
-  let credentials = Buffer.from(matches[1], 'base64').toString()
-
-  const i = credentials.indexOf(':')
-  if (i === -1) {
-    credentials = { token: credentials }
-  } else {
-    // https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.1
-    credentials = {
-      username: credentials.slice(0, i),
-      password: credentials.slice(i + 1),
-    }
-  }
-
-  return credentials
-}

@xen-orchestra/mixins/docs/HttpProxy.md

@@ -1,74 +0,0 @@
-> This module provides an HTTP and HTTPS proxy for `xo-proxy` and `xo-server`.
-
-- [Set up](#set-up)
-- [Usage](#usage)
-  - [`xo-proxy`](#xo-proxy)
-  - [`xo-server`](#xo-server)
-- [Use cases](#use-cases)
-  - [Access hosts in a private network](#access-hosts-in-a-private-network)
-  - [Allow upgrading xo-proxy via xo-server](#allow-upgrading-xo-proxy-via-xo-server)
-
-## Set up
-
-The proxy is disabled by default, to enable it, add the following lines to your config:
-
-```toml
-[http.proxy]
-enabled = true
-```
-
-## Usage
-
-For safety reasons, the proxy requires authentication to be used.
-
-### `xo-proxy`
-
-Use the authentication token:
-
-```
-$ cat ~/.config/xo-proxy/config.z-auto.json
-{"authenticationToken":"J0BgKritQgPxoyZrBJ5ViafQfLk06YoyFwC3fmfO5wU"}
-```
-
-Proxy URL to use:
-
-```
-https://J0BgKritQgPxoyZrBJ5ViafQfLk06YoyFwC3fmfO5wU@xo-proxy.company.lan
-```
-
-### `xo-server`
-
-> Only available for admin users.
-
-You can use your credentials:
-
-```
-https://user:password@xo.company.lan
-```
-
-Or create a dedicated token with `xo-cli`:
-
-```
-$ xo-cli --createToken xoa.company.lan admin@admin.net
-Password: ********
-Successfully logged with admin@admin.net
-Authentication token created
-
-DiYBFavJwf9GODZqQJs23eAx9eh3KlsRhBi8RcoX0KM
-```
-
-And use it in the URL:
-
-```
-https://DiYBFavJwf9GODZqQJs23eAx9eh3KlsRhBi8RcoX0KM@xo.company.lan
-```
-
-## Use cases
-
-### Access hosts in a private network
-
-To access hosts in a private network, deploy an XO Proxy in this network, expose its port 443 and use it as an HTTP proxy to connect to your servers in XO.
-
-### Allow upgrading xo-proxy via xo-server
-
-If your xo-proxy does not have direct Internet access, you can use xo-server as an HTTP proxy to make upgrades possible.

@xen-orchestra/mixins/package.json

@@ -14,18 +14,16 @@
     "url": "https://vates.fr"
   },
   "license": "AGPL-3.0-or-later",
-  "version": "0.5.0",
+  "version": "0.2.0",
   "engines": {
     "node": ">=12"
   },
   "dependencies": {
-    "@vates/event-listeners-manager": "^1.0.1",
     "@vates/parse-duration": "^0.1.1",
-    "@xen-orchestra/emit-async": "^1.0.0",
+    "@xen-orchestra/emit-async": "^0.1.0",
     "@xen-orchestra/log": "^0.3.0",
-    "app-conf": "^2.1.0",
-    "lodash": "^4.17.21",
-    "promise-toolbox": "^0.21.0"
+    "app-conf": "^2.0.0",
+    "lodash": "^4.17.21"
   },
   "scripts": {
     "postversion": "npm publish --access public"

@xen-orchestra/openflow/package.json

@@ -9,7 +9,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.1.2",
+  "version": "0.1.1",
   "engines": {
     "node": ">=8.10"
   },
@@ -30,7 +30,7 @@
     "rimraf": "^3.0.0"
   },
   "dependencies": {
-    "@vates/read-chunk": "^1.0.0"
+    "@vates/read-chunk": "^0.1.2"
   },
   "author": {
     "name": "Vates SAS",

@xen-orchestra/proxy-cli/index.js

@@ -1,23 +1,25 @@
 #!/usr/bin/env node
 
-import assert from 'assert'
-import colors from 'ansi-colors'
-import contentType from 'content-type'
-import CSON from 'cson-parser'
-import fromCallback from 'promise-toolbox/fromCallback'
-import fs from 'fs'
-import getopts from 'getopts'
-import hrp from 'http-request-plus'
-import split2 from 'split2'
-import pumpify from 'pumpify'
-import { extname } from 'path'
-import { format, parse } from 'json-rpc-protocol'
-import { inspect } from 'util'
-import { load as loadConfig } from 'app-conf'
-import { pipeline } from 'stream'
-import { readChunk } from '@vates/read-chunk'
+'use strict'
 
-const pkg = JSON.parse(fs.readFileSync(new URL('package.json', import.meta.url)))
+const assert = require('assert')
+const colors = require('ansi-colors')
+const contentType = require('content-type')
+const CSON = require('cson-parser')
+const fromCallback = require('promise-toolbox/fromCallback')
+const fs = require('fs')
+const getopts = require('getopts')
+const hrp = require('http-request-plus')
+const split2 = require('split2')
+const pumpify = require('pumpify')
+const { extname, join } = require('path')
+const { format, parse } = require('json-rpc-protocol')
+const { inspect } = require('util')
+const { load: loadConfig } = require('app-conf')
+const { pipeline } = require('stream')
+const { readChunk } = require('@vates/read-chunk')
+
+const pkg = require('./package.json')
 
 const FORMATS = {
   __proto__: null,
@@ -30,22 +32,30 @@ const parseValue = value => (value.startsWith('json:') ? JSON.parse(value.slice(
 
 async function main(argv) {
   const config = await loadConfig('xo-proxy', {
+    appDir: join(__dirname, '..'),
     ignoreUnknownFormats: true,
   })
 
-  const opts = getopts(argv, {
+  const { hostname = 'localhost', port } = config?.http?.listen?.https ?? {}
+
+  const {
+    _: args,
+    file,
+    help,
+    host,
+    raw,
+    token,
+  } = getopts(argv, {
     alias: { file: 'f', help: 'h' },
     boolean: ['help', 'raw'],
     default: {
       token: config.authenticationToken,
     },
     stopEarly: true,
-    string: ['file', 'host', 'token', 'url'],
+    string: ['file', 'host', 'token'],
   })
 
-  const { _: args, file } = opts
-
-  if (opts.help || (file === '' && args.length === 0)) {
+  if (help || (file === '' && args.length === 0)) {
     return console.log(
       '%s',
       `Usage:
@@ -70,29 +80,18 @@ ${pkg.name} v${pkg.version}`
   const baseRequest = {
     headers: {
       'content-type': 'application/json',
+      cookie: `authenticationToken=${token}`,
     },
     pathname: '/api/v1',
+    protocol: 'https:',
     rejectUnauthorized: false,
   }
-  let { token } = opts
-  if (opts.url !== '') {
-    const { protocol, host, username } = new URL(opts.url)
-    Object.assign(baseRequest, { protocol, host })
-    if (username !== '') {
-      token = username
-    }
+  if (host !== '') {
+    baseRequest.host = host
   } else {
-    baseRequest.protocol = 'https:'
-    if (opts.host !== '') {
-      baseRequest.host = opts.host
-    } else {
-      const { hostname = 'localhost', port } = config?.http?.listen?.https ?? {}
-      baseRequest.hostname = hostname
-      baseRequest.port = port
-    }
+    baseRequest.hostname = hostname
+    baseRequest.port = port
   }
-  baseRequest.headers.cookie = `authenticationToken=${token}`
-
   const call = async ({ method, params }) => {
     if (callPath.length !== 0) {
       process.stderr.write(`\n${colors.bold(`--- call #${callPath.join('.')}`)} ---\n\n`)
@@ -131,7 +130,7 @@ ${pkg.name} v${pkg.version}`
           stdout.write(inspect(JSON.parse(line), { colors: true, depth: null }))
           stdout.write('\n')
         }
-      } else if (opts.raw && typeof result === 'string') {
+      } else if (raw && typeof result === 'string') {
         stdout.write(result)
       } else {
         stdout.write(inspect(result, { colors: true, depth: null }))

@xen-orchestra/proxy-cli/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/proxy-cli",
-  "version": "0.3.1",
+  "version": "0.2.0",
   "license": "AGPL-3.0-or-later",
   "description": "CLI for @xen-orchestra/proxy",
   "keywords": [
@@ -19,16 +19,16 @@
   },
   "preferGlobal": true,
   "bin": {
-    "xo-proxy-cli": "./index.mjs"
+    "xo-proxy-cli": "./index.js"
   },
   "engines": {
-    "node": ">=14.13"
+    "node": ">=14"
   },
   "dependencies": {
     "@iarna/toml": "^2.2.0",
-    "@vates/read-chunk": "^1.0.0",
+    "@vates/read-chunk": "^0.1.2",
     "ansi-colors": "^4.1.1",
-    "app-conf": "^2.1.0",
+    "app-conf": "^2.0.0",
     "content-type": "^1.0.4",
     "cson-parser": "^4.0.7",
     "getopts": "^2.2.3",

@xen-orchestra/proxy/app/index.mjs

@@ -1,6 +1,5 @@
-import Config from '@xen-orchestra/mixins/Config.mjs'
-import Hooks from '@xen-orchestra/mixins/Hooks.mjs'
-import HttpProxy from '@xen-orchestra/mixins/HttpProxy.mjs'
+import Config from '@xen-orchestra/mixins/Config.js'
+import Hooks from '@xen-orchestra/mixins/Hooks.js'
 import mixin from '@xen-orchestra/mixin'
 import { createDebounceResource } from '@vates/disposable/debounceResource.js'
 
@@ -14,9 +13,7 @@ import ReverseProxy from './mixins/reverseProxy.mjs'
 
 export default class App {
   constructor(opts) {
-    mixin(this, { Api, Appliance, Authentication, Backups, Config, Hooks, HttpProxy, Logs, Remotes, ReverseProxy }, [
-      opts,
-    ])
+    mixin(this, { Api, Appliance, Authentication, Backups, Config, Hooks, Logs, Remotes, ReverseProxy }, [opts])
 
     const debounceResource = createDebounceResource()
     this.config.watchDuration('resourceCacheDelay', delay => {

@xen-orchestra/proxy/app/mixins/api.mjs

@@ -52,7 +52,7 @@ export default class Api {
       ctx.req.setTimeout(0)
 
       const profile = await app.authentication.findProfile({
-        token: ctx.cookies.get('authenticationToken'),
+        authenticationToken: ctx.cookies.get('authenticationToken'),
       })
       if (profile === undefined) {
         ctx.status = 401

@xen-orchestra/proxy/app/mixins/authentication.mjs

@@ -52,7 +52,7 @@ export default class Authentication {
   }
 
   async findProfile(credentials) {
-    if (credentials?.token === this.#token) {
+    if (credentials?.authenticationToken === this.#token) {
       return new Profile()
     }
   }

@xen-orchestra/proxy/app/mixins/backups.mjs

@@ -250,7 +250,7 @@ export default class Backups {
         listPoolMetadataBackups: [
           async ({ remotes }) => {
             const backupsByRemote = {}
-            await pEach(Object.entries(remotes), async ([remoteId, remote]) => {
+            await asyncMap(Object.entries(remotes), async ([remoteId, remote]) => {
               try {
                 await Disposable.use(this.getAdapter(remote), async adapter => {
                   backupsByRemote[remoteId] = await adapter.listPoolMetadataBackups()
@@ -274,7 +274,7 @@ export default class Backups {
         listVmBackups: [
           async ({ remotes }) => {
             const backups = {}
-            await pEach(Object.keys(remotes), async remoteId => {
+            await asyncMap(Object.keys(remotes), async remoteId => {
               try {
                 await Disposable.use(this.getAdapter(remotes[remoteId]), async adapter => {
                   backups[remoteId] = formatVmBackups(await adapter.listAllVmBackups())
@@ -304,7 +304,7 @@ export default class Backups {
         listXoMetadataBackups: [
           async ({ remotes }) => {
             const backupsByRemote = {}
-            await pEach(Object.entries(remotes), async ([remoteId, remote]) => {
+            await asyncMap(Object.entries(remotes), async ([remoteId, remote]) => {
               try {
                 await Disposable.use(this.getAdapter(remote), async adapter => {
                   backupsByRemote[remoteId] = await adapter.listXoMetadataBackups()

@xen-orchestra/proxy/config.toml

@@ -22,6 +22,27 @@ disableMergeWorker = false
 snapshotNameLabelTpl = '[XO Backup {job.name}] {vm.name_label}'
 vhdDirectoryCompression = 'brotli'
 
+[backups.defaultSettings]
+reportWhen = 'failure'
+
+[backups.metadata.defaultSettings]
+retentionPoolMetadata = 0
+retentionXoMetadata = 0
+
+[backups.vm.defaultSettings]
+bypassVdiChainsCheck = false
+checkpointSnapshot = false
+concurrency = 2
+copyRetention = 0
+deleteFirst = false
+exportRetention = 0
+fullInterval = 0
+offlineBackup = false
+offlineSnapshot = false
+snapshotRetention = 0
+timeout = 0
+vmTimeout = 0
+
 # This is a work-around.
 #
 # See https://github.com/vatesfr/xen-orchestra/pull/4674
@@ -60,6 +81,11 @@ timeout = 600e3
 disableFileRemotes = true
 
 [xapiOptions]
+# VDIs with `[NOBAK]` flag can be ignored while snapshotting an halted VM.
+#
+# This is disabled by default for the time being but will be turned on after enough testing.
+ignoreNobakVdis = false
+
 maxUncoalescedVdis = 1
 watchEvents = ['network', 'PIF', 'pool', 'SR', 'task', 'VBD', 'VDI', 'VIF', 'VM']
 

@xen-orchestra/proxy/index.mjs

@@ -6,7 +6,6 @@ import getopts from 'getopts'
 import pRetry from 'promise-toolbox/retry'
 import { catchGlobalErrors } from '@xen-orchestra/log/configure.js'
 import { create as createServer } from 'http-server-plus'
-import { createCachedLookup } from '@vates/cached-dns.lookup'
 import { createLogger } from '@xen-orchestra/log'
 import { createSecureServer } from 'http2'
 import { genSelfSignedCert } from '@xen-orchestra/self-signed'
@@ -16,8 +15,6 @@ import { load as loadConfig } from 'app-conf'
 
 catchGlobalErrors(createLogger('xo:proxy'))
 
-createCachedLookup().patchGlobal()
-
 const { fatal, info, warn } = createLogger('xo:proxy:bootstrap')
 
 const APP_DIR = new URL('.', import.meta.url).pathname

@xen-orchestra/proxy/package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "@xen-orchestra/proxy",
-  "version": "0.23.4",
+  "version": "0.20.1",
   "license": "AGPL-3.0-or-later",
   "description": "XO Proxy used to remotely execute backup jobs",
   "keywords": [
@@ -27,26 +27,25 @@
   "dependencies": {
     "@iarna/toml": "^2.2.0",
     "@koa/router": "^10.0.0",
-    "@vates/cached-dns.lookup": "^1.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.0.0",
     "@vates/disposable": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.26.0",
-    "@xen-orchestra/fs": "^1.1.0",
+    "@xen-orchestra/backups": "^0.21.0",
+    "@xen-orchestra/fs": "^1.0.0",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/mixin": "^0.1.0",
-    "@xen-orchestra/mixins": "^0.5.0",
-    "@xen-orchestra/self-signed": "^0.1.3",
-    "@xen-orchestra/xapi": "^1.4.0",
+    "@xen-orchestra/mixins": "^0.2.0",
+    "@xen-orchestra/self-signed": "^0.1.0",
+    "@xen-orchestra/xapi": "^0.10.0",
     "ajv": "^8.0.3",
-    "app-conf": "^2.1.0",
+    "app-conf": "^2.0.0",
     "async-iterator-to-stream": "^1.1.0",
     "fs-extra": "^10.0.0",
     "get-stream": "^6.0.0",
     "getopts": "^2.2.3",
     "golike-defer": "^0.5.1",
-    "http-server-plus": "^0.11.1",
+    "http-server-plus": "^0.11.0",
     "http2-proxy": "^5.0.53",
     "json-rpc-protocol": "^0.13.1",
     "jsonrpc-websocket-client": "^0.7.2",
@@ -60,7 +59,7 @@
     "source-map-support": "^0.5.16",
     "stoppable": "^1.0.6",
     "xdg-basedir": "^5.1.0",
-    "xen-api": "^1.2.1",
+    "xen-api": "^1.1.0",
     "xo-common": "^0.8.0"
   },
   "devDependencies": {

@xen-orchestra/self-signed/index.js

@@ -2,23 +2,22 @@
 
 const { execFile } = require('child_process')
 
-const RE =
-  /^(-----BEGIN PRIVATE KEY-----.+-----END PRIVATE KEY-----\n)(-----BEGIN CERTIFICATE-----.+-----END CERTIFICATE-----\n)$/s
-exports.genSelfSignedCert = async ({ days = 360 } = {}) =>
+const openssl = (cmd, args, { input, ...opts } = {}) =>
   new Promise((resolve, reject) => {
-    execFile(
-      'openssl',
-      ['req', '-batch', '-new', '-x509', '-days', String(days), '-nodes', '-newkey', 'rsa:2048', '-keyout', '-'],
-      (error, stdout) => {
-        if (error != null) {
-          return reject(error)
-        }
-        const matches = RE.exec(stdout)
-        if (matches === null) {
-          return reject(new Error('stdout does not match regular expression'))
-        }
-        const [, key, cert] = matches
-        resolve({ cert, key })
-      }
+    const child = execFile('openssl', [cmd, ...args], opts, (error, stdout) =>
+      error != null ? reject(error) : resolve(stdout)
     )
+    if (input !== undefined) {
+      child.stdin.end(input)
+    }
   })
+
+exports.genSelfSignedCert = async ({ days = 360 } = {}) => {
+  const key = await openssl('genrsa', ['2048'])
+  return {
+    cert: await openssl('req', ['-batch', '-new', '-key', '-', '-x509', '-days', String(days), '-nodes'], {
+      input: key,
+    }),
+    key,
+  }
+}

@xen-orchestra/self-signed/package.json

@@ -9,7 +9,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.1.3",
+  "version": "0.1.0",
   "engines": {
     "node": ">=8.10"
   },

@xen-orchestra/template/.babelrc.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))

@xen-orchestra/template/.eslintrc.js

@@ -0,0 +1 @@
+../../scripts/babel-eslintrc.js

@xen-orchestra/template/package.json

@@ -14,13 +14,31 @@
     "name": "Vates SAS",
     "url": "https://vates.fr"
   },
+  "preferGlobal": false,
+  "main": "dist/",
+  "browserslist": [
+    ">2%"
+  ],
   "engines": {
     "node": ">=6"
   },
-  "dependencies": {
-    "lodash": "^4.17.15"
+  "devDependencies": {
+    "@babel/cli": "^7.0.0",
+    "@babel/core": "^7.0.0",
+    "@babel/preset-env": "^7.0.0",
+    "cross-env": "^7.0.2",
+    "rimraf": "^3.0.0"
   },
   "scripts": {
+    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
+    "clean": "rimraf dist/",
+    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
+    "prebuild": "yarn run clean",
+    "predev": "yarn run prebuild",
+    "prepublishOnly": "yarn run build",
     "postversion": "npm publish --access public"
+  },
+  "dependencies": {
+    "lodash": "^4.17.15"
   }
 }

@xen-orchestra/template/src/index.js

@@ -1,10 +1,8 @@
-'use strict'
-
-const escapeRegExp = require('lodash/escapeRegExp')
+import escapeRegExp from 'lodash/escapeRegExp'
 
 const compareLengthDesc = (a, b) => b.length - a.length
 
-exports.compileTemplate = function compileTemplate(pattern, rules) {
+export function compileTemplate(pattern, rules) {
   const matches = Object.keys(rules).sort(compareLengthDesc).map(escapeRegExp).join('|')
   const regExp = new RegExp(`\\\\(?:\\\\|${matches})|${matches}`, 'g')
   return (...params) =>

@xen-orchestra/template/src/index.spec.js

@@ -1,8 +1,5 @@
 /* eslint-env jest */
-
-'use strict'
-
-const { compileTemplate } = require('.')
+import { compileTemplate } from '.'
 
 it("correctly replaces the template's variables", () => {
   const replacer = compileTemplate('{property}_\\{property}_\\\\{property}_{constant}_%_FOO', {

@xen-orchestra/upload-ova/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/upload-ova",
-  "version": "0.1.5",
+  "version": "0.1.4",
   "license": "AGPL-3.0-or-later",
   "description": "Basic CLI to upload ova files to Xen-Orchestra",
   "keywords": [
@@ -43,7 +43,7 @@
     "pw": "^0.0.4",
     "xdg-basedir": "^4.0.0",
     "xo-lib": "^0.11.1",
-    "xo-vmdk-to-vhd": "^2.4.2"
+    "xo-vmdk-to-vhd": "^2.2.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

@xen-orchestra/xapi/_AggregateError.js

@@ -1,9 +0,0 @@
-'use strict'
-
-// TODO: remove when Node >=15.0
-module.exports = class AggregateError extends Error {
-  constructor(errors, message) {
-    super(message)
-    this.errors = errors
-  }
-}

@xen-orchestra/xapi/cli.js

@@ -4,5 +4,5 @@
 
 const { Xapi } = require('./')
 require('xen-api/dist/cli.js')
-  .default(opts => new Xapi(opts))
+  .default(opts => new Xapi({ ignoreNobakVdis: true, ...opts }))
   .catch(console.error.bind(console, 'FATAL'))

@xen-orchestra/xapi/index.js

@@ -101,16 +101,20 @@ function removeWatcher(predicate, cb) {
 class Xapi extends Base {
   constructor({
     callRetryWhenTooManyPendingTasks = { delay: 5e3, tries: 10 },
+    ignoreNobakVdis,
     maxUncoalescedVdis,
     vdiDestroyRetryWhenInUse = { delay: 5e3, tries: 10 },
     ...opts
   }) {
+    assert.notStrictEqual(ignoreNobakVdis, undefined)
+
     super(opts)
     this._callRetryWhenTooManyPendingTasks = {
       ...callRetryWhenTooManyPendingTasks,
       onRetry,
       when: { code: 'TOO_MANY_PENDING_TASKS' },
     }
+    this._ignoreNobakVdis = ignoreNobakVdis
     this._maxUncoalescedVdis = maxUncoalescedVdis
     this._vdiDestroyRetryWhenInUse = {
       ...vdiDestroyRetryWhenInUse,
@@ -187,30 +191,7 @@ class Xapi extends Base {
     }
     return removeWatcher.bind(watchers, predicate, cb)
   }
-
-  // wait for an object to be in a specified state
-
-  waitObjectState(refOrUuid, predicate, { timeout } = {}) {
-    return new Promise((resolve, reject) => {
-      let timeoutHandle
-      const stop = this.watchObject(refOrUuid, object => {
-        if (predicate(object)) {
-          clearTimeout(timeoutHandle)
-          stop()
-          resolve(object)
-        }
-      })
-
-      if (timeout !== undefined) {
-        timeoutHandle = setTimeout(() => {
-          stop()
-          reject(new Error(`waitObjectState: timeout reached before ${refOrUuid} in expected state`))
-        }, timeout)
-      }
-    })
-  }
 }
-
 function mixin(mixins) {
   const xapiProto = Xapi.prototype
   const { defineProperties, getOwnPropertyDescriptor, getOwnPropertyNames } = Object
@@ -230,9 +211,8 @@ function mixin(mixins) {
   defineProperties(xapiProto, descriptors)
 }
 mixin({
-  host: require('./host.js'),
-  SR: require('./sr.js'),
   task: require('./task.js'),
+  host: require('./host.js'),
   VBD: require('./vbd.js'),
   VDI: require('./vdi.js'),
   VIF: require('./vif.js'),

@xen-orchestra/xapi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/xapi",
-  "version": "1.4.0",
+  "version": "0.10.0",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/xapi",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
@@ -15,7 +15,7 @@
     "node": ">=14"
   },
   "peerDependencies": {
-    "xen-api": "^1.2.1"
+    "xen-api": "^1.1.0"
   },
   "scripts": {
     "postversion": "npm publish --access public"
@@ -26,10 +26,8 @@
     "@xen-orchestra/log": "^0.3.0",
     "d3-time-format": "^3.0.0",
     "golike-defer": "^0.5.1",
-    "json-rpc-protocol": "^0.13.2",
     "lodash": "^4.17.15",
     "promise-toolbox": "^0.21.0",
-    "vhd-lib": "^3.3.1",
     "xo-common": "^0.8.0"
   },
   "private": false,

@xen-orchestra/xapi/sr.js

@@ -1,172 +0,0 @@
-'use strict'
-
-const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
-const { decorateClass } = require('@vates/decorate-with')
-const { defer } = require('golike-defer')
-const { incorrectState } = require('xo-common/api-errors')
-const { VDI_FORMAT_RAW } = require('./index.js')
-const peekFooterFromStream = require('vhd-lib/peekFooterFromVhdStream')
-
-const AggregateError = require('./_AggregateError.js')
-
-const { warn } = require('@xen-orchestra/log').createLogger('xo:xapi:sr')
-
-const OC_MAINTENANCE = 'xo:maintenanceState'
-
-class Sr {
-  async create({
-    content_type = 'user', // recommended by Citrix
-    device_config,
-    host,
-    name_description = '',
-    name_label,
-    physical_size = 0,
-    shared,
-    sm_config = {},
-    type,
-  }) {
-    const ref = await this.call(
-      'SR.create',
-      host,
-      device_config,
-      physical_size,
-      name_label,
-      name_description,
-      type,
-      content_type,
-      shared,
-      sm_config
-    )
-
-    // https://developer-docs.citrix.com/projects/citrix-hypervisor-sdk/en/latest/xc-api-extensions/#sr
-    this.setFieldEntry('SR', ref, 'other_config', 'auto-scan', 'true').catch(warn)
-
-    return ref
-  }
-
-  // Switch the SR to maintenance mode:
-  // - shutdown all running VMs with a VDI on this SR
-  //   - their UUID is saved into SR.other_config[OC_MAINTENANCE].shutdownVms
-  //   - clean shutdown is attempted, and falls back to a hard shutdown
-  // - unplug all connected hosts from this SR
-  async enableMaintenanceMode($defer, ref, { vmsToShutdown = [] } = {}) {
-    const state = { timestamp: Date.now() }
-
-    // will throw if already in maintenance mode
-    await this.call('SR.add_to_other_config', ref, OC_MAINTENANCE, JSON.stringify(state))
-
-    await $defer.onFailure.call(this, 'call', 'SR.remove_from_other_config', ref, OC_MAINTENANCE)
-
-    const runningVms = new Map()
-    const handleVbd = async ref => {
-      const vmRef = await this.getField('VBD', ref, 'VM')
-      if (!runningVms.has(vmRef)) {
-        const power_state = await this.getField('VM', vmRef, 'power_state')
-        const isPaused = power_state === 'Paused'
-        if (isPaused || power_state === 'Running') {
-          runningVms.set(vmRef, isPaused)
-        }
-      }
-    }
-    await pEach(await this.getField('SR', ref, 'VDIs'), async ref => {
-      await pEach(await this.getField('VDI', ref, 'VBDs'), handleVbd)
-    })
-
-    {
-      const runningVmUuids = await pEach(runningVms.keys(), ref => this.getField('VM', ref, 'uuid'))
-
-      const set = new Set(vmsToShutdown)
-      for (const vmUuid of runningVmUuids) {
-        if (!set.has(vmUuid)) {
-          throw incorrectState({
-            actual: vmsToShutdown,
-            expected: runningVmUuids,
-            property: 'vmsToShutdown',
-          })
-        }
-      }
-    }
-
-    state.shutdownVms = {}
-
-    await pEach(
-      runningVms,
-      async ([ref, isPaused]) => {
-        state.shutdownVms[await this.getField('VM', ref, 'uuid')] = isPaused
-
-        try {
-          await this.callAsync('VM.clean_shutdown', ref)
-        } catch (error) {
-          warn('SR_enableMaintenanceMode, VM clean shutdown', { error })
-          await this.callAsync('VM.hard_shutdown', ref)
-        }
-
-        $defer.onFailure.call(this, 'callAsync', 'VM.start', ref, isPaused, true)
-      },
-      { stopOnError: false }
-    )
-
-    state.unpluggedPbds = []
-    await pEach(
-      await this.getField('SR', ref, 'PBDs'),
-      async ref => {
-        if (await this.getField('PBD', ref, 'currently_attached')) {
-          state.unpluggedPbds.push(await this.getField('PBD', ref, 'uuid'))
-
-          await this.callAsync('PBD.unplug', ref)
-
-          $defer.onFailure.call(this, 'callAsync', 'PBD.plug', ref)
-        }
-      },
-      { stopOnError: false }
-    )
-
-    await this.setFieldEntry('SR', ref, 'other_config', OC_MAINTENANCE, JSON.stringify(state))
-  }
-
-  // this method is best effort and will not stop on first error
-  async disableMaintenanceMode(ref) {
-    const state = JSON.parse((await this.getField('SR', ref, 'other_config'))[OC_MAINTENANCE])
-
-    // will throw if not in maintenance mode
-    await this.call('SR.remove_from_other_config', ref, OC_MAINTENANCE)
-
-    const errors = []
-
-    await pEach(state.unpluggedPbds, async uuid => {
-      try {
-        await this.callAsync('PBD.plug', await this.call('PBD.get_by_uuid', uuid))
-      } catch (error) {
-        errors.push(error)
-      }
-    })
-
-    await pEach(Object.entries(state.shutdownVms), async ([uuid, isPaused]) => {
-      try {
-        await this.callAsync('VM.start', await this.call('VM.get_by_uuid', uuid), isPaused, true)
-      } catch (error) {
-        errors.push(error)
-      }
-    })
-
-    if (errors.length !== 0) {
-      throw new AggregateError(errors)
-    }
-  }
-
-  async importVdi(
-    $defer,
-    ref,
-    stream,
-    { name_label = '[XO] Imported disk - ' + new Date().toISOString(), ...vdiCreateOpts } = {}
-  ) {
-    const footer = await peekFooterFromStream(stream)
-    const vdiRef = await this.VDI_create({ ...vdiCreateOpts, name_label, SR: ref, virtual_size: footer.currentSize })
-    $defer.onFailure.call(this, 'callAsync', 'VDI.destroy', vdiRef)
-    await this.VDI_importContent(vdiRef, stream, { format: VDI_FORMAT_RAW })
-    return vdiRef
-  }
-}
-module.exports = Sr
-
-decorateClass(Sr, { enableMaintenanceMode: defer, importVdi: defer })

@xen-orchestra/xapi/vbd.js

@@ -6,8 +6,6 @@ const { Ref } = require('xen-api')
 
 const isVmRunning = require('./_isVmRunning.js')
 
-const { warn } = require('@xen-orchestra/log').createLogger('xo:xapi:vbd')
-
 const noop = Function.prototype
 
 module.exports = class Vbd {
@@ -68,10 +66,8 @@ module.exports = class Vbd {
     })
 
     if (isVmRunning(powerState)) {
-      this.callAsync('VBD.plug', vbdRef).catch(warn)
+      await this.callAsync('VBD.plug', vbdRef)
     }
-
-    return vbdRef
   }
 
   async unplug(ref) {

@xen-orchestra/xapi/vdi.js

@@ -30,7 +30,8 @@ class Vdi {
       other_config = {},
       read_only = false,
       sharable = false,
-      SR = this.pool.default_SR,
+      sm_config,
+      SR,
       tags,
       type = 'user',
       virtual_size,
@@ -38,10 +39,10 @@ class Vdi {
     },
     {
       // blindly copying `sm_config` from another VDI can create problems,
-      // therefore it should be passed explicitly
+      // therefore it is ignored by default by this method
       //
       // see https://github.com/vatesfr/xen-orchestra/issues/4482
-      sm_config,
+      setSmConfig = false,
     } = {}
   ) {
     return this.call('VDI.create', {
@@ -50,7 +51,7 @@ class Vdi {
       other_config,
       read_only,
       sharable,
-      sm_config,
+      sm_config: setSmConfig ? sm_config : undefined,
       SR,
       tags,
       type,

@xen-orchestra/xapi/vm.js

@@ -11,8 +11,7 @@ const { asyncMap } = require('@xen-orchestra/async-map')
 const { createLogger } = require('@xen-orchestra/log')
 const { decorateClass } = require('@vates/decorate-with')
 const { defer } = require('golike-defer')
-const { incorrectState, forbiddenOperation } = require('xo-common/api-errors.js')
-const { JsonRpcError } = require('json-rpc-protocol')
+const { incorrectState } = require('xo-common/api-errors.js')
 const { Ref } = require('xen-api')
 
 const extractOpaqueRef = require('./_extractOpaqueRef.js')
@@ -46,21 +45,6 @@ const cleanBiosStrings = biosStrings => {
   }
 }
 
-async function listNobakVbds(xapi, vbdRefs) {
-  const vbds = []
-  await pEach(vbdRefs, async vbdRef => {
-    const vbd = await xapi.getRecord('VBD', vbdRef)
-    if (
-      vbd.type === 'Disk' &&
-      Ref.isNotEmpty(vbd.VDI) &&
-      (await xapi.getField('VDI', vbd.VDI, 'name_label')).startsWith('[NOBAK]')
-    ) {
-      vbds.push(vbd)
-    }
-  })
-  return vbds
-}
-
 async function safeGetRecord(xapi, type, ref) {
   try {
     return await xapi.getRecord(type, ref)
@@ -145,25 +129,9 @@ class Vm {
     }
   }
 
-  async checkpoint($defer, vmRef, { cancelToken = CancelToken.none, ignoreNobakVdis = false, name_label } = {}) {
-    const vm = await this.getRecord('VM', vmRef)
-
-    let destroyNobakVdis = false
-
-    if (ignoreNobakVdis) {
-      if (vm.power_state === 'Halted') {
-        await pEach(await listNobakVbds(this, vm.VBDs), async vbd => {
-          await this.VBD_destroy(vbd.$ref)
-          $defer.call(this, 'VBD_create', vbd)
-        })
-      } else {
-        // cannot unplug VBDs on Running, Paused and Suspended VMs
-        destroyNobakVdis = true
-      }
-    }
-
+  async checkpoint(vmRef, { cancelToken = CancelToken.none, name_label } = {}) {
     if (name_label === undefined) {
-      name_label = vm.name_label
+      name_label = await this.getField('VM', vmRef, 'name_label')
     }
     try {
       const ref = await this.callAsync(cancelToken, 'VM.checkpoint', vmRef, name_label).then(extractOpaqueRef)
@@ -180,10 +148,6 @@ class Vm {
         noop
       )
 
-      if (destroyNobakVdis) {
-        await pEach(await listNobakVbds(this, await this.getField('VM', ref, 'VBDs')), vbd => this.VDI_destroy(vbd.VDI))
-      }
-
       return ref
     } catch (error) {
       if (error.code === 'VM_BAD_POWER_STATE') {
@@ -342,13 +306,7 @@ class Vm {
     const vm = await this.getRecord('VM', vmRef)
 
     if (!bypassBlockedOperation && 'destroy' in vm.blocked_operations) {
-      throw forbiddenOperation(
-        `destroy is blocked: ${
-          vm.blocked_operations.destroy === 'true'
-            ? 'protected from accidental deletion'
-            : vm.blocked_operations.destroy
-        }`
-      )
+      throw new Error('destroy is blocked')
     }
 
     if (!forceDeleteDefaultTemplate && isDefaultTemplate(vm)) {
@@ -383,7 +341,7 @@ class Vm {
     await this.call('VM.destroy', vmRef)
 
     await Promise.all([
-      pEach(vm.snapshots, snapshotRef =>
+      asyncMap(vm.snapshots, snapshotRef =>
         this.VM_destroy(snapshotRef).catch(error => {
           warn('VM_destroy: failed to destroy snapshot', {
             error,
@@ -393,7 +351,7 @@ class Vm {
         })
       ),
       deleteDisks &&
-        pEach(disks, async vdiRef => {
+        asyncMap(disks, async vdiRef => {
           try {
             // Dont destroy if attached to other (non control domain) VMs
             for (const vbdRef of await this.getField('VDI', vdiRef, 'VBDs')) {
@@ -508,22 +466,6 @@ class Vm {
       }
       return ref
     } catch (error) {
-      if (
-        // xxhash is the new form consistency hashing in CH 8.1 which uses a faster,
-        // more efficient hashing algorithm to generate the consistency checks
-        // in order to support larger files without the consistency checking process taking an incredibly long time
-        error.code === 'IMPORT_ERROR' &&
-        error.params?.some(
-          param =>
-            param.includes('INTERNAL_ERROR') &&
-            param.includes('Expected to find an inline checksum') &&
-            param.includes('.xxhash')
-        )
-      ) {
-        warn('import', { error })
-        throw new JsonRpcError('Importing this VM requires XCP-ng or Citrix Hypervisor >=8.1')
-      }
-
       // augment the error with as much relevant info as possible
       const [poolMaster, sr] = await Promise.all([
         safeGetRecord(this, 'host', this.pool.master),
@@ -535,39 +477,21 @@ class Vm {
     }
   }
 
-  async snapshot(
-    $defer,
-    vmRef,
-    { cancelToken = CancelToken.none, ignoreNobakVdis = false, name_label, unplugVusbs = false } = {}
-  ) {
+  async snapshot($defer, vmRef, { cancelToken = CancelToken.none, name_label } = {}) {
     const vm = await this.getRecord('VM', vmRef)
-
-    const isHalted = vm.power_state === 'Halted'
-
-    // requires the VM to be halted because it's not possible to re-plug VUSB on a live VM
-    if (unplugVusbs && isHalted) {
-      // vm.VUSBs can be undefined (e.g. on XS 7.0.0)
-      const vusbs = vm.VUSBs
-      if (vusbs !== undefined) {
-        await pEach(vusbs, async ref => {
-          const vusb = await this.getRecord('VUSB', ref)
-          await vusb.$call('destroy')
-          $defer.call(this, 'call', 'VUSB.create', vusb.VM, vusb.USB_group, vusb.other_config)
-        })
-      }
-    }
-
-    let destroyNobakVdis = false
-    if (ignoreNobakVdis) {
-      if (isHalted) {
-        await pEach(await listNobakVbds(this, vm.VBDs), async vbd => {
-          await this.VBD_destroy(vbd.$ref)
+    // cannot unplug VBDs on Running, Paused and Suspended VMs
+    if (vm.power_state === 'Halted' && this._ignoreNobakVdis) {
+      await asyncMap(vm.VBDs, async vbdRef => {
+        const vbd = await this.getRecord('VBD', vbdRef)
+        if (
+          vbd.type === 'Disk' &&
+          Ref.isNotEmpty(vbd.VDI) &&
+          (await this.getField('VDI', vbd.VDI, 'name_label')).startsWith('[NOBAK]')
+        ) {
+          await this.VBD_destroy(vbdRef)
           $defer.call(this, 'VBD_create', vbd)
-        })
-      } else {
-        // cannot unplug VBDs on Running, Paused and Suspended VMs
-        destroyNobakVdis = true
-      }
+        }
+      })
     }
 
     if (name_label === undefined) {
@@ -656,17 +580,12 @@ class Vm {
       noop
     )
 
-    if (destroyNobakVdis) {
-      await pEach(await listNobakVbds(this, await this.getField('VM', ref, 'VBDs')), vbd => this.VDI_destroy(vbd.VDI))
-    }
-
     return ref
   }
 }
 module.exports = Vm
 
 decorateClass(Vm, {
-  checkpoint: defer,
   create: defer,
   export: defer,
   snapshot: defer,

CHANGELOG.md

@@ -1,201 +1,9 @@
 # ChangeLog
 
-## **5.72.0** (2022-06-30)
+## **5.69.2** (2022-04-13)
 
 <img id="latest" src="https://badgen.net/badge/channel/latest/yellow" alt="Channel: latest" />
 
-### Highlights
-
-- [Backup] Merge delta backups without copying data when using VHD directories on NFS/SMB/local remote(https://github.com/vatesfr/xen-orchestra/pull/6271))
-- [Proxies] Ability to copy the proxy access URL (PR [#6287](https://github.com/vatesfr/xen-orchestra/pull/6287))
-- [SR/Advanced] Ability to enable/disable _Maintenance Mode_ [#6215](https://github.com/vatesfr/xen-orchestra/issues/6215) (PRs [#6308](https://github.com/vatesfr/xen-orchestra/pull/6308), [#6297](https://github.com/vatesfr/xen-orchestra/pull/6297))
-- [User] User tokens management through XO interface (PR [#6276](https://github.com/vatesfr/xen-orchestra/pull/6276))
-- [Tasks, VM/General] Self Service users: show tasks related to their pools, hosts, SRs, networks and VMs (PR [#6217](https://github.com/vatesfr/xen-orchestra/pull/6217))
-
-### Enhancements
-
-> Users must be able to say: “Nice enhancement, I'm eager to test it”
-
-- [Backup/Restore] Clearer error message when importing a VM backup requires XCP-n/CH >= 8.1 (PR [#6304](https://github.com/vatesfr/xen-orchestra/pull/6304))
-- [Backup] Users can use VHD directory on any remote type (PR [#6273](https://github.com/vatesfr/xen-orchestra/pull/6273))
-
-### Bug fixes
-
-> Users must be able to say: “I had this issue, happy to know it's fixed”
-
-- [VDI Import] Fix `this._getOrWaitObject is not a function`
-- [VM] Attempting to delete a protected VM should display a modal with the error and the ability to bypass it (PR [#6290](https://github.com/vatesfr/xen-orchestra/pull/6290))
-- [OVA Import] Fix import stuck after first disk
-- [File restore] Ignore symbolic links
-
-### Released packages
-
-- @vates/event-listeners-manager 1.0.1
-- @vates/read-chunk 1.0.0
-- @xen-orchestra/backups 0.26.0
-- @xen-orchestra/backups-cli 0.7.4
-- xo-remote-parser 0.9.1
-- @xen-orchestra/fs 1.1.0
-- @xen-orchestra/openflow 0.1.2
-- @xen-orchestra/xapi 1.4.0
-- @xen-orchestra/proxy 0.23.4
-- @xen-orchestra/proxy-cli 0.3.1
-- vhd-lib 3.3.1
-- vhd-cli 0.8.0
-- xo-vmdk-to-vhd 2.4.2
-- xo-server 5.98.0
-- xo-web 5.99.0
-
-## **5.71.1 (2022-06-13)**
-
-<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
-
-### Enhancements
-
-- Show raw errors to administrators instead of _unknown error from the peer_ (PR [#6260](https://github.com/vatesfr/xen-orchestra/pull/6260))
-
-### Bug fixes
-
-- [New SR] Fix `method.startsWith is not a function` when creating an _ext_ SR
-- Import VDI content now works when there is a HTTP proxy between XO and the host (PR [#6261](https://github.com/vatesfr/xen-orchestra/pull/6261))
-- [Backup] Fix `undefined is not iterable (cannot read property Symbol(Symbol.iterator))` on XS 7.0.0
-- [Backup] Ensure a warning is shown if a target preparation step fails (PR [#6266](https://github.com/vatesfr/xen-orchestra/pull/6266))
-- [OVA Export] Avoid creating a zombie task (PR [#6267](https://github.com/vatesfr/xen-orchestra/pull/6267))
-- [OVA Export] Increase speed by lowering compression to acceptable level (PR [#6267](https://github.com/vatesfr/xen-orchestra/pull/6267))
-- [OVA Export] Fix broken OVAs due to special characters in VM name (PR [#6267](https://github.com/vatesfr/xen-orchestra/pull/6267))
-
-### Released packages
-
-- @xen-orchestra/backups 0.25.0
-- @xen-orchestra/backups-cli 0.7.3
-- xen-api 1.2.1
-- @xen-orchestra/xapi 1.2.0
-- @xen-orchestra/proxy 0.23.2
-- @xen-orchestra/proxy-cli 0.3.0
-- xo-cli 0.14.0
-- xo-vmdk-to-vhd 2.4.1
-- xo-server 5.96.0
-- xo-web 5.97.2
-
-## **5.71.0 (2022-05-31)**
-
-### Highlights
-
-- [Backup] _Restore Health Check_ can now be configured to be run automatically during a backup schedule (PRs [#6227](https://github.com/vatesfr/xen-orchestra/pull/6227), [#6228](https://github.com/vatesfr/xen-orchestra/pull/6228), [#6238](https://github.com/vatesfr/xen-orchestra/pull/6238) & [#6242](https://github.com/vatesfr/xen-orchestra/pull/6242))
-- [Backup] VMs with USB Pass-through devices are now supported! The advanced _Offline Snapshot Mode_ setting must be enabled. For Full Backup or Disaster Recovery jobs, Rolling Snapshot needs to be anabled as well. (PR [#6239](https://github.com/vatesfr/xen-orchestra/pull/6239))
-- [Backup] Implement file cache for listing the backups of a VM (PR [#6220](https://github.com/vatesfr/xen-orchestra/pull/6220))
-- [RPU/Host] If some backup jobs are running on the pool, ask for confirmation before starting an RPU, shutdown/rebooting a host or restarting a host's toolstack (PR [6232](https://github.com/vatesfr/xen-orchestra/pull/6232))
-- [XO Web] Add ability to configure a default filter for Storage [#6236](https://github.com/vatesfr/xen-orchestra/issues/6236) (PR [#6237](https://github.com/vatesfr/xen-orchestra/pull/6237))
-- [REST API] Support VDI creation via VHD import
-
-### Enhancements
-
-- [Backup] Merge multiple VHDs at once which will speed up the merging phase after reducing the retention of a backup job(PR [#6184](https://github.com/vatesfr/xen-orchestra/pull/6184))
-- [Backup] Add setting `backups.metadata.defaultSettings.unconditionalSnapshot` in `xo-server`'s configuration file to force a snapshot even when not required by the backup, this is useful to avoid locking the VM halted during the backup (PR [#6221](https://github.com/vatesfr/xen-orchestra/pull/6221))
-- [VM migration] Ensure the VM can be migrated before performing the migration to avoid issues [#5301](https://github.com/vatesfr/xen-orchestra/issues/5301) (PR [#6245](https://github.com/vatesfr/xen-orchestra/pull/6245))
-- [Backup] Show any detected errors on existing backups instead of fixing them silently (PR [#6207](https://github.com/vatesfr/xen-orchestra/pull/6225))
-- Created SRs will now have auto-scan enabled similarly to what XenCenter does (PR [#6246](https://github.com/vatesfr/xen-orchestra/pull/6246))
-- [RPU] Disable scheduled backup jobs during RPU (PR [#6244](https://github.com/vatesfr/xen-orchestra/pull/6244))
-
-### Bug fixes
-
-- [S3] Fix S3 remote with empty directory not showing anything to restore (PR [#6218](https://github.com/vatesfr/xen-orchestra/pull/6218))
-- [S3] remote fom did not save the `https` and `allow unatuhorized`during remote creation (PR [#6219](https://github.com/vatesfr/xen-orchestra/pull/6219))
-- [VM/advanced] Fix various errors when adding ACLs [#6213](https://github.com/vatesfr/xen-orchestra/issues/6213) (PR [#6230](https://github.com/vatesfr/xen-orchestra/pull/6230))
-- [Home/Self] Don't make VM's resource set name clickable for non admin users as they aren't allowed to view the Self Service page (PR [#6252](https://github.com/vatesfr/xen-orchestra/pull/6252))
-- [load-balancer] Fix density mode failing to shutdown hosts (PR [#6253](https://github.com/vatesfr/xen-orchestra/pull/6253))
-- [Health] Make "Too many snapshots" table sortable by number of snapshots (PR [#6255](https://github.com/vatesfr/xen-orchestra/pull/6255))
-- [Remote] Show complete errors instead of only a potentially missing message (PR [#6216](https://github.com/vatesfr/xen-orchestra/pull/6216))
-
-### Released packages
-
-- @xen-orchestra/self-signed 0.1.3
-- vhd-lib 3.2.0
-- @xen-orchestra/fs 1.0.3
-- vhd-cli 0.7.2
-- xo-vmdk-to-vhd 2.4.0
-- @xen-orchestra/upload-ova 0.1.5
-- @xen-orchestra/xapi 1.1.0
-- @xen-orchestra/backups 0.24.0
-- @xen-orchestra/backups-cli 0.7.2
-- @xen-orchestra/emit-async 1.0.0
-- @xen-orchestra/mixins 0.5.0
-- @xen-orchestra/proxy 0.23.1
-- xo-server 5.95.0
-- xo-web 5.97.1
-- xo-server-backup-reports 0.17.0
-
-## 5.70.2 (2022-05-16)
-
-### Bug fixes
-
-- [Pool/Patches] Fix failure to install patches on Citrix Hypervisor (PR [#6231](https://github.com/vatesfr/xen-orchestra/pull/6231))
-
-### Released packages
-
-- @xen-orchestra/xapi 1.0.0
-- @xen-orchestra/backups 0.23.0
-- @xen-orchestra/mixins 0.4.0
-- @xen-orchestra/proxy 0.22.1
-- xo-server 5.93.1
-
-## 5.70.1 (2022-05-04)
-
-### Enhancement
-
-- [Backup] Support `[NOBAK]` VDI prefix for all backup modes [#2560](https://github.com/vatesfr/xen-orchestra/issues/2560) (PR [#6207](https://github.com/vatesfr/xen-orchestra/pull/6207))
-- [VM/Host Console] Fix fallback for older versions of XCP-ng/XS (PR [#6203](https://github.com/vatesfr/xen-orchestra/pull/6203))
-
-### Bug fixes
-
-- [Backup Health Check] Fix guest tools detection (PR [#6214](https://github.com/vatesfr/xen-orchestra/pull/6214))
-
-### Released packages
-
-- @xen-orchestra/mixins 0.3.1
-- @xen-orchestra/xapi 0.11.0
-- @xen-orchestra/backups 0.22.0
-- @xen-orchestra/proxy 0.22.0
-- xo-server 5.93.0
-
-## 5.70.0 (2022-04-29)
-
-### Highlights
-
-- [VM export] Feat export to `ova` format (PR [#6006](https://github.com/vatesfr/xen-orchestra/pull/6006))
-- [Backup] Add _Restore Health Check_: ensure a backup is viable by doing an automatic test restore (requires guest tools in the VM) (PR [#6148](https://github.com/vatesfr/xen-orchestra/pull/6148))
-- [Import] Feat import `iso` disks (PR [#6180](https://github.com/vatesfr/xen-orchestra/pull/6180))
-- New HTTP/HTTPS proxy implemented in xo-proxy and xo-server, [see the documentation](https://github.com/vatesfr/xen-orchestra/blob/master/@xen-orchestra/mixins/docs/HttpProxy.md) (PR [#6201](https://github.com/vatesfr/xen-orchestra/pull/6201))
-- [Backup job] Cache DNS queries (PR [#6196](https://github.com/vatesfr/xen-orchestra/pull/6196))
-
-### Enhancements
-
-- [VM migrate] Allow to choose a private network for VIFs network (PR [#6200](https://github.com/vatesfr/xen-orchestra/pull/6200))
-- [Proxy] Disable "Deploy proxy" button for source users (PR [#6199](https://github.com/vatesfr/xen-orchestra/pull/6199))
-
-### Bug fixes
-
-- [VM/Host Console] Fix support of older versions of XCP-ng/XS, please not that HTTP proxies are note supported in that case (PR [#6191](https://github.com/vatesfr/xen-orchestra/pull/6191))
-- Fix HTTP proxy support to connect to pools (introduced in XO 5.69.0) (PR [#6204](https://github.com/vatesfr/xen-orchestra/pull/6204))
-- [Backup] Fix failure when sending a backup (Full/Delta/Metadata) to S3 with Object Lock enabled (PR [#6190](https://github.com/vatesfr/xen-orchestra/pull/6190))
-
-### Released packages
-
-- @vates/cached-dns.lookup 1.0.0
-- @vates/event-listeners-manager 1.0.0
-- xen-api 1.2.0
-- @xen-orchestra/mixins 0.3.0
-- xo-vmdk-to-vhd 2.3.0
-- @xen-orchestra/fs 1.0.1
-- @xen-orchestra/backups 0.21.1
-- @xen-orchestra/proxy 0.21.0
-- xo-server 5.92.0
-- xo-web 5.96.0
-- vhd-cli 0.7.1
-- @xen-orchestra/backups-cli 0.7.1
-
-## **5.69.2** (2022-04-13)
-
 ### Enhancements
 
 - [Rolling Pool Update] New algorithm for XCP-ng updates (PR [#6188](https://github.com/vatesfr/xen-orchestra/pull/6188))
@@ -265,6 +73,8 @@
 
 ## **5.68.0** (2022-02-28)
 
+<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
+
 ### Highlights
 
 - [New SR] Add confirmation message before creating local SR (PR [#6121](https://github.com/vatesfr/xen-orchestra/pull/6121))

CHANGELOG.unreleased.md

@@ -7,29 +7,23 @@
 
 > Users must be able to say: “Nice enhancement, I'm eager to test it”
 
-- [SR] When SR is in maintenance, add "Maintenance mode" badge next to its name (PR [#6313](https://github.com/vatesfr/xen-orchestra/pull/6313))
-
 ### Bug fixes
 
 > Users must be able to say: “I had this issue, happy to know it's fixed”
 
 ### Packages to release
 
-> When modifying a package, add it here with its release type.
+> Packages will be released in the order they are here, therefore, they should
+> be listed by inverse order of dependency.
 >
-> The format is the following: - `$packageName` `$releaseType`
+> Rule of thumb: add packages on top.
 >
-> Where `$releaseType` is
+> The format is the following: - `$packageName` `$version`
+>
+> Where `$version` is
 >
 > - patch: if the change is a bug fix or a simple code improvement
 > - minor: if the change is a new feature
 > - major: if the change breaks compatibility
 >
-> Keep this list alphabetically ordered to avoid merge conflicts
-
-<!--packages-start-->
-
-- @vates/async-each minor
-- xo-web minor
-
-<!--packages-end-->
+> In case of conflict, the highest (lowest in previous list) `$version` wins.

SECURITY.md

@@ -1,17 +0,0 @@
-# Security Policy
-
-## Supported Versions
-
-We apply patches and fix security issues for the following versions:
-
-| Version | Supported          |
-| ------- | ------------------ |
-| XOA `latest`     | :white_check_mark: |
-| XOA `stable`     | :white_check_mark: |
-| `master` branch  | :white_check_mark: |
-| anything else   | :x:                |
-
-## Reporting a Vulnerability
-
-If you discover a vulnerability, you should contact us by sending an email to security at vates dot fr
-From there, we'll discuss how to deal with it and prepare a dedicated mitigation.

docker/Dockerfile

@@ -4,7 +4,7 @@ FROM ubuntu:xenial
 # https://qastack.fr/programming/25899912/how-to-install-nvm-in-docker
 
 RUN apt-get update
-RUN apt-get install -y curl qemu-utils blktap-utils vmdk-stream-converter git libxml2-utils
+RUN apt-get install -y curl qemu-utils blktap-utils vmdk-stream-converter git
 ENV NVM_DIR /usr/local/nvm
 RUN mkdir -p /usr/local/nvm
 RUN cd /usr/local/nvm

docs/architecture.md

@@ -138,22 +138,9 @@ This CLI is mainly used as a debug tool, there's no 100% guarantee on its stabil
 > xo-cli --help
 Usage:
 
-  xo-cli --register [--allowUnauthorized] [--expiresIn duration] <XO-Server URL> <username> [<password>]
+  xo-cli --register <XO-Server URL> <username> [<password>]
     Registers the XO instance to use.
 
-    --allowUnauthorized, --au
-      Accept invalid certificate (e.g. self-signed).
-
-    --expiresIn duration
-      Can be used to change the validity duration of the
-      authorization token (default: one month).
-
-  xo-cli --createToken <params>…
-    Create an authentication token for XO API.
-
-    <params>…
-      Accept the same parameters as --register, see its usage.
-
   xo-cli --unregister
     Remove stored credentials.
 
@@ -173,6 +160,7 @@ Usage:
 
   xo-cli <command> [<name>=<value>]...
     Executes a command on the current XO instance.
+
 ```
 
 #### Register your XO instance

docs/backup_troubleshooting.md

@@ -91,46 +91,9 @@ This error can be caused by leaving any removable device (such as USB storage) a
 ## Error: Lock file is already being held
 
 This error message appears in the logs in some instances of a failed backup job. It means that the VM’s folder on the remote is already used by a process. This could be:
-
 - another backup job
 - a merge process on the Virtual Hard Disk (VHD)
 
 To solve this issue, we recommend that you:
-
 - wait until the other backup job is completed/the merge process is done
 - make sure your remote storage is not being overworked
-
-## Error: HTTP connection has timed out
-
-This error occurs when XO tries to fetch data from a host, via the HTTP GET method. This error essentially means that the host (dom0 specifically) isn't responding anymore, after we asked it to expose the disk to be exported. This could be a symptom of having an overloaded dom0 that couldn't respond fast enough. It can also be caused by dom0 having trouble attaching the disk in question to expose it for fetching via HTTP, or just not having enough resources to answer our GET request.
-
-::: warning
-As a temporary workaround you can increase the timeout higher than the default value, to allow the host more time to respond. But you will need to eventually diagnose the root cause of the slow host response or else you risk the issue returning.
-:::
-
-Create the following file:
-```
-/etc/xo-server/config.httpInactivityTimeout.toml
-```
-Add the following lines:
-```
-# XOA Support - Work-around HTTP timeout issue during backups
-[xapiOptions]
-httpInactivityTimeout = 1800000 # 30 mins
-```
-
-## Error: Expected values to be strictly equal
-
-This error occurs at the end of the transfer. XO checks the exported VM disk integrity, to ensure it's a valid VHD file (we check the VHD header as well as the footer of the received file). This error means the header and footage did not match, so the file is incomplete (likely the export from dom0 failed at some point and we only received a partial HD/VM disk).
-
-## Error: the job is already running
-
-This means the same job is still running, typically from the last scheduled run. This happens when you have a backup job scheduled too often. It can also occur if you have a long timeout configured for the job, and a slow VM export or slow transfer to your remote. In either case, you need to adjust your backup schedule to allow time for the job to finish or timeout before the next scheduled run. We consider this an error to ensure you'll be notified that the planned schedule won't run this time because the previous one isn't finished.
-
-## Error: VDI_IO_ERROR
-
-This error comes directly from your host/dom0, and not XO. Essentially, XO asked the host to expose a VM disk to export via HTTP (as usual), XO managed to make the HTTP GET connection, and even start the transfer. But then at some point the host couldn't read the VM disk any further, causing this error on the host side. This might happen if the VDI is corrupted on the storage, or if there's a race condition during snapshots. More rarely, this can also occur if your SR is just too slow to keep up with the export as well as live VM traffic.
-
-## Error: no XAPI associated to <UUID>
-
-This message means that XO had a UUID of a VM to backup, but when the job ran it couldn't find any object matching it. This could be caused by the pool where this VM lived no longer being connected to XO. Double-check that the pool hosting the VM is currently connected under Settings > Servers. You can also search for the VM UUID in the Home > VMs search bar. If you can see it, run the backup job again and it will work. If you cannot, either the VM was removed or the pool is not connected.

docs/configuration.md

@@ -66,13 +66,12 @@ You shouldn't have to change this. It's the path where `xo-web` files are served
 
 ## Custom certificate authority
 
-If you use certificates signed by an in-house CA for your XCP-ng or XenServer hosts, and want to have Xen Orchestra connect to them without rejection, you can use the [`NODE_EXTRA_CA_CERTS`](https://nodejs.org/api/cli.html#cli_node_extra_ca_certs_file) environment variable.
+If you use certificates signed by an in-house CA for your XenServer hosts, and want to have Xen Orchestra connect to them without rejection, you need to add the `--use-openssl-ca` option in Node, but also add the CA to your trust store (`/etc/ssl/certs` via `update-ca-certificates` in your XOA).
 
-To enable this option in your XOA, create `/etc/systemd/system/xo-server.service.d/ca.conf` with the following content:
+To enable this option in your XOA, edit the `/etc/systemd/system/xo-server.service` file and add this:
 
 ```
-[Service]
-Environment=NODE_EXTRA_CA_CERTS=/usr/local/share/ca-certificates/my-cert.crt
+Environment=NODE_OPTIONS=--use-openssl-ca
 ```
 
 Don't forget to reload `systemd` conf and restart `xo-server`:
@@ -82,7 +81,9 @@ Don't forget to reload `systemd` conf and restart `xo-server`:
 # systemctl restart xo-server.service
 ```
 
-> For XO Proxy, the process is almost the same except the file to create is `/etc/systemd/system/xo-proxy.service.d/ca.conf` and the service to restart is `xo-proxy.service`.
+:::tip
+The `--use-openssl-ca` option is ignored by Node if Xen-Orchestra is run with Linux capabilities. Capabilities are commonly used to bind applications to privileged ports (<1024) (i.e. `CAP_NET_BIND_SERVICE`). Local NAT rules (`iptables`) or a reverse proxy would be required to use privileged ports and a custom certficate authority.
+:::
 
 ## Redis server
 

docs/continuous_replication.md

@@ -18,8 +18,6 @@ If you lose your main pool, you can start the copy on the other side, with very
 
 :::warning
 It is normal that you can't boot the copied VM directly: we protect it. The normal workflow is to make a clone and then work on it.
-
-This also affects VMs with "Auto Power On" enabled, because of our protections you can ensure these won't start on your CR destination if you happen to reboot it.
 :::
 
 ## Configure it

docs/disaster_recovery.md

@@ -35,7 +35,3 @@ A higher retention number will lead to huge space occupation on your SR.
 If you boot a copy of your production VM, be careful: if they share the same static IP, you'll have troubles.
 
 A good way to avoid this kind of problem is to remove the network interface on the DR VM and check if the export is correctly done.
-
-:::warning
-For each DR replicated VM, we add "start" as a blocked operation, meaning even VMs with "Auto power on" enabled will not be started on your DR destination if it reboots.
-:::

docs/installation.md

@@ -273,52 +273,6 @@ Don't forget to start redis if you don't reboot now:
 service redis start
 ```
 
-### OpenBSD
-
-If you are using OpenBSD, you need to install these packages:
-
-```
-pkg_add gmake redis python--%2.7 git node autoconf yarn
-```
-
-A few of the npm packages look for system binaries as part of their installation, and if missing will try to build it themselves. Installing these will save some time and allow for easier upgrades later:
-
-```
-pkg_add jpeg optipng gifsicle
-```
-
-Because OpenBSD is shipped with CLANG and not GCC, you need to do this:
-
-```
-export CC=/usr/bin/clang
-export CXX=/usr/bin/clang++
-```
-
-You will need to update the number of allowed open files and make `node` available to `npm` :
-
-```
-ulimit -n 10240
-ln -s /usr/local/bin/node /tmp/node
-```
-
-If `yarn` cannot find Python, give it an hand :
-
-```
-PYTHON=/usr/local/bin/python2 yarn
-```
-
-Enable redis on boot with:
-
-```
-rcctl enable redis
-```
-
-Don't forget to start redis if you don't reboot now:
-
-```
-rcctl start redis
-```
-
 ### sudo
 
 If you are running `xo-server` as a non-root user, you need to use `sudo` to be able to mount NFS remotes. You can do this by editing `xo-server` configuration file and setting `useSudo = true`. It's near the end of the file:

docs/restapi.md

@@ -22,7 +22,7 @@ Cookie: authenticationToken=TN2YBOMYtXB_hHtf4wTzm9p5tTuqq2i15yeuhcz2xXM
 
 The server will respond to an invalid token with a `401 Unauthorized` status.
 
-**[Not implemented at this time]** The server can request that the client updates its token with a `Set-Cookie` header:
+The server can request that the client updates its token with a `Set-Cookie` header:
 
 ```http
 HTTP/1.1 200 OK
@@ -63,6 +63,7 @@ The following query parameters are supported:
 - `filter`: a string that will be used to select only matching objects, see [the syntax documentation](manage_infrastructure.md#live-filter-search)
 - `ndjson`: if specified, the result will be in [NDJSON format](http://ndjson.org/)
 
+
 Simple request:
 
 ```http
@@ -78,6 +79,7 @@ Content-Type: application/json
 ]
 ```
 
+
 Here is an example with `curl`:
 
 ```bash
@@ -121,6 +123,7 @@ Content-Type: application/x-ndjson
 {"name_label":"Debian 10 Cloudinit self-service","power_state":"Halted","url":"/rest/v0/vms/5019156b-f40d-bc57-835b-4a259b177be1"}
 ```
 
+
 ## VM and VDI export
 
 VDI export and VM export are supported by the API. Below is a simple example to export a VM with `zstd` compression into a `myVM.xva` file:
@@ -141,28 +144,6 @@ curl \
   > myDisk.vhd
 ```
 
-## VDI Import
-
-A VHD can be imported on an SR to create a VDI at `/rest/v0/srs/<sr uuid>/vdis`.
-
-```bash
-curl \
-  -X POST \
-  -b authenticationToken=KQxQdm2vMiv7jBIK0hgkmgxKzemd8wSJ7ugFGKFkTbs \
-  -T myDisk.vhd \
-  'https://xo.example.org/rest/v0/srs/357bd56c-71f9-4b2a-83b8-3451dec04b8f/vdis?name_label=my_imported_VDI' \
-  | cat
-```
-
-> Note: the final `| cat` ensures cURL's standard output is not a TTY, which is necessary for upload stats to be dislayed.
-
-This request returns the UUID of the created VDI.
-
-The following query parameters are supported to customize the created VDI:
-
-- `name_label`
-- `name_description`
-
 ## The future
 
 We are adding features and improving the REST API step by step. If you have interesting use cases or feedback, please ask directly at <https://xcp-ng.org/forum/category/12/xen-orchestra>

package.json

@@ -5,10 +5,9 @@
     "@babel/register": "^7.0.0",
     "babel-jest": "^27.3.1",
     "benchmark": "^2.1.4",
-    "deptree": "^1.0.0",
     "eslint": "^8.7.0",
     "eslint-config-prettier": "^8.1.0",
-    "eslint-config-standard": "^17.0.0",
+    "eslint-config-standard": "^17.0.0-1",
     "eslint-config-standard-jsx": "^11.0.0-1",
     "eslint-plugin-eslint-comments": "^3.2.0",
     "eslint-plugin-import": "^2.22.1",
@@ -16,7 +15,7 @@
     "eslint-plugin-promise": "^6.0.0",
     "eslint-plugin-react": "^7.21.5",
     "exec-promise": "^0.7.0",
-    "globby": "^13.1.1",
+    "globby": "^11.0.1",
     "handlebars": "^4.7.6",
     "husky": "^4.2.5",
     "jest": "^27.3.1",
@@ -24,7 +23,6 @@
     "lodash": "^4.17.4",
     "prettier": "^2.0.5",
     "promise-toolbox": "^0.21.0",
-    "semver": "^7.3.6",
     "sorted-object": "^2.0.1",
     "vuepress": "^1.4.1"
   },
@@ -60,11 +58,9 @@
     "testEnvironment": "node",
     "testPathIgnorePatterns": [
       "/@vates/decorate-with/",
-      "/@vates/event-listeners-manager/",
       "/@vates/predicates/",
       "/@xen-orchestra/audit-core/",
       "/dist/",
-      "/xen-api/",
       "/xo-server/",
       "/xo-server-test/",
       "/xo-web/"

packages/complex-matcher/.babelrc.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))

packages/complex-matcher/.eslintrc.js

@@ -0,0 +1 @@
+../../scripts/babel-eslintrc.js

packages/complex-matcher/index.bench.js

@@ -1,14 +0,0 @@
-'use strict'
-
-const { parse } = require('./')
-const { ast, pattern } = require('./index.fixtures')
-
-module.exports = ({ benchmark }) => {
-  benchmark('parse', () => {
-    parse(pattern)
-  })
-
-  benchmark('toString', () => {
-    ast.toString()
-  })
-}

packages/complex-matcher/package.json

@@ -16,6 +16,7 @@
     "url": "https://vates.fr"
   },
   "preferGlobal": false,
+  "main": "dist/",
   "browserslist": [
     ">2%"
   ],
@@ -25,7 +26,21 @@
   "dependencies": {
     "lodash": "^4.17.4"
   },
+  "devDependencies": {
+    "@babel/cli": "^7.0.0",
+    "@babel/core": "^7.0.0",
+    "@babel/preset-env": "^7.0.0",
+    "babel-plugin-lodash": "^3.3.2",
+    "cross-env": "^7.0.2",
+    "rimraf": "^3.0.0"
+  },
   "scripts": {
+    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
+    "clean": "rimraf dist/",
+    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
+    "prebuild": "yarn run clean",
+    "predev": "yarn run prebuild",
+    "prepublishOnly": "yarn run build",
     "postversion": "npm publish"
   }
 }