fix(s3): retry upload even on error 400

See https://xcp-ng.org/forum/topic/6334/coalesce-not-showing-anymore/3
Introduced by a9c1239149

.github/ISSUE_TEMPLATE/feature_request.md

@@ -4,7 +4,6 @@ about: Suggest an idea for this project
 title: ''
 labels: ''
 assignees: ''
-
 ---
 
 **Is your feature request related to a problem? Please describe.**

.gitignore

@@ -10,8 +10,6 @@
 /packages/*/dist/
 /packages/*/node_modules/
 
-/packages/vhd-cli/src/commands/index.js
-
 /packages/xen-api/examples/node_modules/
 /packages/xen-api/plot.dat
 

@vates/async-each/.USAGE.md

@@ -14,7 +14,7 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
 
 `opts` is an object that can contains the following options:
 
-- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
+- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `10`. The value `0` means no concurrency limit.
 - `signal`: an abort signal to stop the iteration
 - `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
 

@vates/async-each/README.md

@@ -32,7 +32,7 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
 
 `opts` is an object that can contains the following options:
 
-- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
+- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `10`. The value `0` means no concurrency limit.
 - `signal`: an abort signal to stop the iteration
 - `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
 

@vates/async-each/index.js

@@ -9,7 +9,16 @@ class AggregateError extends Error {
   }
 }
 
-exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 1, signal, stopOnError = true } = {}) {
+/**
+ * @template Item
+ * @param {Iterable<Item>} iterable
+ * @param {(item: Item, index: number, iterable: Iterable<Item>) => Promise<void>} iteratee
+ * @returns {Promise<void>}
+ */
+exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 10, signal, stopOnError = true } = {}) {
+  if (concurrency === 0) {
+    concurrency = Infinity
+  }
   return new Promise((resolve, reject) => {
     const it = (iterable[Symbol.iterator] || iterable[Symbol.asyncIterator]).call(iterable)
     const errors = []

@vates/async-each/index.spec.js

@@ -36,7 +36,7 @@ describe('asyncEach', () => {
       it('works', async () => {
         const iteratee = jest.fn(async () => {})
 
-        await asyncEach.call(thisArg, iterable, iteratee)
+        await asyncEach.call(thisArg, iterable, iteratee, { concurrency: 1 })
 
         expect(iteratee.mock.instances).toEqual(Array.from(values, () => thisArg))
         expect(iteratee.mock.calls).toEqual(Array.from(values, (value, index) => [value, index, iterable]))
@@ -66,7 +66,7 @@ describe('asyncEach', () => {
           }
         })
 
-        expect(await rejectionOf(asyncEach(iterable, iteratee, { stopOnError: true }))).toBe(error)
+        expect(await rejectionOf(asyncEach(iterable, iteratee, { concurrency: 1, stopOnError: true }))).toBe(error)
         expect(iteratee).toHaveBeenCalledTimes(2)
       })
 
@@ -91,7 +91,9 @@ describe('asyncEach', () => {
           }
         })
 
-        await expect(asyncEach(iterable, iteratee, { signal: ac.signal })).rejects.toThrow('asyncEach aborted')
+        await expect(asyncEach(iterable, iteratee, { concurrency: 1, signal: ac.signal })).rejects.toThrow(
+          'asyncEach aborted'
+        )
         expect(iteratee).toHaveBeenCalledTimes(2)
       })
     })

@vates/async-each/package.json

@@ -24,7 +24,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "0.1.0",
+  "version": "1.0.0",
   "engines": {
     "node": ">=8.10"
   },

@vates/cached-dns.lookup/.USAGE.md

@@ -0,0 +1,30 @@
+Node does not cache queries to `dns.lookup`, which can lead application doing a lot of connections to have perf issues and to saturate Node threads pool.
+
+This library attempts to mitigate these problems by providing a version of this function with a version short cache, applied on both errors and results.
+
+> Limitation: `verbatim: false` option is not supported.
+
+It has exactly the same API as the native method and can be used directly:
+
+```js
+import { createCachedLookup } from '@vates/cached-dns.lookup'
+
+const lookup = createCachedLookup()
+
+lookup('example.net', { all: true, family: 0 }, (error, result) => {
+  if (error != null) {
+    return console.warn(error)
+  }
+  console.log(result)
+})
+```
+
+Or it can be used to replace the native implementation and speed up the whole app:
+
+```js
+// assign our cached implementation to dns.lookup
+const restore = createCachedLookup().patchGlobal()
+
+// to restore the previous implementation
+restore()
+```

@vates/cached-dns.lookup/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/cached-dns.lookup/README.md

@@ -0,0 +1,63 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/cached-dns.lookup
+
+[![Package Version](https://badgen.net/npm/v/@vates/cached-dns.lookup)](https://npmjs.org/package/@vates/cached-dns.lookup) ![License](https://badgen.net/npm/license/@vates/cached-dns.lookup) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/cached-dns.lookup)](https://bundlephobia.com/result?p=@vates/cached-dns.lookup) [![Node compatibility](https://badgen.net/npm/node/@vates/cached-dns.lookup)](https://npmjs.org/package/@vates/cached-dns.lookup)
+
+> Cached implementation of dns.lookup
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/cached-dns.lookup):
+
+```
+> npm install --save @vates/cached-dns.lookup
+```
+
+## Usage
+
+Node does not cache queries to `dns.lookup`, which can lead application doing a lot of connections to have perf issues and to saturate Node threads pool.
+
+This library attempts to mitigate these problems by providing a version of this function with a version short cache, applied on both errors and results.
+
+> Limitation: `verbatim: false` option is not supported.
+
+It has exactly the same API as the native method and can be used directly:
+
+```js
+import { createCachedLookup } from '@vates/cached-dns.lookup'
+
+const lookup = createCachedLookup()
+
+lookup('example.net', { all: true, family: 0 }, (error, result) => {
+  if (error != null) {
+    return console.warn(error)
+  }
+  console.log(result)
+})
+```
+
+Or it can be used to replace the native implementation and speed up the whole app:
+
+```js
+// assign our cached implementation to dns.lookup
+const restore = createCachedLookup().patchGlobal()
+
+// to restore the previous implementation
+restore()
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/cached-dns.lookup/index.js

@@ -0,0 +1,72 @@
+'use strict'
+
+const assert = require('assert')
+const dns = require('dns')
+const LRU = require('lru-cache')
+
+function reportResults(all, results, callback) {
+  if (all) {
+    callback(null, results)
+  } else {
+    const first = results[0]
+    callback(null, first.address, first.family)
+  }
+}
+
+exports.createCachedLookup = function createCachedLookup({ lookup = dns.lookup } = {}) {
+  const cache = new LRU({
+    max: 500,
+
+    // 1 minute: long enough to be effective, short enough so there is no need to bother with DNS TTLs
+    ttl: 60e3,
+  })
+
+  function cachedLookup(hostname, options, callback) {
+    let all = false
+    let family = 0
+    if (typeof options === 'function') {
+      callback = options
+    } else if (typeof options === 'number') {
+      family = options
+    } else if (options != null) {
+      assert.notStrictEqual(options.verbatim, false, 'not supported by this implementation')
+      ;({ all = all, family = family } = options)
+    }
+
+    // cache by family option because there will be an error if there is no
+    // entries for the requestion family so we cannot easily cache all families
+    // and filter on reporting back
+    const key = hostname + '/' + family
+
+    const result = cache.get(key)
+    if (result !== undefined) {
+      setImmediate(reportResults, all, result, callback)
+    } else {
+      lookup(hostname, { all: true, family, verbatim: true }, function onLookup(error, results) {
+        // errors are not cached because this will delay recovery after DNS/network issues
+        //
+        // there are no reliable way to detect if the error is real or simply
+        // that there are no results for the requested hostname
+        //
+        // there should be much fewer errors than success, therefore it should
+        // not be a big deal to not cache them
+        if (error != null) {
+          return callback(error)
+        }
+
+        cache.set(key, results)
+        reportResults(all, results, callback)
+      })
+    }
+  }
+  cachedLookup.patchGlobal = function patchGlobal() {
+    const previous = dns.lookup
+    dns.lookup = cachedLookup
+    return function restoreGlobal() {
+      assert.strictEqual(dns.lookup, cachedLookup)
+      dns.lookup = previous
+    }
+  }
+
+  return cachedLookup
+}

@vates/cached-dns.lookup/package.json

@@ -0,0 +1,32 @@
+{
+  "engines": {
+    "node": ">=8"
+  },
+  "dependencies": {
+    "lru-cache": "^7.0.4"
+  },
+  "private": false,
+  "name": "@vates/cached-dns.lookup",
+  "description": "Cached implementation of dns.lookup",
+  "keywords": [
+    "cache",
+    "dns",
+    "lookup"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/cached-dns.lookup",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/cached-dns.lookup",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "1.0.0",
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@vates/event-listeners-manager/.USAGE.md

@@ -0,0 +1,50 @@
+> This library is compatible with Node's `EventEmitter` and web browsers' `EventTarget` APIs.
+
+### API
+
+```js
+import { EventListenersManager } from '@vates/event-listeners-manager'
+
+const events = new EventListenersManager(emitter)
+
+// adding listeners
+events.add('foo', onFoo).add('bar', onBar).on('baz', onBaz)
+
+// removing a specific listener
+events.remove('foo', onFoo)
+
+// removing all listeners for a specific event
+events.removeAll('foo')
+
+// removing all listeners
+events.removeAll()
+```
+
+### Typical use case
+
+> Removing all listeners when no longer necessary.
+
+Manually:
+
+```js
+const onFoo = () => {}
+const onBar = () => {}
+const onBaz = () => {}
+emitter.on('foo', onFoo).on('bar', onBar).on('baz', onBaz)
+
+// CODE LOGIC
+
+emitter.off('foo', onFoo).off('bar', onBar).off('baz', onBaz)
+```
+
+With this library:
+
+```js
+const events = new EventListenersManager(emitter)
+
+events.add('foo', () => {})).add('bar', () => {})).add('baz', () => {}))
+
+// CODE LOGIC
+
+events.removeAll()
+```

@vates/event-listeners-manager/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/event-listeners-manager/README.md

@@ -0,0 +1,81 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/event-listeners-manager
+
+[![Package Version](https://badgen.net/npm/v/@vates/event-listeners-manager)](https://npmjs.org/package/@vates/event-listeners-manager) ![License](https://badgen.net/npm/license/@vates/event-listeners-manager) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/event-listeners-manager)](https://bundlephobia.com/result?p=@vates/event-listeners-manager) [![Node compatibility](https://badgen.net/npm/node/@vates/event-listeners-manager)](https://npmjs.org/package/@vates/event-listeners-manager)
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/event-listeners-manager):
+
+```
+> npm install --save @vates/event-listeners-manager
+```
+
+## Usage
+
+> This library is compatible with Node's `EventEmitter` and web browsers' `EventTarget` APIs.
+
+### API
+
+```js
+import { EventListenersManager } from '@vates/event-listeners-manager'
+
+const events = new EventListenersManager(emitter)
+
+// adding listeners
+events.add('foo', onFoo).add('bar', onBar).on('baz', onBaz)
+
+// removing a specific listener
+events.remove('foo', onFoo)
+
+// removing all listeners for a specific event
+events.removeAll('foo')
+
+// removing all listeners
+events.removeAll()
+```
+
+### Typical use case
+
+> Removing all listeners when no longer necessary.
+
+Manually:
+
+```js
+const onFoo = () => {}
+const onBar = () => {}
+const onBaz = () => {}
+emitter.on('foo', onFoo).on('bar', onBar).on('baz', onBaz)
+
+// CODE LOGIC
+
+emitter.off('foo', onFoo).off('bar', onBar).off('baz', onBaz)
+```
+
+With this library:
+
+```js
+const events = new EventListenersManager(emitter)
+
+events.add('foo', () => {})).add('bar', () => {})).add('baz', () => {}))
+
+// CODE LOGIC
+
+events.removeAll()
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/event-listeners-manager/index.js

@@ -0,0 +1,56 @@
+'use strict'
+
+exports.EventListenersManager = class EventListenersManager {
+  constructor(emitter) {
+    this._listeners = new Map()
+
+    this._add = (emitter.addListener || emitter.addEventListener).bind(emitter)
+    this._remove = (emitter.removeListener || emitter.removeEventListener).bind(emitter)
+  }
+
+  add(type, listener) {
+    let listeners = this._listeners.get(type)
+    if (listeners === undefined) {
+      listeners = new Set()
+      this._listeners.set(type, listeners)
+    }
+
+    // don't add the same listener multiple times (allowed on Node.js)
+    if (!listeners.has(listener)) {
+      listeners.add(listener)
+      this._add(type, listener)
+    }
+
+    return this
+  }
+
+  remove(type, listener) {
+    const allListeners = this._listeners
+    const listeners = allListeners.get(type)
+    if (listeners !== undefined && listeners.delete(listener)) {
+      this._remove(type, listener)
+      if (listeners.size === 0) {
+        allListeners.delete(type)
+      }
+    }
+
+    return this
+  }
+
+  removeAll(type) {
+    const allListeners = this._listeners
+    const remove = this._remove
+    const types = type !== undefined ? [type] : allListeners.keys()
+    for (const type of types) {
+      const listeners = allListeners.get(type)
+      if (listeners !== undefined) {
+        allListeners.delete(type)
+        for (const listener of listeners) {
+          remove(type, listener)
+        }
+      }
+    }
+
+    return this
+  }
+}

@vates/event-listeners-manager/index.spec.js

@@ -0,0 +1,67 @@
+'use strict'
+
+const t = require('tap')
+const { EventEmitter } = require('events')
+
+const { EventListenersManager } = require('./')
+
+const noop = Function.prototype
+
+// function spy (impl = Function.prototype) {
+//   function spy() {
+//     spy.calls.push([Array.from(arguments), this])
+//   }
+//   spy.calls = []
+//   return spy
+// }
+
+function assertListeners(t, event, listeners) {
+  t.strictSame(t.context.ee.listeners(event), listeners)
+}
+
+t.beforeEach(function (t) {
+  t.context.ee = new EventEmitter()
+  t.context.em = new EventListenersManager(t.context.ee)
+})
+
+t.test('.add adds a listener', function (t) {
+  t.context.em.add('foo', noop)
+
+  assertListeners(t, 'foo', [noop])
+
+  t.end()
+})
+
+t.test('.add does not add a duplicate listener', function (t) {
+  t.context.em.add('foo', noop).add('foo', noop)
+
+  assertListeners(t, 'foo', [noop])
+
+  t.end()
+})
+
+t.test('.remove removes a listener', function (t) {
+  t.context.em.add('foo', noop).remove('foo', noop)
+
+  assertListeners(t, 'foo', [])
+
+  t.end()
+})
+
+t.test('.removeAll removes all listeners of a given type', function (t) {
+  t.context.em.add('foo', noop).add('bar', noop).removeAll('foo')
+
+  assertListeners(t, 'foo', [])
+  assertListeners(t, 'bar', [noop])
+
+  t.end()
+})
+
+t.test('.removeAll removes all listeners', function (t) {
+  t.context.em.add('foo', noop).add('bar', noop).removeAll()
+
+  assertListeners(t, 'foo', [])
+  assertListeners(t, 'bar', [])
+
+  t.end()
+})

@vates/event-listeners-manager/package.json

@@ -0,0 +1,46 @@
+{
+  "engines": {
+    "node": ">=6"
+  },
+  "private": false,
+  "name": "@vates/event-listeners-manager",
+  "descriptions": "Easy way to clean up event listeners",
+  "keywords": [
+    "add",
+    "addEventListener",
+    "addListener",
+    "browser",
+    "clear",
+    "DOM",
+    "emitter",
+    "event",
+    "EventEmitter",
+    "EventTarget",
+    "management",
+    "manager",
+    "node",
+    "remove",
+    "removeEventListener",
+    "removeListener"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/event-listeners-manager",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/event-listeners-manager",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "1.0.1",
+  "scripts": {
+    "postversion": "npm publish --access public",
+    "test": "tap --branches=72"
+  },
+  "devDependencies": {
+    "tap": "^16.2.0"
+  }
+}

@vates/fuse-vhd/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/fuse-vhd/index.js

@@ -0,0 +1,71 @@
+'use strict'
+
+const LRU = require('lru-cache')
+const Fuse = require('fuse-native')
+const { VhdSynthetic } = require('vhd-lib')
+const { Disposable, fromCallback } = require('promise-toolbox')
+const { createLogger } = require('@xen-orchestra/log')
+
+const { warn } = createLogger('vates:fuse-vhd')
+
+// build a s stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js
+const stat = st => ({
+  mtime: st.mtime || new Date(),
+  atime: st.atime || new Date(),
+  ctime: st.ctime || new Date(),
+  size: st.size !== undefined ? st.size : 0,
+  mode: st.mode === 'dir' ? 16877 : st.mode === 'file' ? 33188 : st.mode === 'link' ? 41453 : st.mode,
+  uid: st.uid !== undefined ? st.uid : process.getuid(),
+  gid: st.gid !== undefined ? st.gid : process.getgid(),
+})
+
+exports.mount = Disposable.factory(async function* mount(handler, diskPath, mountDir) {
+  const vhd = yield VhdSynthetic.fromVhdChain(handler, diskPath)
+
+  const cache = new LRU({
+    max: 16, // each cached block is 2MB in size
+  })
+  await vhd.readBlockAllocationTable()
+  const fuse = new Fuse(mountDir, {
+    async readdir(path, cb) {
+      if (path === '/') {
+        return cb(null, ['vhd0'])
+      }
+      cb(Fuse.ENOENT)
+    },
+    async getattr(path, cb) {
+      if (path === '/') {
+        return cb(
+          null,
+          stat({
+            mode: 'dir',
+            size: 4096,
+          })
+        )
+      }
+      if (path === '/vhd0') {
+        return cb(
+          null,
+          stat({
+            mode: 'file',
+            size: vhd.footer.currentSize,
+          })
+        )
+      }
+
+      cb(Fuse.ENOENT)
+    },
+    read(path, fd, buf, len, pos, cb) {
+      if (path === '/vhd0') {
+        return vhd
+          .readRawData(pos, len, cache, buf)
+          .then(cb)
+      }
+      throw new Error(`read file ${path} not exists`)
+    },
+  })
+  return new Disposable(
+    () => fromCallback(() => fuse.unmount()),
+    fromCallback(() => fuse.mount())
+    )
+})

@vates/fuse-vhd/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@vates/fuse-vhd",
+  "version": "0.0.1",
+  "license": "ISC",
+  "private": false,
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/fuse-vhd",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/fuse-vhd",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "engines": {
+    "node": ">=10.0"
+  },
+  "dependencies": {
+    "@xen-orchestra/log": "^0.3.0",
+    "fuse-native": "^2.2.6",
+    "lru-cache": "^7.14.0",
+    "promise-toolbox": "^0.21.0",
+    "vhd-lib": "^4.0.1"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@vates/read-chunk/.USAGE.md

@@ -1,6 +1,9 @@
+### `readChunk(stream, [size])`
+
 - returns the next available chunk of data
 - like `stream.read()`, a number of bytes can be specified
-- returns `null` if the stream has ended
+- returns with less data than expected if stream has ended
+- returns `null` if the stream has ended and no data has been read
 
 ```js
 import { readChunk } from '@vates/read-chunk'
@@ -11,3 +14,13 @@ import { readChunk } from '@vates/read-chunk'
   }
 })()
 ```
+
+### `readChunkStrict(stream, [size])`
+
+Similar behavior to `readChunk` but throws if the stream ended before the requested data could be read.
+
+```js
+import { readChunkStrict } from '@vates/read-chunk'
+
+const chunk = await readChunkStrict(stream, 1024)
+```

@vates/read-chunk/README.md

@@ -16,9 +16,12 @@ Installation of the [npm package](https://npmjs.org/package/@vates/read-chunk):
 
 ## Usage
 
+### `readChunk(stream, [size])`
+
 - returns the next available chunk of data
 - like `stream.read()`, a number of bytes can be specified
-- returns `null` if the stream has ended
+- returns with less data than expected if stream has ended
+- returns `null` if the stream has ended and no data has been read
 
 ```js
 import { readChunk } from '@vates/read-chunk'
@@ -30,6 +33,16 @@ import { readChunk } from '@vates/read-chunk'
 })()
 ```
 
+### `readChunkStrict(stream, [size])`
+
+Similar behavior to `readChunk` but throws if the stream ended before the requested data could be read.
+
+```js
+import { readChunkStrict } from '@vates/read-chunk'
+
+const chunk = await readChunkStrict(stream, 1024)
+```
+
 ## Contributions
 
 Contributions are _very_ welcomed, either on the documentation or on

@vates/read-chunk/index.js

@@ -30,3 +30,22 @@ const readChunk = (stream, size) =>
         onReadable()
       })
 exports.readChunk = readChunk
+
+exports.readChunkStrict = async function readChunkStrict(stream, size) {
+  const chunk = await readChunk(stream, size)
+  if (chunk === null) {
+    throw new Error('stream has ended without data')
+  }
+
+  if (size !== undefined && chunk.length !== size) {
+    const error = new Error('stream has ended with not enough data')
+    Object.defineProperties(error, {
+      chunk: {
+        value: chunk,
+      },
+    })
+    throw error
+  }
+
+  return chunk
+}

@vates/read-chunk/index.spec.js

@@ -4,7 +4,7 @@
 
 const { Readable } = require('stream')
 
-const { readChunk } = require('./')
+const { readChunk, readChunkStrict } = require('./')
 
 const makeStream = it => Readable.from(it, { objectMode: false })
 makeStream.obj = Readable.from
@@ -43,3 +43,27 @@ describe('readChunk', () => {
     })
   })
 })
+
+const rejectionOf = promise =>
+  promise.then(
+    value => {
+      throw value
+    },
+    error => error
+  )
+
+describe('readChunkStrict', function () {
+  it('throws if stream is empty', async () => {
+    const error = await rejectionOf(readChunkStrict(makeStream([])))
+    expect(error).toBeInstanceOf(Error)
+    expect(error.message).toBe('stream has ended without data')
+    expect(error.chunk).toEqual(undefined)
+  })
+
+  it('throws if stream ends with not enough data', async () => {
+    const error = await rejectionOf(readChunkStrict(makeStream(['foo', 'bar']), 10))
+    expect(error).toBeInstanceOf(Error)
+    expect(error.message).toBe('stream has ended with not enough data')
+    expect(error.chunk).toEqual(Buffer.from('foobar'))
+  })
+})

@vates/read-chunk/package.json

@@ -19,7 +19,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.1.2",
+  "version": "1.0.0",
   "engines": {
     "node": ">=8.10"
   },

@xen-orchestra/backups-cli/commands/clean-vms.js

@@ -26,7 +26,13 @@ module.exports = async function main(args) {
   await asyncMap(_, async vmDir => {
     vmDir = resolve(vmDir)
     try {
-      await adapter.cleanVm(vmDir, { fixMetadata: fix, remove, merge, onLog: (...args) => console.warn(...args) })
+      await adapter.cleanVm(vmDir, {
+        fixMetadata: fix,
+        remove,
+        merge,
+        logInfo: (...args) => console.log(...args),
+        logWarn: (...args) => console.warn(...args),
+      })
     } catch (error) {
       console.error('adapter.cleanVm', vmDir, error)
     }

@xen-orchestra/backups-cli/package.json

@@ -7,8 +7,8 @@
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.21.0",
-    "@xen-orchestra/fs": "^1.0.0",
+    "@xen-orchestra/backups": "^0.27.4",
+    "@xen-orchestra/fs": "^3.1.0",
     "filenamify": "^4.1.0",
     "getopts": "^2.2.5",
     "lodash": "^4.17.15",
@@ -27,7 +27,7 @@
   "scripts": {
     "postversion": "npm publish --access public"
   },
-  "version": "0.7.0",
+  "version": "0.7.7",
   "license": "AGPL-3.0-or-later",
   "author": {
     "name": "Vates SAS",

@xen-orchestra/backups/Backup.js

@@ -6,7 +6,7 @@ const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { compileTemplate } = require('@xen-orchestra/template')
 const { limitConcurrency } = require('limit-concurrency-decorator')
 
-const { extractIdsFromSimplePattern } = require('./_extractIdsFromSimplePattern.js')
+const { extractIdsFromSimplePattern } = require('./extractIdsFromSimplePattern.js')
 const { PoolMetadataBackup } = require('./_PoolMetadataBackup.js')
 const { Task } = require('./Task.js')
 const { VmBackup } = require('./_VmBackup.js')
@@ -24,6 +24,34 @@ const getAdaptersByRemote = adapters => {
 
 const runTask = (...args) => Task.run(...args).catch(noop) // errors are handled by logs
 
+const DEFAULT_SETTINGS = {
+  reportWhen: 'failure',
+}
+
+const DEFAULT_VM_SETTINGS = {
+  bypassVdiChainsCheck: false,
+  checkpointSnapshot: false,
+  concurrency: 2,
+  copyRetention: 0,
+  deleteFirst: false,
+  exportRetention: 0,
+  fullInterval: 0,
+  healthCheckSr: undefined,
+  healthCheckVmsWithTags: [],
+  maxMergedDeltasPerRun: 2,
+  offlineBackup: false,
+  offlineSnapshot: false,
+  snapshotRetention: 0,
+  timeout: 0,
+  unconditionalSnapshot: false,
+  vmTimeout: 0,
+}
+
+const DEFAULT_METADATA_SETTINGS = {
+  retentionPoolMetadata: 0,
+  retentionXoMetadata: 0,
+}
+
 exports.Backup = class Backup {
   constructor({ config, getAdapter, getConnectedRecord, job, schedule }) {
     this._config = config
@@ -42,17 +70,22 @@ exports.Backup = class Backup {
       '{job.name}': job.name,
       '{vm.name_label}': vm => vm.name_label,
     })
-  }
 
-  run() {
-    const type = this._job.type
+    const { type } = job
+    const baseSettings = { ...DEFAULT_SETTINGS }
     if (type === 'backup') {
-      return this._runVmBackup()
+      Object.assign(baseSettings, DEFAULT_VM_SETTINGS, config.defaultSettings, config.vm?.defaultSettings)
+      this.run = this._runVmBackup
     } else if (type === 'metadataBackup') {
-      return this._runMetadataBackup()
+      Object.assign(baseSettings, DEFAULT_METADATA_SETTINGS, config.defaultSettings, config.metadata?.defaultSettings)
+      this.run = this._runMetadataBackup
     } else {
       throw new Error(`No runner for the backup type ${type}`)
     }
+    Object.assign(baseSettings, job.settings[''])
+
+    this._baseSettings = baseSettings
+    this._settings = { ...baseSettings, ...job.settings[schedule.id] }
   }
 
   async _runMetadataBackup() {
@@ -64,13 +97,6 @@ exports.Backup = class Backup {
     }
 
     const config = this._config
-    const settings = {
-      ...config.defaultSettings,
-      ...config.metadata.defaultSettings,
-      ...job.settings[''],
-      ...job.settings[schedule.id],
-    }
-
     const poolIds = extractIdsFromSimplePattern(job.pools)
     const isEmptyPools = poolIds.length === 0
     const isXoMetadata = job.xoMetadata !== undefined
@@ -78,6 +104,8 @@ exports.Backup = class Backup {
       throw new Error('no metadata mode found')
     }
 
+    const settings = this._settings
+
     const { retentionPoolMetadata, retentionXoMetadata } = settings
 
     if (
@@ -189,14 +217,7 @@ exports.Backup = class Backup {
     const schedule = this._schedule
 
     const config = this._config
-    const { settings } = job
-    const scheduleSettings = {
-      ...config.defaultSettings,
-      ...config.vm.defaultSettings,
-      ...settings[''],
-      ...settings[schedule.id],
-    }
-
+    const settings = this._settings
     await Disposable.use(
       Disposable.all(
         extractIdsFromSimplePattern(job.srs).map(id =>
@@ -224,14 +245,15 @@ exports.Backup = class Backup {
           })
         )
       ),
-      async (srs, remoteAdapters) => {
+      () => (settings.healthCheckSr !== undefined ? this._getRecord('SR', settings.healthCheckSr) : undefined),
+      async (srs, remoteAdapters, healthCheckSr) => {
         // remove adapters that failed (already handled)
         remoteAdapters = remoteAdapters.filter(_ => _ !== undefined)
 
         // remove srs that failed (already handled)
         srs = srs.filter(_ => _ !== undefined)
 
-        if (remoteAdapters.length === 0 && srs.length === 0 && scheduleSettings.snapshotRetention === 0) {
+        if (remoteAdapters.length === 0 && srs.length === 0 && settings.snapshotRetention === 0) {
           return
         }
 
@@ -241,23 +263,27 @@ exports.Backup = class Backup {
 
         remoteAdapters = getAdaptersByRemote(remoteAdapters)
 
+        const allSettings = this._job.settings
+        const baseSettings = this._baseSettings
+
         const handleVm = vmUuid =>
           runTask({ name: 'backup VM', data: { type: 'VM', id: vmUuid } }, () =>
             Disposable.use(this._getRecord('VM', vmUuid), vm =>
               new VmBackup({
+                baseSettings,
                 config,
                 getSnapshotNameLabel,
+                healthCheckSr,
                 job,
-                // remotes,
                 remoteAdapters,
                 schedule,
-                settings: { ...scheduleSettings, ...settings[vmUuid] },
+                settings: { ...settings, ...allSettings[vm.uuid] },
                 srs,
                 vm,
               }).run()
             )
           )
-        const { concurrency } = scheduleSettings
+        const { concurrency } = settings
         await asyncMapSettled(vmIds, concurrency === 0 ? handleVm : limitConcurrency(concurrency)(handleVm))
       }
     )

@xen-orchestra/backups/HealthCheckVmBackup.js

@@ -0,0 +1,64 @@
+'use strict'
+
+const { Task } = require('./Task')
+
+exports.HealthCheckVmBackup = class HealthCheckVmBackup {
+  #xapi
+  #restoredVm
+
+  constructor({ restoredVm, xapi }) {
+    this.#restoredVm = restoredVm
+    this.#xapi = xapi
+  }
+
+  async run() {
+    return Task.run(
+      {
+        name: 'vmstart',
+      },
+      async () => {
+        let restoredVm = this.#restoredVm
+        const xapi = this.#xapi
+        const restoredId = restoredVm.uuid
+
+        // remove vifs
+        await Promise.all(restoredVm.$VIFs.map(vif => xapi.callAsync('VIF.destroy', vif.$ref)))
+
+        const start = new Date()
+        // start Vm
+
+        await xapi.callAsync(
+          'VM.start',
+          restoredVm.$ref,
+          false, // Start paused?
+          false // Skip pre-boot checks?
+        )
+        const started = new Date()
+        const timeout = 10 * 60 * 1000
+        const startDuration = started - start
+
+        let remainingTimeout = timeout - startDuration
+
+        if (remainingTimeout < 0) {
+          throw new Error(`VM ${restoredId} not started after ${timeout / 1000} second`)
+        }
+
+        // wait for the 'Running' event to be really stored in local xapi object cache
+        restoredVm = await xapi.waitObjectState(restoredVm.$ref, vm => vm.power_state === 'Running', {
+          timeout: remainingTimeout,
+        })
+
+        const running = new Date()
+        remainingTimeout -= running - started
+
+        if (remainingTimeout < 0) {
+          throw new Error(`local xapi  did not get Runnig state for VM ${restoredId} after ${timeout / 1000} second`)
+        }
+        // wait for the guest tool version to be defined
+        await xapi.waitObjectState(restoredVm.guest_metrics, gm => gm?.PV_drivers_version?.major !== undefined, {
+          timeout: remainingTimeout,
+        })
+      }
+    )
+  }
+}

@xen-orchestra/backups/RemoteAdapter.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
+const { synchronized } = require('decorator-synchronized')
 const Disposable = require('promise-toolbox/Disposable')
 const fromCallback = require('promise-toolbox/fromCallback')
 const fromEvent = require('promise-toolbox/fromEvent')
@@ -9,22 +10,27 @@ const groupBy = require('lodash/groupBy.js')
 const pickBy = require('lodash/pickBy.js')
 const { dirname, join, normalize, resolve } = require('path')
 const { createLogger } = require('@xen-orchestra/log')
-const { Constants, createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib')
+const { createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib')
 const { deduped } = require('@vates/disposable/deduped.js')
 const { decorateMethodsWith } = require('@vates/decorate-with')
 const { compose } = require('@vates/compose')
 const { execFile } = require('child_process')
-const { readdir, stat } = require('fs-extra')
+const { readdir, lstat } = require('fs-extra')
 const { v4: uuidv4 } = require('uuid')
 const { ZipFile } = require('yazl')
+const zlib = require('zlib')
 
 const { BACKUP_DIR } = require('./_getVmBackupDir.js')
 const { cleanVm } = require('./_cleanVm.js')
+const { formatFilenameDate } = require('./_filenameDate.js')
 const { getTmpDir } = require('./_getTmpDir.js')
 const { isMetadataFile } = require('./_backupType.js')
 const { isValidXva } = require('./_isValidXva.js')
 const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js')
 const { lvs, pvs } = require('./_lvm.js')
+// @todo : this import is marked extraneous , sould be fixed when lib is published
+const { mount } = require('@vates/fuse-vhd')
+const { asyncEach } = require('@vates/async-each')
 
 const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
 exports.DIR_XO_CONFIG_BACKUPS = DIR_XO_CONFIG_BACKUPS
@@ -32,7 +38,7 @@ exports.DIR_XO_CONFIG_BACKUPS = DIR_XO_CONFIG_BACKUPS
 const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
 exports.DIR_XO_POOL_METADATA_BACKUPS = DIR_XO_POOL_METADATA_BACKUPS
 
-const { warn } = createLogger('xo:backups:RemoteAdapter')
+const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
 
 const compareTimestamp = (a, b) => a.timestamp - b.timestamp
 
@@ -42,16 +48,13 @@ const resolveRelativeFromFile = (file, path) => resolve('/', dirname(file), path
 
 const resolveSubpath = (root, path) => resolve(root, `.${resolve('/', path)}`)
 
-const RE_VHDI = /^vhdi(\d+)$/
-
 async function addDirectory(files, realPath, metadataPath) {
-  try {
-    const subFiles = await readdir(realPath)
-    await asyncMap(subFiles, file => addDirectory(files, realPath + '/' + file, metadataPath + '/' + file))
-  } catch (error) {
-    if (error == null || error.code !== 'ENOTDIR') {
-      throw error
-    }
+  const stats = await lstat(realPath)
+  if (stats.isDirectory()) {
+    await asyncMap(await readdir(realPath), file =>
+      addDirectory(files, realPath + '/' + file, metadataPath + '/' + file)
+    )
+  } else if (stats.isFile()) {
     files.push({
       realPath,
       metadataPath,
@@ -73,11 +76,14 @@ const debounceResourceFactory = factory =>
   }
 
 class RemoteAdapter {
-  constructor(handler, { debounceResource = res => res, dirMode, vhdDirectoryCompression } = {}) {
+  constructor(handler, { debounceResource = res => res, dirMode, vhdDirectoryCompression, useGetDiskLegacy=false } = {}) {
     this._debounceResource = debounceResource
     this._dirMode = dirMode
     this._handler = handler
     this._vhdDirectoryCompression = vhdDirectoryCompression
+    this._readCacheListVmBackups = synchronized.withKey()(this._readCacheListVmBackups)
+    this._useGetDiskLegacy = useGetDiskLegacy
+
   }
 
   get handler() {
@@ -125,7 +131,9 @@ class RemoteAdapter {
   }
 
   async *_getPartition(devicePath, partition) {
-    const options = ['loop', 'ro']
+    // the norecovery option is necessary because if the partition is dirty,
+    // mount will try to fix it which is impossible if because the device is read-only
+    const options = ['loop', 'ro', 'norecovery']
 
     if (partition !== undefined) {
       const { size, start } = partition
@@ -222,11 +230,30 @@ class RemoteAdapter {
     return promise
   }
 
+  #removeVmBackupsFromCache(backups) {
+    for (const [dir, filenames] of Object.entries(
+      groupBy(
+        backups.map(_ => _._filename),
+        dirname
+      )
+    )) {
+      // detached async action, will not reject
+      this._updateCache(dir + '/cache.json.gz', backups => {
+        for (const filename of filenames) {
+          debug('removing cache entry', { entry: filename })
+          delete backups[filename]
+        }
+      })
+    }
+  }
+
   async deleteDeltaVmBackups(backups) {
     const handler = this._handler
 
     // this will delete the json, unused VHDs will be detected by `cleanVm`
     await asyncMapSettled(backups, ({ _filename }) => handler.unlink(_filename))
+
+    this.#removeVmBackupsFromCache(backups)
   }
 
   async deleteMetadataBackup(backupId) {
@@ -254,6 +281,8 @@ class RemoteAdapter {
     await asyncMapSettled(backups, ({ _filename, xva }) =>
       Promise.all([handler.unlink(_filename), handler.unlink(resolveRelativeFromFile(_filename, xva))])
     )
+
+    this.#removeVmBackupsFromCache(backups)
   }
 
   deleteVmBackup(file) {
@@ -261,7 +290,8 @@ class RemoteAdapter {
   }
 
   async deleteVmBackups(files) {
-    const { delta, full, ...others } = groupBy(await asyncMap(files, file => this.readVmBackupMetadata(file)), 'mode')
+    const metadatas = await asyncMap(files, file => this.readVmBackupMetadata(file))
+    const { delta, full, ...others } = groupBy(metadatas, 'mode')
 
     const unsupportedModes = Object.keys(others)
     if (unsupportedModes.length !== 0) {
@@ -273,11 +303,13 @@ class RemoteAdapter {
       full !== undefined && this.deleteFullVmBackups(full),
     ])
 
-    const dirs = new Set(files.map(file => dirname(file)))
-    for (const dir of dirs) {
-      // don't merge in main process, unused VHDs will be merged in the next backup run
-      await this.cleanVm(dir, { remove: true, onLog: warn })
-    }
+    await asyncMap(new Set(files.map(file => dirname(file))), dir =>
+      // - don't merge in main process, unused VHDs will be merged in the next backup run
+      // - don't error in case this fails:
+      //   - if lock is already being held, a backup is running and cleanVm will be ran at the end
+      //   - otherwise, there is nothing more we can do, orphan file will be cleaned in the future
+      this.cleanVm(dir, { remove: true, logWarn: warn }).catch(noop)
+    )
   }
 
   #getCompressionType() {
@@ -285,14 +317,17 @@ class RemoteAdapter {
   }
 
   #useVhdDirectory() {
-    return this.handler.type === 's3'
+    return this.handler.useVhdDirectory()
   }
 
   #useAlias() {
     return this.#useVhdDirectory()
   }
 
-  async *getDisk(diskId) {
+
+  async *#getDiskLegacy(diskId) {
+
+    const RE_VHDI = /^vhdi(\d+)$/
     const handler = this._handler
 
     const diskPath = handler._getFilePath('/' + diskId)
@@ -322,6 +357,20 @@ class RemoteAdapter {
     }
   }
 
+  async *getDisk(diskId) {
+    if(this._useGetDiskLegacy){
+      yield * this.#getDiskLegacy(diskId)
+      return
+    }
+    const handler = this._handler
+    // this is a disposable
+    const mountDir = yield getTmpDir()
+    // this is also a disposable
+    yield mount(handler, diskId, mountDir)
+    // this will yield disk path to caller
+    yield `${mountDir}/vhd0`
+  }
+
   // partitionId values:
   //
   // - undefined: raw disk
@@ -372,18 +421,25 @@ class RemoteAdapter {
   listPartitionFiles(diskId, partitionId, path) {
     return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => {
       path = resolveSubpath(rootPath, path)
-
       const entriesMap = {}
-      await asyncMap(await readdir(path), async name => {
-        try {
-          const stats = await stat(`${path}/${name}`)
-          entriesMap[stats.isDirectory() ? `${name}/` : name] = {}
-        } catch (error) {
-          if (error == null || error.code !== 'ENOENT') {
-            throw error
+      await asyncEach(
+        await readdir(path),
+        async name => {
+          try {
+            const stats = await lstat(`${path}/${name}`)
+            if (stats.isDirectory()) {
+              entriesMap[name + '/'] = {}
+            } else if (stats.isFile()) {
+              entriesMap[name] = {}
+            }
+          } catch (error) {
+            if (error == null || error.code !== 'ENOENT') {
+              throw error
+            }
           }
-        }
-      })
+        },
+        { concurrency: 1 }
+      )
 
       return entriesMap
     })
@@ -448,34 +504,114 @@ class RemoteAdapter {
     return backupsByPool
   }
 
-  async listVmBackups(vmUuid, predicate) {
+  #getVmBackupsCache(vmUuid) {
+    return `${BACKUP_DIR}/${vmUuid}/cache.json.gz`
+  }
+
+  async #readCache(path) {
+    try {
+      return JSON.parse(await fromCallback(zlib.gunzip, await this.handler.readFile(path)))
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        warn('#readCache', { error, path })
+      }
+    }
+  }
+
+  _updateCache = synchronized.withKey()(this._updateCache)
+  // eslint-disable-next-line no-dupe-class-members
+  async _updateCache(path, fn) {
+    const cache = await this.#readCache(path)
+    if (cache !== undefined) {
+      fn(cache)
+
+      await this.#writeCache(path, cache)
+    }
+  }
+
+  async #writeCache(path, data) {
+    try {
+      await this.handler.writeFile(path, await fromCallback(zlib.gzip, JSON.stringify(data)), { flags: 'w' })
+    } catch (error) {
+      warn('#writeCache', { error, path })
+    }
+  }
+
+  async invalidateVmBackupListCache(vmUuid) {
+    await this.handler.unlink(this.#getVmBackupsCache(vmUuid))
+  }
+
+  async #getCachabledDataListVmBackups(dir) {
+    debug('generating cache', { path: dir })
+
     const handler = this._handler
-    const backups = []
+    const backups = {}
 
     try {
-      const files = await handler.list(`${BACKUP_DIR}/${vmUuid}`, {
+      const files = await handler.list(dir, {
         filter: isMetadataFile,
         prependDir: true,
       })
       await asyncMap(files, async file => {
         try {
           const metadata = await this.readVmBackupMetadata(file)
-          if (predicate === undefined || predicate(metadata)) {
-            // inject an id usable by importVmBackupNg()
-            metadata.id = metadata._filename
-
-            backups.push(metadata)
-          }
+          // inject an id usable by importVmBackupNg()
+          metadata.id = metadata._filename
+          backups[file] = metadata
         } catch (error) {
-          warn(`listVmBackups ${file}`, { error })
+          warn(`can't read vm backup metadata`, { error, file, dir })
         }
       })
+      return backups
     } catch (error) {
       let code
       if (error == null || ((code = error.code) !== 'ENOENT' && code !== 'ENOTDIR')) {
         throw error
       }
     }
+  }
+
+  // use _ to mark this method as private by convention
+  // since we decorate it with synchronized.withKey in the constructor
+  // and # function are not writeable.
+  //
+  // read the list of backup of a Vm from cache
+  // if cache is missing  or broken  => regenerate it and return
+
+  async _readCacheListVmBackups(vmUuid) {
+    const path = this.#getVmBackupsCache(vmUuid)
+
+    const cache = await this.#readCache(path)
+    if (cache !== undefined) {
+      debug('found VM backups cache, using it', { path })
+      return cache
+    }
+
+    // nothing cached, or cache unreadable => regenerate it
+    const backups = await this.#getCachabledDataListVmBackups(`${BACKUP_DIR}/${vmUuid}`)
+    if (backups === undefined) {
+      return
+    }
+
+    // detached async action, will not reject
+    this.#writeCache(path, backups)
+
+    return backups
+  }
+
+  async listVmBackups(vmUuid, predicate) {
+    const backups = []
+    const cached = await this._readCacheListVmBackups(vmUuid)
+
+    if (cached === undefined) {
+      return []
+    }
+
+    Object.values(cached).forEach(metadata => {
+      if (predicate === undefined || predicate(metadata)) {
+        backups.push(metadata)
+      }
+    })
 
     return backups.sort(compareTimestamp)
   }
@@ -501,13 +637,35 @@ class RemoteAdapter {
     return backups.sort(compareTimestamp)
   }
 
-  async writeVhd(path, input, { checksum = true, validator = noop } = {}) {
+  async writeVmBackupMetadata(vmUuid, metadata) {
+    const path = `/${BACKUP_DIR}/${vmUuid}/${formatFilenameDate(metadata.timestamp)}.json`
+
+    await this.handler.outputFile(path, JSON.stringify(metadata), {
+      dirMode: this._dirMode,
+    })
+
+    // will not throw
+    this._updateCache(this.#getVmBackupsCache(vmUuid), backups => {
+      debug('adding cache entry', { entry: path })
+      backups[path] = {
+        ...metadata,
+
+        // these values are required in the cache
+        _filename: path,
+        id: path,
+      }
+    })
+
+    return path
+  }
+
+  async writeVhd(path, input, { checksum = true, validator = noop, writeBlockConcurrency } = {}) {
     const handler = this._handler
 
     if (this.#useVhdDirectory()) {
       const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
       await createVhdDirectoryFromStream(handler, dataPath, input, {
-        concurrency: 16,
+        concurrency: writeBlockConcurrency,
         compression: this.#getCompressionType(),
         async validator() {
           await input.task
@@ -531,46 +689,27 @@ class RemoteAdapter {
     })
   }
 
-  async _createSyntheticStream(handler, paths) {
-    let disposableVhds = []
-
-    // if it's a path : open all hierarchy of parent
-    if (typeof paths === 'string') {
-      let vhd
-      let vhdPath = paths
-      do {
-        const disposable = await openVhd(handler, vhdPath)
-        vhd = disposable.value
-        disposableVhds.push(disposable)
-        vhdPath = resolveRelativeFromFile(vhdPath, vhd.header.parentUnicodeName)
-      } while (vhd.footer.diskType !== Constants.DISK_TYPES.DYNAMIC)
-    } else {
-      // only open the list of path given
-      disposableVhds = paths.map(path => openVhd(handler, path))
-    }
-
+  // open the  hierarchy of ancestors until we find a full one
+  async _createSyntheticStream(handler, path) {
+    const disposableSynthetic = await VhdSynthetic.fromVhdChain(handler, path)
     // I don't want the vhds to be disposed on return
     // but only when the stream is done ( or failed )
-    const disposables = await Disposable.all(disposableVhds)
-    const vhds = disposables.value
 
     let disposed = false
     const disposeOnce = async () => {
       if (!disposed) {
         disposed = true
-
         try {
-          await disposables.dispose()
+          await disposableSynthetic.dispose()
         } catch (error) {
-          warn('_createSyntheticStream: failed to dispose VHDs', { error })
+          warn('openVhd: failed to dispose VHDs', { error })
         }
       }
     }
-
-    const synthetic = new VhdSynthetic(vhds)
-    await synthetic.readHeaderAndFooter()
+    const synthetic = disposableSynthetic.value
     await synthetic.readBlockAllocationTable()
     const stream = await synthetic.stream()
+
     stream.on('end', disposeOnce)
     stream.on('close', disposeOnce)
     stream.on('error', disposeOnce)
@@ -603,7 +742,10 @@ class RemoteAdapter {
   }
 
   async readVmBackupMetadata(path) {
-    return Object.defineProperty(JSON.parse(await this._handler.readFile(path)), '_filename', { value: path })
+    // _filename is a private field used to compute the backup id
+    //
+    // it's enumerable to make it cacheable
+    return { ...JSON.parse(await this._handler.readFile(path)), _filename: path }
   }
 }
 

@xen-orchestra/backups/Task.js

@@ -3,8 +3,10 @@
 const CancelToken = require('promise-toolbox/CancelToken')
 const Zone = require('node-zone')
 
-const logAfterEnd = () => {
-  throw new Error('task has already ended')
+const logAfterEnd = log => {
+  const error = new Error('task has already ended')
+  error.log = log
+  throw error
 }
 
 const noop = Function.prototype

@xen-orchestra/backups/_VmBackup.js

@@ -45,7 +45,18 @@ const forkDeltaExport = deltaExport =>
   })
 
 class VmBackup {
-  constructor({ config, getSnapshotNameLabel, job, remoteAdapters, remotes, schedule, settings, srs, vm }) {
+  constructor({
+    config,
+    getSnapshotNameLabel,
+    healthCheckSr,
+    job,
+    remoteAdapters,
+    remotes,
+    schedule,
+    settings,
+    srs,
+    vm,
+  }) {
     if (vm.other_config['xo:backup:job'] === job.id && 'start' in vm.blocked_operations) {
       // don't match replicated VMs created by this very job otherwise they
       // will be replicated again and again
@@ -55,7 +66,6 @@ class VmBackup {
     this.config = config
     this.job = job
     this.remoteAdapters = remoteAdapters
-    this.remotes = remotes
     this.scheduleId = schedule.id
     this.timestamp = undefined
 
@@ -69,6 +79,7 @@ class VmBackup {
     this._fullVdisRequired = undefined
     this._getSnapshotNameLabel = getSnapshotNameLabel
     this._isDelta = job.mode === 'delta'
+    this._healthCheckSr = healthCheckSr
     this._jobId = job.id
     this._jobSnapshots = undefined
     this._xapi = vm.$xapi
@@ -95,7 +106,6 @@ class VmBackup {
         : [FullBackupWriter, FullReplicationWriter]
 
       const allSettings = job.settings
-
       Object.keys(remoteAdapters).forEach(remoteId => {
         const targetSettings = {
           ...settings,
@@ -118,35 +128,49 @@ class VmBackup {
   }
 
   // calls fn for each function, warns of any errors, and throws only if there are no writers left
-  async _callWriters(fn, warnMessage, parallel = true) {
+  async _callWriters(fn, step, parallel = true) {
     const writers = this._writers
     const n = writers.size
     if (n === 0) {
       return
     }
-    if (n === 1) {
-      const [writer] = writers
+
+    async function callWriter(writer) {
+      const { name } = writer.constructor
       try {
+        debug('writer step starting', { step, writer: name })
         await fn(writer)
+        debug('writer step succeeded', { duration: step, writer: name })
       } catch (error) {
         writers.delete(writer)
+
+        warn('writer step failed', { error, step, writer: name })
+
+        // these two steps are the only one that are not already in their own sub tasks
+        if (step === 'writer.checkBaseVdis()' || step === 'writer.beforeBackup()') {
+          Task.warning(
+            `the writer ${name} has failed the step ${step} with error ${error.message}. It won't be used anymore in this job execution.`
+          )
+        }
+
         throw error
       }
-      return
+    }
+    if (n === 1) {
+      const [writer] = writers
+      return callWriter(writer)
     }
 
     const errors = []
     await (parallel ? asyncMap : asyncEach)(writers, async function (writer) {
       try {
-        await fn(writer)
+        await callWriter(writer)
       } catch (error) {
         errors.push(error)
-        this.delete(writer)
-        warn(warnMessage, { error, writer: writer.constructor.name })
       }
     })
     if (writers.size === 0) {
-      throw new AggregateError(errors, 'all targets have failed, step: ' + warnMessage)
+      throw new AggregateError(errors, 'all targets have failed, step: ' + step)
     }
   }
 
@@ -173,7 +197,10 @@ class VmBackup {
     const settings = this._settings
 
     const doSnapshot =
-      this._isDelta || (!settings.offlineBackup && vm.power_state === 'Running') || settings.snapshotRetention !== 0
+      settings.unconditionalSnapshot ||
+      this._isDelta ||
+      (!settings.offlineBackup && vm.power_state === 'Running') ||
+      settings.snapshotRetention !== 0
     if (doSnapshot) {
       await Task.run({ name: 'snapshot' }, async () => {
         if (!settings.bypassVdiChainsCheck) {
@@ -181,7 +208,9 @@ class VmBackup {
         }
 
         const snapshotRef = await vm[settings.checkpointSnapshot ? '$checkpoint' : '$snapshot']({
+          ignoreNobakVdis: true,
           name_label: this._getSnapshotNameLabel(vm),
+          unplugVusbs: true,
         })
         this.timestamp = Date.now()
 
@@ -303,22 +332,17 @@ class VmBackup {
   }
 
   async _removeUnusedSnapshots() {
-    const jobSettings = this.job.settings
+    const allSettings = this.job.settings
+    const baseSettings = this._baseSettings
     const baseVmRef = this._baseVm?.$ref
-    const { config } = this
-    const baseSettings = {
-      ...config.defaultSettings,
-      ...config.metadata.defaultSettings,
-      ...jobSettings[''],
-    }
 
     const snapshotsPerSchedule = groupBy(this._jobSnapshots, _ => _.other_config['xo:backup:schedule'])
     const xapi = this._xapi
     await asyncMap(Object.entries(snapshotsPerSchedule), ([scheduleId, snapshots]) => {
       const settings = {
         ...baseSettings,
-        ...jobSettings[scheduleId],
-        ...jobSettings[this.vm.uuid],
+        ...allSettings[scheduleId],
+        ...allSettings[this.vm.uuid],
       }
       return asyncMap(getOldEntries(settings.snapshotRetention, snapshots), ({ $ref }) => {
         if ($ref !== baseVmRef) {
@@ -397,6 +421,24 @@ class VmBackup {
     this._fullVdisRequired = fullVdisRequired
   }
 
+  async _healthCheck() {
+    const settings = this._settings
+
+    if (this._healthCheckSr === undefined) {
+      return
+    }
+
+    // check if current VM has tags
+    const { tags } = this.vm
+    const intersect = settings.healthCheckVmsWithTags.some(t => tags.includes(t))
+
+    if (settings.healthCheckVmsWithTags.length !== 0 && !intersect) {
+      return
+    }
+
+    await this._callWriters(writer => writer.healthCheck(this._healthCheckSr), 'writer.healthCheck()')
+  }
+
   async run($defer) {
     const settings = this._settings
     assert(
@@ -406,7 +448,9 @@ class VmBackup {
 
     await this._callWriters(async writer => {
       await writer.beforeBackup()
-      $defer(() => writer.afterBackup())
+      $defer(async () => {
+        await writer.afterBackup()
+      })
     }, 'writer.beforeBackup()')
 
     await this._fetchJobSnapshots()
@@ -442,6 +486,7 @@ class VmBackup {
       await this._fetchJobSnapshots()
       await this._removeUnusedSnapshots()
     }
+    await this._healthCheck()
   }
 }
 exports.VmBackup = VmBackup

@xen-orchestra/backups/_backupWorker.js

@@ -4,6 +4,8 @@ require('@xen-orchestra/log/configure.js').catchGlobalErrors(
   require('@xen-orchestra/log').createLogger('xo:backups:worker')
 )
 
+require('@vates/cached-dns.lookup').createCachedLookup().patchGlobal()
+
 const Disposable = require('promise-toolbox/Disposable')
 const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const { compose } = require('@vates/compose')

@xen-orchestra/backups/_cleanVm.integ.spec.js

@@ -5,9 +5,9 @@
 const rimraf = require('rimraf')
 const tmp = require('tmp')
 const fs = require('fs-extra')
+const uuid = require('uuid')
 const { getHandler } = require('@xen-orchestra/fs')
 const { pFromCallback } = require('promise-toolbox')
-const crypto = require('crypto')
 const { RemoteAdapter } = require('./RemoteAdapter')
 const { VHDFOOTER, VHDHEADER } = require('./tests.fixtures.js')
 const { VhdFile, Constants, VhdDirectory, VhdAbstract } = require('vhd-lib')
@@ -34,7 +34,8 @@ afterEach(async () => {
   await handler.forget()
 })
 
-const uniqueId = () => crypto.randomBytes(16).toString('hex')
+const uniqueId = () => uuid.v1()
+const uniqueIdBuffer = () => uuid.v1({}, Buffer.alloc(16))
 
 async function generateVhd(path, opts = {}) {
   let vhd
@@ -53,10 +54,9 @@ async function generateVhd(path, opts = {}) {
   }
 
   vhd.header = { ...VHDHEADER, ...opts.header }
-  vhd.footer = { ...VHDFOOTER, ...opts.footer }
-  vhd.footer.uuid = Buffer.from(crypto.randomBytes(16))
+  vhd.footer = { ...VHDFOOTER, ...opts.footer, uuid: uniqueIdBuffer() }
 
-  if (vhd.header.parentUnicodeName) {
+  if (vhd.header.parentUuid) {
     vhd.footer.diskType = Constants.DISK_TYPES.DIFFERENCING
   } else {
     vhd.footer.diskType = Constants.DISK_TYPES.DYNAMIC
@@ -78,48 +78,53 @@ test('It remove broken vhd', async () => {
   await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
   expect((await handler.list(basePath)).length).toEqual(1)
   let loggued = ''
-  const onLog = message => {
+  const logInfo = message => {
     loggued += message
   }
-  await adapter.cleanVm('/', { remove: false, onLog })
-  expect(loggued).toEqual(`error while checking the VHD with path /${basePath}/notReallyAVhd.vhd`)
+  await adapter.cleanVm('/', { remove: false, logInfo, logWarn: logInfo, lock: false })
+  expect(loggued).toEqual(`VHD check error`)
   // not removed
   expect((await handler.list(basePath)).length).toEqual(1)
   // really remove it
-  await adapter.cleanVm('/', { remove: true, onLog })
+  await adapter.cleanVm('/', { remove: true, logInfo, logWarn: () => {}, lock: false })
   expect((await handler.list(basePath)).length).toEqual(0)
 })
 
 test('it remove vhd with missing or multiple ancestors', async () => {
-  // one with a broken parent
+  // one with a broken parent, should be deleted
   await generateVhd(`${basePath}/abandonned.vhd`, {
     header: {
       parentUnicodeName: 'gone.vhd',
-      parentUid: Buffer.from(crypto.randomBytes(16)),
+      parentUuid: uniqueIdBuffer(),
     },
   })
 
-  // one orphan, which is a full vhd, no parent
+  // one orphan, which is a full vhd, no parent : should stay
   const orphan = await generateVhd(`${basePath}/orphan.vhd`)
-  // a child to the orphan
+  // a child to the orphan in the metadata : should stay
   await generateVhd(`${basePath}/child.vhd`, {
     header: {
       parentUnicodeName: 'orphan.vhd',
-      parentUid: orphan.footer.uuid,
+      parentUuid: orphan.footer.uuid,
     },
   })
-
+  await handler.writeFile(
+    `metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      vhds: [`${basePath}/child.vhd`, `${basePath}/abandonned.vhd`],
+    }),
+    { flags: 'w' }
+  )
   // clean
   let loggued = ''
-  const onLog = message => {
+  const logInfo = message => {
     loggued += message + '\n'
   }
-  await adapter.cleanVm('/', { remove: true, onLog })
+  await adapter.cleanVm('/', { remove: true, logInfo, logWarn: logInfo, lock: false })
 
   const deletedOrphanVhd = loggued.match(/deleting orphan VHD/g) || []
   expect(deletedOrphanVhd.length).toEqual(1) // only one vhd should have been deleted
-  const deletedAbandonnedVhd = loggued.match(/abandonned.vhd is missing/g) || []
-  expect(deletedAbandonnedVhd.length).toEqual(1) // and it must be abandonned.vhd
 
   // we don't test the filew on disk, since they will all be marker as unused and deleted without a metadata.json file
 })
@@ -147,19 +152,17 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy
   await generateVhd(`${basePath}/child.vhd`, {
     header: {
       parentUnicodeName: 'orphan.vhd',
-      parentUid: orphan.footer.uuid,
+      parentUuid: orphan.footer.uuid,
     },
   })
 
   let loggued = ''
-  const onLog = message => {
+  const logInfo = message => {
     loggued += message + '\n'
   }
-  await adapter.cleanVm('/', { remove: true, onLog })
-  let matched = loggued.match(/deleting unused VHD /g) || []
+  await adapter.cleanVm('/', { remove: true, logInfo, logWarn: logInfo, lock: false })
+  let matched = loggued.match(/deleting unused VHD/g) || []
   expect(matched.length).toEqual(1) // only one vhd should have been deleted
-  matched = loggued.match(/abandonned.vhd is unused/g) || []
-  expect(matched.length).toEqual(1) // and it must be abandonned.vhd
 
   // a missing vhd cause clean to remove all vhds
   await handler.writeFile(
@@ -176,8 +179,8 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy
     { flags: 'w' }
   )
   loggued = ''
-  await adapter.cleanVm('/', { remove: true, onLog })
-  matched = loggued.match(/deleting unused VHD /g) || []
+  await adapter.cleanVm('/', { remove: true, logInfo, logWarn: () => {}, lock: false })
+  matched = loggued.match(/deleting unused VHD/g) || []
   expect(matched.length).toEqual(2) // all vhds (orphan and  child  ) should have been deleted
 })
 
@@ -201,30 +204,28 @@ test('it merges delta of non destroyed chain', async () => {
   const child = await generateVhd(`${basePath}/child.vhd`, {
     header: {
       parentUnicodeName: 'orphan.vhd',
-      parentUid: orphan.footer.uuid,
+      parentUuid: orphan.footer.uuid,
     },
   })
   // a grand child
   await generateVhd(`${basePath}/grandchild.vhd`, {
     header: {
       parentUnicodeName: 'child.vhd',
-      parentUid: child.footer.uuid,
+      parentUuid: child.footer.uuid,
     },
   })
 
   let loggued = []
-  const onLog = message => {
+  const logInfo = message => {
     loggued.push(message)
   }
-  await adapter.cleanVm('/', { remove: true, onLog })
-  expect(loggued[0]).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused`)
-  expect(loggued[1]).toEqual(`incorrect size in metadata: 12000 instead of 209920`)
+  await adapter.cleanVm('/', { remove: true, logInfo, logWarn: logInfo, lock: false })
+  expect(loggued[0]).toEqual(`incorrect backup size in metadata`)
 
   loggued = []
-  await adapter.cleanVm('/', { remove: true, merge: true, onLog })
-  const [unused, merging] = loggued
-  expect(unused).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused`)
-  expect(merging).toEqual(`merging /${basePath}/child.vhd into /${basePath}/orphan.vhd`)
+  await adapter.cleanVm('/', { remove: true, merge: true, logInfo, logWarn: () => {}, lock: false })
+  const [merging] = loggued
+  expect(merging).toEqual(`merging VHD chain`)
 
   const metadata = JSON.parse(await handler.readFile(`metadata.json`))
   // size should be the size of children + grand children after the merge
@@ -254,7 +255,7 @@ test('it finish unterminated merge ', async () => {
   const child = await generateVhd(`${basePath}/child.vhd`, {
     header: {
       parentUnicodeName: 'orphan.vhd',
-      parentUid: orphan.footer.uuid,
+      parentUuid: orphan.footer.uuid,
     },
   })
   // a merge in progress file
@@ -270,7 +271,7 @@ test('it finish unterminated merge ', async () => {
     })
   )
 
-  await adapter.cleanVm('/', { remove: true, merge: true })
+  await adapter.cleanVm('/', { remove: true, merge: true, logWarn: () => {}, lock: false })
   // merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
 
   // only check deletion
@@ -310,7 +311,7 @@ describe('tests multiple combination ', () => {
           mode: vhdMode,
           header: {
             parentUnicodeName: 'gone.vhd',
-            parentUid: crypto.randomBytes(16),
+            parentUuid: uniqueIdBuffer(),
           },
         })
 
@@ -324,7 +325,7 @@ describe('tests multiple combination ', () => {
           mode: vhdMode,
           header: {
             parentUnicodeName: 'ancestor.vhd' + (useAlias ? '.alias.vhd' : ''),
-            parentUid: ancestor.footer.uuid,
+            parentUuid: ancestor.footer.uuid,
           },
         })
         // a grand child  vhd in metadata
@@ -333,7 +334,7 @@ describe('tests multiple combination ', () => {
           mode: vhdMode,
           header: {
             parentUnicodeName: 'child.vhd' + (useAlias ? '.alias.vhd' : ''),
-            parentUid: child.footer.uuid,
+            parentUuid: child.footer.uuid,
           },
         })
 
@@ -348,7 +349,7 @@ describe('tests multiple combination ', () => {
           mode: vhdMode,
           header: {
             parentUnicodeName: 'cleanAncestor.vhd' + (useAlias ? '.alias.vhd' : ''),
-            parentUid: cleanAncestor.footer.uuid,
+            parentUuid: cleanAncestor.footer.uuid,
           },
         })
 
@@ -377,7 +378,7 @@ describe('tests multiple combination ', () => {
           })
         )
 
-        await adapter.cleanVm('/', { remove: true, merge: true })
+        await adapter.cleanVm('/', { remove: true, merge: true, logWarn: () => {}, lock: false })
 
         const metadata = JSON.parse(await handler.readFile(`metadata.json`))
         // size should be the size of children + grand children + clean after the merge
@@ -413,7 +414,7 @@ describe('tests multiple combination ', () => {
 test('it cleans orphan merge states ', async () => {
   await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
 
-  await adapter.cleanVm('/', { remove: true })
+  await adapter.cleanVm('/', { remove: true, logWarn: () => {}, lock: false })
 
   expect(await handler.list(basePath)).toEqual([])
 })
@@ -428,7 +429,11 @@ test('check Aliases should work alone', async () => {
 
   await generateVhd(`vhds/data/missingalias.vhd`)
 
-  await checkAliases(['vhds/missingData.alias.vhd', 'vhds/ok.alias.vhd'], 'vhds/data', { remove: true, handler })
+  await checkAliases(['vhds/missingData.alias.vhd', 'vhds/ok.alias.vhd'], 'vhds/data', {
+    remove: true,
+    handler,
+    logWarn: () => {},
+  })
 
   // only ok have suvived
   const alias = (await handler.list('vhds')).filter(f => f.endsWith('.vhd'))

@xen-orchestra/backups/_cleanVm.js

@@ -1,22 +1,27 @@
 'use strict'
 
-const assert = require('assert')
 const sum = require('lodash/sum')
+const UUID = require('uuid')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { Constants, mergeVhd, openVhd, VhdAbstract, VhdFile } = require('vhd-lib')
+const { Constants, openVhd, VhdAbstract, VhdFile } = require('vhd-lib')
 const { isVhdAlias, resolveVhdAlias } = require('vhd-lib/aliases')
 const { dirname, resolve } = require('path')
 const { DISK_TYPES } = Constants
 const { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } = require('./_backupType.js')
 const { limitConcurrency } = require('limit-concurrency-decorator')
+const { mergeVhdChain } = require('vhd-lib/merge')
 
 const { Task } = require('./Task.js')
 const { Disposable } = require('promise-toolbox')
+const handlerPath = require('@xen-orchestra/fs/path')
 
 // checking the size of a vhd directory is costly
 // 1 Http Query per 1000 blocks
 // we only check size of all the vhd are VhdFiles
-function shouldComputeVhdsSize(vhds) {
+function shouldComputeVhdsSize(handler, vhds) {
+  if (handler.isEncrypted) {
+    return false
+  }
   return vhds.every(vhd => vhd instanceof VhdFile)
 }
 
@@ -24,86 +29,49 @@ const computeVhdsSize = (handler, vhdPaths) =>
   Disposable.use(
     vhdPaths.map(vhdPath => openVhd(handler, vhdPath)),
     async vhds => {
-      if (shouldComputeVhdsSize(vhds)) {
+      if (shouldComputeVhdsSize(handler, vhds)) {
         const sizes = await asyncMap(vhds, vhd => vhd.getSize())
         return sum(sizes)
       }
     }
   )
 
-// chain is an array of VHDs from child to parent
-//
-// the whole chain will be merged into parent, parent will be renamed to child
-// and all the others will deleted
-async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
-  assert(chain.length >= 2)
-
-  let child = chain[0]
-  const parent = chain[chain.length - 1]
-  const children = chain.slice(0, -1).reverse()
-
-  chain
-    .slice(1)
-    .reverse()
-    .forEach(parent => {
-      onLog(`the parent ${parent} of the child ${child} is unused`)
-    })
-
+// chain is [ ancestor, child_1, ..., child_n ]
+async function _mergeVhdChain(handler, chain, { logInfo, remove, merge, mergeBlockConcurrency }) {
   if (merge) {
-    // `mergeVhd` does not work with a stream, either
-    // - make it accept a stream
-    // - or create synthetic VHD which is not a stream
-    if (children.length !== 1) {
-      // TODO: implement merging multiple children
-      children.length = 1
-      child = children[0]
-    }
-
-    onLog(`merging ${child} into ${parent}`)
+    logInfo(`merging VHD chain`, { chain })
 
     let done, total
     const handle = setInterval(() => {
       if (done !== undefined) {
-        onLog(`merging ${child}: ${done}/${total}`)
+        logInfo('merge in progress', {
+          done,
+          parent: chain[0],
+          progress: Math.round((100 * done) / total),
+          total,
+        })
       }
     }, 10e3)
-
-    const mergedSize = await mergeVhd(
-      handler,
-      parent,
-      handler,
-      child,
-      // children.length === 1
-      //   ? child
-      //   : await createSyntheticStream(handler, children),
-      {
+    try {
+      return await mergeVhdChain(handler, chain, {
+        logInfo,
+        mergeBlockConcurrency,
         onProgress({ done: d, total: t }) {
           done = d
           total = t
         },
-      }
-    )
-
-    clearInterval(handle)
-    await Promise.all([
-      VhdAbstract.rename(handler, parent, child),
-      asyncMap(children.slice(0, -1), child => {
-        onLog(`the VHD ${child} is unused`)
-        if (remove) {
-          onLog(`deleting unused VHD ${child}`)
-          return VhdAbstract.unlink(handler, child)
-        }
-      }),
-    ])
-
-    return mergedSize
+        removeUnused: remove,
+      })
+    } finally {
+      clearInterval(handle)
+    }
   }
 }
 
 const noop = Function.prototype
 
 const INTERRUPTED_VHDS_REG = /^\.(.+)\.merge.json$/
-const listVhds = async (handler, vmDir) => {
+const listVhds = async (handler, vmDir, logWarn) => {
   const vhds = new Set()
   const aliases = {}
   const interruptedVhds = new Map()
@@ -123,12 +91,23 @@ const listVhds = async (handler, vmDir) => {
             filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file),
           })
           aliases[vdiDir] = list.filter(vhd => isVhdAlias(vhd)).map(file => `${vdiDir}/${file}`)
-          list.forEach(file => {
+
+          await asyncMap(list, async file => {
             const res = INTERRUPTED_VHDS_REG.exec(file)
             if (res === null) {
               vhds.add(`${vdiDir}/${file}`)
             } else {
-              interruptedVhds.set(`${vdiDir}/${res[1]}`, `${vdiDir}/${file}`)
+              try {
+                const mergeState = JSON.parse(await handler.readFile(`${vdiDir}/${file}`))
+                interruptedVhds.set(`${vdiDir}/${res[1]}`, {
+                  statePath: `${vdiDir}/${file}`,
+                  chain: mergeState.chain,
+                })
+              } catch (error) {
+                // fall back to a non resuming merge
+                vhds.add(`${vdiDir}/${file}`)
+                logWarn('failed to read existing merge state', { path: file, error })
+              }
             }
           })
         }
@@ -138,16 +117,21 @@ const listVhds = async (handler, vmDir) => {
   return { vhds, interruptedVhds, aliases }
 }
 
-async function checkAliases(aliasPaths, targetDataRepository, { handler, onLog = noop, remove = false }) {
+async function checkAliases(
+  aliasPaths,
+  targetDataRepository,
+  { handler, logInfo = noop, logWarn = console.warn, remove = false }
+) {
   const aliasFound = []
-  for (const path of aliasPaths) {
-    const target = await resolveVhdAlias(handler, path)
+  for (const alias of aliasPaths) {
+    const target = await resolveVhdAlias(handler, alias)
 
     if (!isVhdFile(target)) {
-      onLog(`Alias ${path} references a non vhd target:  ${target}`)
+      logWarn('alias references non VHD target', { alias, target })
       if (remove) {
+        logInfo('removing alias and non VHD target', { alias, target })
         await handler.unlink(target)
-        await handler.unlink(path)
+        await handler.unlink(alias)
       }
       continue
     }
@@ -160,13 +144,13 @@ async function checkAliases(aliasPaths, targetDataRepository, { handler, onLog =
         // error during dispose should not trigger a deletion
       }
     } catch (error) {
-      onLog(`target ${target} of alias ${path} is missing or broken`, { error })
+      logWarn('missing or broken alias target', { alias, target, error })
       if (remove) {
         try {
-          await VhdAbstract.unlink(handler, path)
-        } catch (e) {
-          if (e.code !== 'ENOENT') {
-            onLog(`Error while deleting target ${target} of alias ${path}`, { error: e })
+          await VhdAbstract.unlink(handler, alias)
+        } catch (error) {
+          if (error.code !== 'ENOENT') {
+            logWarn('error deleting alias target', { alias, target, error })
           }
         }
       }
@@ -176,37 +160,48 @@ async function checkAliases(aliasPaths, targetDataRepository, { handler, onLog =
     aliasFound.push(resolve('/', target))
   }
 
-  const entries = await handler.list(targetDataRepository, {
+  const vhds = await handler.list(targetDataRepository, {
     ignoreMissing: true,
     prependDir: true,
   })
 
-  entries.forEach(async entry => {
-    if (!aliasFound.includes(entry)) {
-      onLog(`the Vhd  ${entry} is not referenced by a an alias`)
+  await asyncMap(vhds, async path => {
+    if (!aliasFound.includes(path)) {
+      logWarn('no alias references VHD', { path })
       if (remove) {
-        await VhdAbstract.unlink(handler, entry)
+        logInfo('deleting unused VHD', { path })
+        await VhdAbstract.unlink(handler, path)
       }
     }
   })
 }
+
 exports.checkAliases = checkAliases
 
 const defaultMergeLimiter = limitConcurrency(1)
 
 exports.cleanVm = async function cleanVm(
   vmDir,
-  { fixMetadata, remove, merge, mergeLimiter = defaultMergeLimiter, onLog = noop }
+  {
+    fixMetadata,
+    remove,
+    merge,
+    mergeBlockConcurrency,
+    mergeLimiter = defaultMergeLimiter,
+    logInfo = noop,
+    logWarn = console.warn,
+  }
 ) {
-  const limitedMergeVhdChain = mergeLimiter(mergeVhdChain)
+  const limitedMergeVhdChain = mergeLimiter(_mergeVhdChain)
 
   const handler = this._handler
 
   const vhdsToJSons = new Set()
+  const vhdById = new Map()
   const vhdParents = { __proto__: null }
   const vhdChildren = { __proto__: null }
 
-  const { vhds, interruptedVhds, aliases } = await listVhds(handler, vmDir)
+  const { vhds, interruptedVhds, aliases } = await listVhds(handler, vmDir, logWarn)
 
   // remove broken VHDs
   await asyncMap(vhds, async path => {
@@ -224,12 +219,31 @@ exports.cleanVm = async function cleanVm(
           }
           vhdChildren[parent] = path
         }
+        // Detect VHDs with the same UUIDs
+        //
+        // Due to a bug introduced in a1bcd35e2
+        const duplicate = vhdById.get(UUID.stringify(vhd.footer.uuid))
+        let vhdKept = vhd
+        if (duplicate !== undefined) {
+          logWarn('uuid is duplicated', { uuid: UUID.stringify(vhd.footer.uuid) })
+          if (duplicate.containsAllDataOf(vhd)) {
+            logWarn(`should delete ${path}`)
+            vhdKept = duplicate
+            vhds.delete(path)
+          } else if (vhd.containsAllDataOf(duplicate)) {
+            logWarn(`should delete ${duplicate._path}`)
+            vhds.delete(duplicate._path)
+          } else {
+            logWarn('same ids but different content')
+          }
+        }
+        vhdById.set(UUID.stringify(vhdKept.footer.uuid), vhdKept)
       })
     } catch (error) {
       vhds.delete(path)
-      onLog(`error while checking the VHD with path ${path}`, { error })
+      logWarn('VHD check error', { path, error })
       if (error?.code === 'ERR_ASSERTION' && remove) {
-        onLog(`deleting broken ${path}`)
+        logInfo('deleting broken VHD', { path })
         return VhdAbstract.unlink(handler, path)
       }
     }
@@ -238,15 +252,15 @@ exports.cleanVm = async function cleanVm(
   // remove interrupted merge states for missing VHDs
   for (const interruptedVhd of interruptedVhds.keys()) {
     if (!vhds.has(interruptedVhd)) {
-      const statePath = interruptedVhds.get(interruptedVhd)
+      const { statePath } = interruptedVhds.get(interruptedVhd)
       interruptedVhds.delete(interruptedVhd)
 
-      onLog('orphan merge state', {
+      logWarn('orphan merge state', {
         mergeStatePath: statePath,
         missingVhdPath: interruptedVhd,
       })
       if (remove) {
-        onLog(`deleting orphan merge state ${statePath}`)
+        logInfo('deleting orphan merge state', { statePath })
         await handler.unlink(statePath)
       }
     }
@@ -255,7 +269,7 @@ exports.cleanVm = async function cleanVm(
   // check if alias are correct
   // check if all vhd in data subfolder have a corresponding alias
   await asyncMap(Object.keys(aliases), async dir => {
-    await checkAliases(aliases[dir], `${dir}/data`, { handler, onLog, remove })
+    await checkAliases(aliases[dir], `${dir}/data`, { handler, logInfo, logWarn, remove })
   })
 
   // remove VHDs with missing ancestors
@@ -277,9 +291,9 @@ exports.cleanVm = async function cleanVm(
       if (!vhds.has(parent)) {
         vhds.delete(vhdPath)
 
-        onLog(`the parent ${parent} of the VHD ${vhdPath} is missing`)
+        logWarn('parent VHD is missing', { parent, child: vhdPath })
         if (remove) {
-          onLog(`deleting orphan VHD ${vhdPath}`)
+          logInfo('deleting orphan VHD', { path: vhdPath })
           deletions.push(VhdAbstract.unlink(handler, vhdPath))
         }
       }
@@ -297,6 +311,7 @@ exports.cleanVm = async function cleanVm(
   }
 
   const jsons = new Set()
+  let mustInvalidateCache = false
   const xvas = new Set()
   const xvaSums = []
   const entries = await handler.list(vmDir, {
@@ -316,7 +331,7 @@ exports.cleanVm = async function cleanVm(
     // check is not good enough to delete the file, the best we can do is report
     // it
     if (!(await this.isValidXva(path))) {
-      onLog(`the XVA with path ${path} is potentially broken`)
+      logWarn('XVA might be broken', { path })
     }
   })
 
@@ -330,7 +345,7 @@ exports.cleanVm = async function cleanVm(
     try {
       metadata = JSON.parse(await handler.readFile(json))
     } catch (error) {
-      onLog(`failed to read metadata file ${json}`, { error })
+      logWarn('failed to read backup metadata', { path: json, error })
       jsons.delete(json)
       return
     }
@@ -341,10 +356,11 @@ exports.cleanVm = async function cleanVm(
       if (xvas.has(linkedXva)) {
         unusedXvas.delete(linkedXva)
       } else {
-        onLog(`the XVA linked to the metadata ${json} is missing`)
+        logWarn('the XVA linked to the backup is missing', { backup: json, xva: linkedXva })
         if (remove) {
-          onLog(`deleting incomplete backup ${json}`)
+          logInfo('deleting incomplete backup', { path: json })
           jsons.delete(json)
+          mustInvalidateCache = true
           await handler.unlink(json)
         }
       }
@@ -364,9 +380,10 @@ exports.cleanVm = async function cleanVm(
           vhdsToJSons[path] = json
         })
       } else {
-        onLog(`Some VHDs linked to the metadata ${json} are missing`, { missingVhds })
+        logWarn('some VHDs linked to the backup are missing', { backup: json, missingVhds })
         if (remove) {
-          onLog(`deleting incomplete backup ${json}`)
+          logInfo('deleting incomplete backup', { path: json })
+          mustInvalidateCache = true
           jsons.delete(json)
           await handler.unlink(json)
         }
@@ -378,7 +395,7 @@ exports.cleanVm = async function cleanVm(
   const unusedVhdsDeletion = []
   const toMerge = []
   {
-    // VHD chains (as list from child to ancestor) to merge indexed by last
+    // VHD chains (as list from oldest to most recent) to merge indexed by most recent
     // ancestor
     const vhdChainsToMerge = { __proto__: null }
 
@@ -402,14 +419,14 @@ exports.cleanVm = async function cleanVm(
       if (child !== undefined) {
         const chain = getUsedChildChainOrDelete(child)
         if (chain !== undefined) {
-          chain.push(vhd)
+          chain.unshift(vhd)
           return chain
         }
       }
 
-      onLog(`the VHD ${vhd} is unused`)
+      logWarn('unused VHD', { path: vhd })
       if (remove) {
-        onLog(`deleting unused VHD ${vhd}`)
+        logInfo('deleting unused VHD', { path: vhd })
         unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
       }
     }
@@ -420,7 +437,13 @@ exports.cleanVm = async function cleanVm(
 
     // merge interrupted VHDs
     for (const parent of interruptedVhds.keys()) {
-      vhdChainsToMerge[parent] = [vhdChildren[parent], parent]
+      // before #6349 the chain wasn't in the mergeState
+      const { chain, statePath } = interruptedVhds.get(parent)
+      if (chain === undefined) {
+        vhdChainsToMerge[parent] = [parent, vhdChildren[parent]]
+      } else {
+        vhdChainsToMerge[parent] = chain.map(vhdPath => handlerPath.resolveFromFile(statePath, vhdPath))
+      }
     }
 
     Object.values(vhdChainsToMerge).forEach(chain => {
@@ -433,9 +456,15 @@ exports.cleanVm = async function cleanVm(
   const metadataWithMergedVhd = {}
   const doMerge = async () => {
     await asyncMap(toMerge, async chain => {
-      const merged = await limitedMergeVhdChain(chain, { handler, onLog, remove, merge })
+      const merged = await limitedMergeVhdChain(handler, chain, {
+        logInfo,
+        logWarn,
+        remove,
+        merge,
+        mergeBlockConcurrency,
+      })
       if (merged !== undefined) {
-        const metadataPath = vhdsToJSons[chain[0]] // all the chain should have the same metada file
+        const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metada file
         metadataWithMergedVhd[metadataPath] = true
       }
     })
@@ -445,18 +474,18 @@ exports.cleanVm = async function cleanVm(
     ...unusedVhdsDeletion,
     toMerge.length !== 0 && (merge ? Task.run({ name: 'merge' }, doMerge) : doMerge()),
     asyncMap(unusedXvas, path => {
-      onLog(`the XVA ${path} is unused`)
+      logWarn('unused XVA', { path })
       if (remove) {
-        onLog(`deleting unused XVA ${path}`)
+        logInfo('deleting unused XVA', { path })
         return handler.unlink(path)
       }
     }),
     asyncMap(xvaSums, path => {
       // no need to handle checksums for XVAs deleted by the script, they will be handled by `unlink()`
       if (!xvas.has(path.slice(0, -'.checksum'.length))) {
-        onLog(`the XVA checksum ${path} is unused`)
+        logInfo('unused XVA checksum', { path })
         if (remove) {
-          onLog(`deleting unused XVA checksum ${path}`)
+          logInfo('deleting unused XVA checksum', { path })
           return handler.unlink(path)
         }
       }
@@ -478,7 +507,11 @@ exports.cleanVm = async function cleanVm(
       if (mode === 'full') {
         // a full backup : check size
         const linkedXva = resolve('/', vmDir, xva)
-        fileSystemSize = await handler.getSize(linkedXva)
+        try {
+          fileSystemSize = await handler.getSize(linkedXva)
+        } catch (error) {
+          // can fail with encrypted remote
+        }
       } else if (mode === 'delta') {
         const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
         fileSystemSize = await computeVhdsSize(handler, linkedVhds)
@@ -490,11 +523,15 @@ exports.cleanVm = async function cleanVm(
 
         // don't warn if the size has changed after a merge
         if (!merged && fileSystemSize !== size) {
-          onLog(`incorrect size in metadata: ${size ?? 'none'} instead of ${fileSystemSize}`)
+          logWarn('incorrect backup size in metadata', {
+            path: metadataPath,
+            actual: size ?? 'none',
+            expected: fileSystemSize,
+          })
         }
       }
     } catch (error) {
-      onLog(`failed to get size of ${metadataPath}`, { error })
+      logWarn('failed to get backup size', { backup: metadataPath, error })
       return
     }
 
@@ -504,11 +541,16 @@ exports.cleanVm = async function cleanVm(
       try {
         await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })
       } catch (error) {
-        onLog(`failed to update size in backup metadata ${metadataPath} after merge`, { error })
+        logWarn('failed to update backup size in metadata', { path: metadataPath, error })
       }
     }
   })
 
+  // purge cache if a metadata file has been deleted
+  if (mustInvalidateCache) {
+    await handler.unlink(vmDir + '/cache.json.gz')
+  }
+
   return {
     // boolean whether some VHDs were merged (or should be merged)
     merge: toMerge.length !== 0,

@xen-orchestra/backups/_deltaVm.js

@@ -1,12 +1,12 @@
 'use strict'
 
-const compareVersions = require('compare-versions')
 const find = require('lodash/find.js')
 const groupBy = require('lodash/groupBy.js')
 const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const omit = require('lodash/omit.js')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { CancelToken } = require('promise-toolbox')
+const { compareVersions } = require('compare-versions')
 const { createVhdStreamWithLength } = require('vhd-lib')
 const { defer } = require('golike-defer')
 
@@ -65,17 +65,6 @@ exports.exportDeltaVm = async function exportDeltaVm(
       return
     }
 
-    // If the VDI name start with `[NOBAK]`, do not export it.
-    if (vdi.name_label.startsWith('[NOBAK]')) {
-      // FIXME: find a way to not create the VDI snapshot in the
-      // first time.
-      //
-      // The snapshot must not exist otherwise it could break the
-      // next export.
-      ignoreErrors.call(vdi.$destroy())
-      return
-    }
-
     vbds[vbd.$ref] = vbd
 
     const vdiRef = vdi.$ref

@xen-orchestra/backups/_forkStreamUnpipe.js

@@ -3,6 +3,8 @@
 const eos = require('end-of-stream')
 const { PassThrough } = require('stream')
 
+const { debug } = require('@xen-orchestra/log').createLogger('xo:backups:forkStreamUnpipe')
+
 // create a new readable stream from an existing one which may be piped later
 //
 // in case of error in the new readable stream, it will simply be unpiped
@@ -11,18 +13,23 @@ exports.forkStreamUnpipe = function forkStreamUnpipe(stream) {
   const { forks = 0 } = stream
   stream.forks = forks + 1
 
+  debug('forking', { forks: stream.forks })
+
   const proxy = new PassThrough()
   stream.pipe(proxy)
   eos(stream, error => {
     if (error !== undefined) {
+      debug('error on original stream, destroying fork', { error })
       proxy.destroy(error)
     }
   })
-  eos(proxy, _ => {
-    stream.forks--
+  eos(proxy, error => {
+    debug('end of stream, unpiping', { error, forks: --stream.forks })
+
     stream.unpipe(proxy)
 
     if (stream.forks === 0) {
+      debug('no more forks, destroying original stream')
       stream.destroy(new Error('no more consumers for this stream'))
     }
   })

@xen-orchestra/backups/_isValidXva.js

@@ -49,6 +49,11 @@ const isValidTar = async (handler, size, fd) => {
 // TODO: find an heuristic for compressed files
 async function isValidXva(path) {
   const handler = this._handler
+
+  // size is longer when encrypted  + reading part of an encrypted file is not implemented
+  if (handler.isEncrypted) {
+    return true
+  }
   try {
     const fd = await handler.openFile(path, 'r')
     try {
@@ -66,7 +71,6 @@ async function isValidXva(path) {
     }
   } catch (error) {
     // never throw, log and report as valid to avoid side effects
-    console.error('isValidXva', path, error)
     return true
   }
 }

@xen-orchestra/backups/docs/VM backups/README.md

@@ -6,15 +6,22 @@
 - [Task logs](#task-logs)
   - [During backup](#during-backup)
   - [During restoration](#during-restoration)
+- [API](#api)
+  - [Run description object](#run-description-object)
+  - [`IdPattern`](#idpattern)
+  - [Settings](#settings)
+- [Writer API](#writer-api)
 
 ## File structure on remote
 
+### with vhd files
+
 ```
 <remote>
 └─ xo-vm-backups
   ├─ index.json // TODO
   └─ <VM UUID>
-     ├─ index.json // TODO
+     ├─ cache.json.gz
      ├─ vdis
      │  └─ <job UUID>
      │     └─ <VDI UUID>
@@ -25,6 +32,31 @@
      └─ <YYYYMMDD>T<HHmmss>.xva.checksum
 ```
 
+### with vhd directories
+
+When `useVhdDirectory` is enabled on the remote, the directory containing the VHDs has a slightly different architecture:
+
+```
+<vdis>/<job UUID>/<VDI UUID>
+  ├─ <YYYYMMDD>T<HHmmss>.alias.vhd // contains the relative path to a VHD directory
+  ├─ <YYYYMMDD>T<HHmmss>.alias.vhd
+  └─ data
+    ├─ <uuid>.vhd // VHD directory format is described in vhd-lib/Vhd/VhdDirectory.js
+    └─ <uuid>.vhd
+```
+
+## Cache for a VM
+
+In a VM directory, if the file `cache.json.gz` exists, it contains the metadata for all the backups for this VM.
+
+Add the following file: `xo-vm-backups/<VM UUID>/cache.json.gz`.
+
+This cache is compressed in Gzip and contains an JSON object with the metadata for all the backups of this VM indexed by their absolute path (i.e. `/xo-vm-backups/<VM UUID>/<timestamp>.json`).
+
+This file is generated on demande when listing the backups, and directly updated on backup creation/deletion.
+
+In case any incoherence is detected, the file is deleted so it will be fully generated when required.
+
 ## Attributes
 
 ### Of created snapshots
@@ -64,24 +96,30 @@ job.start(data: { mode: Mode, reportWhen: ReportWhen })
 ├─ task.warning(message: string)
 ├─ task.start(data: { type: 'VM', id: string })
 │  ├─ task.warning(message: string)
+|  ├─ task.start(message: 'clean-vm')
+│  │  └─ task.end
 │  ├─ task.start(message: 'snapshot')
 │  │  └─ task.end
-│  ├─ task.start(message: 'export', data: { type: 'SR' | 'remote', id: string })
+│  ├─ task.start(message: 'export', data: { type: 'SR' | 'remote', id: string, isFull: boolean })
 │  │  ├─ task.warning(message: string)
 │  │  ├─ task.start(message: 'transfer')
 │  │  │  ├─ task.warning(message: string)
 │  │  │  └─ task.end(result: { size: number })
 │  │  │
+│  │  │  // in case there is a healthcheck scheduled for this vm in this job
+│  │  ├─ task.start(message: 'health check')
+│  │  │  ├─ task.start(message: 'transfer')
+│  │  │  │  └─ task.end(result: { size: number })
+│  │  │  ├─ task.start(message: 'vmstart')
+│  │  │  │  └─ task.end
+│  │  │  └─ task.end
+│  │  │
 │  │  │  // in case of full backup, DR and CR
 │  │  ├─ task.start(message: 'clean')
 │  │  │  ├─ task.warning(message: string)
 │  │  │  └─ task.end
-│  │  │
-│  │  │ // in case of delta backup
-│  │  ├─ task.start(message: 'merge')
-│  │  │  ├─ task.warning(message: string)
-│  │  │  └─ task.end(result: { size: number })
-│  │  │
+│  │  └─ task.end
+|  ├─ task.start(message: 'clean-vm')
 │  │  └─ task.end
 │  └─ task.end
 └─ job.end
@@ -95,3 +133,102 @@ task.start(message: 'restore', data: { jobId: string, srId: string, time: number
 │  └─ task.end(result: { id: string, size: number })
 └─ task.end
 ```
+
+## API
+
+### Run description object
+
+This is a JavaScript object containing all the information necessary to run a backup job.
+
+```coffee
+# Information about the job itself
+job:
+
+  # Unique identifier
+  id: string
+
+  # Human readable identifier
+  name: string
+
+  # Whether this job is doing Full Backup / Disaster Recovery or
+  # Delta Backup / Continuous Replication
+  mode: 'full' | 'delta'
+
+  # For backup jobs, indicates which remotes to use
+  remotes: IdPattern
+
+  settings:
+
+    # Used for the whole job
+    '': Settings
+
+    # Used for a specific schedule
+    [ScheduleId]: Settings
+
+    # Used for a specific VM
+    [VmId]: Settings
+
+  # For replication jobs, indicates which SRs to use
+  srs: IdPattern
+
+  # Here for historical reasons
+  type: 'backup'
+
+  # Indicates which VMs to backup/replicate
+  vms: IdPattern
+
+# Indicates which XAPI to use to connect to a specific VM or SR
+recordToXapi:
+  [ObjectId]: XapiId
+
+# Information necessary to connect to each remote
+remotes:
+  [RemoteId]:
+    url: string
+
+# Indicates which schedule is used for this run
+schedule:
+  id: ScheduleId
+
+# Information necessary to connect to each XAPI
+xapis:
+  [XapiId]:
+    allowUnauthorized: boolean
+    credentials:
+      password: string
+      username: string
+    url: string
+```
+
+### `IdPattern`
+
+For a single object:
+
+```
+{ id: string }
+```
+
+For multiple objects:
+
+```
+{ id: { __or: string[] } }
+```
+
+> This syntax is compatible with [`value-matcher`](https://github.com/vatesfr/xen-orchestra/tree/master/packages/value-matcher).
+
+### Settings
+
+Settings are described in [`@xen-orchestra/backups/Backup.js](https://github.com/vatesfr/xen-orchestra/blob/master/%40xen-orchestra/backups/Backup.js).
+
+## Writer API
+
+- `beforeBackup()`
+  - **Delta**
+    - `checkBaseVdis(baseUuidToSrcVdi, baseVm)`
+    - `prepare({ isFull })`
+    - `transfer({ timestamp, deltaExport, sizeContainers })`
+    - `cleanup()`
+    - `healthCheck(sr)`
+  - **Full**
+    - `run({ timestamp, sizeContainer, stream })`
+- `afterBackup()`

@xen-orchestra/backups/merge-worker/cli.js

@@ -1,4 +1,6 @@
 #!/usr/bin/env node
+// eslint-disable-next-line eslint-comments/disable-enable-pair
+/* eslint-disable n/shebang */
 
 'use strict'
 
@@ -62,7 +64,7 @@ const main = Disposable.wrap(async function* main(args) {
     try {
       const vmDir = getVmBackupDir(String(await handler.readFile(taskFile)))
       try {
-        await adapter.cleanVm(vmDir, { merge: true, onLog: info, remove: true })
+        await adapter.cleanVm(vmDir, { merge: true, logInfo: info, logWarn: warn, remove: true })
       } catch (error) {
         // consider the clean successful if the VM dir is missing
         if (error.code !== 'ENOENT') {

@xen-orchestra/backups/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.21.0",
+  "version": "0.27.4",
   "engines": {
     "node": ">=14.6"
   },
@@ -16,16 +16,20 @@
     "postversion": "npm publish --access public"
   },
   "dependencies": {
+    "@vates/async-each": "^1.0.0",
+    "@vates/cached-dns.lookup": "^1.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.0.0",
     "@vates/disposable": "^0.1.1",
+    "@vates/fuse-vhd": "^0.0.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^1.0.0",
+    "@xen-orchestra/fs": "^3.1.0",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/template": "^0.1.0",
-    "compare-versions": "^4.0.1",
+    "compare-versions": "^5.0.1",
     "d3-time-format": "^3.0.0",
+    "decorator-synchronized": "^0.6.0",
     "end-of-stream": "^1.4.4",
     "fs-extra": "^10.0.0",
     "golike-defer": "^0.5.1",
@@ -35,8 +39,8 @@
     "parse-pairs": "^1.1.0",
     "promise-toolbox": "^0.21.0",
     "proper-lockfile": "^4.1.2",
-    "uuid": "^8.3.2",
-    "vhd-lib": "^3.1.0",
+    "uuid": "^9.0.0",
+    "vhd-lib": "^4.0.1",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
@@ -44,7 +48,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^0.10.0"
+    "@xen-orchestra/xapi": "^1.4.2"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

@xen-orchestra/backups/writers/DeltaBackupWriter.js

@@ -36,6 +36,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
       try {
         const vhds = await handler.list(`${vdisDir}/${srcVdi.uuid}`, {
           filter: _ => _[0] !== '.' && _.endsWith('.vhd'),
+          ignoreMissing: true,
           prependDir: true,
         })
         const packedBaseUuid = packUuid(baseUuid)
@@ -80,7 +81,9 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
       },
     })
     this.transfer = task.wrapFn(this.transfer)
-    this.cleanup = task.wrapFn(this.cleanup, true)
+    this.healthCheck = task.wrapFn(this.healthCheck)
+    this.cleanup = task.wrapFn(this.cleanup)
+    this.afterBackup = task.wrapFn(this.afterBackup, true)
 
     return task.run(() => this._prepare())
   }
@@ -156,7 +159,6 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
         }/${adapter.getVhdFileName(basename)}`
     )
 
-    const metadataFilename = `${backupDir}/${basename}.json`
     const metadataContent = {
       jobId,
       mode: job.mode,
@@ -202,6 +204,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
             // merges and chainings
             checksum: false,
             validator: tmpPath => checkVhd(handler, tmpPath),
+            writeBlockConcurrency: this._backup.config.writeBlockConcurrency,
           })
 
           if (isDelta) {
@@ -221,9 +224,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
       }
     })
     metadataContent.size = size
-    await handler.outputFile(metadataFilename, JSON.stringify(metadataContent), {
-      dirMode: backup.config.dirMode,
-    })
+    this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadataContent)
 
     // TODO: run cleanup?
   }

@xen-orchestra/backups/writers/FullBackupWriter.js

@@ -34,7 +34,6 @@ exports.FullBackupWriter = class FullBackupWriter extends MixinBackupWriter(Abst
     const { job, scheduleId, vm } = backup
 
     const adapter = this._adapter
-    const handler = adapter.handler
     const backupDir = getVmBackupDir(vm.uuid)
 
     // TODO: clean VM backup directory
@@ -74,9 +73,7 @@ exports.FullBackupWriter = class FullBackupWriter extends MixinBackupWriter(Abst
       return { size: sizeContainer.size }
     })
     metadata.size = sizeContainer.size
-    await handler.outputFile(metadataFilename, JSON.stringify(metadata), {
-      dirMode: backup.config.dirMode,
-    })
+    this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadata)
 
     if (!deleteFirst) {
       await deleteOldBackups()

@xen-orchestra/backups/writers/_AbstractWriter.js

@@ -9,4 +9,6 @@ exports.AbstractWriter = class AbstractWriter {
   beforeBackup() {}
 
   afterBackup() {}
+
+  healthCheck(sr) {}
 }

@xen-orchestra/backups/writers/_MixinBackupWriter.js

@@ -3,11 +3,15 @@
 const { createLogger } = require('@xen-orchestra/log')
 const { join } = require('path')
 
-const { getVmBackupDir } = require('../_getVmBackupDir.js')
-const MergeWorker = require('../merge-worker/index.js')
+const assert = require('assert')
 const { formatFilenameDate } = require('../_filenameDate.js')
+const { getVmBackupDir } = require('../_getVmBackupDir.js')
+const { HealthCheckVmBackup } = require('../HealthCheckVmBackup.js')
+const { ImportVmBackup } = require('../ImportVmBackup.js')
+const { Task } = require('../Task.js')
+const MergeWorker = require('../merge-worker/index.js')
 
-const { warn } = createLogger('xo:backups:MixinBackupWriter')
+const { info, warn } = createLogger('xo:backups:MixinBackupWriter')
 
 exports.MixinBackupWriter = (BaseClass = Object) =>
   class MixinBackupWriter extends BaseClass {
@@ -25,11 +29,18 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
 
     async _cleanVm(options) {
       try {
-        return await this._adapter.cleanVm(this.#vmBackupDir, {
-          ...options,
-          fixMetadata: true,
-          onLog: warn,
-          lock: false,
+        return await Task.run({ name: 'clean-vm' }, () => {
+          return this._adapter.cleanVm(this.#vmBackupDir, {
+            ...options,
+            fixMetadata: true,
+            logInfo: info,
+            logWarn: (message, data) => {
+              warn(message, data)
+              Task.warning(message, data)
+            },
+            lock: false,
+            mergeBlockConcurrency: this._backup.config.mergeBlockConcurrency,
+          })
         })
       } catch (error) {
         warn(error)
@@ -65,4 +76,38 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
         await MergeWorker.run(remotePath)
       }
     }
+
+    healthCheck(sr) {
+      assert.notStrictEqual(
+        this._metadataFileName,
+        undefined,
+        'Metadata file name should be defined before making a healthcheck'
+      )
+      return Task.run(
+        {
+          name: 'health check',
+        },
+        async () => {
+          const xapi = sr.$xapi
+          const srUuid = sr.uuid
+          const adapter = this._adapter
+          const metadata = await adapter.readVmBackupMetadata(this._metadataFileName)
+          const { id: restoredId } = await new ImportVmBackup({
+            adapter,
+            metadata,
+            srUuid,
+            xapi,
+          }).run()
+          const restoredVm = xapi.getObject(restoredId)
+          try {
+            await new HealthCheckVmBackup({
+              restoredVm,
+              xapi,
+            }).run()
+          } finally {
+            await xapi.VM_destroy(restoredVm.$ref)
+          }
+        }
+      )
+    }
   }

@xen-orchestra/cr-seed-cli/package.json

@@ -18,7 +18,7 @@
   "preferGlobal": true,
   "dependencies": {
     "golike-defer": "^0.5.1",
-    "xen-api": "^1.1.0"
+    "xen-api": "^1.2.2"
   },
   "scripts": {
     "postversion": "npm publish"

@xen-orchestra/emit-async/.USAGE.md

@@ -22,7 +22,7 @@ await ee.emitAsync('start')
 // error handling though:
 await ee.emitAsync(
   {
-    onError(error) {
+    onError(error, event, listener) {
       console.warn(error)
     },
   },

@xen-orchestra/emit-async/README.md

@@ -40,7 +40,7 @@ await ee.emitAsync('start')
 // error handling though:
 await ee.emitAsync(
   {
-    onError(error) {
+    onError(error, event, listener) {
       console.warn(error)
     },
   },

@xen-orchestra/emit-async/index.js

@@ -1,5 +1,7 @@
 'use strict'
 
+const identity = v => v
+
 module.exports = function emitAsync(event) {
   let opts
   let i = 1
@@ -17,12 +19,18 @@ module.exports = function emitAsync(event) {
   }
 
   const onError = opts != null && opts.onError
+  const addErrorHandler = onError
+    ? (promise, listener) => promise.catch(error => onError(error, event, listener))
+    : identity
 
   return Promise.all(
     this.listeners(event).map(listener =>
-      new Promise(resolve => {
-        resolve(listener.apply(this, args))
-      }).catch(onError)
+      addErrorHandler(
+        new Promise(resolve => {
+          resolve(listener.apply(this, args))
+        }),
+        listener
+      )
     )
   )
 }

@xen-orchestra/emit-async/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/emit-async",
-  "version": "0.1.0",
+  "version": "1.0.0",
   "license": "ISC",
   "description": "Emit an event for async listeners to settle",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/emit-async",

@xen-orchestra/fs/docs/encryption.md

@@ -0,0 +1,19 @@
+## metadata files
+
+- Older remotes dont have any metadata file
+- Remote used since 5.75 have two files : encryption.json and metadata.json
+
+The metadata files are checked by the sync() method. If the check fails it MUST throw an error and dismount.
+
+If the remote is empty, the `sync` method creates them
+
+### encryption.json
+
+A non encrypted file contain the algorithm and parameters used for this remote.
+This MUST NOT contains the key.
+
+### metadata.json
+
+An encrypted JSON file containing the settings of a remote. Today this is an empty JSON file ( `{random: <randomuuid>}` ), it serves to check if the encryption key set in the remote is valid, but in the future will be able to store some remote settings to ease disaster recovery.
+
+If this file can't be read (decrypted, decompressed, .. ), that means that the remote settings have been updated. If the remote is empty, update the `encryption.json` and `metadata.json` files , else raise an error.

@xen-orchestra/fs/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/fs",
-  "version": "1.0.0",
+  "version": "3.1.0",
   "license": "AGPL-3.0-or-later",
   "description": "The File System for Xen Orchestra backups.",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
@@ -17,19 +17,20 @@
     "xo-fs": "./cli.js"
   },
   "engines": {
-    "node": ">=14"
+    "node": ">=14.13"
   },
   "dependencies": {
-    "@marsaud/smb2": "^0.18.0",
-    "@sindresorhus/df": "^3.1.1",
-    "@vates/async-each": "^0.1.0",
-    "@vates/coalesce-calls": "^0.1.0",
-    "@vates/decorate-with": "^2.0.0",
-    "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/log": "^0.3.0",
     "@aws-sdk/client-s3": "^3.54.0",
     "@aws-sdk/lib-storage": "^3.54.0",
+    "@aws-sdk/middleware-apply-body-checksum": "^3.58.0",
     "@aws-sdk/node-http-handler": "^3.54.0",
+    "@sindresorhus/df": "^3.1.1",
+    "@vates/async-each": "^1.0.0",
+    "@vates/coalesce-calls": "^0.1.0",
+    "@vates/decorate-with": "^2.0.0",
+    "@vates/read-chunk": "^1.0.0",
+    "@xen-orchestra/async-map": "^0.1.2",
+    "@xen-orchestra/log": "^0.3.0",
     "bind-property-descriptor": "^2.0.0",
     "decorator-synchronized": "^0.6.0",
     "execa": "^5.0.0",
@@ -39,9 +40,10 @@
     "lodash": "^4.17.4",
     "promise-toolbox": "^0.21.0",
     "proper-lockfile": "^4.1.2",
-    "readable-stream": "^3.0.6",
+    "pumpify": "^2.0.1",
+    "readable-stream": "^4.1.0",
     "through2": "^4.0.2",
-    "xo-remote-parser": "^0.8.0"
+    "xo-remote-parser": "^0.9.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",
@@ -49,10 +51,9 @@
     "@babel/plugin-proposal-decorators": "^7.1.6",
     "@babel/plugin-proposal-function-bind": "^7.0.0",
     "@babel/preset-env": "^7.8.0",
-    "async-iterator-to-stream": "^1.1.0",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^7.0.2",
-    "dotenv": "^15.0.0",
+    "dotenv": "^16.0.0",
     "rimraf": "^3.0.0"
   },
   "scripts": {
@@ -67,5 +68,9 @@
   "author": {
     "name": "Vates SAS",
     "url": "https://vates.fr"
+  },
+  "exports": {
+    ".": "./dist/index.js",
+    "./path": "./dist/path.js"
   }
 }

@xen-orchestra/fs/src/_encryptor.js

@@ -0,0 +1,71 @@
+const { readChunk } = require('@vates/read-chunk')
+const crypto = require('crypto')
+const pumpify = require('pumpify')
+
+function getEncryptor(key) {
+  if (key === undefined) {
+    return {
+      id: 'NULL_ENCRYPTOR',
+      algorithm: 'none',
+      key: 'none',
+      ivLength: 0,
+      encryptData: buffer => buffer,
+      encryptStream: stream => stream,
+      decryptData: buffer => buffer,
+      decryptStream: stream => stream,
+    }
+  }
+  const algorithm = 'aes-256-cbc'
+  const ivLength = 16
+
+  function encryptStream(input) {
+    const iv = crypto.randomBytes(ivLength)
+    const cipher = crypto.createCipheriv(algorithm, Buffer.from(key), iv)
+
+    const encrypted = pumpify(input, cipher)
+    encrypted.unshift(iv)
+    return encrypted
+  }
+
+  async function decryptStream(encryptedStream) {
+    const iv = await readChunk(encryptedStream, ivLength)
+    const cipher = crypto.createDecipheriv(algorithm, Buffer.from(key), iv)
+    /**
+     * WARNING
+     *
+     * the crytped size has an initializtion vector + a padding at the end
+     * whe can't predict the decrypted size from the start of the encrypted size
+     * thus, we can't set decrypted.length reliably
+     *
+     */
+    return pumpify(encryptedStream, cipher)
+  }
+
+  function encryptData(buffer) {
+    const iv = crypto.randomBytes(ivLength)
+    const cipher = crypto.createCipheriv(algorithm, Buffer.from(key), iv)
+    const encrypted = cipher.update(buffer)
+    return Buffer.concat([iv, encrypted, cipher.final()])
+  }
+
+  function decryptData(buffer) {
+    const iv = buffer.slice(0, ivLength)
+    const encrypted = buffer.slice(ivLength)
+    const decipher = crypto.createDecipheriv(algorithm, Buffer.from(key), iv)
+    const decrypted = decipher.update(encrypted)
+    return Buffer.concat([decrypted, decipher.final()])
+  }
+
+  return {
+    id: algorithm,
+    algorithm,
+    key,
+    ivLength,
+    encryptData,
+    encryptStream,
+    decryptData,
+    decryptStream,
+  }
+}
+
+exports._getEncryptor = getEncryptor

@xen-orchestra/fs/src/abstract.js

@@ -1,15 +1,20 @@
 import asyncMapSettled from '@xen-orchestra/async-map/legacy'
+import assert from 'assert'
 import getStream from 'get-stream'
 import { coalesceCalls } from '@vates/coalesce-calls'
+import { createLogger } from '@xen-orchestra/log'
 import { fromCallback, fromEvent, ignoreErrors, timeout } from 'promise-toolbox'
 import { limitConcurrency } from 'limit-concurrency-decorator'
 import { parse } from 'xo-remote-parser'
 import { pipeline } from 'stream'
-import { randomBytes } from 'crypto'
+import { randomBytes, randomUUID } from 'crypto'
 import { synchronized } from 'decorator-synchronized'
 
-import { basename, dirname, normalize as normalizePath } from './_path'
+import { basename, dirname, normalize as normalizePath } from './path'
 import { createChecksumStream, validChecksumOfReadStream } from './checksum'
+import { _getEncryptor } from './_encryptor'
+
+const { info, warn } = createLogger('@xen-orchestra:fs')
 
 const checksumFile = file => file + '.checksum'
 const computeRate = (hrtime, size) => {
@@ -20,6 +25,9 @@ const computeRate = (hrtime, size) => {
 const DEFAULT_TIMEOUT = 6e5 // 10 min
 const DEFAULT_MAX_PARALLEL_OPERATIONS = 10
 
+const ENCRYPTION_DESC_FILENAME = 'encryption.json'
+const ENCRYPTION_METADATA_FILENAME = 'metadata.json'
+
 const ignoreEnoent = error => {
   if (error == null || error.code !== 'ENOENT') {
     throw error
@@ -60,6 +68,7 @@ class PrefixWrapper {
 }
 
 export default class RemoteHandlerAbstract {
+  _encryptor
   constructor(remote, options = {}) {
     if (remote.url === 'test://') {
       this._remote = remote
@@ -70,6 +79,7 @@ export default class RemoteHandlerAbstract {
       }
     }
     ;({ highWaterMark: this._highWaterMark, timeout: this._timeout = DEFAULT_TIMEOUT } = options)
+    this._encryptor = _getEncryptor(this._remote.encryptionKey)
 
     const sharedLimit = limitConcurrency(options.maxParallelOperations ?? DEFAULT_MAX_PARALLEL_OPERATIONS)
     this.closeFile = sharedLimit(this.closeFile)
@@ -108,90 +118,51 @@ export default class RemoteHandlerAbstract {
     await this.__closeFile(fd)
   }
 
-  // TODO: remove method
-  async createOutputStream(file, { checksum = false, dirMode, ...options } = {}) {
+  async createReadStream(file, { checksum = false, ignoreMissingChecksum = false, ...options } = {}) {
+    if (options.end !== undefined || options.start !== undefined) {
+      assert.strictEqual(this.isEncrypted, false, `Can't read part of a file when encryption is active ${file}`)
+    }
     if (typeof file === 'string') {
       file = normalizePath(file)
     }
-    const path = typeof file === 'string' ? file : file.path
-    const streamP = timeout.call(
-      this._createOutputStream(file, {
-        dirMode,
-        flags: 'wx',
-        ...options,
-      }),
+
+    let stream = await timeout.call(
+      this._createReadStream(file, { ...options, highWaterMark: this._highWaterMark }),
       this._timeout
     )
 
-    if (!checksum) {
-      return streamP
-    }
+    // detect early errors
+    await fromEvent(stream, 'readable')
 
-    const checksumStream = createChecksumStream()
-    const forwardError = error => {
-      checksumStream.emit('error', error)
-    }
+    if (checksum) {
+      try {
+        const path = typeof file === 'string' ? file : file.path
+        const checksum = await this._readFile(checksumFile(path), { flags: 'r' })
 
-    const stream = await streamP
-    stream.on('error', forwardError)
-    checksumStream.pipe(stream)
-
-    checksumStream.checksumWritten = checksumStream.checksum
-      .then(value => this._outputFile(checksumFile(path), value, { flags: 'wx' }))
-      .catch(forwardError)
-
-    return checksumStream
-  }
-
-  createReadStream(file, { checksum = false, ignoreMissingChecksum = false, ...options } = {}) {
-    if (typeof file === 'string') {
-      file = normalizePath(file)
-    }
-    const path = typeof file === 'string' ? file : file.path
-    const streamP = timeout
-      .call(this._createReadStream(file, { ...options, highWaterMark: this._highWaterMark }), this._timeout)
-      .then(stream => {
-        // detect early errors
-        let promise = fromEvent(stream, 'readable')
-
-        // try to add the length prop if missing and not a range stream
-        if (stream.length === undefined && options.end === undefined && options.start === undefined) {
-          promise = Promise.all([
-            promise,
-            ignoreErrors.call(
-              this._getSize(file).then(size => {
-                stream.length = size
-              })
-            ),
-          ])
+        const { length } = stream
+        stream = validChecksumOfReadStream(stream, String(checksum).trim())
+        stream.length = length
+      } catch (error) {
+        if (!(ignoreMissingChecksum && error.code === 'ENOENT')) {
+          throw error
         }
-
-        return promise.then(() => stream)
-      })
-
-    if (!checksum) {
-      return streamP
-    }
-
-    // avoid a unhandled rejection warning
-    ignoreErrors.call(streamP)
-
-    return this._readFile(checksumFile(path), { flags: 'r' }).then(
-      checksum =>
-        streamP.then(stream => {
-          const { length } = stream
-          stream = validChecksumOfReadStream(stream, String(checksum).trim())
-          stream.length = length
-
-          return stream
-        }),
-      error => {
-        if (ignoreMissingChecksum && error && error.code === 'ENOENT') {
-          return streamP
-        }
-        throw error
       }
-    )
+    }
+
+    if (this.isEncrypted) {
+      stream = this._encryptor.decryptStream(stream)
+    } else {
+      // try to add the length prop if missing and not a range stream
+      if (stream.length === undefined && options.end === undefined && options.start === undefined) {
+        try {
+          stream.length = await this._getSize(file)
+        } catch (error) {
+          // ignore errors
+        }
+      }
+    }
+
+    return stream
   }
 
   /**
@@ -207,6 +178,8 @@ export default class RemoteHandlerAbstract {
   async outputStream(path, input, { checksum = true, dirMode, validator } = {}) {
     path = normalizePath(path)
     let checksumStream
+
+    input = this._encryptor.encryptStream(input)
     if (checksum) {
       checksumStream = createChecksumStream()
       pipeline(input, checksumStream, noop)
@@ -217,6 +190,8 @@ export default class RemoteHandlerAbstract {
       validator,
     })
     if (checksum) {
+      // using _outpuFile means the checksum will NOT be encrypted
+      // it is by design to allow checking of encrypted files without the key
       await this._outputFile(checksumFile(path), await checksumStream.checksum, { dirMode, flags: 'wx' })
     }
   }
@@ -236,8 +211,13 @@ export default class RemoteHandlerAbstract {
     return timeout.call(this._getInfo(), this._timeout)
   }
 
+  // when using encryption, the file size is aligned with the encryption block size ( 16 bytes )
+  // that means that the size will be 1 to 16 bytes more than the content size + the initialized vector length (16 bytes)
   async getSize(file) {
-    return timeout.call(this._getSize(typeof file === 'string' ? normalizePath(file) : file), this._timeout)
+    assert.strictEqual(this.isEncrypted, false, `Can't compute size of an encrypted file ${file}`)
+
+    const size = await timeout.call(this._getSize(typeof file === 'string' ? normalizePath(file) : file), this._timeout)
+    return size - this._encryptor.ivLength
   }
 
   async list(dir, { filter, ignoreMissing = false, prependDir = false } = {}) {
@@ -283,15 +263,18 @@ export default class RemoteHandlerAbstract {
   }
 
   async outputFile(file, data, { dirMode, flags = 'wx' } = {}) {
-    await this._outputFile(normalizePath(file), data, { dirMode, flags })
+    const encryptedData = this._encryptor.encryptData(data)
+    await this._outputFile(normalizePath(file), encryptedData, { dirMode, flags })
   }
 
   async read(file, buffer, position) {
+    assert.strictEqual(this.isEncrypted, false, `Can't read part of an encrypted file ${file}`)
     return this._read(typeof file === 'string' ? normalizePath(file) : file, buffer, position)
   }
 
   async readFile(file, { flags = 'r' } = {}) {
-    return this._readFile(normalizePath(file), { flags })
+    const data = await this._readFile(normalizePath(file), { flags })
+    return this._encryptor.decryptData(data)
   }
 
   async rename(oldPath, newPath, { checksum = false } = {}) {
@@ -331,6 +314,61 @@ export default class RemoteHandlerAbstract {
   @synchronized()
   async sync() {
     await this._sync()
+    try {
+      await this._checkMetadata()
+    } catch (error) {
+      await this._forget()
+      throw error
+    }
+  }
+
+  async _canWriteMetadata() {
+    const list = await this.list('/', {
+      filter: e => !e.startsWith('.') && e !== ENCRYPTION_DESC_FILENAME && e !== ENCRYPTION_METADATA_FILENAME,
+    })
+    return list.length === 0
+  }
+
+  async _createMetadata() {
+    await Promise.all([
+      this._writeFile(
+        normalizePath(ENCRYPTION_DESC_FILENAME),
+        JSON.stringify({ algorithm: this._encryptor.algorithm }),
+        {
+          flags: 'w',
+        }
+      ), // not encrypted
+      this.writeFile(ENCRYPTION_METADATA_FILENAME, `{"random":"${randomUUID()}"}`, { flags: 'w' }), // encrypted
+    ])
+  }
+
+  async _checkMetadata() {
+    try {
+      // this file is not encrypted
+      const data = await this._readFile(normalizePath(ENCRYPTION_DESC_FILENAME))
+      JSON.parse(data)
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        throw error
+      }
+    }
+
+    try {
+      // this file is encrypted
+      const data = await this.readFile(ENCRYPTION_METADATA_FILENAME)
+      JSON.parse(data)
+    } catch (error) {
+      if (error.code === 'ENOENT' || (await this._canWriteMetadata())) {
+        info('will update metadata of this remote')
+        return this._createMetadata()
+      }
+      warn(
+        `The encryptionKey settings of this remote does not match the key used to create it. You won't be able to read any data from this remote`,
+        { error }
+      )
+      // will probably send a ERR_OSSL_EVP_BAD_DECRYPT if key is incorrect
+      throw error
+    }
   }
 
   async test() {
@@ -357,11 +395,12 @@ export default class RemoteHandlerAbstract {
         readRate: computeRate(readDuration, SIZE),
       }
     } catch (error) {
+      warn(`error while testing the remote at step ${step}`, { error })
       return {
         success: false,
         step,
         file: testFileName,
-        error: error.message || String(error),
+        error,
       }
     } finally {
       ignoreErrors.call(this._unlink(testFileName))
@@ -383,11 +422,13 @@ export default class RemoteHandlerAbstract {
   }
 
   async write(file, buffer, position) {
+    assert.strictEqual(this.isEncrypted, false, `Can't write part of a file with encryption ${file}`)
     await this._write(typeof file === 'string' ? normalizePath(file) : file, buffer, position)
   }
 
   async writeFile(file, data, { flags = 'wx' } = {}) {
-    await this._writeFile(normalizePath(file), data, { flags })
+    const encryptedData = this._encryptor.encryptData(data)
+    await this._writeFile(normalizePath(file), encryptedData, { flags })
   }
 
   // Methods that can be called by private methods to avoid parallel limit on public methods
@@ -420,6 +461,10 @@ export default class RemoteHandlerAbstract {
 
   // Methods that can be implemented by inheriting classes
 
+  useVhdDirectory() {
+    return this._remote.useVhdDirectory ?? false
+  }
+
   async _closeFile(fd) {
     throw new Error('Not implemented')
   }
@@ -502,9 +547,13 @@ export default class RemoteHandlerAbstract {
 
   async _outputStream(path, input, { dirMode, validator }) {
     const tmpPath = `${dirname(path)}/.${basename(path)}`
-    const output = await this.createOutputStream(tmpPath, {
-      dirMode,
-    })
+    const output = await timeout.call(
+      this._createOutputStream(tmpPath, {
+        dirMode,
+        flags: 'wx',
+      }),
+      this._timeout
+    )
     try {
       await fromCallback(pipeline, input, output)
       if (validator !== undefined) {
@@ -587,6 +636,10 @@ export default class RemoteHandlerAbstract {
   async _writeFile(file, data, options) {
     throw new Error('Not implemented')
   }
+
+  get isEncrypted() {
+    return this._encryptor.id !== 'NULL_ENCRYPTOR'
+  }
 }
 
 function createPrefixWrapperMethods() {

@xen-orchestra/fs/src/abstract.spec.js

@@ -30,18 +30,6 @@ describe('closeFile()', () => {
   })
 })
 
-describe('createOutputStream()', () => {
-  it(`throws in case of timeout`, async () => {
-    const testHandler = new TestHandler({
-      createOutputStream: () => new Promise(() => {}),
-    })
-
-    const promise = testHandler.createOutputStream('File')
-    jest.advanceTimersByTime(TIMEOUT)
-    await expect(promise).rejects.toThrowError(TimeoutError)
-  })
-})
-
 describe('getInfo()', () => {
   it('throws in case of timeout', async () => {
     const testHandler = new TestHandler({

@xen-orchestra/fs/src/fs.spec.js

@@ -1,10 +1,7 @@
 /* eslint-env jest */
 
 import 'dotenv/config'
-import asyncIteratorToStream from 'async-iterator-to-stream'
 import { forOwn, random } from 'lodash'
-import { fromCallback } from 'promise-toolbox'
-import { pipeline } from 'readable-stream'
 import { tmpdir } from 'os'
 
 import { getHandler } from '.'
@@ -27,9 +24,6 @@ const unsecureRandomBytes = n => {
 
 const TEST_DATA_LEN = 1024
 const TEST_DATA = unsecureRandomBytes(TEST_DATA_LEN)
-const createTestDataStream = asyncIteratorToStream(function* () {
-  yield TEST_DATA
-})
 
 const rejectionOf = p =>
   p.then(
@@ -82,14 +76,6 @@ handlers.forEach(url => {
       })
     })
 
-    describe('#createOutputStream()', () => {
-      it('creates parent dir if missing', async () => {
-        const stream = await handler.createOutputStream('dir/file')
-        await fromCallback(pipeline, createTestDataStream(), stream)
-        await expect(await handler.readFile('dir/file')).toEqual(TEST_DATA)
-      })
-    })
-
     describe('#getInfo()', () => {
       let info
       beforeAll(async () => {

@xen-orchestra/fs/src/index.js

@@ -5,7 +5,6 @@ import RemoteHandlerLocal from './local'
 import RemoteHandlerNfs from './nfs'
 import RemoteHandlerS3 from './s3'
 import RemoteHandlerSmb from './smb'
-import RemoteHandlerSmbMount from './smb-mount'
 
 const HANDLERS = {
   file: RemoteHandlerLocal,
@@ -15,10 +14,8 @@ const HANDLERS = {
 
 try {
   execa.sync('mount.cifs', ['-V'])
-  HANDLERS.smb = RemoteHandlerSmbMount
-} catch (_) {
   HANDLERS.smb = RemoteHandlerSmb
-}
+} catch (_) {}
 
 export const getHandler = (remote, ...rest) => {
   const Handler = HANDLERS[parse(remote.url).type]

@xen-orchestra/fs/src/local.js

@@ -1,13 +1,38 @@
 import df from '@sindresorhus/df'
 import fs from 'fs-extra'
 import lockfile from 'proper-lockfile'
+import { createLogger } from '@xen-orchestra/log'
 import { fromEvent, retry } from 'promise-toolbox'
 
 import RemoteHandlerAbstract from './abstract'
 
+const { info, warn } = createLogger('xo:fs:local')
+
+// save current stack trace and add it to any rejected error
+//
+// This is especially useful when the resolution is separate from the initial
+// call, which is often the case with RPC libs.
+//
+// There is a perf impact and it should be avoided in production.
+async function addSyncStackTrace(fn, ...args) {
+  const stackContainer = new Error()
+  try {
+    return await fn.apply(this, args)
+  } catch (error) {
+    error.syncStack = stackContainer.stack
+    throw error
+  }
+}
+
+function dontAddSyncStackTrace(fn, ...args) {
+  return fn.apply(this, args)
+}
+
 export default class LocalHandler extends RemoteHandlerAbstract {
   constructor(remote, opts = {}) {
     super(remote)
+
+    this._addSyncStackTrace = opts.syncStackTraces ?? true ? addSyncStackTrace : dontAddSyncStackTrace
     this._retriesOnEagain = {
       delay: 1e3,
       retries: 9,
@@ -30,17 +55,17 @@ export default class LocalHandler extends RemoteHandlerAbstract {
   }
 
   async _closeFile(fd) {
-    return fs.close(fd)
+    return this._addSyncStackTrace(fs.close, fd)
   }
 
   async _copy(oldPath, newPath) {
-    return fs.copy(this._getFilePath(oldPath), this._getFilePath(newPath))
+    return this._addSyncStackTrace(fs.copy, this._getFilePath(oldPath), this._getFilePath(newPath))
   }
 
   async _createReadStream(file, options) {
     if (typeof file === 'string') {
       const stream = fs.createReadStream(this._getFilePath(file), options)
-      await fromEvent(stream, 'open')
+      await this._addSyncStackTrace(fromEvent, stream, 'open')
       return stream
     }
     return fs.createReadStream('', {
@@ -53,7 +78,7 @@ export default class LocalHandler extends RemoteHandlerAbstract {
   async _createWriteStream(file, options) {
     if (typeof file === 'string') {
       const stream = fs.createWriteStream(this._getFilePath(file), options)
-      await fromEvent(stream, 'open')
+      await this._addSyncStackTrace(fromEvent, stream, 'open')
       return stream
     }
     return fs.createWriteStream('', {
@@ -79,71 +104,98 @@ export default class LocalHandler extends RemoteHandlerAbstract {
   }
 
   async _getSize(file) {
-    const stats = await fs.stat(this._getFilePath(typeof file === 'string' ? file : file.path))
+    const stats = await this._addSyncStackTrace(fs.stat, this._getFilePath(typeof file === 'string' ? file : file.path))
     return stats.size
   }
 
   async _list(dir) {
-    return fs.readdir(this._getFilePath(dir))
+    return this._addSyncStackTrace(fs.readdir, this._getFilePath(dir))
   }
 
-  _lock(path) {
-    return lockfile.lock(this._getFilePath(path))
+  async _lock(path) {
+    const acquire = lockfile.lock.bind(undefined, this._getFilePath(path), {
+      async onCompromised(error) {
+        warn('lock compromised', { error })
+        try {
+          release = await acquire()
+          info('compromised lock was reacquired')
+        } catch (error) {
+          warn('compromised lock could not be reacquired', { error })
+        }
+      },
+    })
+
+    let release = await this._addSyncStackTrace(acquire)
+
+    return async () => {
+      try {
+        await this._addSyncStackTrace(release)
+      } catch (error) {
+        warn('lock could not be released', { error })
+      }
+    }
   }
 
   _mkdir(dir, { mode }) {
-    return fs.mkdir(this._getFilePath(dir), { mode })
+    return this._addSyncStackTrace(fs.mkdir, this._getFilePath(dir), { mode })
   }
 
   async _openFile(path, flags) {
-    return fs.open(this._getFilePath(path), flags)
+    return this._addSyncStackTrace(fs.open, this._getFilePath(path), flags)
   }
 
   async _read(file, buffer, position) {
     const needsClose = typeof file === 'string'
-    file = needsClose ? await fs.open(this._getFilePath(file), 'r') : file.fd
+    file = needsClose ? await this._addSyncStackTrace(fs.open, this._getFilePath(file), 'r') : file.fd
     try {
-      return await fs.read(file, buffer, 0, buffer.length, position === undefined ? null : position)
+      return await this._addSyncStackTrace(
+        fs.read,
+        file,
+        buffer,
+        0,
+        buffer.length,
+        position === undefined ? null : position
+      )
     } finally {
       if (needsClose) {
-        await fs.close(file)
+        await this._addSyncStackTrace(fs.close, file)
       }
     }
   }
 
   async _readFile(file, options) {
     const filePath = this._getFilePath(file)
-    return await retry(() => fs.readFile(filePath, options), this._retriesOnEagain)
+    return await this._addSyncStackTrace(retry, () => fs.readFile(filePath, options), this._retriesOnEagain)
   }
 
   async _rename(oldPath, newPath) {
-    return fs.rename(this._getFilePath(oldPath), this._getFilePath(newPath))
+    return this._addSyncStackTrace(fs.rename, this._getFilePath(oldPath), this._getFilePath(newPath))
   }
 
   async _rmdir(dir) {
-    return fs.rmdir(this._getFilePath(dir))
+    return this._addSyncStackTrace(fs.rmdir, this._getFilePath(dir))
   }
 
   async _sync() {
     const path = this._getRealPath('/')
-    await fs.ensureDir(path)
-    await fs.access(path, fs.R_OK | fs.W_OK)
+    await this._addSyncStackTrace(fs.ensureDir, path)
+    await this._addSyncStackTrace(fs.access, path, fs.R_OK | fs.W_OK)
   }
 
   _truncate(file, len) {
-    return fs.truncate(this._getFilePath(file), len)
+    return this._addSyncStackTrace(fs.truncate, this._getFilePath(file), len)
   }
 
   async _unlink(file) {
     const filePath = this._getFilePath(file)
-    return await retry(() => fs.unlink(filePath), this._retriesOnEagain)
+    return await this._addSyncStackTrace(retry, () => fs.unlink(filePath), this._retriesOnEagain)
   }
 
   _writeFd(file, buffer, position) {
-    return fs.write(file.fd, buffer, 0, buffer.length, position)
+    return this._addSyncStackTrace(fs.write, file.fd, buffer, 0, buffer.length, position)
   }
 
   _writeFile(file, data, { flags }) {
-    return fs.writeFile(this._getFilePath(file), data, { flag: flags })
+    return this._addSyncStackTrace(fs.writeFile, this._getFilePath(file), data, { flag: flags })
   }
 }

@xen-orchestra/fs/src/path.js

@@ -1,6 +1,6 @@
 import path from 'path'
 
-const { basename, dirname, join, resolve, sep } = path.posix
+const { basename, dirname, join, resolve, relative, sep } = path.posix
 
 export { basename, dirname, join }
 
@@ -19,3 +19,6 @@ export function split(path) {
 
   return parts
 }
+
+export const relativeFromFile = (file, path) => relative(dirname(file), path)
+export const resolveFromFile = (file, path) => resolve('/', dirname(file), path).slice(1)

@xen-orchestra/fs/src/s3.js

@@ -14,6 +14,7 @@ import {
 } from '@aws-sdk/client-s3'
 import { Upload } from '@aws-sdk/lib-storage'
 import { NodeHttpHandler } from '@aws-sdk/node-http-handler'
+import { getApplyMd5BodyChecksumPlugin } from '@aws-sdk/middleware-apply-body-checksum'
 import assert from 'assert'
 import { Agent as HttpAgent } from 'http'
 import { Agent as HttpsAgent } from 'https'
@@ -26,7 +27,7 @@ import copyStreamToBuffer from './_copyStreamToBuffer.js'
 import createBufferFromStream from './_createBufferFromStream.js'
 import guessAwsRegion from './_guessAwsRegion.js'
 import RemoteHandlerAbstract from './abstract'
-import { basename, join, split } from './_path'
+import { basename, join, split } from './path'
 import { asyncEach } from '@vates/async-each'
 
 // endpoints https://docs.aws.amazon.com/general/latest/gr/s3.html
@@ -75,6 +76,9 @@ export default class S3Handler extends RemoteHandlerAbstract {
       }),
     })
 
+    // Workaround for https://github.com/aws/aws-sdk-js-v3/issues/2673
+    this._s3.middlewareStack.use(getApplyMd5BodyChecksumPlugin(this._s3.config))
+
     const parts = split(path)
     this._bucket = parts.shift()
     this._dir = join(...parts)
@@ -93,7 +97,12 @@ export default class S3Handler extends RemoteHandlerAbstract {
   }
 
   _makePrefix(dir) {
-    return join(this._dir, dir, '/')
+    const prefix = join(this._dir, dir, '/')
+
+    // no prefix for root
+    if (prefix !== './') {
+      return prefix
+    }
   }
 
   _createParams(file) {
@@ -146,6 +155,14 @@ export default class S3Handler extends RemoteHandlerAbstract {
       if (e.name === 'EntityTooLarge') {
         return this._multipartCopy(oldPath, newPath)
       }
+      // normalize this error code
+      if (e.name === 'NoSuchKey') {
+        const error = new Error(`ENOENT: no such file or directory '${oldPath}'`)
+        error.cause = e
+        error.code = 'ENOENT'
+        error.path = oldPath
+        throw error
+      }
       throw e
     }
   }
@@ -206,7 +223,7 @@ export default class S3Handler extends RemoteHandlerAbstract {
   // https://www.backblaze.com/b2/docs/calling.html#error_handling
   @decorateWith(pRetry.wrap, {
     delays: [100, 200, 500, 1000, 2000],
-    when: e => e.$metadata?.httpStatusCode === 500,
+    when: e => [500, 400].includes(e.$metadata?.httpStatusCode),
     onRetry(error) {
       warn('retrying writing file', {
         attemptNumber: this.attemptNumber,
@@ -226,14 +243,17 @@ export default class S3Handler extends RemoteHandlerAbstract {
   }
 
   async _createReadStream(path, options) {
-    if (!(await this._isFile(path))) {
-      const error = new Error(`ENOENT: no such file '${path}'`)
-      error.code = 'ENOENT'
-      error.path = path
-      throw error
+    try {
+      return (await this._s3.send(new GetObjectCommand(this._createParams(path)))).Body
+    } catch (e) {
+      if (e.name === 'NoSuchKey') {
+        const error = new Error(`ENOENT: no such file '${path}'`)
+        error.code = 'ENOENT'
+        error.path = path
+        throw error
+      }
+      throw e
     }
-
-    return (await this._s3.send(new GetObjectCommand(this._createParams(path)))).Body
   }
 
   async _unlink(path) {
@@ -513,4 +533,8 @@ export default class S3Handler extends RemoteHandlerAbstract {
   }
 
   async _closeFile(fd) {}
+
+  useVhdDirectory() {
+    return true
+  }
 }

@xen-orchestra/fs/src/smb-mount.js

@@ -1,23 +0,0 @@
-import { parse } from 'xo-remote-parser'
-
-import MountHandler from './_mount'
-import { normalize } from './_path'
-
-export default class SmbMountHandler extends MountHandler {
-  constructor(remote, opts) {
-    const { domain = 'WORKGROUP', host, password, path, username } = parse(remote.url)
-    super(remote, opts, {
-      type: 'cifs',
-      device: '//' + host + normalize(path),
-      options: `domain=${domain}`,
-      env: {
-        USER: username,
-        PASSWD: password,
-      },
-    })
-  }
-
-  get type() {
-    return 'smb'
-  }
-}

@xen-orchestra/fs/src/smb.js

@@ -1,163 +1,23 @@
-import Smb2 from '@marsaud/smb2'
+import { parse } from 'xo-remote-parser'
 
-import RemoteHandlerAbstract from './abstract'
+import MountHandler from './_mount'
+import { normalize } from './path'
 
-// Normalize the error code for file not found.
-const wrapError = (error, code) => ({
-  __proto__: error,
-  cause: error,
-  code,
-})
-const normalizeError = (error, shouldBeDirectory) => {
-  const { code } = error
-
-  throw code === 'STATUS_DIRECTORY_NOT_EMPTY'
-    ? wrapError(error, 'ENOTEMPTY')
-    : code === 'STATUS_FILE_IS_A_DIRECTORY'
-    ? wrapError(error, 'EISDIR')
-    : code === 'STATUS_NOT_A_DIRECTORY'
-    ? wrapError(error, 'ENOTDIR')
-    : code === 'STATUS_OBJECT_NAME_NOT_FOUND' || code === 'STATUS_OBJECT_PATH_NOT_FOUND'
-    ? wrapError(error, 'ENOENT')
-    : code === 'STATUS_OBJECT_NAME_COLLISION'
-    ? wrapError(error, 'EEXIST')
-    : code === 'STATUS_NOT_SUPPORTED' || code === 'STATUS_INVALID_PARAMETER'
-    ? wrapError(error, shouldBeDirectory ? 'ENOTDIR' : 'EISDIR')
-    : error
-}
-const normalizeDirError = error => normalizeError(error, true)
-
-export default class SmbHandler extends RemoteHandlerAbstract {
+export default class SmbHandler extends MountHandler {
   constructor(remote, opts) {
-    super(remote, opts)
-
-    // defined in _sync()
-    this._client = undefined
-
-    const prefix = this._remote.path
-    this._prefix = prefix !== '' ? prefix + '\\' : prefix
+    const { domain = 'WORKGROUP', host, password, path, username } = parse(remote.url)
+    super(remote, opts, {
+      type: 'cifs',
+      device: '//' + host + normalize(path),
+      options: `domain=${domain}`,
+      env: {
+        USER: username,
+        PASSWD: password,
+      },
+    })
   }
 
   get type() {
     return 'smb'
   }
-
-  _getFilePath(file) {
-    return this._prefix + (typeof file === 'string' ? file : file.path).slice(1).replace(/\//g, '\\')
-  }
-
-  _dirname(file) {
-    const parts = file.split('\\')
-    parts.pop()
-    return parts.join('\\')
-  }
-
-  _closeFile(file) {
-    return this._client.close(file).catch(normalizeError)
-  }
-
-  _createReadStream(file, options) {
-    if (typeof file === 'string') {
-      file = this._getFilePath(file)
-    } else {
-      options = { autoClose: false, ...options, fd: file.fd }
-      file = ''
-    }
-    return this._client.createReadStream(file, options).catch(normalizeError)
-  }
-
-  _createWriteStream(file, options) {
-    if (typeof file === 'string') {
-      file = this._getFilePath(file)
-    } else {
-      options = { autoClose: false, ...options, fd: file.fd }
-      file = ''
-    }
-    return this._client.createWriteStream(file, options).catch(normalizeError)
-  }
-
-  _forget() {
-    const client = this._client
-    this._client = undefined
-    return client.disconnect()
-  }
-
-  _getSize(file) {
-    return this._client.getSize(this._getFilePath(file)).catch(normalizeError)
-  }
-
-  _list(dir) {
-    return this._client.readdir(this._getFilePath(dir)).catch(normalizeDirError)
-  }
-
-  _mkdir(dir, { mode }) {
-    return this._client.mkdir(this._getFilePath(dir), mode).catch(normalizeDirError)
-  }
-
-  // TODO: add flags
-  _openFile(path, flags) {
-    return this._client.open(this._getFilePath(path), flags).catch(normalizeError)
-  }
-
-  async _read(file, buffer, position) {
-    const client = this._client
-    const needsClose = typeof file === 'string'
-    file = needsClose ? await client.open(this._getFilePath(file)) : file.fd
-    try {
-      return await client.read(file, buffer, 0, buffer.length, position)
-    } catch (error) {
-      normalizeError(error)
-    } finally {
-      if (needsClose) {
-        await client.close(file)
-      }
-    }
-  }
-
-  _readFile(file, options) {
-    return this._client.readFile(this._getFilePath(file), options).catch(normalizeError)
-  }
-
-  _rename(oldPath, newPath) {
-    return this._client
-      .rename(this._getFilePath(oldPath), this._getFilePath(newPath), {
-        replace: true,
-      })
-      .catch(normalizeError)
-  }
-
-  _rmdir(dir) {
-    return this._client.rmdir(this._getFilePath(dir)).catch(normalizeDirError)
-  }
-
-  _sync() {
-    const remote = this._remote
-
-    this._client = new Smb2({
-      share: `\\\\${remote.host}`,
-      domain: remote.domain,
-      username: remote.username,
-      password: remote.password,
-      autoCloseTimeout: 0,
-    })
-
-    // Check access (smb2 does not expose connect in public so far...)
-    return this.list('.')
-  }
-
-  _truncate(file, len) {
-    return this._client.truncate(this._getFilePath(file), len).catch(normalizeError)
-  }
-
-  _unlink(file) {
-    return this._client.unlink(this._getFilePath(file)).catch(normalizeError)
-  }
-
-  _writeFd(file, buffer, position) {
-    return this._client.write(file.fd, buffer, 0, buffer.length, position)
-  }
-
-  _writeFile(file, data, options) {
-    return this._client.writeFile(this._getFilePath(file), data, options).catch(normalizeError)
-  }
 }

@xen-orchestra/mixins/Config.mjs

@@ -1,15 +1,16 @@
-'use strict'
-
-const get = require('lodash/get')
-const identity = require('lodash/identity')
-const isEqual = require('lodash/isEqual')
-const { createLogger } = require('@xen-orchestra/log')
-const { parseDuration } = require('@vates/parse-duration')
-const { watch } = require('app-conf')
+import get from 'lodash/get.js'
+import identity from 'lodash/identity.js'
+import isEqual from 'lodash/isEqual.js'
+import { createLogger } from '@xen-orchestra/log'
+import { parseDuration } from '@vates/parse-duration'
+import { watch } from 'app-conf'
 
 const { warn } = createLogger('xo:mixins:config')
 
-module.exports = class Config {
+// if path is undefined, an empty string or an empty array, returns the root value
+const niceGet = (value, path) => (path === undefined || path.length === 0 ? value : get(value, path))
+
+export default class Config {
   constructor(app, { appDir, appName, config }) {
     this._config = config
     const watchers = (this._watchers = new Set())
@@ -32,7 +33,7 @@ module.exports = class Config {
   }
 
   get(path) {
-    const value = get(this._config, path)
+    const value = niceGet(this._config, path)
     if (value === undefined) {
       throw new TypeError('missing config entry: ' + path)
     }
@@ -44,20 +45,27 @@ module.exports = class Config {
   }
 
   getOptional(path) {
-    return get(this._config, path)
+    return niceGet(this._config, path)
   }
 
   watch(path, cb) {
+    // short syntax for the whole config: watch(cb)
+    if (typeof path === 'function') {
+      cb = path
+      path = undefined
+    }
+
     // internal arg
     const processor = arguments.length > 2 ? arguments[2] : identity
 
     let prev
     const watcher = config => {
       try {
-        const value = processor(get(config, path))
+        const value = processor(niceGet(config, path))
         if (!isEqual(value, prev)) {
+          const previous = prev
           prev = value
-          cb(value)
+          cb(value, previous, path)
         }
       } catch (error) {
         warn('watch', { error, path })

@xen-orchestra/mixins/Hooks.js

@@ -1,51 +0,0 @@
-'use strict'
-
-const assert = require('assert')
-const emitAsync = require('@xen-orchestra/emit-async')
-const EventEmitter = require('events')
-const { createLogger } = require('@xen-orchestra/log')
-
-const { debug, warn } = createLogger('xo:mixins:hooks')
-
-const runHook = async (emitter, hook) => {
-  debug(`${hook} start…`)
-  await emitAsync.call(
-    emitter,
-    {
-      onError: error => warn(`${hook} failure`, { error }),
-    },
-    hook
-  )
-  debug(`${hook} finished`)
-}
-
-module.exports = class Hooks extends EventEmitter {
-  // Run *clean* async listeners.
-  //
-  // They normalize existing data, clear invalid entries, etc.
-  clean() {
-    return runHook(this, 'clean')
-  }
-
-  _status = 'stopped'
-
-  // Run *start* async listeners.
-  //
-  // They initialize the application.
-  async start() {
-    assert.strictEqual(this._status, 'stopped')
-    this._status = 'starting'
-    await runHook(this, 'start')
-    this.emit((this._status = 'started'))
-  }
-
-  // Run *stop* async listeners.
-  //
-  // They close connections, unmount file systems, save states, etc.
-  async stop() {
-    assert.strictEqual(this._status, 'started')
-    this._status = 'stopping'
-    await runHook(this, 'stop')
-    this.emit((this._status = 'stopped'))
-  }
-}

@xen-orchestra/mixins/Hooks.mjs

@@ -0,0 +1,70 @@
+import assert from 'assert'
+import emitAsync from '@xen-orchestra/emit-async'
+import EventEmitter from 'events'
+import { createLogger } from '@xen-orchestra/log'
+
+const { debug, warn } = createLogger('xo:mixins:hooks')
+
+const runHook = async (emitter, hook) => {
+  debug(`${hook} start…`)
+  await emitAsync.call(
+    emitter,
+    {
+      onError: error => warn(`${hook} failure`, { error }),
+    },
+    hook
+  )
+  debug(`${hook} finished`)
+}
+
+export default class Hooks extends EventEmitter {
+  // Run *clean* async listeners.
+  //
+  // They normalize existing data, clear invalid entries, etc.
+  clean() {
+    return runHook(this, 'clean')
+  }
+
+  _status = 'stopped'
+
+  // Run *start* async listeners.
+  //
+  // They initialize the application.
+  //
+  // *startCore* is automatically called if necessary.
+  async start() {
+    if (this._status === 'stopped') {
+      await this.startCore()
+    } else {
+      assert.strictEqual(this._status, 'core started')
+    }
+    this._status = 'starting'
+    await runHook(this, 'start')
+    this.emit((this._status = 'started'))
+  }
+
+  // Run *start core* async listeners.
+  //
+  // They initialize core features of the application (connect to databases,
+  // etc.) and should be fast and side-effects free.
+  async startCore() {
+    assert.strictEqual(this._status, 'stopped')
+    this._status = 'starting core'
+    await runHook(this, 'start core')
+    this.emit((this._status = 'core started'))
+  }
+
+  // Run *stop*  async listeners if necessary and *stop core* listeners.
+  //
+  // They close connections, unmount file systems, save states, etc.
+  async stop() {
+    if (this._status !== 'core started') {
+      assert.strictEqual(this._status, 'started')
+      this._status = 'stopping'
+      await runHook(this, 'stop')
+      this._status = 'core started'
+    }
+    await runHook(this, 'stop core')
+    this.emit((this._status = 'stopped'))
+  }
+}

@xen-orchestra/mixins/HttpProxy.mjs

@@ -0,0 +1,144 @@
+import { createLogger } from '@xen-orchestra/log'
+import { EventListenersManager } from '@vates/event-listeners-manager'
+import { pipeline } from 'stream'
+import { ServerResponse, request } from 'http'
+import assert from 'assert'
+import fromCallback from 'promise-toolbox/fromCallback'
+import fromEvent from 'promise-toolbox/fromEvent'
+import net from 'net'
+
+import { parseBasicAuth } from './_parseBasicAuth.mjs'
+
+const { debug, warn } = createLogger('xo:mixins:HttpProxy')
+
+const IGNORED_HEADERS = new Set([
+  // https://datatracker.ietf.org/doc/html/rfc2616#section-13.5.1
+  'connection',
+  'keep-alive',
+  'proxy-authenticate',
+  'proxy-authorization',
+  'te',
+  'trailers',
+  'transfer-encoding',
+  'upgrade',
+
+  // don't forward original host
+  'host',
+])
+
+export default class HttpProxy {
+  #app
+
+  constructor(app, { httpServer }) {
+    // don't setup the proxy if httpServer is not present
+    //
+    // that can happen when the app is instanciated in another context like xo-server-recover-account
+    if (httpServer === undefined) {
+      return
+    }
+
+    this.#app = app
+
+    const events = new EventListenersManager(httpServer)
+    app.config.watch('http.proxy.enabled', (enabled = true) => {
+      events.removeAll()
+      if (enabled) {
+        events.add('connect', this.#handleConnect.bind(this)).add('request', this.#handleRequest.bind(this))
+      }
+    })
+  }
+
+  async #handleAuthentication(req, res, next) {
+    const auth = parseBasicAuth(req.headers['proxy-authorization'])
+
+    let authenticated = false
+
+    if (auth !== undefined) {
+      const app = this.#app
+
+      if (app.authenticateUser !== undefined) {
+        // xo-server
+        try {
+          const { user } = await app.authenticateUser(auth)
+          authenticated = user.permission === 'admin'
+        } catch (error) {}
+      } else {
+        // xo-proxy
+        authenticated = (await app.authentication.findProfile(auth)) !== undefined
+      }
+    }
+
+    if (authenticated) {
+      return next()
+    }
+
+    // https://datatracker.ietf.org/doc/html/rfc7235#section-3.2
+    res.statusCode = '407'
+    res.setHeader('proxy-authenticate', 'Basic realm="proxy"')
+    return res.end('Proxy Authentication Required')
+  }
+
+  // https://nodejs.org/api/http.html#event-connect
+  async #handleConnect(req, clientSocket, head) {
+    const { url } = req
+
+    debug('CONNECT proxy', { url })
+
+    // https://github.com/TooTallNate/proxy/blob/d677ef31fd4ca9f7e868b34c18b9cb22b0ff69da/proxy.js#L391-L398
+    const res = new ServerResponse(req)
+    res.assignSocket(clientSocket)
+
+    try {
+      await this.#handleAuthentication(req, res, async () => {
+        const { port, hostname } = new URL('http://' + req.url)
+        const serverSocket = net.connect(port || 80, hostname)
+
+        await fromEvent(serverSocket, 'connect')
+
+        clientSocket.write('HTTP/1.1 200 Connection Established\r\n\r\n')
+        serverSocket.write(head)
+        fromCallback(pipeline, clientSocket, serverSocket).catch(warn)
+        fromCallback(pipeline, serverSocket, clientSocket).catch(warn)
+      })
+    } catch (error) {
+      warn(error)
+      clientSocket.end()
+    }
+  }
+
+  async #handleRequest(req, res) {
+    const { url } = req
+
+    if (url.startsWith('/')) {
+      // not a proxy request
+      return
+    }
+
+    debug('HTTP proxy', { url })
+
+    try {
+      assert(url.startsWith('http:'), 'HTTPS should use connect')
+
+      await this.#handleAuthentication(req, res, async () => {
+        const { headers } = req
+        const pHeaders = {}
+        for (const key of Object.keys(headers)) {
+          if (!IGNORED_HEADERS.has(key)) {
+            pHeaders[key] = headers[key]
+          }
+        }
+
+        const pReq = request(url, { headers: pHeaders, method: req.method })
+        fromCallback(pipeline, req, pReq).catch(warn)
+
+        const pRes = await fromEvent(pReq, 'response')
+        res.writeHead(pRes.statusCode, pRes.statusMessage, pRes.headers)
+        await fromCallback(pipeline, pRes, res)
+      })
+    } catch (error) {
+      res.statusCode = 500
+      res.end('Internal Server Error')
+      warn(error)
+    }
+  }
+}

@xen-orchestra/mixins/SslCertificate.mjs

@@ -0,0 +1,214 @@
+import { createLogger } from '@xen-orchestra/log'
+import { createSecureContext } from 'tls'
+import { dirname } from 'node:path'
+import { X509Certificate } from 'node:crypto'
+import acme from 'acme-client'
+import fs from 'node:fs/promises'
+import get from 'lodash/get.js'
+
+const { debug, info, warn } = createLogger('xo:mixins:sslCertificate')
+
+acme.setLogger(message => {
+  debug(message)
+})
+
+// - create any missing parent directories
+// - replace existing files
+// - secure permissions (read-only for the owner)
+async function outputFile(path, content) {
+  await fs.mkdir(dirname(path), { recursive: true })
+  try {
+    await fs.unlink(path)
+  } catch (error) {
+    if (error.code !== 'ENOENT') {
+      throw error
+    }
+  }
+  await fs.writeFile(path, content, { flag: 'wx', mode: 0o400 })
+}
+
+// from https://github.com/publishlab/node-acme-client/blob/master/examples/auto.js
+class SslCertificate {
+  #cert
+  #challengeCreateFn
+  #challengeRemoveFn
+  #delayBeforeRenewal = 30 * 24 * 60 * 60 * 1000 // 30 days
+  #secureContext
+  #updateSslCertificatePromise
+
+  constructor({ challengeCreateFn, challengeRemoveFn }, cert, key) {
+    this.#challengeCreateFn = challengeCreateFn
+    this.#challengeRemoveFn = challengeRemoveFn
+
+    this.#set(cert, key)
+  }
+
+  get #isValid() {
+    const cert = this.#cert
+    return cert !== undefined && Date.parse(cert.validTo) > Date.now() && cert.issuer !== cert.subject
+  }
+
+  get #shouldBeRenewed() {
+    return !(this.#isValid && Date.parse(this.#cert.validTo) > Date.now() + this.#delayBeforeRenewal)
+  }
+
+  #set(cert, key) {
+    this.#cert = new X509Certificate(cert)
+    this.#secureContext = createSecureContext({ cert, key })
+  }
+
+  async getSecureContext(config) {
+    if (!this.#shouldBeRenewed) {
+      return this.#secureContext
+    }
+
+    if (this.#updateSslCertificatePromise === undefined) {
+      // not currently updating certificate
+      //
+      // ensure we only refresh certificate once at a time
+      //
+      // promise is cleaned by #updateSslCertificate itself
+      this.#updateSslCertificatePromise = this.#updateSslCertificate(config)
+    }
+
+    // old certificate is still here, return it while updating
+    if (this.#isValid) {
+      return this.#secureContext
+    }
+
+    return this.#updateSslCertificatePromise
+  }
+
+  async #save(certPath, cert, keyPath, key) {
+    try {
+      await Promise.all([outputFile(keyPath, key), outputFile(certPath, cert)])
+      info('new certificate generated', { cert: certPath, key: keyPath })
+    } catch (error) {
+      warn(`couldn't write let's encrypt certificates to disk `, { error })
+    }
+  }
+
+  async #updateSslCertificate(config) {
+    const { cert: certPath, key: keyPath, acmeEmail, acmeDomain } = config
+    try {
+      let { acmeCa = 'letsencrypt/production' } = config
+      if (!(acmeCa.startsWith('http:') || acmeCa.startsWith('https:'))) {
+        acmeCa = get(acme.directory, acmeCa.split('/'))
+      }
+
+      /* Init client */
+      const client = new acme.Client({
+        directoryUrl: acmeCa,
+        accountKey: await acme.crypto.createPrivateKey(),
+      })
+
+      /* Create CSR */
+      let [key, csr] = await acme.crypto.createCsr({
+        commonName: acmeDomain,
+      })
+      csr = csr.toString()
+      key = key.toString()
+      debug('Successfully generated key and csr')
+
+      /* Certificate */
+      const cert = await client.auto({
+        challengeCreateFn: this.#challengeCreateFn,
+        challengePriority: ['http-01'],
+        challengeRemoveFn: this.#challengeRemoveFn,
+        csr,
+        email: acmeEmail,
+        skipChallengeVerification: true,
+        termsOfServiceAgreed: true,
+      })
+      debug('Successfully generated certificate')
+
+      this.#set(cert, key)
+
+      // don't wait for this
+      this.#save(certPath, cert, keyPath, key)
+
+      return this.#secureContext
+    } catch (error) {
+      warn(`couldn't renew ssl certificate`, { acmeDomain, error })
+    } finally {
+      this.#updateSslCertificatePromise = undefined
+    }
+  }
+}
+
+export default class SslCertificates {
+  #app
+  #challenges = new Map()
+  #challengeHandlers = {
+    challengeCreateFn: (authz, challenge, keyAuthorization) => {
+      this.#challenges.set(challenge.token, keyAuthorization)
+    },
+    challengeRemoveFn: (authz, challenge, keyAuthorization) => {
+      this.#challenges.delete(challenge.token)
+    },
+  }
+  #handlers = new Map()
+
+  constructor(app, { httpServer }) {
+    // don't setup the proxy if httpServer is not present
+    //
+    // that can happen when the app is instanciated in another context like xo-server-recover-account
+    if (httpServer === undefined) {
+      return
+    }
+    const prefix = '/.well-known/acme-challenge/'
+    httpServer.on('request', (req, res) => {
+      const { url } = req
+      if (url.startsWith(prefix)) {
+        const token = url.slice(prefix.length)
+        this.#acmeChallendMiddleware(req, res, token)
+      }
+    })
+
+    this.#app = app
+
+    httpServer.getSecureContext = this.getSecureContext.bind(this)
+  }
+
+  async getSecureContext(httpsDomainName, configKey, initialCert, initialKey) {
+    const config = this.#app.config.get(['http', 'listen', configKey])
+    const handlers = this.#handlers
+
+    const { acmeDomain } = config
+
+    // not a let's encrypt protected end point, sommething changed in the configuration
+    if (acmeDomain === undefined) {
+      handlers.delete(configKey)
+      return
+    }
+
+    // server has been access with another domain, don't use the certificate
+    if (acmeDomain !== httpsDomainName) {
+      return
+    }
+
+    let handler = handlers.get(configKey)
+    if (handler === undefined) {
+      // register the handler for this domain
+      handler = new SslCertificate(this.#challengeHandlers, initialCert, initialKey)
+      handlers.set(configKey, handler)
+    }
+    return handler.getSecureContext(config)
+  }
+
+  // middleware that will serve the http challenge to let's encrypt servers
+  #acmeChallendMiddleware(req, res, token) {
+    debug('fetching challenge for token ', token)
+    const challenge = this.#challenges.get(token)
+    debug('challenge content is ', challenge)
+    if (challenge === undefined) {
+      res.statusCode = 404
+      res.end()
+      return
+    }
+
+    res.write(challenge)
+    res.end()
+    debug('successfully answered challenge ')
+  }
+}

@xen-orchestra/mixins/_parseBasicAuth.mjs

@@ -0,0 +1,27 @@
+const RE = /^\s*basic\s+(.+?)\s*$/i
+
+export function parseBasicAuth(header) {
+  if (header === undefined) {
+    return
+  }
+
+  const matches = RE.exec(header)
+  if (matches === null) {
+    return
+  }
+
+  let credentials = Buffer.from(matches[1], 'base64').toString()
+
+  const i = credentials.indexOf(':')
+  if (i === -1) {
+    credentials = { token: credentials }
+  } else {
+    // https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.1
+    credentials = {
+      username: credentials.slice(0, i),
+      password: credentials.slice(i + 1),
+    }
+  }
+
+  return credentials
+}

@xen-orchestra/mixins/docs/HttpProxy.md

@@ -0,0 +1,74 @@
+> This module provides an HTTP and HTTPS proxy for `xo-proxy` and `xo-server`.
+
+- [Set up](#set-up)
+- [Usage](#usage)
+  - [`xo-proxy`](#xo-proxy)
+  - [`xo-server`](#xo-server)
+- [Use cases](#use-cases)
+  - [Access hosts in a private network](#access-hosts-in-a-private-network)
+  - [Allow upgrading xo-proxy via xo-server](#allow-upgrading-xo-proxy-via-xo-server)
+
+## Set up
+
+The proxy is enabled by default, to disable it, add the following lines to your config:
+
+```toml
+[http.proxy]
+enabled = false
+```
+
+## Usage
+
+For safety reasons, the proxy requires authentication to be used.
+
+### `xo-proxy`
+
+Use the authentication token:
+
+```
+$ cat ~/.config/xo-proxy/config.z-auto.json
+{"authenticationToken":"J0BgKritQgPxoyZrBJ5ViafQfLk06YoyFwC3fmfO5wU"}
+```
+
+Proxy URL to use:
+
+```
+https://J0BgKritQgPxoyZrBJ5ViafQfLk06YoyFwC3fmfO5wU@xo-proxy.company.lan
+```
+
+### `xo-server`
+
+> Only available for admin users.
+
+You can use your credentials:
+
+```
+https://user:password@xo.company.lan
+```
+
+Or create a dedicated token with `xo-cli`:
+
+```
+$ xo-cli --createToken xoa.company.lan admin@admin.net
+Password: ********
+Successfully logged with admin@admin.net
+Authentication token created
+
+DiYBFavJwf9GODZqQJs23eAx9eh3KlsRhBi8RcoX0KM
+```
+
+And use it in the URL:
+
+```
+https://DiYBFavJwf9GODZqQJs23eAx9eh3KlsRhBi8RcoX0KM@xo.company.lan
+```
+
+## Use cases
+
+### Access hosts in a private network
+
+To access hosts in a private network, deploy an XO Proxy in this network, expose its port 443 and use it as an HTTP proxy to connect to your servers in XO.
+
+### Allow upgrading xo-proxy via xo-server
+
+If your xo-proxy does not have direct Internet access, you can use xo-server as an HTTP proxy to make upgrades possible.

@xen-orchestra/mixins/docs/SslCertificate.md

@@ -0,0 +1,49 @@
+> This module provides [Let's Encrypt](https://letsencrypt.org/) integration to `xo-proxy` and `xo-server`.
+
+First of all, make sure your server is listening on HTTP on port 80 and on HTTPS 443.
+
+In `xo-server`, to avoid HTTP access, enable the redirection to HTTPs:
+
+```toml
+[http]
+redirectToHttps = true
+```
+
+Your server must be reachable with the configured domain to the certificate provider (e.g. Let's Encrypt), it usually means publicly reachable.
+
+Finally, add the following entries to your HTTPS configuration.
+
+```toml
+# Must be set to true for this feature
+autoCert = true
+
+# These entries are required and indicates where the certificate and the
+# private key will be saved.
+cert = 'path/to/cert.pem'
+key = 'path/to/key.pem'
+
+# ACME (e.g. Let's Encrypt, ZeroSSL) CA directory
+#
+# Specifies the URL to the ACME CA's directory.
+#
+# A identifier `provider/directory` can be passed instead of a URL, see the
+# list of supported directories here: https://www.npmjs.com/package/acme-client#directory-urls
+#
+# Note that the application cannot detect that this value has changed.
+#
+# In that case delete the certificate and the key files, and restart the
+# application to generate new ones.
+#
+# Default is 'letsencrypt/production'
+acmeCa = 'zerossl/production'
+
+# Domain for which the certificate should be created.
+#
+# This entry is required.
+acmeDomain = 'my.domain.net'
+
+# Optional email address which will be used for the certificate creation.
+#
+# It will be notified of any issues.
+acmeEmail = 'admin@my.domain.net'
+```

@xen-orchestra/mixins/package.json

@@ -14,16 +14,19 @@
     "url": "https://vates.fr"
   },
   "license": "AGPL-3.0-or-later",
-  "version": "0.2.0",
+  "version": "0.8.0",
   "engines": {
-    "node": ">=12"
+    "node": ">=15.6"
   },
   "dependencies": {
+    "@vates/event-listeners-manager": "^1.0.1",
     "@vates/parse-duration": "^0.1.1",
-    "@xen-orchestra/emit-async": "^0.1.0",
+    "@xen-orchestra/emit-async": "^1.0.0",
     "@xen-orchestra/log": "^0.3.0",
-    "app-conf": "^2.0.0",
-    "lodash": "^4.17.21"
+    "acme-client": "^5.0.0",
+    "app-conf": "^2.3.0",
+    "lodash": "^4.17.21",
+    "promise-toolbox": "^0.21.0"
   },
   "scripts": {
     "postversion": "npm publish --access public"

@xen-orchestra/openflow/package.json

@@ -9,7 +9,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.1.1",
+  "version": "0.1.2",
   "engines": {
     "node": ">=8.10"
   },
@@ -30,7 +30,7 @@
     "rimraf": "^3.0.0"
   },
   "dependencies": {
-    "@vates/read-chunk": "^0.1.2"
+    "@vates/read-chunk": "^1.0.0"
   },
   "author": {
     "name": "Vates SAS",

@xen-orchestra/proxy-cli/index.mjs

@@ -1,25 +1,23 @@
 #!/usr/bin/env node
 
-'use strict'
+import assert from 'assert'
+import colors from 'ansi-colors'
+import contentType from 'content-type'
+import CSON from 'cson-parser'
+import fromCallback from 'promise-toolbox/fromCallback'
+import fs from 'fs'
+import getopts from 'getopts'
+import hrp from 'http-request-plus'
+import split2 from 'split2'
+import pumpify from 'pumpify'
+import { extname } from 'path'
+import { format, parse } from 'json-rpc-protocol'
+import { inspect } from 'util'
+import { load as loadConfig } from 'app-conf'
+import { pipeline } from 'stream'
+import { readChunk } from '@vates/read-chunk'
 
-const assert = require('assert')
-const colors = require('ansi-colors')
-const contentType = require('content-type')
-const CSON = require('cson-parser')
-const fromCallback = require('promise-toolbox/fromCallback')
-const fs = require('fs')
-const getopts = require('getopts')
-const hrp = require('http-request-plus')
-const split2 = require('split2')
-const pumpify = require('pumpify')
-const { extname, join } = require('path')
-const { format, parse } = require('json-rpc-protocol')
-const { inspect } = require('util')
-const { load: loadConfig } = require('app-conf')
-const { pipeline } = require('stream')
-const { readChunk } = require('@vates/read-chunk')
-
-const pkg = require('./package.json')
+const pkg = JSON.parse(fs.readFileSync(new URL('package.json', import.meta.url)))
 
 const FORMATS = {
   __proto__: null,
@@ -32,30 +30,22 @@ const parseValue = value => (value.startsWith('json:') ? JSON.parse(value.slice(
 
 async function main(argv) {
   const config = await loadConfig('xo-proxy', {
-    appDir: join(__dirname, '..'),
     ignoreUnknownFormats: true,
   })
 
-  const { hostname = 'localhost', port } = config?.http?.listen?.https ?? {}
-
-  const {
-    _: args,
-    file,
-    help,
-    host,
-    raw,
-    token,
-  } = getopts(argv, {
+  const opts = getopts(argv, {
     alias: { file: 'f', help: 'h' },
     boolean: ['help', 'raw'],
     default: {
       token: config.authenticationToken,
     },
     stopEarly: true,
-    string: ['file', 'host', 'token'],
+    string: ['file', 'host', 'token', 'url'],
   })
 
-  if (help || (file === '' && args.length === 0)) {
+  const { _: args, file } = opts
+
+  if (opts.help || (file === '' && args.length === 0)) {
     return console.log(
       '%s',
       `Usage:
@@ -80,18 +70,29 @@ ${pkg.name} v${pkg.version}`
   const baseRequest = {
     headers: {
       'content-type': 'application/json',
-      cookie: `authenticationToken=${token}`,
     },
     pathname: '/api/v1',
-    protocol: 'https:',
     rejectUnauthorized: false,
   }
-  if (host !== '') {
-    baseRequest.host = host
+  let { token } = opts
+  if (opts.url !== '') {
+    const { protocol, host, username } = new URL(opts.url)
+    Object.assign(baseRequest, { protocol, host })
+    if (username !== '') {
+      token = username
+    }
   } else {
-    baseRequest.hostname = hostname
-    baseRequest.port = port
+    baseRequest.protocol = 'https:'
+    if (opts.host !== '') {
+      baseRequest.host = opts.host
+    } else {
+      const { hostname = 'localhost', port } = config?.http?.listen?.https ?? {}
+      baseRequest.hostname = hostname
+      baseRequest.port = port
+    }
   }
+  baseRequest.headers.cookie = `authenticationToken=${token}`
+
   const call = async ({ method, params }) => {
     if (callPath.length !== 0) {
       process.stderr.write(`\n${colors.bold(`--- call #${callPath.join('.')}`)} ---\n\n`)
@@ -130,7 +131,7 @@ ${pkg.name} v${pkg.version}`
           stdout.write(inspect(JSON.parse(line), { colors: true, depth: null }))
           stdout.write('\n')
         }
-      } else if (raw && typeof result === 'string') {
+      } else if (opts.raw && typeof result === 'string') {
         stdout.write(result)
       } else {
         stdout.write(inspect(result, { colors: true, depth: null }))

@xen-orchestra/proxy-cli/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/proxy-cli",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "license": "AGPL-3.0-or-later",
   "description": "CLI for @xen-orchestra/proxy",
   "keywords": [
@@ -19,16 +19,16 @@
   },
   "preferGlobal": true,
   "bin": {
-    "xo-proxy-cli": "./index.js"
+    "xo-proxy-cli": "./index.mjs"
   },
   "engines": {
-    "node": ">=14"
+    "node": ">=14.13"
   },
   "dependencies": {
     "@iarna/toml": "^2.2.0",
-    "@vates/read-chunk": "^0.1.2",
+    "@vates/read-chunk": "^1.0.0",
     "ansi-colors": "^4.1.1",
-    "app-conf": "^2.0.0",
+    "app-conf": "^2.3.0",
     "content-type": "^1.0.4",
     "cson-parser": "^4.0.7",
     "getopts": "^2.2.3",

@xen-orchestra/proxy/app/index.mjs

@@ -1,5 +1,7 @@
-import Config from '@xen-orchestra/mixins/Config.js'
-import Hooks from '@xen-orchestra/mixins/Hooks.js'
+import Config from '@xen-orchestra/mixins/Config.mjs'
+import Hooks from '@xen-orchestra/mixins/Hooks.mjs'
+import HttpProxy from '@xen-orchestra/mixins/HttpProxy.mjs'
+import SslCertificate from '@xen-orchestra/mixins/SslCertificate.mjs'
 import mixin from '@xen-orchestra/mixin'
 import { createDebounceResource } from '@vates/disposable/debounceResource.js'
 
@@ -13,7 +15,23 @@ import ReverseProxy from './mixins/reverseProxy.mjs'
 
 export default class App {
   constructor(opts) {
-    mixin(this, { Api, Appliance, Authentication, Backups, Config, Hooks, Logs, Remotes, ReverseProxy }, [opts])
+    mixin(
+      this,
+      {
+        Api,
+        Appliance,
+        Authentication,
+        Backups,
+        Config,
+        Hooks,
+        HttpProxy,
+        Logs,
+        Remotes,
+        ReverseProxy,
+        SslCertificate,
+      },
+      [opts]
+    )
 
     const debounceResource = createDebounceResource()
     this.config.watchDuration('resourceCacheDelay', delay => {

@xen-orchestra/proxy/app/mixins/api.mjs

@@ -1,4 +1,4 @@
-import { format, parse, MethodNotFound } from 'json-rpc-protocol'
+import { format, parse, MethodNotFound, JsonRpcError } from 'json-rpc-protocol'
 import * as errors from 'xo-common/api-errors.js'
 import Ajv from 'ajv'
 import asyncIteratorToStream from 'async-iterator-to-stream'
@@ -9,11 +9,26 @@ import helmet from 'koa-helmet'
 import Koa from 'koa'
 import once from 'lodash/once.js'
 import Router from '@koa/router'
+import stubTrue from 'lodash/stubTrue.js'
 import Zone from 'node-zone'
 import { createLogger } from '@xen-orchestra/log'
 
 const { debug, warn } = createLogger('xo:proxy:api')
 
+// format an error to JSON-RPC but do not hide non JSON-RPC errors
+function formatError(responseId, error) {
+  if (error != null && typeof error.toJsonRpcError !== 'function') {
+    const { message, ...data } = error
+
+    // force these entries even if they are not enumerable
+    data.code = error.code
+    data.stack = error.stack
+
+    error = new JsonRpcError(error.message, undefined, data)
+  }
+  return format.error(responseId, error)
+}
+
 const ndJsonStream = asyncIteratorToStream(async function* (responseId, iterable) {
   try {
     let cursor, iterator
@@ -24,7 +39,7 @@ const ndJsonStream = asyncIteratorToStream(async function* (responseId, iterable
       cursor = await iterator.next()
       yield format.response(responseId, { $responseType: 'ndjson' }) + '\n'
     } catch (error) {
-      yield format.error(responseId, error)
+      yield formatError(responseId, error)
       throw error
     }
 
@@ -52,7 +67,7 @@ export default class Api {
       ctx.req.setTimeout(0)
 
       const profile = await app.authentication.findProfile({
-        authenticationToken: ctx.cookies.get('authenticationToken'),
+        token: ctx.cookies.get('authenticationToken'),
       })
       if (profile === undefined) {
         ctx.status = 401
@@ -63,7 +78,7 @@ export default class Api {
       try {
         body = parse(body)
       } catch (error) {
-        ctx.body = format.error(null, error)
+        ctx.body = formatError(null, error)
         return
       }
 
@@ -77,7 +92,7 @@ export default class Api {
         const { method, params } = body
         warn('call error', { method, params, error })
         ctx.set('Content-Type', 'application/json')
-        ctx.body = format.error(body.id, error)
+        ctx.body = formatError(body.id, error)
         return
       }
 
@@ -166,14 +181,20 @@ export default class Api {
               throw errors.noSuchObject('method', name)
             }
 
-            const { description, params = {} } = method
-            return { description, name, params }
+            const { description, params = {}, result = {} } = method
+            return { description, name, params, result }
           },
           {
             description: 'returns the signature of an API method',
             params: {
               method: { type: 'string' },
             },
+            result: {
+              description: { type: 'string' },
+              name: { type: 'string' },
+              params: { type: 'object' },
+              result: { type: 'object' },
+            },
           },
         ],
       },
@@ -205,40 +226,29 @@ export default class Api {
     })
   }
 
-  addMethod(name, method, { description, params = {} } = {}) {
+  addMethod(name, method, { description, params = {}, result: resultSchema } = {}) {
     const methods = this._methods
 
     if (name in methods) {
       throw new Error(`API method ${name} already exists`)
     }
 
-    const ajv = this._ajv
-    const validate = ajv.compile({
-      // we want additional properties to be disabled by default
-      additionalProperties: params['*'] || false,
+    const validateParams = this.#compileSchema(params)
+    const validateResult = this.#compileSchema(resultSchema)
 
-      properties: params,
-
-      // we want params to be required by default unless explicitly marked so
-      // we use property `optional` instead of object `required`
-      required: Object.keys(params).filter(name => {
-        const param = params[name]
-        const required = !param.optional
-        delete param.optional
-        return required
-      }),
-
-      type: 'object',
-    })
-
-    const m = params => {
-      if (!validate(params)) {
-        throw errors.invalidParameters(validate.errors)
+    const m = async params => {
+      if (!validateParams(params)) {
+        throw errors.invalidParameters(validateParams.errors)
       }
-      return method(params)
+      const result = await method(params)
+      if (!validateResult(result)) {
+        warn('invalid API method result', { errors: validateResult.error, result })
+      }
+      return result
     }
     m.description = description
     m.params = params
+    m.result = resultSchema
 
     methods[name] = m
 
@@ -289,4 +299,43 @@ export default class Api {
     }
     return fn(params)
   }
+
+  #compileSchema(schema) {
+    if (schema === undefined) {
+      return stubTrue
+    }
+
+    if (schema.type === undefined) {
+      schema = { type: 'object', properties: schema }
+    }
+
+    const { type } = schema
+    if (Array.isArray(type) ? type.includes('object') : type === 'object') {
+      const { properties = {} } = schema
+
+      if (schema.additionalProperties === undefined) {
+        const wildCard = properties['*']
+        if (wildCard === undefined) {
+          // we want additional properties to be disabled by default
+          schema.additionalProperties = false
+        } else {
+          delete properties['*']
+          schema.additionalProperties = wildCard
+        }
+      }
+
+      // we want properties to be required by default unless explicitly marked so
+      // we use property `optional` instead of object `required`
+      if (schema.required === undefined) {
+        schema.required = Object.keys(properties).filter(name => {
+          const param = properties[name]
+          const required = !param.optional
+          delete param.optional
+          return required
+        })
+      }
+    }
+
+    return this._ajv.compile(schema)
+  }
 }

@xen-orchestra/proxy/app/mixins/authentication.mjs

@@ -52,7 +52,7 @@ export default class Authentication {
   }
 
   async findProfile(credentials) {
-    if (credentials?.authenticationToken === this.#token) {
+    if (credentials?.token === this.#token) {
       return new Profile()
     }
   }

@xen-orchestra/proxy/app/mixins/backups.mjs

@@ -407,6 +407,7 @@ export default class Backups {
       debounceResource: app.debounceResource.bind(app),
       dirMode: app.config.get('backups.dirMode'),
       vhdDirectoryCompression: app.config.get('backups.vhdDirectoryCompression'),
+      useGetDiskLegacy: app.config.getOptional('backups.useGetDiskLegacy'),
     })
   }
 

@xen-orchestra/proxy/config.toml

@@ -22,27 +22,6 @@ disableMergeWorker = false
 snapshotNameLabelTpl = '[XO Backup {job.name}] {vm.name_label}'
 vhdDirectoryCompression = 'brotli'
 
-[backups.defaultSettings]
-reportWhen = 'failure'
-
-[backups.metadata.defaultSettings]
-retentionPoolMetadata = 0
-retentionXoMetadata = 0
-
-[backups.vm.defaultSettings]
-bypassVdiChainsCheck = false
-checkpointSnapshot = false
-concurrency = 2
-copyRetention = 0
-deleteFirst = false
-exportRetention = 0
-fullInterval = 0
-offlineBackup = false
-offlineSnapshot = false
-snapshotRetention = 0
-timeout = 0
-vmTimeout = 0
-
 # This is a work-around.
 #
 # See https://github.com/vatesfr/xen-orchestra/pull/4674
@@ -81,11 +60,6 @@ timeout = 600e3
 disableFileRemotes = true
 
 [xapiOptions]
-# VDIs with `[NOBAK]` flag can be ignored while snapshotting an halted VM.
-#
-# This is disabled by default for the time being but will be turned on after enough testing.
-ignoreNobakVdis = false
-
 maxUncoalescedVdis = 1
 watchEvents = ['network', 'PIF', 'pool', 'SR', 'task', 'VBD', 'VDI', 'VIF', 'VM']
 

@xen-orchestra/proxy/index.mjs

@@ -6,6 +6,7 @@ import getopts from 'getopts'
 import pRetry from 'promise-toolbox/retry'
 import { catchGlobalErrors } from '@xen-orchestra/log/configure.js'
 import { create as createServer } from 'http-server-plus'
+import { createCachedLookup } from '@vates/cached-dns.lookup'
 import { createLogger } from '@xen-orchestra/log'
 import { createSecureServer } from 'http2'
 import { genSelfSignedCert } from '@xen-orchestra/self-signed'
@@ -15,6 +16,8 @@ import { load as loadConfig } from 'app-conf'
 
 catchGlobalErrors(createLogger('xo:proxy'))
 
+createCachedLookup().patchGlobal()
+
 const { fatal, info, warn } = createLogger('xo:proxy:bootstrap')
 
 const APP_DIR = new URL('.', import.meta.url).pathname
@@ -53,11 +56,32 @@ ${APP_NAME} v${APP_VERSION}
     createSecureServer: opts => createSecureServer({ ...opts, allowHTTP1: true }),
   })
 
-  forOwn(config.http.listen, async ({ autoCert, cert, key, ...opts }) => {
+  forOwn(config.http.listen, async ({ autoCert, cert, key, ...opts }, configKey) => {
+    const useAcme = autoCert && opts.acmeDomain !== undefined
+
+    // don't pass these entries to httpServer.listen(opts)
+    for (const key of Object.keys(opts).filter(_ => _.startsWith('acme'))) {
+      delete opts[key]
+    }
+
     try {
-      const niceAddress = await pRetry(
-        async () => {
-          if (cert !== undefined && key !== undefined) {
+      let niceAddress
+      if (cert !== undefined && key !== undefined) {
+        if (useAcme) {
+          opts.SNICallback = async (serverName, callback) => {
+            try {
+              // injected by mixins/SslCertificate
+              const secureContext = await httpServer.getSecureContext(serverName, configKey, opts.cert, opts.key)
+              callback(null, secureContext)
+            } catch (error) {
+              warn(error)
+              callback(error, null)
+            }
+          }
+        }
+
+        niceAddress = await pRetry(
+          async () => {
             try {
               opts.cert = fse.readFileSync(cert)
               opts.key = fse.readFileSync(key)
@@ -73,20 +97,22 @@ ${APP_NAME} v${APP_VERSION}
               opts.cert = pems.cert
               opts.key = pems.key
             }
-          }
 
-          return httpServer.listen(opts)
-        },
-        {
-          tries: 2,
-          when: e => autoCert && e.code === 'ERR_SSL_EE_KEY_TOO_SMALL',
-          onRetry: () => {
-            warn('deleting invalid certificate')
-            fse.unlinkSync(cert)
-            fse.unlinkSync(key)
+            return httpServer.listen(opts)
           },
-        }
-      )
+          {
+            tries: 2,
+            when: e => autoCert && e.code === 'ERR_SSL_EE_KEY_TOO_SMALL',
+            onRetry: () => {
+              warn('deleting invalid certificate')
+              fse.unlinkSync(cert)
+              fse.unlinkSync(key)
+            },
+          }
+        )
+      } else {
+        niceAddress = await httpServer.listen(opts)
+      }
 
       info(`Web server listening on ${niceAddress}`)
     } catch (error) {
@@ -143,6 +169,7 @@ ${APP_NAME} v${APP_VERSION}
     process.on(signal, () => {
       if (alreadyCalled) {
         warn('forced exit')
+        // eslint-disable-next-line n/no-process-exit
         process.exit(1)
       }
       alreadyCalled = true
@@ -161,6 +188,7 @@ main(process.argv.slice(2)).then(
   error => {
     fatal(error)
 
+    // eslint-disable-next-line n/no-process-exit
     process.exit(1)
   }
 )

@xen-orchestra/proxy/package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "@xen-orchestra/proxy",
-  "version": "0.20.1",
+  "version": "0.26.1",
   "license": "AGPL-3.0-or-later",
   "description": "XO Proxy used to remotely execute backup jobs",
   "keywords": [
@@ -26,32 +26,33 @@
   },
   "dependencies": {
     "@iarna/toml": "^2.2.0",
-    "@koa/router": "^10.0.0",
+    "@koa/router": "^12.0.0",
+    "@vates/cached-dns.lookup": "^1.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.0.0",
     "@vates/disposable": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.21.0",
-    "@xen-orchestra/fs": "^1.0.0",
+    "@xen-orchestra/backups": "^0.27.4",
+    "@xen-orchestra/fs": "^3.1.0",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/mixin": "^0.1.0",
-    "@xen-orchestra/mixins": "^0.2.0",
-    "@xen-orchestra/self-signed": "^0.1.0",
-    "@xen-orchestra/xapi": "^0.10.0",
+    "@xen-orchestra/mixins": "^0.8.0",
+    "@xen-orchestra/self-signed": "^0.1.3",
+    "@xen-orchestra/xapi": "^1.4.2",
     "ajv": "^8.0.3",
-    "app-conf": "^2.0.0",
+    "app-conf": "^2.3.0",
     "async-iterator-to-stream": "^1.1.0",
     "fs-extra": "^10.0.0",
     "get-stream": "^6.0.0",
     "getopts": "^2.2.3",
     "golike-defer": "^0.5.1",
-    "http-server-plus": "^0.11.0",
+    "http-server-plus": "^0.12.0",
     "http2-proxy": "^5.0.53",
     "json-rpc-protocol": "^0.13.1",
     "jsonrpc-websocket-client": "^0.7.2",
     "koa": "^2.5.1",
     "koa-compress": "^5.0.1",
-    "koa-helmet": "^5.1.0",
+    "koa-helmet": "^6.1.0",
     "lodash": "^4.17.10",
     "node-zone": "^0.4.0",
     "parse-pairs": "^1.0.0",
@@ -59,7 +60,7 @@
     "source-map-support": "^0.5.16",
     "stoppable": "^1.0.6",
     "xdg-basedir": "^5.1.0",
-    "xen-api": "^1.1.0",
+    "xen-api": "^1.2.2",
     "xo-common": "^0.8.0"
   },
   "devDependencies": {

@xen-orchestra/self-signed/index.js

@@ -2,22 +2,23 @@
 
 const { execFile } = require('child_process')
 
-const openssl = (cmd, args, { input, ...opts } = {}) =>
+const RE =
+  /^(-----BEGIN PRIVATE KEY-----.+-----END PRIVATE KEY-----\n)(-----BEGIN CERTIFICATE-----.+-----END CERTIFICATE-----\n)$/s
+exports.genSelfSignedCert = async ({ days = 360 } = {}) =>
   new Promise((resolve, reject) => {
-    const child = execFile('openssl', [cmd, ...args], opts, (error, stdout) =>
-      error != null ? reject(error) : resolve(stdout)
+    execFile(
+      'openssl',
+      ['req', '-batch', '-new', '-x509', '-days', String(days), '-nodes', '-newkey', 'rsa:2048', '-keyout', '-'],
+      (error, stdout) => {
+        if (error != null) {
+          return reject(error)
+        }
+        const matches = RE.exec(stdout)
+        if (matches === null) {
+          return reject(new Error('stdout does not match regular expression'))
+        }
+        const [, key, cert] = matches
+        resolve({ cert, key })
+      }
     )
-    if (input !== undefined) {
-      child.stdin.end(input)
-    }
   })
-
-exports.genSelfSignedCert = async ({ days = 360 } = {}) => {
-  const key = await openssl('genrsa', ['2048'])
-  return {
-    cert: await openssl('req', ['-batch', '-new', '-key', '-', '-x509', '-days', String(days), '-nodes'], {
-      input: key,
-    }),
-    key,
-  }
-}

@xen-orchestra/self-signed/package.json

@@ -9,7 +9,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.1.0",
+  "version": "0.1.3",
   "engines": {
     "node": ">=8.10"
   },

@xen-orchestra/template/.babelrc.js

@@ -1,3 +0,0 @@
-'use strict'
-
-module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))

@xen-orchestra/template/.eslintrc.js

@@ -1 +0,0 @@
-../../scripts/babel-eslintrc.js

@xen-orchestra/template/index.js

@@ -1,8 +1,10 @@
-import escapeRegExp from 'lodash/escapeRegExp'
+'use strict'
+
+const escapeRegExp = require('lodash/escapeRegExp')
 
 const compareLengthDesc = (a, b) => b.length - a.length
 
-export function compileTemplate(pattern, rules) {
+exports.compileTemplate = function compileTemplate(pattern, rules) {
   const matches = Object.keys(rules).sort(compareLengthDesc).map(escapeRegExp).join('|')
   const regExp = new RegExp(`\\\\(?:\\\\|${matches})|${matches}`, 'g')
   return (...params) =>

@xen-orchestra/template/index.spec.js

@@ -1,5 +1,8 @@
 /* eslint-env jest */
-import { compileTemplate } from '.'
+
+'use strict'
+
+const { compileTemplate } = require('.')
 
 it("correctly replaces the template's variables", () => {
   const replacer = compileTemplate('{property}_\\{property}_\\\\{property}_{constant}_%_FOO', {

@xen-orchestra/template/package.json

@@ -14,31 +14,13 @@
     "name": "Vates SAS",
     "url": "https://vates.fr"
   },
-  "preferGlobal": false,
-  "main": "dist/",
-  "browserslist": [
-    ">2%"
-  ],
   "engines": {
     "node": ">=6"
   },
-  "devDependencies": {
-    "@babel/cli": "^7.0.0",
-    "@babel/core": "^7.0.0",
-    "@babel/preset-env": "^7.0.0",
-    "cross-env": "^7.0.2",
-    "rimraf": "^3.0.0"
-  },
-  "scripts": {
-    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
-    "clean": "rimraf dist/",
-    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
-    "prebuild": "yarn run clean",
-    "predev": "yarn run prebuild",
-    "prepublishOnly": "yarn run build",
-    "postversion": "npm publish --access public"
-  },
   "dependencies": {
     "lodash": "^4.17.15"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
   }
 }

@xen-orchestra/upload-ova/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/upload-ova",
-  "version": "0.1.4",
+  "version": "0.1.5",
   "license": "AGPL-3.0-or-later",
   "description": "Basic CLI to upload ova files to Xen-Orchestra",
   "keywords": [
@@ -43,7 +43,7 @@
     "pw": "^0.0.4",
     "xdg-basedir": "^4.0.0",
     "xo-lib": "^0.11.1",
-    "xo-vmdk-to-vhd": "^2.2.0"
+    "xo-vmdk-to-vhd": "^2.4.3"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

@xen-orchestra/xapi/_AggregateError.js

@@ -0,0 +1,9 @@
+'use strict'
+
+// TODO: remove when Node >=15.0
+module.exports = class AggregateError extends Error {
+  constructor(errors, message) {
+    super(message)
+    this.errors = errors
+  }
+}

@xen-orchestra/xapi/cli.js

@@ -4,5 +4,5 @@
 
 const { Xapi } = require('./')
 require('xen-api/dist/cli.js')
-  .default(opts => new Xapi({ ignoreNobakVdis: true, ...opts }))
+  .default(opts => new Xapi(opts))
   .catch(console.error.bind(console, 'FATAL'))

@xen-orchestra/xapi/index.js

@@ -101,20 +101,16 @@ function removeWatcher(predicate, cb) {
 class Xapi extends Base {
   constructor({
     callRetryWhenTooManyPendingTasks = { delay: 5e3, tries: 10 },
-    ignoreNobakVdis,
     maxUncoalescedVdis,
     vdiDestroyRetryWhenInUse = { delay: 5e3, tries: 10 },
     ...opts
   }) {
-    assert.notStrictEqual(ignoreNobakVdis, undefined)
-
     super(opts)
     this._callRetryWhenTooManyPendingTasks = {
       ...callRetryWhenTooManyPendingTasks,
       onRetry,
       when: { code: 'TOO_MANY_PENDING_TASKS' },
     }
-    this._ignoreNobakVdis = ignoreNobakVdis
     this._maxUncoalescedVdis = maxUncoalescedVdis
     this._vdiDestroyRetryWhenInUse = {
       ...vdiDestroyRetryWhenInUse,
@@ -191,7 +187,30 @@ class Xapi extends Base {
     }
     return removeWatcher.bind(watchers, predicate, cb)
   }
+
+  // wait for an object to be in a specified state
+
+  waitObjectState(refOrUuid, predicate, { timeout } = {}) {
+    return new Promise((resolve, reject) => {
+      let timeoutHandle
+      const stop = this.watchObject(refOrUuid, object => {
+        if (predicate(object)) {
+          clearTimeout(timeoutHandle)
+          stop()
+          resolve(object)
+        }
+      })
+
+      if (timeout !== undefined) {
+        timeoutHandle = setTimeout(() => {
+          stop()
+          reject(new Error(`waitObjectState: timeout reached before ${refOrUuid} in expected state`))
+        }, timeout)
+      }
+    })
+  }
 }
+
 function mixin(mixins) {
   const xapiProto = Xapi.prototype
   const { defineProperties, getOwnPropertyDescriptor, getOwnPropertyNames } = Object
@@ -211,8 +230,9 @@ function mixin(mixins) {
   defineProperties(xapiProto, descriptors)
 }
 mixin({
-  task: require('./task.js'),
   host: require('./host.js'),
+  SR: require('./sr.js'),
+  task: require('./task.js'),
   VBD: require('./vbd.js'),
   VDI: require('./vdi.js'),
   VIF: require('./vif.js'),

@xen-orchestra/xapi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/xapi",
-  "version": "0.10.0",
+  "version": "1.4.2",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/xapi",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
@@ -15,7 +15,7 @@
     "node": ">=14"
   },
   "peerDependencies": {
-    "xen-api": "^1.1.0"
+    "xen-api": "^1.2.2"
   },
   "scripts": {
     "postversion": "npm publish --access public"
@@ -26,8 +26,10 @@
     "@xen-orchestra/log": "^0.3.0",
     "d3-time-format": "^3.0.0",
     "golike-defer": "^0.5.1",
+    "json-rpc-protocol": "^0.13.2",
     "lodash": "^4.17.15",
     "promise-toolbox": "^0.21.0",
+    "vhd-lib": "^4.0.1",
     "xo-common": "^0.8.0"
   },
   "private": false,

@xen-orchestra/xapi/sr.js

@@ -0,0 +1,179 @@
+'use strict'
+
+const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
+const { decorateClass } = require('@vates/decorate-with')
+const { defer } = require('golike-defer')
+const { incorrectState } = require('xo-common/api-errors')
+const { VDI_FORMAT_VHD } = require('./index.js')
+const assert = require('node:assert').strict
+const peekFooterFromStream = require('vhd-lib/peekFooterFromVhdStream')
+
+const AggregateError = require('./_AggregateError.js')
+
+const { warn } = require('@xen-orchestra/log').createLogger('xo:xapi:sr')
+
+const OC_MAINTENANCE = 'xo:maintenanceState'
+
+class Sr {
+  async create({
+    content_type = 'user', // recommended by Citrix
+    device_config,
+    host,
+    name_description = '',
+    name_label,
+    physical_size = 0,
+    shared,
+    sm_config = {},
+    type,
+  }) {
+    const ref = await this.call(
+      'SR.create',
+      host,
+      device_config,
+      physical_size,
+      name_label,
+      name_description,
+      type,
+      content_type,
+      shared,
+      sm_config
+    )
+
+    // https://developer-docs.citrix.com/projects/citrix-hypervisor-sdk/en/latest/xc-api-extensions/#sr
+    this.setFieldEntry('SR', ref, 'other_config', 'auto-scan', 'true').catch(warn)
+
+    return ref
+  }
+
+  // Switch the SR to maintenance mode:
+  // - shutdown all running VMs with a VDI on this SR
+  //   - their UUID is saved into SR.other_config[OC_MAINTENANCE].shutdownVms
+  //   - clean shutdown is attempted, and falls back to a hard shutdown
+  // - unplug all connected hosts from this SR
+  async enableMaintenanceMode($defer, ref, { vmsToShutdown = [] } = {}) {
+    const state = { timestamp: Date.now() }
+
+    // will throw if already in maintenance mode
+    await this.call('SR.add_to_other_config', ref, OC_MAINTENANCE, JSON.stringify(state))
+
+    await $defer.onFailure.call(this, 'call', 'SR.remove_from_other_config', ref, OC_MAINTENANCE)
+
+    const runningVms = new Map()
+    const handleVbd = async ref => {
+      const vmRef = await this.getField('VBD', ref, 'VM')
+      if (!runningVms.has(vmRef)) {
+        const power_state = await this.getField('VM', vmRef, 'power_state')
+        const isPaused = power_state === 'Paused'
+        if (isPaused || power_state === 'Running') {
+          runningVms.set(vmRef, isPaused)
+        }
+      }
+    }
+    await asyncMap(await this.getField('SR', ref, 'VDIs'), async ref => {
+      await asyncMap(await this.getField('VDI', ref, 'VBDs'), handleVbd)
+    })
+
+    {
+      const runningVmUuids = await asyncMap(runningVms.keys(), ref => this.getField('VM', ref, 'uuid'))
+
+      const set = new Set(vmsToShutdown)
+      for (const vmUuid of runningVmUuids) {
+        if (!set.has(vmUuid)) {
+          throw incorrectState({
+            actual: vmsToShutdown,
+            expected: runningVmUuids,
+            property: 'vmsToShutdown',
+          })
+        }
+      }
+    }
+
+    state.shutdownVms = {}
+
+    await asyncMapSettled(runningVms, async ([ref, isPaused]) => {
+      state.shutdownVms[await this.getField('VM', ref, 'uuid')] = isPaused
+
+      try {
+        await this.callAsync('VM.clean_shutdown', ref)
+      } catch (error) {
+        warn('SR_enableMaintenanceMode, VM clean shutdown', { error })
+        await this.callAsync('VM.hard_shutdown', ref)
+      }
+
+      $defer.onFailure.call(this, 'callAsync', 'VM.start', ref, isPaused, true)
+    })
+
+    state.unpluggedPbds = []
+    await asyncMapSettled(await this.getField('SR', ref, 'PBDs'), async ref => {
+      if (await this.getField('PBD', ref, 'currently_attached')) {
+        state.unpluggedPbds.push(await this.getField('PBD', ref, 'uuid'))
+
+        await this.callAsync('PBD.unplug', ref)
+
+        $defer.onFailure.call(this, 'callAsync', 'PBD.plug', ref)
+      }
+    })
+
+    await this.setFieldEntry('SR', ref, 'other_config', OC_MAINTENANCE, JSON.stringify(state))
+  }
+
+  // this method is best effort and will not stop on first error
+  async disableMaintenanceMode(ref) {
+    const state = JSON.parse((await this.getField('SR', ref, 'other_config'))[OC_MAINTENANCE])
+
+    // will throw if not in maintenance mode
+    await this.call('SR.remove_from_other_config', ref, OC_MAINTENANCE)
+
+    const errors = []
+
+    await asyncMap(state.unpluggedPbds, async uuid => {
+      try {
+        await this.callAsync('PBD.plug', await this.call('PBD.get_by_uuid', uuid))
+      } catch (error) {
+        errors.push(error)
+      }
+    })
+
+    await asyncMap(Object.entries(state.shutdownVms), async ([uuid, isPaused]) => {
+      try {
+        await this.callAsync('VM.start', await this.call('VM.get_by_uuid', uuid), isPaused, true)
+      } catch (error) {
+        errors.push(error)
+      }
+    })
+
+    if (errors.length !== 0) {
+      throw new AggregateError(errors)
+    }
+  }
+
+  async importVdi(
+    $defer,
+    ref,
+    stream,
+    {
+      format = VDI_FORMAT_VHD,
+      name_label = '[XO] Imported disk - ' + new Date().toISOString(),
+      virtual_size,
+      ...vdiCreateOpts
+    } = {}
+  ) {
+    if (virtual_size === undefined) {
+      if (format === VDI_FORMAT_VHD) {
+        const footer = await peekFooterFromStream(stream)
+        virtual_size = footer.currentSize
+      } else {
+        virtual_size = stream.length
+        assert.notEqual(virtual_size, undefined)
+      }
+    }
+
+    const vdiRef = await this.VDI_create({ ...vdiCreateOpts, name_label, SR: ref, virtual_size })
+    $defer.onFailure.call(this, 'callAsync', 'VDI.destroy', vdiRef)
+    await this.VDI_importContent(vdiRef, stream, { format })
+    return vdiRef
+  }
+}
+module.exports = Sr
+
+decorateClass(Sr, { enableMaintenanceMode: defer, importVdi: defer })

@xen-orchestra/xapi/vbd.js

@@ -6,6 +6,8 @@ const { Ref } = require('xen-api')
 
 const isVmRunning = require('./_isVmRunning.js')
 
+const { warn } = require('@xen-orchestra/log').createLogger('xo:xapi:vbd')
+
 const noop = Function.prototype
 
 module.exports = class Vbd {
@@ -66,8 +68,10 @@ module.exports = class Vbd {
     })
 
     if (isVmRunning(powerState)) {
-      await this.callAsync('VBD.plug', vbdRef)
+      this.callAsync('VBD.plug', vbdRef).catch(warn)
     }
+
+    return vbdRef
   }
 
   async unplug(ref) {

@xen-orchestra/xapi/vdi.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const assert = require('node:assert').strict
 const CancelToken = require('promise-toolbox/CancelToken')
 const pCatch = require('promise-toolbox/catch')
 const pRetry = require('promise-toolbox/retry')
@@ -30,8 +31,7 @@ class Vdi {
       other_config = {},
       read_only = false,
       sharable = false,
-      sm_config,
-      SR,
+      SR = this.pool.default_SR,
       tags,
       type = 'user',
       virtual_size,
@@ -39,10 +39,10 @@ class Vdi {
     },
     {
       // blindly copying `sm_config` from another VDI can create problems,
-      // therefore it is ignored by default by this method
+      // therefore it should be passed explicitly
       //
       // see https://github.com/vatesfr/xen-orchestra/issues/4482
-      setSmConfig = false,
+      sm_config,
     } = {}
   ) {
     return this.call('VDI.create', {
@@ -51,7 +51,7 @@ class Vdi {
       other_config,
       read_only,
       sharable,
-      sm_config: setSmConfig ? sm_config : undefined,
+      sm_config,
       SR,
       tags,
       type,
@@ -87,6 +87,8 @@ class Vdi {
   }
 
   async importContent(ref, stream, { cancelToken = CancelToken.none, format }) {
+    assert.notEqual(format, undefined)
+
     if (stream.length === undefined) {
       throw new Error('Trying to import a VDI without a length field. Please report this error to Xen Orchestra.')
     }

@xen-orchestra/xapi/vm.js

@@ -11,7 +11,8 @@ const { asyncMap } = require('@xen-orchestra/async-map')
 const { createLogger } = require('@xen-orchestra/log')
 const { decorateClass } = require('@vates/decorate-with')
 const { defer } = require('golike-defer')
-const { incorrectState } = require('xo-common/api-errors.js')
+const { incorrectState, forbiddenOperation } = require('xo-common/api-errors.js')
+const { JsonRpcError } = require('json-rpc-protocol')
 const { Ref } = require('xen-api')
 
 const extractOpaqueRef = require('./_extractOpaqueRef.js')
@@ -45,6 +46,21 @@ const cleanBiosStrings = biosStrings => {
   }
 }
 
+async function listNobakVbds(xapi, vbdRefs) {
+  const vbds = []
+  await asyncMap(vbdRefs, async vbdRef => {
+    const vbd = await xapi.getRecord('VBD', vbdRef)
+    if (
+      vbd.type === 'Disk' &&
+      Ref.isNotEmpty(vbd.VDI) &&
+      (await xapi.getField('VDI', vbd.VDI, 'name_label')).startsWith('[NOBAK]')
+    ) {
+      vbds.push(vbd)
+    }
+  })
+  return vbds
+}
+
 async function safeGetRecord(xapi, type, ref) {
   try {
     return await xapi.getRecord(type, ref)
@@ -129,9 +145,25 @@ class Vm {
     }
   }
 
-  async checkpoint(vmRef, { cancelToken = CancelToken.none, name_label } = {}) {
+  async checkpoint($defer, vmRef, { cancelToken = CancelToken.none, ignoreNobakVdis = false, name_label } = {}) {
+    const vm = await this.getRecord('VM', vmRef)
+
+    let destroyNobakVdis = false
+
+    if (ignoreNobakVdis) {
+      if (vm.power_state === 'Halted') {
+        await asyncMap(await listNobakVbds(this, vm.VBDs), async vbd => {
+          await this.VBD_destroy(vbd.$ref)
+          $defer.call(this, 'VBD_create', vbd)
+        })
+      } else {
+        // cannot unplug VBDs on Running, Paused and Suspended VMs
+        destroyNobakVdis = true
+      }
+    }
+
     if (name_label === undefined) {
-      name_label = await this.getField('VM', vmRef, 'name_label')
+      name_label = vm.name_label
     }
     try {
       const ref = await this.callAsync(cancelToken, 'VM.checkpoint', vmRef, name_label).then(extractOpaqueRef)
@@ -148,6 +180,12 @@ class Vm {
         noop
       )
 
+      if (destroyNobakVdis) {
+        await asyncMap(await listNobakVbds(this, await this.getField('VM', ref, 'VBDs')), vbd =>
+          this.VDI_destroy(vbd.VDI)
+        )
+      }
+
       return ref
     } catch (error) {
       if (error.code === 'VM_BAD_POWER_STATE') {
@@ -306,7 +344,13 @@ class Vm {
     const vm = await this.getRecord('VM', vmRef)
 
     if (!bypassBlockedOperation && 'destroy' in vm.blocked_operations) {
-      throw new Error('destroy is blocked')
+      throw forbiddenOperation(
+        `destroy is blocked: ${
+          vm.blocked_operations.destroy === 'true'
+            ? 'protected from accidental deletion'
+            : vm.blocked_operations.destroy
+        }`
+      )
     }
 
     if (!forceDeleteDefaultTemplate && isDefaultTemplate(vm)) {
@@ -466,6 +510,22 @@ class Vm {
       }
       return ref
     } catch (error) {
+      if (
+        // xxhash is the new form consistency hashing in CH 8.1 which uses a faster,
+        // more efficient hashing algorithm to generate the consistency checks
+        // in order to support larger files without the consistency checking process taking an incredibly long time
+        error.code === 'IMPORT_ERROR' &&
+        error.params?.some(
+          param =>
+            param.includes('INTERNAL_ERROR') &&
+            param.includes('Expected to find an inline checksum') &&
+            param.includes('.xxhash')
+        )
+      ) {
+        warn('import', { error })
+        throw new JsonRpcError('Importing this VM requires XCP-ng or Citrix Hypervisor >=8.1')
+      }
+
       // augment the error with as much relevant info as possible
       const [poolMaster, sr] = await Promise.all([
         safeGetRecord(this, 'host', this.pool.master),
@@ -477,21 +537,39 @@ class Vm {
     }
   }
 
-  async snapshot($defer, vmRef, { cancelToken = CancelToken.none, name_label } = {}) {
+  async snapshot(
+    $defer,
+    vmRef,
+    { cancelToken = CancelToken.none, ignoreNobakVdis = false, name_label, unplugVusbs = false } = {}
+  ) {
     const vm = await this.getRecord('VM', vmRef)
-    // cannot unplug VBDs on Running, Paused and Suspended VMs
-    if (vm.power_state === 'Halted' && this._ignoreNobakVdis) {
-      await asyncMap(vm.VBDs, async vbdRef => {
-        const vbd = await this.getRecord('VBD', vbdRef)
-        if (
-          vbd.type === 'Disk' &&
-          Ref.isNotEmpty(vbd.VDI) &&
-          (await this.getField('VDI', vbd.VDI, 'name_label')).startsWith('[NOBAK]')
-        ) {
-          await this.VBD_destroy(vbdRef)
+
+    const isHalted = vm.power_state === 'Halted'
+
+    // requires the VM to be halted because it's not possible to re-plug VUSB on a live VM
+    if (unplugVusbs && isHalted) {
+      // vm.VUSBs can be undefined (e.g. on XS 7.0.0)
+      const vusbs = vm.VUSBs
+      if (vusbs !== undefined) {
+        await asyncMap(vusbs, async ref => {
+          const vusb = await this.getRecord('VUSB', ref)
+          await vusb.$call('destroy')
+          $defer.call(this, 'call', 'VUSB.create', vusb.VM, vusb.USB_group, vusb.other_config)
+        })
+      }
+    }
+
+    let destroyNobakVdis = false
+    if (ignoreNobakVdis) {
+      if (isHalted) {
+        await asyncMap(await listNobakVbds(this, vm.VBDs), async vbd => {
+          await this.VBD_destroy(vbd.$ref)
           $defer.call(this, 'VBD_create', vbd)
-        }
-      })
+        })
+      } else {
+        // cannot unplug VBDs on Running, Paused and Suspended VMs
+        destroyNobakVdis = true
+      }
     }
 
     if (name_label === undefined) {
@@ -580,12 +658,19 @@ class Vm {
       noop
     )
 
+    if (destroyNobakVdis) {
+      await asyncMap(await listNobakVbds(this, await this.getField('VM', ref, 'VBDs')), vbd =>
+        this.VDI_destroy(vbd.VDI)
+      )
+    }
+
     return ref
   }
 }
 module.exports = Vm
 
 decorateClass(Vm, {
+  checkpoint: defer,
   create: defer,
   export: defer,
   snapshot: defer,