wip

Introduced by 083483645e

Fixes Zammad#18120
Fixes Zammad#18266

- Always assume that data can be `null`
- Handle edge cases where all values are `null`
- Properly handle nested RRD collections: collections have different depths (`memory`: 1, `cpus[0]`: 2, `pifs.rx[0]`: 3, ...). This PR replaces `getLastDays` which wouldn't handle those depths properly, with `getDeepLastValues` which is run on the whole stat object and doesn't assume the depth of the collections. It finds any Array at any depth and slices it to only keep the last N values.

.prettierrc.js

@@ -1,8 +1,11 @@
+'use strict'
+
 module.exports = {
   arrowParens: 'avoid',
   jsxSingleQuote: true,
   semi: false,
   singleQuote: true,
+  trailingComma: 'es5',
 
   // 2020-11-24: Requested by nraynaud and approved by the rest of the team
   //

@vates/async-each/package.json

@@ -33,7 +33,7 @@
     "test": "node--test"
   },
   "devDependencies": {
-    "sinon": "^15.0.1",
+    "sinon": "^16.0.0",
     "tap": "^16.3.0",
     "test": "^3.2.1"
   }

@vates/decorate-with/index.test.js

@@ -13,12 +13,15 @@ describe('decorateWith', () => {
     const expectedFn = Function.prototype
     const newFn = () => {}
 
-    const decorator = decorateWith(function wrapper(fn, ...args) {
-      assert.deepStrictEqual(fn, expectedFn)
-      assert.deepStrictEqual(args, expectedArgs)
+    const decorator = decorateWith(
+      function wrapper(fn, ...args) {
+        assert.deepStrictEqual(fn, expectedFn)
+        assert.deepStrictEqual(args, expectedArgs)
 
-      return newFn
-    }, ...expectedArgs)
+        return newFn
+      },
+      ...expectedArgs
+    )
 
     const descriptor = {
       configurable: true,

@vates/disposable/package.json

@@ -29,7 +29,7 @@
     "ensure-array": "^1.0.0"
   },
   "devDependencies": {
-    "sinon": "^15.0.1",
+    "sinon": "^16.0.0",
     "test": "^3.2.1"
   }
 }

@vates/fuse-vhd/index.mjs

@@ -1,9 +1,7 @@
-'use strict'
-
-const LRU = require('lru-cache')
-const Fuse = require('fuse-native')
-const { VhdSynthetic } = require('vhd-lib')
-const { Disposable, fromCallback } = require('promise-toolbox')
+import LRU from 'lru-cache'
+import Fuse from 'fuse-native'
+import { VhdSynthetic } from 'vhd-lib'
+import { Disposable, fromCallback } from 'promise-toolbox'
 
 // build a s stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js
 const stat = st => ({
@@ -16,7 +14,7 @@ const stat = st => ({
   gid: st.gid !== undefined ? st.gid : process.getgid(),
 })
 
-exports.mount = Disposable.factory(async function* mount(handler, diskPath, mountDir) {
+export const mount = Disposable.factory(async function* mount(handler, diskPath, mountDir) {
   const vhd = yield VhdSynthetic.fromVhdChain(handler, diskPath)
 
   const cache = new LRU({

@vates/fuse-vhd/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vates/fuse-vhd",
-  "version": "1.0.0",
+  "version": "2.0.0",
   "license": "ISC",
   "private": false,
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/fuse-vhd",
@@ -15,13 +15,14 @@
     "url": "https://vates.fr"
   },
   "engines": {
-    "node": ">=10.0"
+    "node": ">=14"
   },
+  "main": "./index.mjs",
   "dependencies": {
     "fuse-native": "^2.2.6",
     "lru-cache": "^7.14.0",
     "promise-toolbox": "^0.21.0",
-    "vhd-lib": "^4.5.0"
+    "vhd-lib": "^4.6.1"
   },
   "scripts": {
     "postversion": "npm publish --access public"

@vates/nbd-client/constants.js

@@ -1,42 +0,0 @@
-'use strict'
-exports.INIT_PASSWD = Buffer.from('NBDMAGIC') // "NBDMAGIC" ensure we're connected to a nbd server
-exports.OPTS_MAGIC = Buffer.from('IHAVEOPT') // "IHAVEOPT" start an option block
-exports.NBD_OPT_REPLY_MAGIC = 1100100111001001n // magic received during negociation
-exports.NBD_OPT_EXPORT_NAME = 1
-exports.NBD_OPT_ABORT = 2
-exports.NBD_OPT_LIST = 3
-exports.NBD_OPT_STARTTLS = 5
-exports.NBD_OPT_INFO = 6
-exports.NBD_OPT_GO = 7
-
-exports.NBD_FLAG_HAS_FLAGS = 1 << 0
-exports.NBD_FLAG_READ_ONLY = 1 << 1
-exports.NBD_FLAG_SEND_FLUSH = 1 << 2
-exports.NBD_FLAG_SEND_FUA = 1 << 3
-exports.NBD_FLAG_ROTATIONAL = 1 << 4
-exports.NBD_FLAG_SEND_TRIM = 1 << 5
-
-exports.NBD_FLAG_FIXED_NEWSTYLE = 1 << 0
-
-exports.NBD_CMD_FLAG_FUA = 1 << 0
-exports.NBD_CMD_FLAG_NO_HOLE = 1 << 1
-exports.NBD_CMD_FLAG_DF = 1 << 2
-exports.NBD_CMD_FLAG_REQ_ONE = 1 << 3
-exports.NBD_CMD_FLAG_FAST_ZERO = 1 << 4
-
-exports.NBD_CMD_READ = 0
-exports.NBD_CMD_WRITE = 1
-exports.NBD_CMD_DISC = 2
-exports.NBD_CMD_FLUSH = 3
-exports.NBD_CMD_TRIM = 4
-exports.NBD_CMD_CACHE = 5
-exports.NBD_CMD_WRITE_ZEROES = 6
-exports.NBD_CMD_BLOCK_STATUS = 7
-exports.NBD_CMD_RESIZE = 8
-
-exports.NBD_REQUEST_MAGIC = 0x25609513 // magic number to create a new NBD request to send to the server
-exports.NBD_REPLY_MAGIC = 0x67446698 // magic number received from the server when reading response to a nbd request
-exports.NBD_REPLY_ACK = 1
-
-exports.NBD_DEFAULT_PORT = 10809
-exports.NBD_DEFAULT_BLOCK_SIZE = 64 * 1024

@vates/nbd-client/constants.mjs

@@ -0,0 +1,41 @@
+export const INIT_PASSWD = Buffer.from('NBDMAGIC') // "NBDMAGIC" ensure we're connected to a nbd server
+export const OPTS_MAGIC = Buffer.from('IHAVEOPT') // "IHAVEOPT" start an option block
+export const NBD_OPT_REPLY_MAGIC = 1100100111001001n // magic received during negociation
+export const NBD_OPT_EXPORT_NAME = 1
+export const NBD_OPT_ABORT = 2
+export const NBD_OPT_LIST = 3
+export const NBD_OPT_STARTTLS = 5
+export const NBD_OPT_INFO = 6
+export const NBD_OPT_GO = 7
+
+export const NBD_FLAG_HAS_FLAGS = 1 << 0
+export const NBD_FLAG_READ_ONLY = 1 << 1
+export const NBD_FLAG_SEND_FLUSH = 1 << 2
+export const NBD_FLAG_SEND_FUA = 1 << 3
+export const NBD_FLAG_ROTATIONAL = 1 << 4
+export const NBD_FLAG_SEND_TRIM = 1 << 5
+
+export const NBD_FLAG_FIXED_NEWSTYLE = 1 << 0
+
+export const NBD_CMD_FLAG_FUA = 1 << 0
+export const NBD_CMD_FLAG_NO_HOLE = 1 << 1
+export const NBD_CMD_FLAG_DF = 1 << 2
+export const NBD_CMD_FLAG_REQ_ONE = 1 << 3
+export const NBD_CMD_FLAG_FAST_ZERO = 1 << 4
+
+export const NBD_CMD_READ = 0
+export const NBD_CMD_WRITE = 1
+export const NBD_CMD_DISC = 2
+export const NBD_CMD_FLUSH = 3
+export const NBD_CMD_TRIM = 4
+export const NBD_CMD_CACHE = 5
+export const NBD_CMD_WRITE_ZEROES = 6
+export const NBD_CMD_BLOCK_STATUS = 7
+export const NBD_CMD_RESIZE = 8
+
+export const NBD_REQUEST_MAGIC = 0x25609513 // magic number to create a new NBD request to send to the server
+export const NBD_REPLY_MAGIC = 0x67446698 // magic number received from the server when reading response to a nbd request
+export const NBD_REPLY_ACK = 1
+
+export const NBD_DEFAULT_PORT = 10809
+export const NBD_DEFAULT_BLOCK_SIZE = 64 * 1024

@vates/nbd-client/index.mjs

@@ -1,8 +1,11 @@
-'use strict'
-const assert = require('node:assert')
-const { Socket } = require('node:net')
-const { connect } = require('node:tls')
-const {
+import assert from 'node:assert'
+import { Socket } from 'node:net'
+import { connect } from 'node:tls'
+import { fromCallback, pRetry, pDelay, pTimeout } from 'promise-toolbox'
+import { readChunkStrict } from '@vates/read-chunk'
+import { createLogger } from '@xen-orchestra/log'
+
+import {
   INIT_PASSWD,
   NBD_CMD_READ,
   NBD_DEFAULT_BLOCK_SIZE,
@@ -17,16 +20,13 @@ const {
   NBD_REQUEST_MAGIC,
   OPTS_MAGIC,
   NBD_CMD_DISC,
-} = require('./constants.js')
-const { fromCallback, pRetry, pDelay, pTimeout } = require('promise-toolbox')
-const { readChunkStrict } = require('@vates/read-chunk')
-const { createLogger } = require('@xen-orchestra/log')
+} from './constants.mjs'
 
 const { warn } = createLogger('vates:nbd-client')
 
 // documentation is here : https://github.com/NetworkBlockDevice/nbd/blob/master/doc/proto.md
 
-module.exports = class NbdClient {
+export default class NbdClient {
   #serverAddress
   #serverCert
   #serverPort

@vates/nbd-client/package.json

@@ -13,17 +13,18 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "1.2.1",
+  "version": "2.0.0",
   "engines": {
     "node": ">=14.0"
   },
+  "main": "./index.mjs",
   "dependencies": {
     "@vates/async-each": "^1.0.0",
-    "@vates/read-chunk": "^1.1.1",
+    "@vates/read-chunk": "^1.2.0",
     "@xen-orchestra/async-map": "^0.1.2",
     "@xen-orchestra/log": "^0.6.0",
     "promise-toolbox": "^0.21.0",
-    "xen-api": "^1.3.3"
+    "xen-api": "^1.3.6"
   },
   "devDependencies": {
     "tap": "^16.3.0",
@@ -31,6 +32,6 @@
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test-integration": "tap --lines 97 --functions 95 --branches 74 --statements 97 tests/*.integ.js"
+    "test-integration": "tap --lines 97 --functions 95 --branches 74 --statements 97 tests/*.integ.mjs"
   }
 }

@vates/nbd-client/tests/nbdclient.integ.mjs

@@ -1,13 +1,12 @@
-'use strict'
-const NbdClient = require('../index.js')
-const { spawn, exec } = require('node:child_process')
-const fs = require('node:fs/promises')
-const { test } = require('tap')
-const tmp = require('tmp')
-const { pFromCallback } = require('promise-toolbox')
-const { Socket } = require('node:net')
-const { NBD_DEFAULT_PORT } = require('../constants.js')
-const assert = require('node:assert')
+import NbdClient from '../index.mjs'
+import { spawn, exec } from 'node:child_process'
+import fs from 'node:fs/promises'
+import { test } from 'tap'
+import tmp from 'tmp'
+import { pFromCallback } from 'promise-toolbox'
+import { Socket } from 'node:net'
+import { NBD_DEFAULT_PORT } from '../constants.mjs'
+import assert from 'node:assert'
 
 const FILE_SIZE = 10 * 1024 * 1024
 

@vates/node-vsphere-soap/lib/client.mjs

@@ -1,4 +1,3 @@
-'use strict'
 /*
 
   node-vsphere-soap
@@ -12,17 +11,18 @@
 
 */
 
-const EventEmitter = require('events').EventEmitter
-const axios = require('axios')
-const https = require('node:https')
-const util = require('util')
-const soap = require('soap')
-const Cookie = require('soap-cookie') // required for session persistence
+import { EventEmitter } from 'events'
+import axios from 'axios'
+import https from 'node:https'
+import util from 'util'
+import soap from 'soap'
+import Cookie from 'soap-cookie' // required for session persistence
+
 // Client class
 // inherits from EventEmitter
 // possible events: connect, error, ready
 
-function Client(vCenterHostname, username, password, sslVerify) {
+export function Client(vCenterHostname, username, password, sslVerify) {
   this.status = 'disconnected'
   this.reconnectCount = 0
 
@@ -228,4 +228,3 @@ function _soapErrorHandler(self, emitter, command, args, err) {
 }
 
 // end
-exports.Client = Client

@vates/node-vsphere-soap/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@vates/node-vsphere-soap",
-  "version": "1.0.0",
+  "version": "2.0.0",
   "description": "interface to vSphere SOAP/WSDL from node for interfacing with vCenter or ESXi, forked from node-vsphere-soap",
-  "main": "lib/client.js",
+  "main": "lib/client.mjs",
   "author": "reedog117",
   "repository": {
     "directory": "@vates/node-vsphere-soap",
@@ -30,7 +30,7 @@
   "private": false,
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/node-vsphere-soap",
   "engines": {
-    "node": ">=8.10"
+    "node": ">=14"
   },
   "scripts": {
     "postversion": "npm publish --access public"

@vates/node-vsphere-soap/test/config-test.stub.mjs

@@ -1,15 +1,11 @@
-'use strict'
-
 // place your own credentials here for a vCenter or ESXi server
 // this information will be used for connecting to a vCenter instance
 // for module testing
 // name the file config-test.js
 
-const vCenterTestCreds = {
+export const vCenterTestCreds = {
   vCenterIP: 'vcsa',
   vCenterUser: 'vcuser',
   vCenterPassword: 'vcpw',
   vCenter: true,
 }
-
-exports.vCenterTestCreds = vCenterTestCreds

@vates/node-vsphere-soap/test/vsphere-soap.test.mjs

@@ -1,18 +1,16 @@
-'use strict'
-
 /*
   vsphere-soap.test.js
 
   tests for the vCenterConnectionInstance class
 */
 
-const assert = require('assert')
-const { describe, it } = require('test')
+import assert from 'assert'
+import { describe, it } from 'test'
 
-const vc = require('../lib/client')
+import * as vc from '../lib/client.mjs'
 
-// eslint-disable-next-line n/no-missing-require
-const TestCreds = require('../config-test.js').vCenterTestCreds
+// eslint-disable-next-line n/no-missing-import
+import { vCenterTestCreds as TestCreds } from '../config-test.mjs'
 
 const VItest = new vc.Client(TestCreds.vCenterIP, TestCreds.vCenterUser, TestCreds.vCenterPassword, false)
 

@vates/read-chunk/index.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const assert = require('assert')
+const isUtf8 = require('isutf8')
 
 /**
  * Read a chunk of data from a stream.
@@ -81,6 +82,13 @@ exports.readChunkStrict = async function readChunkStrict(stream, size) {
 
   if (size !== undefined && chunk.length !== size) {
     const error = new Error(`stream has ended with not enough data (actual: ${chunk.length}, expected: ${size})`)
+
+    // Buffer.isUtf8 is too recent for now
+    // @todo : replace external package by Buffer.isUtf8 when the supported version of node reach 18
+
+    if (chunk.length < 1024 && isUtf8(chunk)) {
+      error.text = chunk.toString('utf8')
+    }
     Object.defineProperties(error, {
       chunk: {
         value: chunk,

@vates/read-chunk/index.test.js

@@ -102,12 +102,37 @@ describe('readChunkStrict', function () {
     assert.strictEqual(error.chunk, undefined)
   })
 
-  it('throws if stream ends with not enough data', async () => {
+  it('throws if stream ends with not enough data, utf8', async () => {
     const error = await rejectionOf(readChunkStrict(makeStream(['foo', 'bar']), 10))
     assert(error instanceof Error)
     assert.strictEqual(error.message, 'stream has ended with not enough data (actual: 6, expected: 10)')
+    assert.strictEqual(error.text, 'foobar')
     assert.deepEqual(error.chunk, Buffer.from('foobar'))
   })
+
+  it('throws if stream ends with not enough data, non utf8 ', async () => {
+    const source = [Buffer.alloc(10, 128), Buffer.alloc(10, 128)]
+    const error = await rejectionOf(readChunkStrict(makeStream(source), 30))
+    assert(error instanceof Error)
+    assert.strictEqual(error.message, 'stream has ended with not enough data (actual: 20, expected: 30)')
+    assert.strictEqual(error.text, undefined)
+    assert.deepEqual(error.chunk, Buffer.concat(source))
+  })
+
+  it('throws if stream ends with not enough data,  utf8 , long data', async () => {
+    const source = Buffer.from('a'.repeat(1500))
+    const error = await rejectionOf(readChunkStrict(makeStream([source]), 2000))
+    assert(error instanceof Error)
+    assert.strictEqual(error.message, `stream has ended with not enough data (actual: 1500, expected: 2000)`)
+    assert.strictEqual(error.text, undefined)
+    assert.deepEqual(error.chunk, source)
+  })
+
+  it('succeed', async () => {
+    const source = Buffer.from('a'.repeat(20))
+    const chunk = await readChunkStrict(makeStream([source]), 10)
+    assert.deepEqual(source.subarray(10), chunk)
+  })
 })
 
 describe('skip', function () {
@@ -134,6 +159,16 @@ describe('skip', function () {
   it('returns less size if stream ends', async () => {
     assert.deepEqual(await skip(makeStream('foo bar'), 10), 7)
   })
+
+  it('put back if it read too much', async () => {
+    let source = makeStream(['foo', 'bar'])
+    await skip(source, 1) // read part of data chunk
+    const chunk = (await readChunkStrict(source, 2)).toString('utf-8')
+    assert.strictEqual(chunk, 'oo')
+
+    source = makeStream(['foo', 'bar'])
+    assert.strictEqual(await skip(source, 3), 3) // read aligned with data chunk
+  })
 })
 
 describe('skipStrict', function () {
@@ -144,4 +179,9 @@ describe('skipStrict', function () {
     assert.strictEqual(error.message, 'stream has ended with not enough data (actual: 7, expected: 10)')
     assert.deepEqual(error.bytesSkipped, 7)
   })
+  it('succeed', async () => {
+    const source = makeStream(['foo', 'bar', 'baz'])
+    const res = await skipStrict(source, 4)
+    assert.strictEqual(res, undefined)
+  })
 })

@vates/read-chunk/package.json

@@ -19,7 +19,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "1.1.1",
+  "version": "1.2.0",
   "engines": {
     "node": ">=8.10"
   },
@@ -33,5 +33,8 @@
   },
   "devDependencies": {
     "test": "^3.2.1"
+  },
+  "dependencies": {
+    "isutf8": "^4.0.0"
   }
 }

@vates/stream-reader/package.json

@@ -27,7 +27,7 @@
   "license": "ISC",
   "version": "0.1.0",
   "engines": {
-    "node": ">=10"
+    "node": ">=12.3"
   },
   "scripts": {
     "postversion": "npm publish --access public",

@xen-orchestra/async-map/package.json

@@ -35,7 +35,7 @@
     "test": "node--test"
   },
   "devDependencies": {
-    "sinon": "^15.0.1",
+    "sinon": "^16.0.0",
     "test": "^3.2.1"
   }
 }

@xen-orchestra/backups-cli/commands/clean-vms.mjs

@@ -1,5 +1,5 @@
 import { asyncMap } from '@xen-orchestra/async-map'
-import { RemoteAdapter } from '@xen-orchestra/backups/RemoteAdapter.js'
+import { RemoteAdapter } from '@xen-orchestra/backups/RemoteAdapter.mjs'
 import { getSyncedHandler } from '@xen-orchestra/fs'
 import getopts from 'getopts'
 import { basename, dirname } from 'path'

@xen-orchestra/backups-cli/package.json

@@ -7,9 +7,9 @@
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.39.0",
-    "@xen-orchestra/fs": "^4.0.1",
-    "filenamify": "^4.1.0",
+    "@xen-orchestra/backups": "^0.43.0",
+    "@xen-orchestra/fs": "^4.1.0",
+    "filenamify": "^6.0.0",
     "getopts": "^2.2.5",
     "lodash": "^4.17.15",
     "promise-toolbox": "^0.21.0"
@@ -27,7 +27,7 @@
   "scripts": {
     "postversion": "npm publish --access public"
   },
-  "version": "1.0.9",
+  "version": "1.0.13",
   "license": "AGPL-3.0-or-later",
   "author": {
     "name": "Vates SAS",

@xen-orchestra/backups/Backup.mjs

@@ -1,10 +1,8 @@
-'use strict'
+import { Metadata } from './_runners/Metadata.mjs'
+import { VmsRemote } from './_runners/VmsRemote.mjs'
+import { VmsXapi } from './_runners/VmsXapi.mjs'
 
-const { Metadata } = require('./_runners/Metadata.js')
-const { VmsRemote } = require('./_runners/VmsRemote.js')
-const { VmsXapi } = require('./_runners/VmsXapi.js')
-
-exports.createRunner = function createRunner(opts) {
+export function createRunner(opts) {
   const { type } = opts.job
   switch (type) {
     case 'backup':

@xen-orchestra/backups/DurablePartition.mjs

@@ -1,8 +1,6 @@
-'use strict'
+import { asyncMap } from '@xen-orchestra/async-map'
 
-const { asyncMap } = require('@xen-orchestra/async-map')
-
-exports.DurablePartition = class DurablePartition {
+export class DurablePartition {
   // private resource API is used exceptionally to be able to separate resource creation and release
   #partitionDisposers = {}
 

@xen-orchestra/backups/HealthCheckVmBackup.mjs

@@ -1,8 +1,6 @@
-'use strict'
+import { Task } from './Task.mjs'
 
-const { Task } = require('./Task')
-
-exports.HealthCheckVmBackup = class HealthCheckVmBackup {
+export class HealthCheckVmBackup {
   #restoredVm
   #timeout
   #xapi

@xen-orchestra/backups/ImportVmBackup.mjs

@@ -1,13 +1,11 @@
-'use strict'
+import assert from 'node:assert'
 
-const assert = require('assert')
+import { formatFilenameDate } from './_filenameDate.mjs'
+import { importIncrementalVm } from './_incrementalVm.mjs'
+import { Task } from './Task.mjs'
+import { watchStreamSize } from './_watchStreamSize.mjs'
 
-const { formatFilenameDate } = require('./_filenameDate.js')
-const { importIncrementalVm } = require('./_incrementalVm.js')
-const { Task } = require('./Task.js')
-const { watchStreamSize } = require('./_watchStreamSize.js')
-
-exports.ImportVmBackup = class ImportVmBackup {
+export class ImportVmBackup {
   constructor({ adapter, metadata, srUuid, xapi, settings: { newMacAddresses, mapVdisSrs = {} } = {} }) {
     this._adapter = adapter
     this._importIncrementalVmSettings = { newMacAddresses, mapVdisSrs }
@@ -16,7 +14,26 @@ exports.ImportVmBackup = class ImportVmBackup {
     this._xapi = xapi
   }
 
+  async #detectBaseVdis(){
+    const vmUuid = this._metadata.vm.uuid
+    const vm = await this._xapi.getRecordByUuid('VM', vmUuid)
+    const disks = vm.$getDisks()
+    const snapshots = {}
+    console.log({disks})
+    for (const disk of Object.values(disks)){
+      console.log({snapshots: disk.snapshots})
+      for(const snapshotRef of disk.snapshots){
+        
+        const snapshot = await this._xapi.getRecordByUuid('VDI', snapshotRef)
+        snapshots[snapshot.uuid] = disk.uuid
+      }
+    }
+    console.log({snapshots})
+    return snapshots
+  }
+
   async run() {
+    console.log('RUN')
     const adapter = this._adapter
     const metadata = this._metadata
     const isFull = metadata.mode === 'full'
@@ -24,18 +41,23 @@ exports.ImportVmBackup = class ImportVmBackup {
     const sizeContainer = { size: 0 }
 
     let backup
+    
     if (isFull) {
       backup = await adapter.readFullVmBackup(metadata)
       watchStreamSize(backup, sizeContainer)
     } else {
+      console.log('restore delta')
       assert.strictEqual(metadata.mode, 'delta')
-
       const ignoredVdis = new Set(
         Object.entries(this._importIncrementalVmSettings.mapVdisSrs)
           .filter(([_, srUuid]) => srUuid === null)
           .map(([vdiUuid]) => vdiUuid)
       )
-      backup = await adapter.readIncrementalVmBackup(metadata, ignoredVdis)
+      //const vdiSnap = await this._xapi.getRecord('VDI-snapshot','83c96977-9bc5-483d-b816-4c96622fb5e6') 
+      //console.log({vdiSnap})
+      const baseVdis = this.#detectBaseVdis()
+      backup = await adapter.readIncrementalVmBackup(metadata, ignoredVdis, { baseVdis })
+      
       Object.values(backup.streams).forEach(stream => watchStreamSize(stream, sizeContainer))
     }
 
@@ -51,7 +73,7 @@ exports.ImportVmBackup = class ImportVmBackup {
           ? await xapi.VM_import(backup, srRef)
           : await importIncrementalVm(backup, await xapi.getRecord('SR', srRef), {
               ...this._importIncrementalVmSettings,
-              detectBase: false,
+              baseVdis
             })
 
         await Promise.all([

@xen-orchestra/backups/RemoteAdapter.mjs

@@ -1,43 +1,39 @@
-'use strict'
+import { asyncEach } from '@vates/async-each'
+import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
+import { compose } from '@vates/compose'
+import { createLogger } from '@xen-orchestra/log'
+import { createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic ,Constants} from 'vhd-lib'
+import { decorateMethodsWith } from '@vates/decorate-with'
+import { deduped } from '@vates/disposable/deduped.js'
+import { dirname, join, resolve } from 'node:path'
+import { execFile } from 'child_process'
+import { mount } from '@vates/fuse-vhd'
+import { readdir, lstat } from 'node:fs/promises'
+import { synchronized } from 'decorator-synchronized'
+import { v4 as uuidv4 } from 'uuid'
+import { ZipFile } from 'yazl'
+import Disposable from 'promise-toolbox/Disposable'
+import fromCallback from 'promise-toolbox/fromCallback'
+import fromEvent from 'promise-toolbox/fromEvent'
+import groupBy from 'lodash/groupBy.js'
+import pDefer from 'promise-toolbox/defer'
+import pickBy from 'lodash/pickBy.js'
+import tar from 'tar'
+import zlib from 'zlib'
 
-const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
-const { synchronized } = require('decorator-synchronized')
-const Disposable = require('promise-toolbox/Disposable')
-const fromCallback = require('promise-toolbox/fromCallback')
-const fromEvent = require('promise-toolbox/fromEvent')
-const pDefer = require('promise-toolbox/defer')
-const groupBy = require('lodash/groupBy.js')
-const pickBy = require('lodash/pickBy.js')
-const { dirname, join, normalize, resolve } = require('path')
-const { createLogger } = require('@xen-orchestra/log')
-const { createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib')
-const { deduped } = require('@vates/disposable/deduped.js')
-const { decorateMethodsWith } = require('@vates/decorate-with')
-const { compose } = require('@vates/compose')
-const { execFile } = require('child_process')
-const { readdir, lstat } = require('fs-extra')
-const { v4: uuidv4 } = require('uuid')
-const { ZipFile } = require('yazl')
-const zlib = require('zlib')
+import { BACKUP_DIR } from './_getVmBackupDir.mjs'
+import { cleanVm } from './_cleanVm.mjs'
+import { formatFilenameDate } from './_filenameDate.mjs'
+import { getTmpDir } from './_getTmpDir.mjs'
+import { isMetadataFile } from './_backupType.mjs'
+import { isValidXva } from './_isValidXva.mjs'
+import { listPartitions, LVM_PARTITION_TYPE } from './_listPartitions.mjs'
+import { lvs, pvs } from './_lvm.mjs'
+import { watchStreamSize } from './_watchStreamSize.mjs'
 
-const { BACKUP_DIR } = require('./_getVmBackupDir.js')
-const { cleanVm } = require('./_cleanVm.js')
-const { formatFilenameDate } = require('./_filenameDate.js')
-const { getTmpDir } = require('./_getTmpDir.js')
-const { isMetadataFile } = require('./_backupType.js')
-const { isValidXva } = require('./_isValidXva.js')
-const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js')
-const { lvs, pvs } = require('./_lvm.js')
-const { watchStreamSize } = require('./_watchStreamSize')
-// @todo : this import is marked extraneous , sould be fixed when lib is published
-const { mount } = require('@vates/fuse-vhd')
-const { asyncEach } = require('@vates/async-each')
+export const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
 
-const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
-exports.DIR_XO_CONFIG_BACKUPS = DIR_XO_CONFIG_BACKUPS
-
-const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
-exports.DIR_XO_POOL_METADATA_BACKUPS = DIR_XO_POOL_METADATA_BACKUPS
+export const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
 
 const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
 
@@ -46,20 +42,23 @@ const compareTimestamp = (a, b) => a.timestamp - b.timestamp
 const noop = Function.prototype
 
 const resolveRelativeFromFile = (file, path) => resolve('/', dirname(file), path).slice(1)
+const makeRelative = path => resolve('/', path).slice(1)
+const resolveSubpath = (root, path) => resolve(root, makeRelative(path))
 
-const resolveSubpath = (root, path) => resolve(root, `.${resolve('/', path)}`)
+async function addZipEntries(zip, realBasePath, virtualBasePath, relativePaths) {
+  for (const relativePath of relativePaths) {
+    const realPath = join(realBasePath, relativePath)
+    const virtualPath = join(virtualBasePath, relativePath)
 
-async function addDirectory(files, realPath, metadataPath) {
-  const stats = await lstat(realPath)
-  if (stats.isDirectory()) {
-    await asyncMap(await readdir(realPath), file =>
-      addDirectory(files, realPath + '/' + file, metadataPath + '/' + file)
-    )
-  } else if (stats.isFile()) {
-    files.push({
-      realPath,
-      metadataPath,
-    })
+    const stats = await lstat(realPath)
+    const { mode, mtime } = stats
+    const opts = { mode, mtime }
+    if (stats.isDirectory()) {
+      zip.addEmptyDirectory(virtualPath, opts)
+      await addZipEntries(zip, realPath, virtualPath, await readdir(realPath))
+    } else if (stats.isFile()) {
+      zip.addFile(realPath, virtualPath, opts)
+    }
   }
 }
 
@@ -76,7 +75,7 @@ const debounceResourceFactory = factory =>
     return this._debounceResource(factory.apply(this, arguments))
   }
 
-class RemoteAdapter {
+export class RemoteAdapter {
   constructor(
     handler,
     { debounceResource = res => res, dirMode, vhdDirectoryCompression, useGetDiskLegacy = false } = {}
@@ -187,17 +186,6 @@ class RemoteAdapter {
     })
   }
 
-  async *_usePartitionFiles(diskId, partitionId, paths) {
-    const path = yield this.getPartition(diskId, partitionId)
-
-    const files = []
-    await asyncMap(paths, file =>
-      addDirectory(files, resolveSubpath(path, file), normalize('./' + file).replace(/\/+$/, ''))
-    )
-
-    return files
-  }
-
   // check if we will be allowed to merge a a vhd created in this adapter
   // with the vhd at path `path`
   async isMergeableParent(packedParentUid, path) {
@@ -214,15 +202,24 @@ class RemoteAdapter {
     })
   }
 
-  fetchPartitionFiles(diskId, partitionId, paths) {
+  fetchPartitionFiles(diskId, partitionId, paths, format) {
     const { promise, reject, resolve } = pDefer()
     Disposable.use(
       async function* () {
-        const files = yield this._usePartitionFiles(diskId, partitionId, paths)
-        const zip = new ZipFile()
-        files.forEach(({ realPath, metadataPath }) => zip.addFile(realPath, metadataPath))
-        zip.end()
-        const { outputStream } = zip
+        const path = yield this.getPartition(diskId, partitionId)
+        let outputStream
+
+        if (format === 'tgz') {
+          outputStream = tar.c({ cwd: path, gzip: true }, paths.map(makeRelative))
+        } else if (format === 'zip') {
+          const zip = new ZipFile()
+          await addZipEntries(zip, path, '', paths.map(makeRelative))
+          zip.end()
+          ;({ outputStream } = zip)
+        } else {
+          throw new Error('unsupported format ' + format)
+        }
+
         resolve(outputStream)
         await fromEvent(outputStream, 'end')
       }.bind(this)
@@ -669,7 +666,7 @@ class RemoteAdapter {
     const handler = this._handler
     if (this.useVhdDirectory()) {
       const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
-      const sizes = await createVhdDirectoryFromStream(handler, dataPath, input, {
+      const size = await createVhdDirectoryFromStream(handler, dataPath, input, {
         concurrency: writeBlockConcurrency,
         compression: this.#getCompressionType(),
         async validator() {
@@ -678,14 +675,9 @@ class RemoteAdapter {
         },
       })
       await VhdAbstract.createAlias(handler, path, dataPath)
-      return sizes
+      return size
     } else {
-      const size = this.outputStream(path, input, { checksum, validator })
-      return {
-        compressedSize: size,
-        sourceSize: size,
-        writtenSize: size,
-      }
+      return this.outputStream(path, input, { checksum, validator })
     }
   }
 
@@ -702,8 +694,8 @@ class RemoteAdapter {
     return container.size
   }
 
-  // open the  hierarchy of ancestors until we find a full one
-  async _createVhdStream(handler, path, { useChain }) {
+  // open the  hierarchy of ancestors until we find a usable one
+  async _createVhdStream(handler, path, { useChain, snapshotedVdis }) {
     const disposableSynthetic = useChain ? await VhdSynthetic.fromVhdChain(handler, path) : await openVhd(handler, path)
     // I don't want the vhds to be disposed on return
     // but only when the stream is done ( or failed )
@@ -721,7 +713,67 @@ class RemoteAdapter {
     }
     const synthetic = disposableSynthetic.value
     await synthetic.readBlockAllocationTable()
-    const stream = await synthetic.stream()
+    let stream
+    // try to create a stream that will reuse any data already present on the host storage 
+    // by looking for an existing snapshot matching one of the vhd in the chain
+    // and transfer only the differential 
+    if (snapshotedVdis) {
+      try{
+
+        let vhdPaths = await handler.list(dirname(path), {filter: path=>path.endsWith('.vhd')})
+        stream = await Disposable.use(async function *(){
+          const vhdChilds = {}
+          const vhds = yield Disposable.all(vhdPaths.map(path => openVhd(handler, path, opts)))
+          for(const vhd of vhds){
+            vhdChilds[vhd.header.parentUuid] = vhdChilds[vhd.header.parentUuid] ?? []
+            vhdChilds[vhd.header.parentUuid].push(vhd)
+          } 
+  
+          let chain = []
+          let current = synthetic
+
+          // @todo : special case : we want to restore a vdi 
+          // that still have its snapshot => nothing to transfer
+
+          while(current != undefined){
+
+            // find the child VDI of path
+            const childs = vhdChilds[current.footer.uuid]
+  
+            //   more than one => break
+            //   inexistant => break
+            if(childs.length !== 1){
+              break
+            }
+            const child = childs[0]
+  
+            // not a differential => we won't have a list of block changed
+            // no need to continue looking
+            if(child.footer.diskType !== Constants.DISK_TYPES.DIFFERENCING){
+              break
+            }
+            // we have a snapshot
+            if(snapshotedVdis[current.footer.uuid] !== undefined){
+              const descendants = VhdSynthetic.open(handler)
+              negativeVhd = new NegativeVhd(synthetic, descendants)
+              return negative.stream()
+            } else {
+              // continue to look into the chain
+              // hoping we'll found a match deeper
+              current  = child 
+              chain.unshift(current)
+            }
+          }
+        })
+      }catch(error){
+        warn("error while trying to reuse a snapshot, fallback to legacy restore", {error})
+      }
+      
+    }
+    // fallback
+    if (stream === undefined) {
+      stream = await synthetic.stream()
+    }
 
     stream.on('end', disposeOnce)
     stream.on('close', disposeOnce)
@@ -729,7 +781,7 @@ class RemoteAdapter {
     return stream
   }
 
-  async readIncrementalVmBackup(metadata, ignoredVdis, { useChain = true } = {}) {
+  async readIncrementalVmBackup(metadata, ignoredVdis, { useChain = true, snapshotedVdis } = {}) {
     const handler = this._handler
     const { vbds, vhds, vifs, vm, vmSnapshot } = metadata
     const dir = dirname(metadata._filename)
@@ -737,7 +789,7 @@ class RemoteAdapter {
 
     const streams = {}
     await asyncMapSettled(Object.keys(vdis), async ref => {
-      streams[`${ref}.vhd`] = await this._createVhdStream(handler, join(dir, vhds[ref]), { useChain })
+      streams[`${ref}.vhd`] = await this._createVhdStream(handler, join(dir, vhds[ref]), { useChain, snapshotedVdis })
     })
 
     return {
@@ -834,11 +886,7 @@ decorateMethodsWith(RemoteAdapter, {
     debounceResourceFactory,
   ]),
 
-  _usePartitionFiles: Disposable.factory,
-
   getDisk: compose([Disposable.factory, [deduped, diskId => [diskId]], debounceResourceFactory]),
 
   getPartition: Disposable.factory,
 })
-
-exports.RemoteAdapter = RemoteAdapter

@xen-orchestra/backups/RestoreMetadataBackup.js

@@ -1,29 +0,0 @@
-'use strict'
-
-const { join, resolve } = require('node:path/posix')
-
-const { DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')
-const { PATH_DB_DUMP } = require('./_runners/_PoolMetadataBackup.js')
-
-exports.RestoreMetadataBackup = class RestoreMetadataBackup {
-  constructor({ backupId, handler, xapi }) {
-    this._backupId = backupId
-    this._handler = handler
-    this._xapi = xapi
-  }
-
-  async run() {
-    const backupId = this._backupId
-    const handler = this._handler
-    const xapi = this._xapi
-
-    if (backupId.split('/')[0] === DIR_XO_POOL_METADATA_BACKUPS) {
-      return xapi.putResource(await handler.createReadStream(`${backupId}/data`), PATH_DB_DUMP, {
-        task: xapi.task_create('Import pool metadata'),
-      })
-    } else {
-      const metadata = JSON.parse(await handler.readFile(join(backupId, 'metadata.json')))
-      return String(await handler.readFile(resolve(backupId, metadata.data ?? 'data.json')))
-    }
-  }
-}

@xen-orchestra/backups/RestoreMetadataBackup.mjs

@@ -0,0 +1,32 @@
+import { join, resolve } from 'node:path/posix'
+
+import { DIR_XO_POOL_METADATA_BACKUPS } from './RemoteAdapter.mjs'
+import { PATH_DB_DUMP } from './_runners/_PoolMetadataBackup.mjs'
+
+export class RestoreMetadataBackup {
+  constructor({ backupId, handler, xapi }) {
+    this._backupId = backupId
+    this._handler = handler
+    this._xapi = xapi
+  }
+
+  async run() {
+    const backupId = this._backupId
+    const handler = this._handler
+    const xapi = this._xapi
+
+    if (backupId.split('/')[0] === DIR_XO_POOL_METADATA_BACKUPS) {
+      return xapi.putResource(await handler.createReadStream(`${backupId}/data`), PATH_DB_DUMP, {
+        task: xapi.task_create('Import pool metadata'),
+      })
+    } else {
+      const metadata = JSON.parse(await handler.readFile(join(backupId, 'metadata.json')))
+      const dataFileName = resolve(backupId, metadata.data ?? 'data.json')
+      const data = await handler.readFile(dataFileName)
+
+      // if data is JSON, sent it as a plain string, otherwise, consider the data as binary and encode it
+      const isJson = dataFileName.endsWith('.json')
+      return isJson ? data.toString() : { encoding: 'base64', data: data.toString('base64') }
+    }
+  }
+}

@xen-orchestra/backups/Task.mjs

@@ -1,7 +1,5 @@
-'use strict'
-
-const CancelToken = require('promise-toolbox/CancelToken')
-const Zone = require('node-zone')
+import CancelToken from 'promise-toolbox/CancelToken'
+import Zone from 'node-zone'
 
 const logAfterEnd = log => {
   const error = new Error('task has already ended')
@@ -30,7 +28,7 @@ const serializeError = error =>
 
 const $$task = Symbol('@xen-orchestra/backups/Task')
 
-class Task {
+export class Task {
   static get cancelToken() {
     const task = Zone.current.data[$$task]
     return task !== undefined ? task.#cancelToken : CancelToken.none
@@ -151,7 +149,6 @@ class Task {
     })
   }
 }
-exports.Task = Task
 
 for (const method of ['info', 'warning']) {
   Task[method] = (...args) => Zone.current.data[$$task]?.[method](...args)

@xen-orchestra/backups/_backupType.js

@@ -1,6 +0,0 @@
-'use strict'
-
-exports.isMetadataFile = filename => filename.endsWith('.json')
-exports.isVhdFile = filename => filename.endsWith('.vhd')
-exports.isXvaFile = filename => filename.endsWith('.xva')
-exports.isXvaSumFile = filename => filename.endsWith('.xva.checksum')

@xen-orchestra/backups/_backupType.mjs

@@ -0,0 +1,4 @@
+export const isMetadataFile = filename => filename.endsWith('.json')
+export const isVhdFile = filename => filename.endsWith('.vhd')
+export const isXvaFile = filename => filename.endsWith('.xva')
+export const isXvaSumFile = filename => filename.endsWith('.xva.checksum')

@xen-orchestra/backups/_backupWorker.mjs

@@ -1,25 +1,25 @@
-'use strict'
+import { createLogger } from '@xen-orchestra/log'
+import { catchGlobalErrors } from '@xen-orchestra/log/configure'
 
-const logger = require('@xen-orchestra/log').createLogger('xo:backups:worker')
+import Disposable from 'promise-toolbox/Disposable'
+import ignoreErrors from 'promise-toolbox/ignoreErrors'
+import { compose } from '@vates/compose'
+import { createCachedLookup } from '@vates/cached-dns.lookup'
+import { createDebounceResource } from '@vates/disposable/debounceResource.js'
+import { createRunner } from './Backup.mjs'
+import { decorateMethodsWith } from '@vates/decorate-with'
+import { deduped } from '@vates/disposable/deduped.js'
+import { getHandler } from '@xen-orchestra/fs'
+import { parseDuration } from '@vates/parse-duration'
+import { Xapi } from '@xen-orchestra/xapi'
 
-require('@xen-orchestra/log/configure').catchGlobalErrors(logger)
+import { RemoteAdapter } from './RemoteAdapter.mjs'
+import { Task } from './Task.mjs'
 
-require('@vates/cached-dns.lookup').createCachedLookup().patchGlobal()
-
-const Disposable = require('promise-toolbox/Disposable')
-const ignoreErrors = require('promise-toolbox/ignoreErrors')
-const { compose } = require('@vates/compose')
-const { createDebounceResource } = require('@vates/disposable/debounceResource.js')
-const { decorateMethodsWith } = require('@vates/decorate-with')
-const { deduped } = require('@vates/disposable/deduped.js')
-const { getHandler } = require('@xen-orchestra/fs')
-const { createRunner } = require('./Backup.js')
-const { parseDuration } = require('@vates/parse-duration')
-const { Xapi } = require('@xen-orchestra/xapi')
-
-const { RemoteAdapter } = require('./RemoteAdapter.js')
-const { Task } = require('./Task.js')
+createCachedLookup().patchGlobal()
 
+const logger = createLogger('xo:backups:worker')
+catchGlobalErrors(logger)
 const { debug } = logger
 
 class BackupWorker {

@xen-orchestra/backups/_cancelableMap.mjs

@@ -1,13 +1,11 @@
-'use strict'
-
-const cancelable = require('promise-toolbox/cancelable')
-const CancelToken = require('promise-toolbox/CancelToken')
+import cancelable from 'promise-toolbox/cancelable'
+import CancelToken from 'promise-toolbox/CancelToken'
 
 // Similar to `Promise.all` + `map` but pass a cancel token to the callback
 //
 // If any of the executions fails, the cancel token will be triggered and the
 // first reason will be rejected.
-exports.cancelableMap = cancelable(async function cancelableMap($cancelToken, iterable, callback) {
+export const cancelableMap = cancelable(async function cancelableMap($cancelToken, iterable, callback) {
   const { cancel, token } = CancelToken.source([$cancelToken])
   try {
     return await Promise.all(

@xen-orchestra/backups/_cleanVm.integ.mjs

@@ -1,19 +1,19 @@
-'use strict'
+import test from 'test'
+import { strict as assert } from 'node:assert'
 
-const { beforeEach, afterEach, test, describe } = require('test')
-const assert = require('assert').strict
+import tmp from 'tmp'
+import fs from 'fs-extra'
+import * as uuid from 'uuid'
+import { getHandler } from '@xen-orchestra/fs'
+import { pFromCallback } from 'promise-toolbox'
+import { RemoteAdapter } from './RemoteAdapter.mjs'
+import { VHDFOOTER, VHDHEADER } from './tests.fixtures.mjs'
+import { VhdFile, Constants, VhdDirectory, VhdAbstract } from 'vhd-lib'
+import { checkAliases } from './_cleanVm.mjs'
+import { dirname, basename } from 'node:path'
+import { rimraf } from 'rimraf'
 
-const tmp = require('tmp')
-const fs = require('fs-extra')
-const uuid = require('uuid')
-const { getHandler } = require('@xen-orchestra/fs')
-const { pFromCallback } = require('promise-toolbox')
-const { RemoteAdapter } = require('./RemoteAdapter')
-const { VHDFOOTER, VHDHEADER } = require('./tests.fixtures.js')
-const { VhdFile, Constants, VhdDirectory, VhdAbstract } = require('vhd-lib')
-const { checkAliases } = require('./_cleanVm')
-const { dirname, basename } = require('path')
-const { rimraf } = require('rimraf')
+const { beforeEach, afterEach, describe } = test
 
 let tempDir, adapter, handler, jobId, vdiId, basePath, relativePath
 const rootPath = 'xo-vm-backups/VMUUID/'

@xen-orchestra/backups/_cleanVm.mjs

@@ -1,28 +1,37 @@
-'use strict'
+import * as UUID from 'uuid'
+import sum from 'lodash/sum.js'
+import { asyncMap } from '@xen-orchestra/async-map'
+import { Constants, openVhd, VhdAbstract, VhdFile } from 'vhd-lib'
+import { isVhdAlias, resolveVhdAlias } from 'vhd-lib/aliases.js'
+import { dirname, resolve } from 'node:path'
+import { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } from './_backupType.mjs'
+import { limitConcurrency } from 'limit-concurrency-decorator'
+import { mergeVhdChain } from 'vhd-lib/merge.js'
+
+import { Task } from './Task.mjs'
+import { Disposable } from 'promise-toolbox'
+import handlerPath from '@xen-orchestra/fs/path'
 
-const sum = require('lodash/sum')
-const UUID = require('uuid')
-const { asyncMap } = require('@xen-orchestra/async-map')
-const { Constants, openVhd, VhdAbstract } = require('vhd-lib')
-const { isVhdAlias, resolveVhdAlias } = require('vhd-lib/aliases')
-const { dirname, resolve } = require('path')
 const { DISK_TYPES } = Constants
-const { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } = require('./_backupType.js')
-const { limitConcurrency } = require('limit-concurrency-decorator')
-const { mergeVhdChain } = require('vhd-lib/merge')
 
-const { Task } = require('./Task.js')
-const { Disposable } = require('promise-toolbox')
-const handlerPath = require('@xen-orchestra/fs/path')
+// checking the size of a vhd directory is costly
+// 1 Http Query per 1000 blocks
+// we only check size of all the vhd are VhdFiles
+function shouldComputeVhdsSize(handler, vhds) {
+  if (handler.isEncrypted) {
+    return false
+  }
+  return vhds.every(vhd => vhd instanceof VhdFile)
+}
 
 const computeVhdsSize = (handler, vhdPaths) =>
   Disposable.use(
     vhdPaths.map(vhdPath => openVhd(handler, vhdPath)),
     async vhds => {
-      await Promise.all(vhds.map(vhd => vhd.readBlockAllocationTable()))
-      // get file size for vhdfile, computed size from bat for vhd directory
-      const sizes = await asyncMap(vhds, vhd => vhd.streamSize())
-      return sum(sizes)
+      if (shouldComputeVhdsSize(handler, vhds)) {
+        const sizes = await asyncMap(vhds, vhd => vhd.getSize())
+        return sum(sizes)
+      }
     }
   )
 
@@ -107,7 +116,7 @@ const listVhds = async (handler, vmDir, logWarn) => {
   return { vhds, interruptedVhds, aliases }
 }
 
-async function checkAliases(
+export async function checkAliases(
   aliasPaths,
   targetDataRepository,
   { handler, logInfo = noop, logWarn = console.warn, remove = false }
@@ -166,11 +175,9 @@ async function checkAliases(
   })
 }
 
-exports.checkAliases = checkAliases
-
 const defaultMergeLimiter = limitConcurrency(1)
 
-exports.cleanVm = async function cleanVm(
+export async function cleanVm(
   vmDir,
   {
     fixMetadata,
@@ -524,6 +531,11 @@ exports.cleanVm = async function cleanVm(
         const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
         fileSystemSize = await computeVhdsSize(handler, linkedVhds)
 
+        // the size is not computed in some cases (e.g. VhdDirectory)
+        if (fileSystemSize === undefined) {
+          return
+        }
+
         // don't warn if the size has changed after a merge
         if (!merged && fileSystemSize !== size) {
           // FIXME: figure out why it occurs so often and, once fixed, log the real problems with `logWarn`
@@ -541,8 +553,6 @@ exports.cleanVm = async function cleanVm(
 
     // systematically update size after a merge
     if ((merged || fixMetadata) && size !== fileSystemSize) {
-      // @todo add a cumulatedTransferSize property ?
-      // @todo update writtenSize, compressedSize
       metadata.size = fileSystemSize
       mustRegenerateCache = true
       try {

@xen-orchestra/backups/_filenameDate.js

@@ -1,8 +0,0 @@
-'use strict'
-
-const { utcFormat, utcParse } = require('d3-time-format')
-
-// Format a date in ISO 8601 in a safe way to be used in filenames
-// (even on Windows).
-exports.formatFilenameDate = utcFormat('%Y%m%dT%H%M%SZ')
-exports.parseFilenameDate = utcParse('%Y%m%dT%H%M%SZ')

@xen-orchestra/backups/_filenameDate.mjs

@@ -0,0 +1,6 @@
+import { utcFormat, utcParse } from 'd3-time-format'
+
+// Format a date in ISO 8601 in a safe way to be used in filenames
+// (even on Windows).
+export const formatFilenameDate = utcFormat('%Y%m%dT%H%M%SZ')
+export const parseFilenameDate = utcParse('%Y%m%dT%H%M%SZ')

@xen-orchestra/backups/_getOldEntries.mjs

@@ -1,6 +1,4 @@
-'use strict'
-
 // returns all entries but the last retention-th
-exports.getOldEntries = function getOldEntries(retention, entries) {
+export function getOldEntries(retention, entries) {
   return entries === undefined ? [] : retention > 0 ? entries.slice(0, -retention) : entries
 }

@xen-orchestra/backups/_getTmpDir.mjs

@@ -1,13 +1,11 @@
-'use strict'
-
-const Disposable = require('promise-toolbox/Disposable')
-const { join } = require('path')
-const { mkdir, rmdir } = require('fs-extra')
-const { tmpdir } = require('os')
+import Disposable from 'promise-toolbox/Disposable'
+import { join } from 'node:path'
+import { mkdir, rmdir } from 'node:fs/promises'
+import { tmpdir } from 'os'
 
 const MAX_ATTEMPTS = 3
 
-exports.getTmpDir = async function getTmpDir() {
+export async function getTmpDir() {
   for (let i = 0; true; ++i) {
     const path = join(tmpdir(), Math.random().toString(36).slice(2))
     try {

@xen-orchestra/backups/_getVmBackupDir.js

@@ -1,8 +0,0 @@
-'use strict'
-
-const BACKUP_DIR = 'xo-vm-backups'
-exports.BACKUP_DIR = BACKUP_DIR
-
-exports.getVmBackupDir = function getVmBackupDir(uuid) {
-  return `${BACKUP_DIR}/${uuid}`
-}

@xen-orchestra/backups/_getVmBackupDir.mjs

@@ -0,0 +1,5 @@
+export const BACKUP_DIR = 'xo-vm-backups'
+
+export function getVmBackupDir(uuid) {
+  return `${BACKUP_DIR}/${uuid}`
+}

@xen-orchestra/backups/_incrementalVm.mjs

@@ -1,24 +1,22 @@
-'use strict'
+import find from 'lodash/find.js'
+import groupBy from 'lodash/groupBy.js'
+import ignoreErrors from 'promise-toolbox/ignoreErrors'
+import omit from 'lodash/omit.js'
+import { asyncMap } from '@xen-orchestra/async-map'
+import { CancelToken } from 'promise-toolbox'
+import { compareVersions } from 'compare-versions'
+import { createVhdStreamWithLength } from 'vhd-lib'
+import { defer } from 'golike-defer'
 
-const find = require('lodash/find.js')
-const groupBy = require('lodash/groupBy.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors')
-const omit = require('lodash/omit.js')
-const { asyncMap } = require('@xen-orchestra/async-map')
-const { CancelToken } = require('promise-toolbox')
-const { compareVersions } = require('compare-versions')
-const { createVhdStreamWithLength } = require('vhd-lib')
-const { defer } = require('golike-defer')
+import { cancelableMap } from './_cancelableMap.mjs'
+import { Task } from './Task.mjs'
+import pick from 'lodash/pick.js'
 
-const { cancelableMap } = require('./_cancelableMap.js')
-const { Task } = require('./Task.js')
-const pick = require('lodash/pick.js')
+export const TAG_BASE_DELTA = 'xo:base_delta'
 
-const TAG_BASE_DELTA = 'xo:base_delta'
-exports.TAG_BASE_DELTA = TAG_BASE_DELTA
+export const TAG_COPY_SRC = 'xo:copy_of'
 
-const TAG_COPY_SRC = 'xo:copy_of'
-exports.TAG_COPY_SRC = TAG_COPY_SRC
+const TAG_BACKUP_SR = 'xo:backup:sr'
 
 const ensureArray = value => (value === undefined ? [] : Array.isArray(value) ? value : [value])
 const resolveUuid = async (xapi, cache, uuid, type) => {
@@ -33,7 +31,7 @@ const resolveUuid = async (xapi, cache, uuid, type) => {
   return ref
 }
 
-exports.exportIncrementalVm = async function exportIncrementalVm(
+export async function exportIncrementalVm(
   vm,
   baseVm,
   {
@@ -43,6 +41,7 @@ exports.exportIncrementalVm = async function exportIncrementalVm(
     fullVdisRequired = new Set(),
 
     disableBaseTags = false,
+    preferNbd,
   } = {}
 ) {
   // refs of VM's VDIs → base's VDIs.
@@ -90,6 +89,7 @@ exports.exportIncrementalVm = async function exportIncrementalVm(
       baseRef: baseVdi?.$ref,
       cancelToken,
       format: 'vhd',
+      preferNbd,
     })
   })
 
@@ -143,11 +143,38 @@ exports.exportIncrementalVm = async function exportIncrementalVm(
   )
 }
 
-exports.importIncrementalVm = defer(async function importIncrementalVm(
+
+// @todo  movve this to incremental replication
+async function detectBaseVdis(vmRecord, sr) {
+  let baseVm
+  const xapi = sr.$xapi
+  const remoteBaseVmUuid = vmRecord.other_config[TAG_BASE_DELTA]
+  if (remoteBaseVmUuid) {
+    baseVm = find(
+      xapi.objects.all,
+      obj => (obj = obj.other_config) && obj[TAG_COPY_SRC] === remoteBaseVmUuid && obj[TAG_BACKUP_SR] === sr.$id
+    )
+
+    if (!baseVm) {
+      throw new Error(`could not find the base VM (copy of ${remoteBaseVmUuid})`)
+    }
+  }
+  const baseVdis = {}
+  baseVm &&
+    baseVm.$VBDs.forEach(vbd => {
+      const vdi = vbd.$VDI
+      if (vdi !== undefined) {
+        baseVdis[vdi.other_config[TAG_COPY_SRC]] = vbd.$VDI
+      }
+    })
+  return baseVdis
+}
+
+export const importIncrementalVm = defer(async function importIncrementalVm(
   $defer,
   incrementalVm,
   sr,
-  { cancelToken = CancelToken.none, detectBase = true, mapVdisSrs = {}, newMacAddresses = false } = {}
+  { baseVdis = {}, cancelToken = CancelToken.none, detectBase = true, mapVdisSrs = {}, newMacAddresses = false } = {}
 ) {
   const { version } = incrementalVm
   if (compareVersions(version, '1.0.0') < 0) {
@@ -157,32 +184,15 @@ exports.importIncrementalVm = defer(async function importIncrementalVm(
   const vmRecord = incrementalVm.vm
   const xapi = sr.$xapi
 
-  let baseVm
-  if (detectBase) {
-    const remoteBaseVmUuid = vmRecord.other_config[TAG_BASE_DELTA]
-    if (remoteBaseVmUuid) {
-      baseVm = find(xapi.objects.all, obj => (obj = obj.other_config) && obj[TAG_COPY_SRC] === remoteBaseVmUuid)
-
-      if (!baseVm) {
-        throw new Error(`could not find the base VM (copy of ${remoteBaseVmUuid})`)
-      }
-    }
-  }
-
   const cache = new Map()
   const mapVdisSrRefs = {}
   for (const [vdiUuid, srUuid] of Object.entries(mapVdisSrs)) {
     mapVdisSrRefs[vdiUuid] = await resolveUuid(xapi, cache, srUuid, 'SR')
   }
 
-  const baseVdis = {}
-  baseVm &&
-    baseVm.$VBDs.forEach(vbd => {
-      const vdi = vbd.$VDI
-      if (vdi !== undefined) {
-        baseVdis[vbd.VDI] = vbd.$VDI
-      }
-    })
+  if (detectBase) {
+    baseVdis = await detectBaseVdis(vmRecord, sr)
+  }
   const vdiRecords = incrementalVm.vdis
 
   // 0. Create suspend_VDI
@@ -246,10 +256,11 @@ exports.importIncrementalVm = defer(async function importIncrementalVm(
   await asyncMap(Object.keys(vdiRecords), async vdiRef => {
     const vdi = vdiRecords[vdiRef]
     let newVdi
-
+    // @todo how to rewrite this condition when giving directly a baseVdi  ?
     const remoteBaseVdiUuid = detectBase && vdi.other_config[TAG_BASE_DELTA]
     if (remoteBaseVdiUuid) {
-      const baseVdi = find(baseVdis, vdi => vdi.other_config[TAG_COPY_SRC] === remoteBaseVdiUuid)
+      const baseVdi = baseVdis[vdi.other_config[TAG_COPY_SRC]]
+      // @todo : should be an error only for detectBase
       if (!baseVdi) {
         throw new Error(`missing base VDI (copy of ${remoteBaseVdiUuid})`)
       }

@xen-orchestra/backups/_isValidXva.mjs

@@ -1,6 +1,4 @@
-'use strict'
-
-const assert = require('assert')
+import assert from 'node:assert'
 
 const COMPRESSED_MAGIC_NUMBERS = [
   // https://tools.ietf.org/html/rfc1952.html#page-5
@@ -47,7 +45,7 @@ const isValidTar = async (handler, size, fd) => {
 }
 
 // TODO: find an heuristic for compressed files
-async function isValidXva(path) {
+export async function isValidXva(path) {
   const handler = this._handler
 
   // size is longer when encrypted  + reading part of an encrypted file is not implemented
@@ -74,6 +72,5 @@ async function isValidXva(path) {
     return true
   }
 }
-exports.isValidXva = isValidXva
 
 const noop = Function.prototype

@xen-orchestra/backups/_listPartitions.mjs

@@ -1,9 +1,7 @@
-'use strict'
-
-const fromCallback = require('promise-toolbox/fromCallback')
-const { createLogger } = require('@xen-orchestra/log')
-const { createParser } = require('parse-pairs')
-const { execFile } = require('child_process')
+import fromCallback from 'promise-toolbox/fromCallback'
+import { createLogger } from '@xen-orchestra/log'
+import { createParser } from 'parse-pairs'
+import { execFile } from 'child_process'
 
 const { debug } = createLogger('xo:backups:listPartitions')
 
@@ -24,8 +22,7 @@ const IGNORED_PARTITION_TYPES = {
   0x82: true, // swap
 }
 
-const LVM_PARTITION_TYPE = 0x8e
-exports.LVM_PARTITION_TYPE = LVM_PARTITION_TYPE
+export const LVM_PARTITION_TYPE = 0x8e
 
 const parsePartxLine = createParser({
   keyTransform: key => (key === 'UUID' ? 'id' : key.toLowerCase()),
@@ -33,7 +30,7 @@ const parsePartxLine = createParser({
 })
 
 // returns an empty array in case of a non-partitioned disk
-exports.listPartitions = async function listPartitions(devicePath) {
+export async function listPartitions(devicePath) {
   const parts = await fromCallback(execFile, 'partx', [
     '--bytes',
     '--output=NR,START,SIZE,NAME,UUID,TYPE',

@xen-orchestra/backups/_lvm.mjs

@@ -1,8 +1,6 @@
-'use strict'
-
-const fromCallback = require('promise-toolbox/fromCallback')
-const { createParser } = require('parse-pairs')
-const { execFile } = require('child_process')
+import fromCallback from 'promise-toolbox/fromCallback'
+import { createParser } from 'parse-pairs'
+import { execFile } from 'child_process'
 
 // ===================================================================
 
@@ -29,5 +27,5 @@ const makeFunction =
       .map(Array.isArray(fields) ? parse : line => parse(line)[fields])
   }
 
-exports.lvs = makeFunction('lvs')
-exports.pvs = makeFunction('pvs')
+export const lvs = makeFunction('lvs')
+export const pvs = makeFunction('pvs')

@xen-orchestra/backups/_runners/Metadata.mjs

@@ -1,22 +1,20 @@
-'use strict'
+import { asyncMap } from '@xen-orchestra/async-map'
+import Disposable from 'promise-toolbox/Disposable'
+import ignoreErrors from 'promise-toolbox/ignoreErrors'
 
-const { asyncMap } = require('@xen-orchestra/async-map')
-const Disposable = require('promise-toolbox/Disposable')
-const ignoreErrors = require('promise-toolbox/ignoreErrors')
-
-const { extractIdsFromSimplePattern } = require('../extractIdsFromSimplePattern.js')
-const { PoolMetadataBackup } = require('./_PoolMetadataBackup.js')
-const { XoMetadataBackup } = require('./_XoMetadataBackup.js')
-const { DEFAULT_SETTINGS, Abstract } = require('./_Abstract.js')
-const { runTask } = require('./_runTask.js')
-const { getAdaptersByRemote } = require('./_getAdaptersByRemote.js')
+import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
+import { PoolMetadataBackup } from './_PoolMetadataBackup.mjs'
+import { XoMetadataBackup } from './_XoMetadataBackup.mjs'
+import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
+import { runTask } from './_runTask.mjs'
+import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
 
 const DEFAULT_METADATA_SETTINGS = {
   retentionPoolMetadata: 0,
   retentionXoMetadata: 0,
 }
 
-exports.Metadata = class MetadataBackupRunner extends Abstract {
+export const Metadata = class MetadataBackupRunner extends Abstract {
   _computeBaseSettings(config, job) {
     const baseSettings = { ...DEFAULT_SETTINGS }
     Object.assign(baseSettings, DEFAULT_METADATA_SETTINGS, config.defaultSettings, config.metadata?.defaultSettings)

@xen-orchestra/backups/_runners/VmsRemote.mjs

@@ -1,17 +1,15 @@
-'use strict'
+import { asyncMapSettled } from '@xen-orchestra/async-map'
+import Disposable from 'promise-toolbox/Disposable'
+import { limitConcurrency } from 'limit-concurrency-decorator'
 
-const { asyncMapSettled } = require('@xen-orchestra/async-map')
-const Disposable = require('promise-toolbox/Disposable')
-const { limitConcurrency } = require('limit-concurrency-decorator')
-
-const { extractIdsFromSimplePattern } = require('../extractIdsFromSimplePattern.js')
-const { Task } = require('../Task.js')
-const createStreamThrottle = require('./_createStreamThrottle.js')
-const { DEFAULT_SETTINGS, Abstract } = require('./_Abstract.js')
-const { runTask } = require('./_runTask.js')
-const { getAdaptersByRemote } = require('./_getAdaptersByRemote.js')
-const { FullRemote } = require('./_vmRunners/FullRemote.js')
-const { IncrementalRemote } = require('./_vmRunners/IncrementalRemote.js')
+import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
+import { Task } from '../Task.mjs'
+import createStreamThrottle from './_createStreamThrottle.mjs'
+import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
+import { runTask } from './_runTask.mjs'
+import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
+import { FullRemote } from './_vmRunners/FullRemote.mjs'
+import { IncrementalRemote } from './_vmRunners/IncrementalRemote.mjs'
 
 const DEFAULT_REMOTE_VM_SETTINGS = {
   concurrency: 2,
@@ -27,7 +25,7 @@ const DEFAULT_REMOTE_VM_SETTINGS = {
   vmTimeout: 0,
 }
 
-exports.VmsRemote = class RemoteVmsBackupRunner extends Abstract {
+export const VmsRemote = class RemoteVmsBackupRunner extends Abstract {
   _computeBaseSettings(config, job) {
     const baseSettings = { ...DEFAULT_SETTINGS }
     Object.assign(baseSettings, DEFAULT_REMOTE_VM_SETTINGS, config.defaultSettings, config.vm?.defaultSettings)

@xen-orchestra/backups/_runners/VmsXapi.mjs

@@ -1,17 +1,15 @@
-'use strict'
+import { asyncMapSettled } from '@xen-orchestra/async-map'
+import Disposable from 'promise-toolbox/Disposable'
+import { limitConcurrency } from 'limit-concurrency-decorator'
 
-const { asyncMapSettled } = require('@xen-orchestra/async-map')
-const Disposable = require('promise-toolbox/Disposable')
-const { limitConcurrency } = require('limit-concurrency-decorator')
-
-const { extractIdsFromSimplePattern } = require('../extractIdsFromSimplePattern.js')
-const { Task } = require('../Task.js')
-const createStreamThrottle = require('./_createStreamThrottle.js')
-const { DEFAULT_SETTINGS, Abstract } = require('./_Abstract.js')
-const { runTask } = require('./_runTask.js')
-const { getAdaptersByRemote } = require('./_getAdaptersByRemote.js')
-const { IncrementalXapi } = require('./_vmRunners/IncrementalXapi.js')
-const { FullXapi } = require('./_vmRunners/FullXapi.js')
+import { extractIdsFromSimplePattern } from '../extractIdsFromSimplePattern.mjs'
+import { Task } from '../Task.mjs'
+import createStreamThrottle from './_createStreamThrottle.mjs'
+import { DEFAULT_SETTINGS, Abstract } from './_Abstract.mjs'
+import { runTask } from './_runTask.mjs'
+import { getAdaptersByRemote } from './_getAdaptersByRemote.mjs'
+import { IncrementalXapi } from './_vmRunners/IncrementalXapi.mjs'
+import { FullXapi } from './_vmRunners/FullXapi.mjs'
 
 const DEFAULT_XAPI_VM_SETTINGS = {
   bypassVdiChainsCheck: false,
@@ -36,7 +34,7 @@ const DEFAULT_XAPI_VM_SETTINGS = {
   vmTimeout: 0,
 }
 
-exports.VmsXapi = class VmsXapiBackupRunner extends Abstract {
+export const VmsXapi = class VmsXapiBackupRunner extends Abstract {
   _computeBaseSettings(config, job) {
     const baseSettings = { ...DEFAULT_SETTINGS }
     Object.assign(baseSettings, DEFAULT_XAPI_VM_SETTINGS, config.defaultSettings, config.vm?.defaultSettings)

@xen-orchestra/backups/_runners/_Abstract.mjs

@@ -1,17 +1,15 @@
-'use strict'
+import Disposable from 'promise-toolbox/Disposable'
+import pTimeout from 'promise-toolbox/timeout'
+import { compileTemplate } from '@xen-orchestra/template'
+import { runTask } from './_runTask.mjs'
+import { RemoteTimeoutError } from './_RemoteTimeoutError.mjs'
 
-const Disposable = require('promise-toolbox/Disposable')
-const pTimeout = require('promise-toolbox/timeout')
-const { compileTemplate } = require('@xen-orchestra/template')
-const { runTask } = require('./_runTask.js')
-const { RemoteTimeoutError } = require('./_RemoteTimeoutError.js')
-
-exports.DEFAULT_SETTINGS = {
+export const DEFAULT_SETTINGS = {
   getRemoteTimeout: 300e3,
   reportWhen: 'failure',
 }
 
-exports.Abstract = class AbstractRunner {
+export const Abstract = class AbstractRunner {
   constructor({ config, getAdapter, getConnectedRecord, job, schedule }) {
     this._config = config
     this._getRecord = getConnectedRecord

@xen-orchestra/backups/_runners/_PoolMetadataBackup.mjs

@@ -1,16 +1,13 @@
-'use strict'
+import { asyncMap } from '@xen-orchestra/async-map'
 
-const { asyncMap } = require('@xen-orchestra/async-map')
+import { DIR_XO_POOL_METADATA_BACKUPS } from '../RemoteAdapter.mjs'
+import { forkStreamUnpipe } from './_forkStreamUnpipe.mjs'
+import { formatFilenameDate } from '../_filenameDate.mjs'
+import { Task } from '../Task.mjs'
 
-const { DIR_XO_POOL_METADATA_BACKUPS } = require('../RemoteAdapter.js')
-const { forkStreamUnpipe } = require('./_forkStreamUnpipe.js')
-const { formatFilenameDate } = require('../_filenameDate.js')
-const { Task } = require('../Task.js')
+export const PATH_DB_DUMP = '/pool/xmldbdump'
 
-const PATH_DB_DUMP = '/pool/xmldbdump'
-exports.PATH_DB_DUMP = PATH_DB_DUMP
-
-exports.PoolMetadataBackup = class PoolMetadataBackup {
+export class PoolMetadataBackup {
   constructor({ config, job, pool, remoteAdapters, schedule, settings }) {
     this._config = config
     this._job = job

@xen-orchestra/backups/_runners/_RemoteTimeoutError.mjs

@@ -1,8 +1,6 @@
-'use strict'
-class RemoteTimeoutError extends Error {
+export class RemoteTimeoutError extends Error {
   constructor(remoteId) {
     super('timeout while getting the remote ' + remoteId)
     this.remoteId = remoteId
   }
 }
-exports.RemoteTimeoutError = RemoteTimeoutError

@xen-orchestra/backups/_runners/_XoMetadataBackup.mjs

@@ -1,13 +1,11 @@
-'use strict'
+import { asyncMap } from '@xen-orchestra/async-map'
+import { join } from '@xen-orchestra/fs/path'
 
-const { asyncMap } = require('@xen-orchestra/async-map')
-const { join } = require('@xen-orchestra/fs/path')
+import { DIR_XO_CONFIG_BACKUPS } from '../RemoteAdapter.mjs'
+import { formatFilenameDate } from '../_filenameDate.mjs'
+import { Task } from '../Task.mjs'
 
-const { DIR_XO_CONFIG_BACKUPS } = require('../RemoteAdapter.js')
-const { formatFilenameDate } = require('../_filenameDate.js')
-const { Task } = require('../Task.js')
-
-exports.XoMetadataBackup = class XoMetadataBackup {
+export class XoMetadataBackup {
   constructor({ config, job, remoteAdapters, schedule, settings }) {
     this._config = config
     this._job = job
@@ -24,7 +22,13 @@ exports.XoMetadataBackup = class XoMetadataBackup {
     const dir = `${scheduleDir}/${formatFilenameDate(timestamp)}`
 
     const data = job.xoMetadata
-    const dataBaseName = './data.json'
+    let dataBaseName = './data'
+
+    // JSON data is sent as plain string, binary data is sent as an object with `data` and `encoding properties
+    const isJson = typeof data === 'string'
+    if (isJson) {
+      dataBaseName += '.json'
+    }
 
     const metadata = JSON.stringify(
       {
@@ -56,7 +60,7 @@ exports.XoMetadataBackup = class XoMetadataBackup {
           async () => {
             const handler = adapter.handler
             const dirMode = this._config.dirMode
-            await handler.outputFile(dataFileName, data, { dirMode })
+            await handler.outputFile(dataFileName, isJson ? data : Buffer.from(data.data, data.encoding), { dirMode })
             await handler.outputFile(metaDataFileName, metadata, {
               dirMode,
             })

@xen-orchestra/backups/_runners/_createStreamThrottle.mjs

@@ -1,12 +1,10 @@
-'use strict'
-
-const { pipeline } = require('node:stream')
-const { ThrottleGroup } = require('@kldzj/stream-throttle')
-const identity = require('lodash/identity.js')
+import { pipeline } from 'node:stream'
+import { ThrottleGroup } from '@kldzj/stream-throttle'
+import identity from 'lodash/identity.js'
 
 const noop = Function.prototype
 
-module.exports = function createStreamThrottle(rate) {
+export default function createStreamThrottle(rate) {
   if (rate === 0) {
     return identity
   }

@xen-orchestra/backups/_runners/_forkStreamUnpipe.mjs

@@ -1,14 +1,13 @@
-'use strict'
+import { createLogger } from '@xen-orchestra/log'
+import { finished, PassThrough } from 'node:stream'
 
-const { finished, PassThrough } = require('node:stream')
-
-const { debug } = require('@xen-orchestra/log').createLogger('xo:backups:forkStreamUnpipe')
+const { debug } = createLogger('xo:backups:forkStreamUnpipe')
 
 // create a new readable stream from an existing one which may be piped later
 //
 // in case of error in the new readable stream, it will simply be unpiped
 // from the original one
-exports.forkStreamUnpipe = function forkStreamUnpipe(source) {
+export function forkStreamUnpipe(source) {
   const { forks = 0 } = source
   source.forks = forks + 1
 

@xen-orchestra/backups/_runners/_getAdaptersByRemote.mjs

@@ -1,9 +1,7 @@
-'use strict'
-const getAdaptersByRemote = adapters => {
+export function getAdaptersByRemote(adapters) {
   const adaptersByRemote = {}
   adapters.forEach(({ adapter, remoteId }) => {
     adaptersByRemote[remoteId] = adapter
   })
   return adaptersByRemote
 }
-exports.getAdaptersByRemote = getAdaptersByRemote

@xen-orchestra/backups/_runners/_runTask.js

@@ -1,6 +0,0 @@
-'use strict'
-const { Task } = require('../Task.js')
-const noop = Function.prototype
-const runTask = (...args) => Task.run(...args).catch(noop) // errors are handled by logs
-
-exports.runTask = runTask

@xen-orchestra/backups/_runners/_runTask.mjs

@@ -0,0 +1,5 @@
+import { Task } from '../Task.mjs'
+
+const noop = Function.prototype
+
+export const runTask = (...args) => Task.run(...args).catch(noop) // errors are handled by logs

@xen-orchestra/backups/_runners/_vmRunners/FullRemote.mjs

@@ -1,14 +1,12 @@
-'use strict'
+import { decorateMethodsWith } from '@vates/decorate-with'
+import { defer } from 'golike-defer'
+import { AbstractRemote } from './_AbstractRemote.mjs'
+import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
+import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
+import { watchStreamSize } from '../../_watchStreamSize.mjs'
+import { Task } from '../../Task.mjs'
 
-const { decorateMethodsWith } = require('@vates/decorate-with')
-const { defer } = require('golike-defer')
-const { AbstractRemote } = require('./_AbstractRemote')
-const { FullRemoteWriter } = require('../_writers/FullRemoteWriter')
-const { forkStreamUnpipe } = require('../_forkStreamUnpipe')
-const { watchStreamSize } = require('../../_watchStreamSize')
-const { Task } = require('../../Task')
-
-class FullRemoteVmBackupRunner extends AbstractRemote {
+export const FullRemote = class FullRemoteVmBackupRunner extends AbstractRemote {
   _getRemoteWriter() {
     return FullRemoteWriter
   }
@@ -47,7 +45,6 @@ class FullRemoteVmBackupRunner extends AbstractRemote {
   }
 }
 
-exports.FullRemote = FullRemoteVmBackupRunner
-decorateMethodsWith(FullRemoteVmBackupRunner, {
+decorateMethodsWith(FullRemote, {
   _run: defer,
 })

@xen-orchestra/backups/_runners/_vmRunners/FullXapi.mjs

@@ -1,16 +1,14 @@
-'use strict'
+import { createLogger } from '@xen-orchestra/log'
 
-const { createLogger } = require('@xen-orchestra/log')
-
-const { forkStreamUnpipe } = require('../_forkStreamUnpipe.js')
-const { FullRemoteWriter } = require('../_writers/FullRemoteWriter.js')
-const { FullXapiWriter } = require('../_writers/FullXapiWriter.js')
-const { watchStreamSize } = require('../../_watchStreamSize.js')
-const { AbstractXapi } = require('./_AbstractXapi.js')
+import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
+import { FullRemoteWriter } from '../_writers/FullRemoteWriter.mjs'
+import { FullXapiWriter } from '../_writers/FullXapiWriter.mjs'
+import { watchStreamSize } from '../../_watchStreamSize.mjs'
+import { AbstractXapi } from './_AbstractXapi.mjs'
 
 const { debug } = createLogger('xo:backups:FullXapiVmBackup')
 
-exports.FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
+export const FullXapi = class FullXapiVmBackupRunner extends AbstractXapi {
   _getWriters() {
     return [FullRemoteWriter, FullXapiWriter]
   }

@xen-orchestra/backups/_runners/_vmRunners/IncrementalRemote.mjs

@@ -1,15 +1,14 @@
-'use strict'
-const assert = require('node:assert')
+import { asyncEach } from '@vates/async-each'
+import { decorateMethodsWith } from '@vates/decorate-with'
+import { defer } from 'golike-defer'
+import assert from 'node:assert'
+import isVhdDifferencingDisk from 'vhd-lib/isVhdDifferencingDisk.js'
+import mapValues from 'lodash/mapValues.js'
 
-const { decorateMethodsWith } = require('@vates/decorate-with')
-const { defer } = require('golike-defer')
-const { mapValues } = require('lodash')
-const { Task } = require('../../Task')
-const { AbstractRemote } = require('./_AbstractRemote')
-const { IncrementalRemoteWriter } = require('../_writers/IncrementalRemoteWriter')
-const { forkDeltaExport } = require('./_forkDeltaExport')
-const isVhdDifferencingDisk = require('vhd-lib/isVhdDifferencingDisk')
-const { asyncEach } = require('@vates/async-each')
+import { AbstractRemote } from './_AbstractRemote.mjs'
+import { forkDeltaExport } from './_forkDeltaExport.mjs'
+import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs'
+import { Task } from '../../Task.mjs'
 
 class IncrementalRemoteVmBackupRunner extends AbstractRemote {
   _getRemoteWriter() {
@@ -61,7 +60,7 @@ class IncrementalRemoteVmBackupRunner extends AbstractRemote {
   }
 }
 
-exports.IncrementalRemote = IncrementalRemoteVmBackupRunner
+export const IncrementalRemote = IncrementalRemoteVmBackupRunner
 decorateMethodsWith(IncrementalRemoteVmBackupRunner, {
   _run: defer,
 })

@xen-orchestra/backups/_runners/_vmRunners/IncrementalXapi.mjs

@@ -1,28 +1,26 @@
-'use strict'
+import { asyncEach } from '@vates/async-each'
+import { asyncMap } from '@xen-orchestra/async-map'
+import { createLogger } from '@xen-orchestra/log'
+import { pipeline } from 'node:stream'
+import findLast from 'lodash/findLast.js'
+import isVhdDifferencingDisk from 'vhd-lib/isVhdDifferencingDisk.js'
+import keyBy from 'lodash/keyBy.js'
+import mapValues from 'lodash/mapValues.js'
+import vhdStreamValidator from 'vhd-lib/vhdStreamValidator.js'
 
-const findLast = require('lodash/findLast.js')
-const keyBy = require('lodash/keyBy.js')
-const mapValues = require('lodash/mapValues.js')
-const vhdStreamValidator = require('vhd-lib/vhdStreamValidator.js')
-const { asyncMap } = require('@xen-orchestra/async-map')
-const { createLogger } = require('@xen-orchestra/log')
-const { pipeline } = require('node:stream')
-
-const { IncrementalRemoteWriter } = require('../_writers/IncrementalRemoteWriter.js')
-const { IncrementalXapiWriter } = require('../_writers/IncrementalXapiWriter.js')
-const { exportIncrementalVm } = require('../../_incrementalVm.js')
-const { Task } = require('../../Task.js')
-const { watchStreamSize } = require('../../_watchStreamSize.js')
-const { AbstractXapi } = require('./_AbstractXapi.js')
-const { forkDeltaExport } = require('./_forkDeltaExport.js')
-const isVhdDifferencingDisk = require('vhd-lib/isVhdDifferencingDisk')
-const { asyncEach } = require('@vates/async-each')
+import { AbstractXapi } from './_AbstractXapi.mjs'
+import { exportIncrementalVm } from '../../_incrementalVm.mjs'
+import { forkDeltaExport } from './_forkDeltaExport.mjs'
+import { IncrementalRemoteWriter } from '../_writers/IncrementalRemoteWriter.mjs'
+import { IncrementalXapiWriter } from '../_writers/IncrementalXapiWriter.mjs'
+import { Task } from '../../Task.mjs'
+import { watchStreamSize } from '../../_watchStreamSize.mjs'
 
 const { debug } = createLogger('xo:backups:IncrementalXapiVmBackup')
 
 const noop = Function.prototype
 
-exports.IncrementalXapi = class IncrementalXapiVmBackupRunner extends AbstractXapi {
+export const IncrementalXapi = class IncrementalXapiVmBackupRunner extends AbstractXapi {
   _getWriters() {
     return [IncrementalRemoteWriter, IncrementalXapiWriter]
   }
@@ -43,6 +41,7 @@ exports.IncrementalXapi = class IncrementalXapiVmBackupRunner extends AbstractXa
 
     const deltaExport = await exportIncrementalVm(exportedVm, baseVm, {
       fullVdisRequired,
+      preferNbd: this._settings.preferNbd,
     })
     // since NBD is network based, if one disk use nbd , all the disk use them
     // except the suspended VDI

@xen-orchestra/backups/_runners/_vmRunners/_Abstract.mjs

@@ -1,8 +1,6 @@
-'use strict'
-
-const { asyncMap } = require('@xen-orchestra/async-map')
-const { createLogger } = require('@xen-orchestra/log')
-const { Task } = require('../../Task.js')
+import { asyncMap } from '@xen-orchestra/async-map'
+import { createLogger } from '@xen-orchestra/log'
+import { Task } from '../../Task.mjs'
 
 const { debug, warn } = createLogger('xo:backups:AbstractVmRunner')
 
@@ -19,7 +17,7 @@ const asyncEach = async (iterable, fn, thisArg = iterable) => {
   }
 }
 
-exports.Abstract = class AbstractVmBackupRunner {
+export const Abstract = class AbstractVmBackupRunner {
   // calls fn for each function, warns of any errors, and throws only if there are no writers left
   async _callWriters(fn, step, parallel = true) {
     const writers = this._writers

@xen-orchestra/backups/_runners/_vmRunners/_AbstractRemote.mjs

@@ -1,11 +1,12 @@
-'use strict'
-const { Abstract } = require('./_Abstract')
+import { asyncEach } from '@vates/async-each'
+import { Disposable } from 'promise-toolbox'
 
-const { getVmBackupDir } = require('../../_getVmBackupDir')
-const { asyncEach } = require('@vates/async-each')
-const { Disposable } = require('promise-toolbox')
+import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
 
-exports.AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstract {
+import { Abstract } from './_Abstract.mjs'
+import { extractIdsFromSimplePattern } from '../../extractIdsFromSimplePattern.mjs'
+
+export const AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstract {
   constructor({
     config,
     job,
@@ -34,7 +35,8 @@ exports.AbstractRemote = class AbstractRemoteVmBackupRunner extends Abstract {
     this._writers = writers
 
     const RemoteWriter = this._getRemoteWriter()
-    Object.entries(remoteAdapters).forEach(([remoteId, adapter]) => {
+    extractIdsFromSimplePattern(job.remotes).forEach(remoteId => {
+      const adapter = remoteAdapters[remoteId]
       const targetSettings = {
         ...settings,
         ...allSettings[remoteId],

@xen-orchestra/backups/_runners/_vmRunners/_AbstractXapi.mjs

@@ -1,18 +1,16 @@
-'use strict'
+import assert from 'node:assert'
+import groupBy from 'lodash/groupBy.js'
+import ignoreErrors from 'promise-toolbox/ignoreErrors'
+import { asyncMap } from '@xen-orchestra/async-map'
+import { decorateMethodsWith } from '@vates/decorate-with'
+import { defer } from 'golike-defer'
+import { formatDateTime } from '@xen-orchestra/xapi'
 
-const assert = require('assert')
-const groupBy = require('lodash/groupBy.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors')
-const { asyncMap } = require('@xen-orchestra/async-map')
-const { decorateMethodsWith } = require('@vates/decorate-with')
-const { defer } = require('golike-defer')
-const { formatDateTime } = require('@xen-orchestra/xapi')
+import { getOldEntries } from '../../_getOldEntries.mjs'
+import { Task } from '../../Task.mjs'
+import { Abstract } from './_Abstract.mjs'
 
-const { getOldEntries } = require('../../_getOldEntries.js')
-const { Task } = require('../../Task.js')
-const { Abstract } = require('./_Abstract.js')
-
-class AbstractXapiVmBackupRunner extends Abstract {
+export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
   constructor({
     config,
     getSnapshotNameLabel,
@@ -33,6 +31,11 @@ class AbstractXapiVmBackupRunner extends Abstract {
       throw new Error('cannot backup a VM created by this very job')
     }
 
+    const currentOperations = Object.values(vm.current_operations)
+    if (currentOperations.some(_ => _ === 'migrate_send' || _ === 'pool_migrate')) {
+      throw new Error('cannot backup a VM currently being migrated')
+    }
+
     this.config = config
     this.job = job
     this.remoteAdapters = remoteAdapters
@@ -258,7 +261,15 @@ class AbstractXapiVmBackupRunner extends Abstract {
       }
 
       if (this._writers.size !== 0) {
-        await this._copy()
+        const { pool_migrate = null, migrate_send = null } = this._exportedVm.blocked_operations
+
+        const reason = 'VM migration is blocked during backup'
+        await this._exportedVm.update_blocked_operations({ pool_migrate: reason, migrate_send: reason })
+        try {
+          await this._copy()
+        } finally {
+          await this._exportedVm.update_blocked_operations({ pool_migrate, migrate_send })
+        }
       }
     } finally {
       if (startAfter) {
@@ -271,8 +282,7 @@ class AbstractXapiVmBackupRunner extends Abstract {
     await this._healthCheck()
   }
 }
-exports.AbstractXapi = AbstractXapiVmBackupRunner
 
-decorateMethodsWith(AbstractXapiVmBackupRunner, {
+decorateMethodsWith(AbstractXapi, {
   run: defer,
 })

@xen-orchestra/backups/_runners/_vmRunners/_forkDeltaExport.js

@@ -1,12 +0,0 @@
-'use strict'
-
-const { mapValues } = require('lodash')
-const { forkStreamUnpipe } = require('../_forkStreamUnpipe')
-
-exports.forkDeltaExport = function forkDeltaExport(deltaExport) {
-  return Object.create(deltaExport, {
-    streams: {
-      value: mapValues(deltaExport.streams, forkStreamUnpipe),
-    },
-  })
-}

@xen-orchestra/backups/_runners/_vmRunners/_forkDeltaExport.mjs

@@ -0,0 +1,11 @@
+import mapValues from 'lodash/mapValues.js'
+
+import { forkStreamUnpipe } from '../_forkStreamUnpipe.mjs'
+
+export function forkDeltaExport(deltaExport) {
+  return Object.create(deltaExport, {
+    streams: {
+      value: mapValues(deltaExport.streams, forkStreamUnpipe),
+    },
+  })
+}

@xen-orchestra/backups/_runners/_writers/FullRemoteWriter.mjs

@@ -1,13 +1,11 @@
-'use strict'
+import { formatFilenameDate } from '../../_filenameDate.mjs'
+import { getOldEntries } from '../../_getOldEntries.mjs'
+import { Task } from '../../Task.mjs'
 
-const { formatFilenameDate } = require('../../_filenameDate.js')
-const { getOldEntries } = require('../../_getOldEntries.js')
-const { Task } = require('../../Task.js')
+import { MixinRemoteWriter } from './_MixinRemoteWriter.mjs'
+import { AbstractFullWriter } from './_AbstractFullWriter.mjs'
 
-const { MixinRemoteWriter } = require('./_MixinRemoteWriter.js')
-const { AbstractFullWriter } = require('./_AbstractFullWriter.js')
-
-exports.FullRemoteWriter = class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
+export class FullRemoteWriter extends MixinRemoteWriter(AbstractFullWriter) {
   constructor(props) {
     super(props)
 

@xen-orchestra/backups/_runners/_writers/FullXapiWriter.mjs

@@ -1,18 +1,16 @@
-'use strict'
+import ignoreErrors from 'promise-toolbox/ignoreErrors'
+import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
+import { formatDateTime } from '@xen-orchestra/xapi'
 
-const ignoreErrors = require('promise-toolbox/ignoreErrors')
-const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
-const { formatDateTime } = require('@xen-orchestra/xapi')
+import { formatFilenameDate } from '../../_filenameDate.mjs'
+import { getOldEntries } from '../../_getOldEntries.mjs'
+import { Task } from '../../Task.mjs'
 
-const { formatFilenameDate } = require('../../_filenameDate.js')
-const { getOldEntries } = require('../../_getOldEntries.js')
-const { Task } = require('../../Task.js')
+import { AbstractFullWriter } from './_AbstractFullWriter.mjs'
+import { MixinXapiWriter } from './_MixinXapiWriter.mjs'
+import { listReplicatedVms } from './_listReplicatedVms.mjs'
 
-const { AbstractFullWriter } = require('./_AbstractFullWriter.js')
-const { MixinXapiWriter } = require('./_MixinXapiWriter.js')
-const { listReplicatedVms } = require('./_listReplicatedVms.js')
-
-exports.FullXapiWriter = class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
+export class FullXapiWriter extends MixinXapiWriter(AbstractFullWriter) {
   constructor(props) {
     super(props)
 

@xen-orchestra/backups/_runners/_writers/IncrementalRemoteWriter.mjs

@@ -1,29 +1,27 @@
-'use strict'
+import assert from 'node:assert'
+import mapValues from 'lodash/mapValues.js'
+import ignoreErrors from 'promise-toolbox/ignoreErrors'
+import { asyncEach } from '@vates/async-each'
+import { asyncMap } from '@xen-orchestra/async-map'
+import { chainVhd, checkVhdChain, openVhd, VhdAbstract } from 'vhd-lib'
+import { createLogger } from '@xen-orchestra/log'
+import { decorateClass } from '@vates/decorate-with'
+import { defer } from 'golike-defer'
+import { dirname } from 'node:path'
 
-const assert = require('assert')
-const mapValues = require('lodash/mapValues.js')
-const ignoreErrors = require('promise-toolbox/ignoreErrors')
-const { asyncEach } = require('@vates/async-each')
-const { asyncMap } = require('@xen-orchestra/async-map')
-const { chainVhd, checkVhdChain, openVhd, VhdAbstract } = require('vhd-lib')
-const { createLogger } = require('@xen-orchestra/log')
-const { decorateClass } = require('@vates/decorate-with')
-const { defer } = require('golike-defer')
-const { dirname } = require('path')
+import { formatFilenameDate } from '../../_filenameDate.mjs'
+import { getOldEntries } from '../../_getOldEntries.mjs'
+import { Task } from '../../Task.mjs'
 
-const { formatFilenameDate } = require('../../_filenameDate.js')
-const { getOldEntries } = require('../../_getOldEntries.js')
-const { Task } = require('../../Task.js')
-
-const { MixinRemoteWriter } = require('./_MixinRemoteWriter.js')
-const { AbstractIncrementalWriter } = require('./_AbstractIncrementalWriter.js')
-const { checkVhd } = require('./_checkVhd.js')
-const { packUuid } = require('./_packUuid.js')
-const { Disposable } = require('promise-toolbox')
+import { MixinRemoteWriter } from './_MixinRemoteWriter.mjs'
+import { AbstractIncrementalWriter } from './_AbstractIncrementalWriter.mjs'
+import { checkVhd } from './_checkVhd.mjs'
+import { packUuid } from './_packUuid.mjs'
+import { Disposable } from 'promise-toolbox'
 
 const { warn } = createLogger('xo:backups:DeltaBackupWriter')
 
-class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrementalWriter) {
+export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrementalWriter) {
   async checkBaseVdis(baseUuidToSrcVdi) {
     const { handler } = this._adapter
     const adapter = this._adapter
@@ -205,7 +203,7 @@ class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrementalWrite
             // TODO remove when this has been done before the export
             await checkVhd(handler, parentPath)
           }
-          // @todo : sum per property
+
           transferSize += await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
             // no checksum for VHDs, because they will be invalidated by
             // merges and chainings
@@ -232,12 +230,12 @@ class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrementalWrite
 
       return { size: transferSize }
     })
-    metadataContent.size = size // @todo: transferSize
+    metadataContent.size = size
     this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadataContent)
 
     // TODO: run cleanup?
   }
 }
-exports.IncrementalRemoteWriter = decorateClass(IncrementalRemoteWriter, {
+decorateClass(IncrementalRemoteWriter, {
   _transfer: defer,
 })

@xen-orchestra/backups/_runners/_writers/IncrementalXapiWriter.mjs

@@ -1,19 +1,17 @@
-'use strict'
+import { asyncMap, asyncMapSettled } from '@xen-orchestra/async-map'
+import ignoreErrors from 'promise-toolbox/ignoreErrors'
+import { formatDateTime } from '@xen-orchestra/xapi'
 
-const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
-const ignoreErrors = require('promise-toolbox/ignoreErrors')
-const { formatDateTime } = require('@xen-orchestra/xapi')
+import { formatFilenameDate } from '../../_filenameDate.mjs'
+import { getOldEntries } from '../../_getOldEntries.mjs'
+import { importIncrementalVm, TAG_COPY_SRC } from '../../_incrementalVm.mjs'
+import { Task } from '../../Task.mjs'
 
-const { formatFilenameDate } = require('../../_filenameDate.js')
-const { getOldEntries } = require('../../_getOldEntries.js')
-const { importIncrementalVm, TAG_COPY_SRC } = require('../../_incrementalVm.js')
-const { Task } = require('../../Task.js')
+import { AbstractIncrementalWriter } from './_AbstractIncrementalWriter.mjs'
+import { MixinXapiWriter } from './_MixinXapiWriter.mjs'
+import { listReplicatedVms } from './_listReplicatedVms.mjs'
 
-const { AbstractIncrementalWriter } = require('./_AbstractIncrementalWriter.js')
-const { MixinXapiWriter } = require('./_MixinXapiWriter.js')
-const { listReplicatedVms } = require('./_listReplicatedVms.js')
-
-exports.IncrementalXapiWriter = class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWriter) {
+export class IncrementalXapiWriter extends MixinXapiWriter(AbstractIncrementalWriter) {
   async checkBaseVdis(baseUuidToSrcVdi, baseVm) {
     const sr = this._sr
     const replicatedVm = listReplicatedVms(sr.$xapi, this._job.id, sr.uuid, this._vmUuid).find(

@xen-orchestra/backups/_runners/_writers/_AbstractFullWriter.mjs

@@ -1,8 +1,6 @@
-'use strict'
+import { AbstractWriter } from './_AbstractWriter.mjs'
 
-const { AbstractWriter } = require('./_AbstractWriter.js')
-
-exports.AbstractFullWriter = class AbstractFullWriter extends AbstractWriter {
+export class AbstractFullWriter extends AbstractWriter {
   async run({ timestamp, sizeContainer, stream, vm, vmSnapshot }) {
     try {
       return await this._run({ timestamp, sizeContainer, stream, vm, vmSnapshot })

@xen-orchestra/backups/_runners/_writers/_AbstractIncrementalWriter.mjs

@@ -1,8 +1,6 @@
-'use strict'
+import { AbstractWriter } from './_AbstractWriter.mjs'
 
-const { AbstractWriter } = require('./_AbstractWriter.js')
-
-exports.AbstractIncrementalWriter = class AbstractIncrementalWriter extends AbstractWriter {
+export class AbstractIncrementalWriter extends AbstractWriter {
   checkBaseVdis(baseUuidToSrcVdi, baseVm) {
     throw new Error('Not implemented')
   }

@xen-orchestra/backups/_runners/_writers/_AbstractWriter.mjs

@@ -1,9 +1,7 @@
-'use strict'
+import { formatFilenameDate } from '../../_filenameDate.mjs'
+import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
 
-const { formatFilenameDate } = require('../../_filenameDate')
-const { getVmBackupDir } = require('../../_getVmBackupDir')
-
-exports.AbstractWriter = class AbstractWriter {
+export class AbstractWriter {
   constructor({ config, healthCheckSr, job, vmUuid, scheduleId, settings }) {
     this._config = config
     this._healthCheckSr = healthCheckSr

@xen-orchestra/backups/_runners/_writers/_MixinRemoteWriter.mjs

@@ -1,19 +1,17 @@
-'use strict'
+import { createLogger } from '@xen-orchestra/log'
+import { join } from 'node:path'
+import assert from 'node:assert'
 
-const { createLogger } = require('@xen-orchestra/log')
-const { join } = require('path')
-
-const assert = require('assert')
-const { formatFilenameDate } = require('../../_filenameDate.js')
-const { getVmBackupDir } = require('../../_getVmBackupDir.js')
-const { HealthCheckVmBackup } = require('../../HealthCheckVmBackup.js')
-const { ImportVmBackup } = require('../../ImportVmBackup.js')
-const { Task } = require('../../Task.js')
-const MergeWorker = require('../../merge-worker/index.js')
+import { formatFilenameDate } from '../../_filenameDate.mjs'
+import { getVmBackupDir } from '../../_getVmBackupDir.mjs'
+import { HealthCheckVmBackup } from '../../HealthCheckVmBackup.mjs'
+import { ImportVmBackup } from '../../ImportVmBackup.mjs'
+import { Task } from '../../Task.mjs'
+import * as MergeWorker from '../../merge-worker/index.mjs'
 
 const { info, warn } = createLogger('xo:backups:MixinBackupWriter')
 
-exports.MixinRemoteWriter = (BaseClass = Object) =>
+export const MixinRemoteWriter = (BaseClass = Object) =>
   class MixinRemoteWriter extends BaseClass {
     #lock
 

@xen-orchestra/backups/_runners/_writers/_MixinXapiWriter.mjs

@@ -1,12 +1,10 @@
-'use strict'
+import { extractOpaqueRef } from '@xen-orchestra/xapi'
+import assert from 'node:assert/strict'
 
-const { extractOpaqueRef } = require('@xen-orchestra/xapi')
+import { HealthCheckVmBackup } from '../../HealthCheckVmBackup.mjs'
+import { Task } from '../../Task.mjs'
 
-const { Task } = require('../../Task')
-const assert = require('node:assert/strict')
-const { HealthCheckVmBackup } = require('../../HealthCheckVmBackup')
-
-exports.MixinXapiWriter = (BaseClass = Object) =>
+export const MixinXapiWriter = (BaseClass = Object) =>
   class MixinXapiWriter extends BaseClass {
     constructor({ sr, ...rest }) {
       super(rest)
@@ -20,7 +18,7 @@ exports.MixinXapiWriter = (BaseClass = Object) =>
       const vdiRefs = await xapi.VM_getDisks(baseVm.$ref)
       for (const vdiRef of vdiRefs) {
         const vdi = xapi.getObject(vdiRef)
-        if (vdi.$SR.uuid !== this._heathCheckSr.uuid) {
+        if (vdi.$SR.uuid !== this._healthCheckSr.uuid) {
           return false
         }
       }

@xen-orchestra/backups/_runners/_writers/_checkVhd.js

@@ -1,8 +0,0 @@
-'use strict'
-
-const openVhd = require('vhd-lib').openVhd
-const Disposable = require('promise-toolbox/Disposable')
-
-exports.checkVhd = async function checkVhd(handler, path) {
-  await Disposable.use(openVhd(handler, path), () => {})
-}

@xen-orchestra/backups/_runners/_writers/_checkVhd.mjs

@@ -0,0 +1,6 @@
+import { openVhd } from 'vhd-lib'
+import Disposable from 'promise-toolbox/Disposable'
+
+export async function checkVhd(handler, path) {
+  await Disposable.use(openVhd(handler, path), () => {})
+}

@xen-orchestra/backups/_runners/_writers/_listReplicatedVms.mjs

@@ -1,5 +1,3 @@
-'use strict'
-
 const getReplicatedVmDatetime = vm => {
   const { 'xo:backup:datetime': datetime = vm.name_label.slice(-17, -1) } = vm.other_config
   return datetime
@@ -7,7 +5,7 @@ const getReplicatedVmDatetime = vm => {
 
 const compareReplicatedVmDatetime = (a, b) => (getReplicatedVmDatetime(a) < getReplicatedVmDatetime(b) ? -1 : 1)
 
-exports.listReplicatedVms = function listReplicatedVms(xapi, scheduleOrJobId, srUuid, vmUuid) {
+export function listReplicatedVms(xapi, scheduleOrJobId, srUuid, vmUuid) {
   const { all } = xapi.objects
   const vms = {}
   for (const key in all) {

@xen-orchestra/backups/_runners/_writers/_packUuid.mjs

@@ -1,7 +1,5 @@
-'use strict'
-
 const PARSE_UUID_RE = /-/g
 
-exports.packUuid = function packUuid(uuid) {
+export function packUuid(uuid) {
   return Buffer.from(uuid.replace(PARSE_UUID_RE, ''), 'hex')
 }

@xen-orchestra/backups/_watchStreamSize.mjs

@@ -1,6 +1,4 @@
-'use strict'
-
-exports.watchStreamSize = function watchStreamSize(stream, container = { size: 0 }) {
+export function watchStreamSize(stream, container = { size: 0 }) {
   stream.on('data', data => {
     container.size += data.length
   })

@xen-orchestra/backups/extractIdsFromSimplePattern.mjs

@@ -1,6 +1,4 @@
-'use strict'
-
-exports.extractIdsFromSimplePattern = function extractIdsFromSimplePattern(pattern) {
+export function extractIdsFromSimplePattern(pattern) {
   if (pattern === undefined) {
     return []
   }

@xen-orchestra/backups/formatVmBackups.mjs

@@ -1,7 +1,5 @@
-'use strict'
-
-const mapValues = require('lodash/mapValues.js')
-const { dirname } = require('path')
+import mapValues from 'lodash/mapValues.js'
+import { dirname } from 'node:path'
 
 function formatVmBackup(backup) {
   return {
@@ -31,6 +29,6 @@ function formatVmBackup(backup) {
 }
 
 // format all backups as returned by RemoteAdapter#listAllVmBackups()
-exports.formatVmBackups = function formatVmBackups(backupsByVM) {
+export function formatVmBackups(backupsByVM) {
   return mapValues(backupsByVM, backups => backups.map(formatVmBackup))
 }

@xen-orchestra/backups/merge-worker/cli.js

@@ -1,92 +0,0 @@
-#!/usr/bin/env node
-// eslint-disable-next-line eslint-comments/disable-enable-pair
-/* eslint-disable n/shebang */
-
-'use strict'
-
-const { catchGlobalErrors } = require('@xen-orchestra/log/configure')
-const { createLogger } = require('@xen-orchestra/log')
-const { getSyncedHandler } = require('@xen-orchestra/fs')
-const { join } = require('path')
-const Disposable = require('promise-toolbox/Disposable')
-const min = require('lodash/min')
-
-const { getVmBackupDir } = require('../_getVmBackupDir.js')
-const { RemoteAdapter } = require('../RemoteAdapter.js')
-
-const { CLEAN_VM_QUEUE } = require('./index.js')
-
-// -------------------------------------------------------------------
-
-catchGlobalErrors(createLogger('xo:backups:mergeWorker'))
-
-const { fatal, info, warn } = createLogger('xo:backups:mergeWorker')
-
-// -------------------------------------------------------------------
-
-const main = Disposable.wrap(async function* main(args) {
-  const handler = yield getSyncedHandler({ url: 'file://' + process.cwd() })
-
-  yield handler.lock(CLEAN_VM_QUEUE)
-
-  const adapter = new RemoteAdapter(handler)
-
-  const listRetry = async () => {
-    const timeoutResolver = resolve => setTimeout(resolve, 10e3)
-    for (let i = 0; i < 10; ++i) {
-      const entries = await handler.list(CLEAN_VM_QUEUE)
-      if (entries.length !== 0) {
-        return entries
-      }
-      await new Promise(timeoutResolver)
-    }
-  }
-
-  let taskFiles
-  while ((taskFiles = await listRetry()) !== undefined) {
-    const taskFileBasename = min(taskFiles)
-    const previousTaskFile = join(CLEAN_VM_QUEUE, taskFileBasename)
-    const taskFile = join(CLEAN_VM_QUEUE, '_' + taskFileBasename)
-
-    // move this task to the end
-    try {
-      await handler.rename(previousTaskFile, taskFile)
-    } catch (error) {
-      // this error occurs if the task failed too many times (i.e. too many `_` prefixes)
-      // there is nothing more that can be done
-      if (error.code === 'ENAMETOOLONG') {
-        await handler.unlink(previousTaskFile)
-      }
-
-      throw error
-    }
-
-    try {
-      const vmDir = getVmBackupDir(String(await handler.readFile(taskFile)))
-      try {
-        await adapter.cleanVm(vmDir, { merge: true, logInfo: info, logWarn: warn, remove: true })
-      } catch (error) {
-        // consider the clean successful if the VM dir is missing
-        if (error.code !== 'ENOENT') {
-          throw error
-        }
-      }
-
-      handler.unlink(taskFile).catch(error => warn('deleting task failure', { error }))
-    } catch (error) {
-      warn('failure handling task', { error })
-    }
-  }
-})
-
-info('starting')
-main(process.argv.slice(2)).then(
-  () => {
-    info('bye :-)')
-  },
-  error => {
-    fatal(error)
-
-    process.exit(1)
-  }
-)

@xen-orchestra/backups/merge-worker/cli.mjs

@@ -0,0 +1,103 @@
+#!/usr/bin/env node
+// eslint-disable-next-line eslint-comments/disable-enable-pair
+/* eslint-disable n/shebang */
+
+import { asyncEach } from '@vates/async-each'
+import { catchGlobalErrors } from '@xen-orchestra/log/configure'
+import { createLogger } from '@xen-orchestra/log'
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { join } from 'node:path'
+import { load as loadConfig } from 'app-conf'
+import Disposable from 'promise-toolbox/Disposable'
+
+import { getVmBackupDir } from '../_getVmBackupDir.mjs'
+import { RemoteAdapter } from '../RemoteAdapter.mjs'
+
+import { CLEAN_VM_QUEUE } from './index.mjs'
+
+const APP_NAME = 'xo-merge-worker'
+const APP_DIR = new URL('.', import.meta.url).pathname
+// -------------------------------------------------------------------
+
+catchGlobalErrors(createLogger('xo:backups:mergeWorker'))
+
+const { fatal, info, warn } = createLogger('xo:backups:mergeWorker')
+
+// -------------------------------------------------------------------
+
+const main = Disposable.wrap(async function* main(args) {
+  const handler = yield getSyncedHandler({ url: 'file://' + process.cwd() })
+
+  yield handler.lock(CLEAN_VM_QUEUE)
+
+  const adapter = new RemoteAdapter(handler)
+
+  const listRetry = async () => {
+    const timeoutResolver = resolve => setTimeout(resolve, 10e3)
+    for (let i = 0; i < 10; ++i) {
+      const entries = await handler.list(CLEAN_VM_QUEUE)
+      if (entries.length !== 0) {
+        entries.sort()
+        return entries
+      }
+      await new Promise(timeoutResolver)
+    }
+  }
+
+  let taskFiles
+  while ((taskFiles = await listRetry()) !== undefined) {
+    const { concurrency } = await loadConfig(APP_NAME, {
+      appDir: APP_DIR,
+      ignoreUnknownFormats: true,
+    })
+    await asyncEach(
+      taskFiles,
+      async taskFileBasename => {
+        const previousTaskFile = join(CLEAN_VM_QUEUE, taskFileBasename)
+        const taskFile = join(CLEAN_VM_QUEUE, '_' + taskFileBasename)
+
+        // move this task to the end
+        try {
+          await handler.rename(previousTaskFile, taskFile)
+        } catch (error) {
+          // this error occurs if the task failed too many times (i.e. too many `_` prefixes)
+          // there is nothing more that can be done
+          if (error.code === 'ENAMETOOLONG') {
+            await handler.unlink(previousTaskFile)
+          }
+
+          throw error
+        }
+
+        try {
+          const vmDir = getVmBackupDir(String(await handler.readFile(taskFile)))
+          try {
+            await adapter.cleanVm(vmDir, { merge: true, logInfo: info, logWarn: warn, remove: true })
+          } catch (error) {
+            // consider the clean successful if the VM dir is missing
+            if (error.code !== 'ENOENT') {
+              throw error
+            }
+          }
+
+          handler.unlink(taskFile).catch(error => warn('deleting task failure', { error }))
+        } catch (error) {
+          warn('failure handling task', { error })
+        }
+      },
+      { concurrency }
+    )
+  }
+})
+
+info('starting')
+main(process.argv.slice(2)).then(
+  () => {
+    info('bye :-)')
+  },
+  error => {
+    fatal(error)
+
+    process.exit(1)
+  }
+)

@xen-orchestra/backups/merge-worker/config.toml

@@ -0,0 +1 @@
+concurrency = 1

@xen-orchestra/backups/merge-worker/index.mjs

@@ -1,13 +1,12 @@
-'use strict'
+import { join } from 'node:path'
+import { spawn } from 'child_process'
+import { check } from 'proper-lockfile'
 
-const { join, resolve } = require('path')
-const { spawn } = require('child_process')
-const { check } = require('proper-lockfile')
+export const CLEAN_VM_QUEUE = '/xo-vm-backups/.queue/clean-vm/'
 
-const CLEAN_VM_QUEUE = (exports.CLEAN_VM_QUEUE = '/xo-vm-backups/.queue/clean-vm/')
+const CLI_PATH = new URL('cli.mjs', import.meta.url).pathname
 
-const CLI_PATH = resolve(__dirname, 'cli.js')
-exports.run = async function runMergeWorker(remotePath) {
+export const run = async function runMergeWorker(remotePath) {
   try {
     // TODO: find a way to pass the acquire the lock and then pass it down the worker
     if (await check(join(remotePath, CLEAN_VM_QUEUE))) {

@xen-orchestra/backups/package.json

@@ -8,32 +8,33 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.39.0",
+  "version": "0.43.0",
   "engines": {
     "node": ">=14.18"
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test-integration": "node--test *.integ.js"
+    "test-integration": "node--test *.integ.mjs"
   },
   "dependencies": {
+    "@iarna/toml": "^2.2.5",
     "@kldzj/stream-throttle": "^1.1.1",
     "@vates/async-each": "^1.0.0",
     "@vates/cached-dns.lookup": "^1.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.0.0",
     "@vates/disposable": "^0.1.4",
-    "@vates/fuse-vhd": "^1.0.0",
-    "@vates/nbd-client": "^1.2.1",
+    "@vates/fuse-vhd": "^2.0.0",
+    "@vates/nbd-client": "^2.0.0",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^4.0.1",
+    "@xen-orchestra/fs": "^4.1.0",
     "@xen-orchestra/log": "^0.6.0",
     "@xen-orchestra/template": "^0.1.0",
-    "compare-versions": "^5.0.1",
-    "d3-time-format": "^3.0.0",
+    "app-conf": "^2.3.0",
+    "compare-versions": "^6.0.0",
+    "d3-time-format": "^4.1.0",
     "decorator-synchronized": "^0.6.0",
-    "fs-extra": "^11.1.0",
     "golike-defer": "^0.5.1",
     "limit-concurrency-decorator": "^0.5.0",
     "lodash": "^4.17.20",
@@ -41,19 +42,21 @@
     "parse-pairs": "^2.0.0",
     "promise-toolbox": "^0.21.0",
     "proper-lockfile": "^4.1.2",
+    "tar": "^6.1.15",
     "uuid": "^9.0.0",
-    "vhd-lib": "^4.5.0",
-    "xen-api": "^1.3.3",
+    "vhd-lib": "^4.6.1",
+    "xen-api": "^1.3.6",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
+    "fs-extra": "^11.1.0",
     "rimraf": "^5.0.1",
-    "sinon": "^15.0.1",
+    "sinon": "^16.0.0",
     "test": "^3.2.1",
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^2.2.1"
+    "@xen-orchestra/xapi": "^3.2.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

@xen-orchestra/backups/parseMetadataBackupId.mjs

@@ -1,8 +1,6 @@
-'use strict'
+import { DIR_XO_CONFIG_BACKUPS, DIR_XO_POOL_METADATA_BACKUPS } from './RemoteAdapter.mjs'
 
-const { DIR_XO_CONFIG_BACKUPS, DIR_XO_POOL_METADATA_BACKUPS } = require('./RemoteAdapter.js')
-
-exports.parseMetadataBackupId = function parseMetadataBackupId(backupId) {
+export function parseMetadataBackupId(backupId) {
   const [dir, ...rest] = backupId.split('/')
   if (dir === DIR_XO_CONFIG_BACKUPS) {
     const [scheduleId, timestamp] = rest

@xen-orchestra/backups/runBackupWorker.mjs

@@ -1,14 +1,11 @@
-'use strict'
-
-const path = require('path')
-const { createLogger } = require('@xen-orchestra/log')
-const { fork } = require('child_process')
+import { createLogger } from '@xen-orchestra/log'
+import { fork } from 'child_process'
 
 const { warn } = createLogger('xo:backups:backupWorker')
 
-const PATH = path.resolve(__dirname, '_backupWorker.js')
+const PATH = new URL('_backupWorker.mjs', import.meta.url).pathname
 
-exports.runBackupWorker = function runBackupWorker(params, onLog) {
+export function runBackupWorker(params, onLog) {
   return new Promise((resolve, reject) => {
     const worker = fork(PATH)
 

@xen-orchestra/backups/tests.fixtures.mjs

@@ -1,7 +1,5 @@
-'use strict'
-
 // a valid footer of a 2
-exports.VHDFOOTER = {
+export const VHDFOOTER = {
   cookie: 'conectix',
   features: 2,
   fileFormatVersion: 65536,
@@ -20,7 +18,7 @@ exports.VHDFOOTER = {
   hidden: '',
   reserved: '',
 }
-exports.VHDHEADER = {
+export const VHDHEADER = {
   cookie: 'cxsparse',
   dataOffset: undefined,
   tableOffset: 2048,

@xen-orchestra/cr-seed-cli/package.json

@@ -18,7 +18,7 @@
   "preferGlobal": true,
   "dependencies": {
     "golike-defer": "^0.5.1",
-    "xen-api": "^1.3.3"
+    "xen-api": "^1.3.6"
   },
   "scripts": {
     "postversion": "npm publish"

@xen-orchestra/cron/package.json

@@ -42,7 +42,7 @@
     "test": "node--test"
   },
   "devDependencies": {
-    "sinon": "^15.0.1",
+    "sinon": "^16.0.0",
     "test": "^3.2.1"
   }
 }

@xen-orchestra/fs/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/fs",
-  "version": "4.0.1",
+  "version": "4.1.0",
   "license": "AGPL-3.0-or-later",
   "description": "The File System for Xen Orchestra backups.",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
@@ -29,7 +29,7 @@
     "@vates/async-each": "^1.0.0",
     "@vates/coalesce-calls": "^0.1.0",
     "@vates/decorate-with": "^2.0.0",
-    "@vates/read-chunk": "^1.1.1",
+    "@vates/read-chunk": "^1.2.0",
     "@xen-orchestra/log": "^0.6.0",
     "bind-property-descriptor": "^2.0.0",
     "decorator-synchronized": "^0.6.0",
@@ -53,7 +53,7 @@
     "cross-env": "^7.0.2",
     "dotenv": "^16.0.0",
     "rimraf": "^5.0.1",
-    "sinon": "^15.0.4",
+    "sinon": "^16.0.0",
     "test": "^3.3.0",
     "tmp": "^0.2.1"
   },

@xen-orchestra/fs/src/abstract.js

@@ -624,14 +624,18 @@ export default class RemoteHandlerAbstract {
 
     const files = await this._list(dir)
     await asyncEach(files, file =>
-      this._unlink(`${dir}/${file}`).catch(error => {
-        // Unlink dir behavior is not consistent across platforms
-        // https://github.com/nodejs/node-v0.x-archive/issues/5791
-        if (error.code === 'EISDIR' || error.code === 'EPERM') {
-          return this._rmtree(`${dir}/${file}`)
-        }
-        throw error
-      })
+      this._unlink(`${dir}/${file}`).catch(
+        error => {
+          // Unlink dir behavior is not consistent across platforms
+          // https://github.com/nodejs/node-v0.x-archive/issues/5791
+          if (error.code === 'EISDIR' || error.code === 'EPERM') {
+            return this._rmtree(`${dir}/${file}`)
+          }
+          throw error
+        },
+       // real unlink concurrency will be 2**max directory depth 
+        { concurrency: 2 }
+      )
     )
     return this._rmtree(dir)
   }

@xen-orchestra/fs/src/abstract.test.js

@@ -209,7 +209,7 @@ describe('encryption', () => {
     // encrypt with a non default algorithm
     const encryptor = _getEncryptor('aes-256-cbc', '73c1838d7d8a6088ca2317fb5f29cd91')
 
-    await fs.writeFile(`${dir}/encryption.json`, `{"algorithm": "aes-256-gmc"}`)
+    await fs.writeFile(`${dir}/encryption.json`, `{"algorithm": "aes-256-gcm"}`)
     await fs.writeFile(`${dir}/metadata.json`, encryptor.encryptData(`{"random": "NOTSORANDOM"}`))
 
     // remote is now non empty : can't modify key anymore

@xen-orchestra/fs/src/s3.js

@@ -16,9 +16,7 @@ import { NodeHttpHandler } from '@aws-sdk/node-http-handler'
 import { getApplyMd5BodyChecksumPlugin } from '@aws-sdk/middleware-apply-body-checksum'
 import { Agent as HttpAgent } from 'http'
 import { Agent as HttpsAgent } from 'https'
-import pRetry from 'promise-toolbox/retry'
 import { createLogger } from '@xen-orchestra/log'
-import { decorateWith } from '@vates/decorate-with'
 import { PassThrough, pipeline } from 'stream'
 import { parse } from 'xo-remote-parser'
 import copyStreamToBuffer from './_copyStreamToBuffer.js'
@@ -26,6 +24,7 @@ import guessAwsRegion from './_guessAwsRegion.js'
 import RemoteHandlerAbstract from './abstract'
 import { basename, join, split } from './path'
 import { asyncEach } from '@vates/async-each'
+import { pRetry } from 'promise-toolbox'
 
 // endpoints https://docs.aws.amazon.com/general/latest/gr/s3.html
 
@@ -78,6 +77,44 @@ export default class S3Handler extends RemoteHandlerAbstract {
     const parts = split(path)
     this.#bucket = parts.shift()
     this.#dir = join(...parts)
+    const WITH_RETRY = [
+      '_closeFile',
+      '_copy',
+      '_getInfo',
+      '_getSize',
+      '_list',
+      '_mkdir',
+      '_openFile',
+      '_outputFile',
+      '_read',
+      '_readFile',
+      '_rename',
+      '_rmdir',
+      '_truncate',
+      '_unlink',
+      '_write',
+      '_writeFile',
+    ]
+    WITH_RETRY.forEach(functionName => {
+      if (this[functionName] !== undefined) {
+        // adding the retry on the top level mtehod won't
+        // cover when _functionName are called internally
+        this[functionName] = pRetry.wrap(this[functionName], {
+          delays: [100, 200, 500, 1000, 2000],
+          // these errors should not change on retry
+          when: err => !['EEXIST', 'EISDIR', 'ENOTEMPTY', 'ENOENT', 'ENOTDIR', 'EISDIR'].includes(err?.code),
+          onRetry(error) {
+            warn('retrying method on fs ', {
+              method: functionName,
+              attemptNumber: this.attemptNumber,
+              delay: this.delay,
+              error,
+              file: this.arguments?.[0],
+            })
+          },
+        })
+      }
+    })
   }
 
   get type() {
@@ -212,21 +249,6 @@ export default class S3Handler extends RemoteHandlerAbstract {
     }
   }
 
-  // some objectstorage provider like backblaze, can answer a 500/503 routinely
-  // in this case we should retry,  and let their load balancing do its magic
-  // https://www.backblaze.com/b2/docs/calling.html#error_handling
-  @decorateWith(pRetry.wrap, {
-    delays: [100, 200, 500, 1000, 2000],
-    when: e => e.$metadata?.httpStatusCode === 500,
-    onRetry(error) {
-      warn('retrying writing file', {
-        attemptNumber: this.attemptNumber,
-        delay: this.delay,
-        error,
-        file: this.arguments[0],
-      })
-    },
-  })
   async _writeFile(file, data, options) {
     return this.#s3.send(
       new PutObjectCommand({

@xen-orchestra/lite/.prettierrc.cjs

@@ -1,2 +1,4 @@
 // Keeping this file to prevent applying the global monorepo config for now
-module.exports = {};
+module.exports = {
+  trailingComma: "es5",
+};

@xen-orchestra/lite/CHANGELOG.md

@@ -1,6 +1,27 @@
 # ChangeLog
 
-## **0.2.0**
+## **next**
+
+- Ability to snapshot/copy a VM from its view (PR [#7087](https://github.com/vatesfr/xen-orchestra/pull/7087))
+
+## **0.1.4** (2023-10-03)
+
+- Ability to migrate selected VMs to another host (PR [#7040](https://github.com/vatesfr/xen-orchestra/pull/7040))
+- Ability to snapshot selected VMs (PR [#7021](https://github.com/vatesfr/xen-orchestra/pull/7021))
+- Add Patches to Pool Dashboard (PR [#6709](https://github.com/vatesfr/xen-orchestra/pull/6709))
+- Add remember me checkbox on the login page (PR [#7030](https://github.com/vatesfr/xen-orchestra/pull/7030))
+
+## **0.1.3** (2023-09-01)
+
+- Add Alarms to Pool Dashboard (PR [#6976](https://github.com/vatesfr/xen-orchestra/pull/6976))
+
+## **0.1.2** (2023-07-28)
+
+- Ability to export selected VMs as CSV file (PR [#6915](https://github.com/vatesfr/xen-orchestra/pull/6915))
+- [Pool/VMs] Ability to export selected VMs as JSON file (PR [#6911](https://github.com/vatesfr/xen-orchestra/pull/6911))
+- Add Tasks to Pool Dashboard (PR [#6713](https://github.com/vatesfr/xen-orchestra/pull/6713))
+
+## **0.1.1** (2023-07-03)
 
 - Invalidate sessionId token after logout (PR [#6480](https://github.com/vatesfr/xen-orchestra/pull/6480))
 - Settings page (PR [#6418](https://github.com/vatesfr/xen-orchestra/pull/6418))

@xen-orchestra/lite/docs/xen-api-collections.md

@@ -0,0 +1,144 @@
+<!-- TOC -->
+
+- [XenApiCollection](#xenapicollection)
+  - [Get the collection](#get-the-collection)
+  - [Defer the subscription](#defer-the-subscription)
+  - [Create a dedicated collection](#create-a-dedicated-collection)
+  - [Alter the collection](#alter-the-collection)
+  _ [Example 1: Adding props to records](#example-1-adding-props-to-records)
+  _ [Example 2: Adding props to the collection](#example-2-adding-props-to-the-collection) \* [Example 3, filtering and sorting the collection](#example-3-filtering-and-sorting-the-collection)
+  <!-- TOC -->
+
+# XenApiCollection
+
+## Get the collection
+
+To retrieve a collection, invoke `useXenApiCollection("VM")`.
+
+By doing this, the current component will be automatically subscribed to the collection and will be updated when the
+collection changes.
+
+When the component is unmounted, the subscription will be automatically stopped.
+
+## Defer the subscription
+
+If you don't want to fetch the data of the collection when the component is mounted, you can pass `{ immediate: false }`
+as options: `const { start, isStarted } = useXenApiCollection("VM", { immediate: false })`.
+
+Then you subscribe to the collection by calling `start()`.
+
+## Create a dedicated collection
+
+It is recommended to create a dedicated collection composable for each type of record you want to use.
+
+They are stored in `src/composables/xen-api-collection/*-collection.composable.ts`.
+
+```typescript
+// src/composables/xen-api-collection/console-collection.composable.ts
+
+export const useConsoleCollection = () => useXenApiCollection("console");
+```
+
+If you want to allow the user to defer the subscription, you can propagate the options to `useXenApiCollection`.
+
+```typescript
+// console-collection.composable.ts
+
+export const useConsoleCollection = <
+  Immediate extends boolean = true,
+>(options?: {
+  immediate?: Immediate;
+}) => useXenApiCollection("console", options);
+```
+
+```typescript
+// MyComponent.vue
+
+const collection = useConsoleCollection({ immediate: false });
+
+setTimeout(() => collection.start(), 10000);
+```
+
+## Alter the collection
+
+You can alter the collection by overriding parts of it.
+
+### Example 1: Adding props to records
+
+```typescript
+// xen-api.ts
+
+export interface XenApiConsole extends XenApiRecord<"console"> {
+  // ... existing props
+  someProp: string;
+  someOtherProp: number;
+}
+```
+
+```typescript
+// console-collection.composable.ts
+
+export const useConsoleCollection = () => {
+  const collection = useXenApiCollection("console");
+
+  const records = computed(() => {
+    return collection.records.value.map((console) => ({
+      ...console,
+      someProp: "Some value",
+      someOtherProp: 42,
+    }));
+  });
+
+  return {
+    ...collection,
+    records,
+  };
+};
+```
+
+```typescript
+const consoleCollection = useConsoleCollection();
+
+consoleCollection.getByUuid("...").someProp; // "Some value"
+```
+
+### Example 2: Adding props to the collection
+
+```typescript
+// vm-collection.composable.ts
+
+export const useVmCollection = () => {
+  const collection = useXenApiCollection("VM");
+
+  return {
+    ...collection,
+    runningVms: computed(() =>
+      collection.records.value.filter(
+        (vm) => vm.power_state === POWER_STATE.RUNNING
+      )
+    ),
+  };
+};
+```
+
+### Example 3, filtering and sorting the collection
+
+```typescript
+// vm-collection.composable.ts
+
+export const useVmCollection = () => {
+  const collection = useXenApiCollection("VM");
+
+  return {
+    ...collection,
+    records: computed(() =>
+      collection.records.value
+        .filter(
+          (vm) =>
+            !vm.is_a_snapshot && !vm.is_a_template && !vm.is_control_domain
+        )
+        .sort((vm1, vm2) => vm1.name_label.localeCompare(vm2.name_label))
+    ),
+  };
+};
+```