fix

.github/ISSUE_TEMPLATE/feature_request.md

@@ -4,6 +4,7 @@ about: Suggest an idea for this project
 title: ''
 labels: ''
 assignees: ''
+
 ---
 
 **Is your feature request related to a problem? Please describe.**

@vates/async-each/.USAGE.md

@@ -14,7 +14,7 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
 
 `opts` is an object that can contains the following options:
 
-- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `10`. The value `0` means no concurrency limit.
+- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
 - `signal`: an abort signal to stop the iteration
 - `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
 

@vates/async-each/README.md

@@ -32,7 +32,7 @@ Returns a promise wich rejects as soon as a call to `iteratee` throws or a promi
 
 `opts` is an object that can contains the following options:
 
-- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `10`. The value `0` means no concurrency limit.
+- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
 - `signal`: an abort signal to stop the iteration
 - `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
 

@vates/async-each/index.js

@@ -9,16 +9,7 @@ class AggregateError extends Error {
   }
 }
 
-/**
- * @template Item
- * @param {Iterable<Item>} iterable
- * @param {(item: Item, index: number, iterable: Iterable<Item>) => Promise<void>} iteratee
- * @returns {Promise<void>}
- */
-exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 10, signal, stopOnError = true } = {}) {
-  if (concurrency === 0) {
-    concurrency = Infinity
-  }
+exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 1, signal, stopOnError = true } = {}) {
   return new Promise((resolve, reject) => {
     const it = (iterable[Symbol.iterator] || iterable[Symbol.asyncIterator]).call(iterable)
     const errors = []

@vates/async-each/index.spec.js

@@ -36,7 +36,7 @@ describe('asyncEach', () => {
       it('works', async () => {
         const iteratee = jest.fn(async () => {})
 
-        await asyncEach.call(thisArg, iterable, iteratee, { concurrency: 1 })
+        await asyncEach.call(thisArg, iterable, iteratee)
 
         expect(iteratee.mock.instances).toEqual(Array.from(values, () => thisArg))
         expect(iteratee.mock.calls).toEqual(Array.from(values, (value, index) => [value, index, iterable]))
@@ -66,7 +66,7 @@ describe('asyncEach', () => {
           }
         })
 
-        expect(await rejectionOf(asyncEach(iterable, iteratee, { concurrency: 1, stopOnError: true }))).toBe(error)
+        expect(await rejectionOf(asyncEach(iterable, iteratee, { stopOnError: true }))).toBe(error)
         expect(iteratee).toHaveBeenCalledTimes(2)
       })
 
@@ -91,9 +91,7 @@ describe('asyncEach', () => {
           }
         })
 
-        await expect(asyncEach(iterable, iteratee, { concurrency: 1, signal: ac.signal })).rejects.toThrow(
-          'asyncEach aborted'
-        )
+        await expect(asyncEach(iterable, iteratee, { signal: ac.signal })).rejects.toThrow('asyncEach aborted')
         expect(iteratee).toHaveBeenCalledTimes(2)
       })
     })

@vates/async-each/package.json

@@ -24,7 +24,7 @@
     "url": "https://vates.fr"
   },
   "license": "ISC",
-  "version": "1.0.0",
+  "version": "0.1.0",
   "engines": {
     "node": ">=8.10"
   },

@vates/fuse-vhd/.npmignore

@@ -1 +0,0 @@
-../../scripts/npmignore

@vates/fuse-vhd/index.js

@@ -1,71 +0,0 @@
-'use strict'
-
-const LRU = require('lru-cache')
-const Fuse = require('fuse-native')
-const { VhdSynthetic } = require('vhd-lib')
-const { Disposable, fromCallback } = require('promise-toolbox')
-const { createLogger } = require('@xen-orchestra/log')
-
-const { warn } = createLogger('vates:fuse-vhd')
-
-// build a s stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js
-const stat = st => ({
-  mtime: st.mtime || new Date(),
-  atime: st.atime || new Date(),
-  ctime: st.ctime || new Date(),
-  size: st.size !== undefined ? st.size : 0,
-  mode: st.mode === 'dir' ? 16877 : st.mode === 'file' ? 33188 : st.mode === 'link' ? 41453 : st.mode,
-  uid: st.uid !== undefined ? st.uid : process.getuid(),
-  gid: st.gid !== undefined ? st.gid : process.getgid(),
-})
-
-exports.mount = Disposable.factory(async function* mount(handler, diskPath, mountDir) {
-  const vhd = yield VhdSynthetic.fromVhdChain(handler, diskPath)
-
-  const cache = new LRU({
-    max: 16, // each cached block is 2MB in size
-  })
-  await vhd.readBlockAllocationTable()
-  const fuse = new Fuse(mountDir, {
-    async readdir(path, cb) {
-      if (path === '/') {
-        return cb(null, ['vhd0'])
-      }
-      cb(Fuse.ENOENT)
-    },
-    async getattr(path, cb) {
-      if (path === '/') {
-        return cb(
-          null,
-          stat({
-            mode: 'dir',
-            size: 4096,
-          })
-        )
-      }
-      if (path === '/vhd0') {
-        return cb(
-          null,
-          stat({
-            mode: 'file',
-            size: vhd.footer.currentSize,
-          })
-        )
-      }
-
-      cb(Fuse.ENOENT)
-    },
-    read(path, fd, buf, len, pos, cb) {
-      if (path === '/vhd0') {
-        return vhd
-          .readRawData(pos, len, cache, buf)
-          .then(cb)
-      }
-      throw new Error(`read file ${path} not exists`)
-    },
-  })
-  return new Disposable(
-    () => fromCallback(() => fuse.unmount()),
-    fromCallback(() => fuse.mount())
-    )
-})

@vates/fuse-vhd/package.json

@@ -1,30 +0,0 @@
-{
-  "name": "@vates/fuse-vhd",
-  "version": "1.0.0",
-  "license": "ISC",
-  "private": false,
-  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/fuse-vhd",
-  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
-  "repository": {
-    "directory": "@vates/fuse-vhd",
-    "type": "git",
-    "url": "https://github.com/vatesfr/xen-orchestra.git"
-  },
-  "author": {
-    "name": "Vates SAS",
-    "url": "https://vates.fr"
-  },
-  "engines": {
-    "node": ">=10.0"
-  },
-  "dependencies": {
-    "@xen-orchestra/log": "^0.3.0",
-    "fuse-native": "^2.2.6",
-    "lru-cache": "^7.14.0",
-    "promise-toolbox": "^0.21.0",
-    "vhd-lib": "^4.1.0"
-  },
-  "scripts": {
-    "postversion": "npm publish --access public"
-  }
-}

@vates/nbd-client/.npmignore

@@ -1 +0,0 @@
-../../scripts/npmignore

@vates/nbd-client/constants.mjs

@@ -1,41 +0,0 @@
-export const INIT_PASSWD = 'NBDMAGIC' // "NBDMAGIC" ensure we're connected to a nbd server
-export const OPTS_MAGIC = 'IHAVEOPT' // "IHAVEOPT" start an option block
-export const NBD_OPT_REPLY_MAGIC = 0x3e889045565a9 // magic received during negociation
-export const NBD_OPT_EXPORT_NAME = 1
-export const NBD_OPT_ABORT = 2
-export const NBD_OPT_LIST = 3
-export const NBD_OPT_STARTTLS = 5
-export const NBD_OPT_INFO = 6
-export const NBD_OPT_GO = 7
-
-export const NBD_FLAG_HAS_FLAGS = 1 << 0
-export const NBD_FLAG_READ_ONLY = 1 << 1
-export const NBD_FLAG_SEND_FLUSH = 1 << 2
-export const NBD_FLAG_SEND_FUA = 1 << 3
-export const NBD_FLAG_ROTATIONAL = 1 << 4
-export const NBD_FLAG_SEND_TRIM = 1 << 5
-
-export const NBD_FLAG_FIXED_NEWSTYLE = 1 << 0
-
-export const NBD_CMD_FLAG_FUA = 1 << 0
-export const NBD_CMD_FLAG_NO_HOLE = 1 << 1
-export const NBD_CMD_FLAG_DF = 1 << 2
-export const NBD_CMD_FLAG_REQ_ONE = 1 << 3
-export const NBD_CMD_FLAG_FAST_ZERO = 1 << 4
-
-export const NBD_CMD_READ = 0
-export const NBD_CMD_WRITE = 1
-export const NBD_CMD_DISC = 2
-export const NBD_CMD_FLUSH = 3
-export const NBD_CMD_TRIM = 4
-export const NBD_CMD_CACHE = 5
-export const NBD_CMD_WRITE_ZEROES = 6
-export const NBD_CMD_BLOCK_STATUS = 7
-export const NBD_CMD_RESIZE = 8
-
-export const NBD_REQUEST_MAGIC = 0x25609513 // magic number to create a new NBD request to send to the server
-export const NBD_REPLY_MAGIC = 0x67446698 // magic number received from the server when reading response to a nbd request
-
-export const NBD_DEFAULT_PORT = 10809
-export const NBD_DEFAULT_BLOCK_SIZE = 64 * 1024
-export const MAX_BUFFER_LENGTH = 10 * 1024 * 1024

@vates/nbd-client/index.mjs

@@ -1,272 +0,0 @@
-import assert from 'node:assert'
-import { Socket } from 'node:net'
-import { connect } from 'node:tls'
-import {
-  INIT_PASSWD,
-  MAX_BUFFER_LENGTH,
-  NBD_CMD_READ,
-  NBD_DEFAULT_BLOCK_SIZE,
-  NBD_DEFAULT_PORT,
-  NBD_FLAG_FIXED_NEWSTYLE,
-  NBD_FLAG_HAS_FLAGS,
-  NBD_OPT_EXPORT_NAME,
-  NBD_OPT_REPLY_MAGIC,
-  NBD_OPT_STARTTLS,
-  NBD_REPLY_MAGIC,
-  NBD_REQUEST_MAGIC,
-  OPTS_MAGIC,
-} from './constants.mjs'
-
-// documentation is here : https://github.com/NetworkBlockDevice/nbd/blob/master/doc/proto.md
-
-export default class NbdClient {
-  _serverAddress
-  _serverCert
-  _serverPort
-  _serverSocket
-  _useSecureConnection = false
-
-  _exportname
-  _nbDiskBlocks = 0
-
-  _receptionBuffer = Buffer.alloc(0)
-  _sendingBuffer = Buffer.alloc(0)
-
-  // ensure the read are resolved in the right order
-  _rawReadResolve = []
-  _rawReadLength = []
-
-  // AFAIK, there is no guaranty the server answer in the same order as the query
-  _nextCommandQueryId = BigInt(0)
-  _commandQueries = {} // map of queries waiting for an answer
-
-  constructor({ address, port = NBD_DEFAULT_PORT, exportname, cert, secure = true }) {
-    this._address = address
-    this._serverPort = port
-    this._exportname = exportname
-    this._serverCert = cert
-    this._useSecureConnection = secure
-  }
-
-  get nbBlocks() {
-    return this._nbDiskBlocks
-  }
-
-  _handleData(data) {
-    if (data !== undefined) {
-      this._receptionBuffer = Buffer.concat([this._receptionBuffer, Buffer.from(data)])
-    }
-    if (this._receptionBuffer.length > MAX_BUFFER_LENGTH) {
-      throw new Error(
-        `Buffer grown too much with a total size of  ${this._receptionBuffer.length} bytes (last chunk is ${data.length})`
-      )
-    }
-    // if we're waiting for a specific bit length (in the handshake for example or a block data)
-    while (this._rawReadResolve.length > 0 && this._receptionBuffer.length >= this._rawReadLength[0]) {
-      const resolve = this._rawReadResolve.shift()
-      const waitingForLength = this._rawReadLength.shift()
-      resolve(this._takeFromBuffer(waitingForLength))
-    }
-    if (this._rawReadResolve.length === 0 && this._receptionBuffer.length > 4) {
-      if (this._receptionBuffer.readInt32BE(0) === NBD_REPLY_MAGIC) {
-        this._readBlockResponse()
-      }
-      // keep the received bits in the buffer for subsequent use
-    }
-
-  }
-
-  async _addListenners() {
-    const serverSocket = this._serverSocket
-    serverSocket.on('data', data => this._handleData(data))
-
-    serverSocket.on('close', function () {
-      console.log('Connection closed')
-    })
-    serverSocket.on('error', function (err) {
-      throw err
-    })
-  }
-
-  async _tlsConnect() {
-    return new Promise(resolve => {
-      this._serverSocket = connect(
-        {
-          socket: this._serverSocket,
-          rejectUnauthorized: false,
-          cert: this._serverCert,
-        },
-        resolve
-      )
-      this._addListenners()
-    })
-  }
-  async _unsecureConnect() {
-    this._serverSocket = new Socket()
-    this._addListenners()
-    return new Promise((resolve, reject) => {
-      this._serverSocket.connect(this._serverPort, this._serverAddress, () => {
-        resolve()
-      })
-    })
-  }
-  async connect() {
-    await this._unsecureConnect()
-    await this._handshake()
-  }
-
-  async disconnect() {
-    await this._serverSocket.destroy()
-  }
-  async _sendOption(option, buffer = Buffer.alloc(0)) {
-    await this._writeToSocket(OPTS_MAGIC)
-    await this._writeToSocketInt32(option)
-    await this._writeToSocketInt32(buffer.length)
-    await this._writeToSocket(buffer)
-    assert(await this._readFromSocketInt64(), NBD_OPT_REPLY_MAGIC) // magic number everywhere
-    assert(await this._readFromSocketInt32(), option) // the option passed
-    assert(await this._readFromSocketInt32(), 1) // ACK
-    const length = await this._readFromSocketInt32()
-    assert(length === 0) // length
-  }
-
-  async _handshake() {
-    assert(await this._readFromSocket(8), INIT_PASSWD)
-    assert(await this._readFromSocket(8), OPTS_MAGIC)
-    const flagsBuffer = await this._readFromSocket(2)
-    const flags = flagsBuffer.readInt16BE(0)
-    assert(flags | NBD_FLAG_FIXED_NEWSTYLE) // only FIXED_NEWSTYLE one is supported from the server options
-    await this._writeToSocketInt32(NBD_FLAG_FIXED_NEWSTYLE) // client also support  NBD_FLAG_C_FIXED_NEWSTYLE
-
-    if (this._useSecureConnection) {
-      // upgrade socket to TLS
-      await this._sendOption(NBD_OPT_STARTTLS)
-      await this._tlsConnect()
-    }
-
-    // send export name required it also implictly closes the negociation phase
-    await this._writeToSocket(Buffer.from(OPTS_MAGIC))
-    await this._writeToSocketInt32(NBD_OPT_EXPORT_NAME)
-    await this._writeToSocketInt32(this._exportname.length)
-
-    await this._writeToSocket(Buffer.from(this._exportname))
-    // 8 + 2 + 124
-    const answer = await this._readFromSocket(134)
-    const exportSize = answer.readBigUInt64BE(0)
-    const transmissionFlags = answer.readInt16BE(8)
-    assert(transmissionFlags & NBD_FLAG_HAS_FLAGS, 'NBD_FLAG_HAS_FLAGS') // must always be 1 by the norm
-
-    // xapi server always send NBD_FLAG_READ_ONLY (3) as a flag
-
-    this._nbDiskBlocks = Number(exportSize / BigInt(NBD_DEFAULT_BLOCK_SIZE))
-    this._exportSize = exportSize
-  }
-
-  _takeFromBuffer(length) {
-    const res = Buffer.from(this._receptionBuffer.slice(0, length))
-    this._receptionBuffer = this._receptionBuffer.slice(length)
-    return res
-  }
-
-  _readFromSocket(length) {
-    if (this._receptionBuffer.length >= length) {
-      return this._takeFromBuffer(length)
-    }
-    return new Promise(resolve => {
-      this._rawReadResolve.push(resolve)
-      this._rawReadLength.push(length)
-    })
-  }
-
-  _writeToSocket(buffer) {
-    return new Promise(resolve => {
-      this._serverSocket.write(buffer, resolve)
-    })
-  }
-
-  async _readFromSocketInt32() {
-    const buffer = await this._readFromSocket(4)
-
-    return buffer.readInt32BE(0)
-  }
-
-  async _readFromSocketInt64() {
-    const buffer = await this._readFromSocket(8)
-    return buffer.readBigUInt64BE(0)
-  }
-
-  _writeToSocketUInt32(int) {
-    const buffer = Buffer.alloc(4)
-    buffer.writeUInt32BE(int)
-    return this._writeToSocket(buffer)
-  }
-  _writeToSocketInt32(int) {
-    const buffer = Buffer.alloc(4)
-    buffer.writeInt32BE(int)
-    return this._writeToSocket(buffer)
-  }
-
-  _writeToSocketInt16(int) {
-    const buffer = Buffer.alloc(2)
-    buffer.writeInt16BE(int)
-    return this._writeToSocket(buffer)
-  }
-  _writeToSocketInt64(int) {
-    const buffer = Buffer.alloc(8)
-    buffer.writeBigUInt64BE(BigInt(int))
-    return this._writeToSocket(buffer)
-  }
-
-  async _readBlockResponse() {
-    const magic = await this._readFromSocketInt32()
-
-    if (magic !== NBD_REPLY_MAGIC) {
-      throw new Error(`magic number for block answer is wrong : ${magic}`)
-    }
-    // error
-    const error = await this._readFromSocketInt32()
-    if (error !== 0) {
-      throw new Error(`GOT ERROR CODE  : ${error}`)
-    }
-
-    const blockQueryId = await this._readFromSocketInt64()
-    const query = this._commandQueries[blockQueryId]
-    if (!query) {
-      throw new Error(` no query associated with id ${blockQueryId} ${Object.keys(this._commandQueries)}`)
-    }
-    delete this._commandQueries[blockQueryId]
-    const data = await this._readFromSocket(query.size)
-    assert.strictEqual(data.length, query.size)
-    query.resolve(data)
-    this._handleData()
-  }
-
-  async readBlock(index, size = NBD_DEFAULT_BLOCK_SIZE) {
-    const queryId = this._nextCommandQueryId
-    this._nextCommandQueryId++
-
-    const buffer = Buffer.alloc(28)
-    buffer.writeInt32BE(NBD_REQUEST_MAGIC, 0)
-    buffer.writeInt16BE(0, 4) // no command flags for a simple block read
-    buffer.writeInt16BE(NBD_CMD_READ, 6)
-    buffer.writeBigUInt64BE(queryId, 8)
-    // byte offset in the raw disk
-    const offset = BigInt(index) * BigInt(size)
-    buffer.writeBigUInt64BE(offset, 16)
-    // ensure we do not read after the end of the export (which immediatly disconnect us)
-
-    const maxSize = Math.min(Number(this._exportSize - offset), size)
-    // size wanted
-    buffer.writeInt32BE(maxSize, 24)
-
-    return new Promise(resolve => {
-      this._commandQueries[queryId] = {
-        size: maxSize,
-        resolve,
-      }
-
-      // write command at once to ensure no concurrency issue
-      this._writeToSocket(buffer)
-    })
-  }
-}

@vates/nbd-client/nbdclient.spec.mjs

@@ -1,75 +0,0 @@
-import assert from 'assert'
-import NbdClient from './index.mjs'
-import { spawn } from 'node:child_process'
-import fs from 'node:fs/promises'
-import { test } from 'tap'
-import tmp from 'tmp'
-import { pFromCallback } from 'promise-toolbox'
-import { asyncEach } from '@vates/async-each'
-
-const FILE_SIZE = 2 * 1024 * 1024
-
-async function createTempFile(size) {
-  const tmpPath = await pFromCallback(cb => tmp.file(cb))
-  const data = Buffer.alloc(size, 0)
-  for (let i = 0; i < size; i += 4) {
-    data.writeUInt32BE(i, i)
-  }
-  await fs.writeFile(tmpPath, data)
-
-  return tmpPath
-}
-
-test('it works with unsecured network', async tap => {
-  const path = await createTempFile(FILE_SIZE)
-
-  const nbdServer = spawn(
-    'nbdkit',
-    [
-      'file',
-      path,
-      '--newstyle', //
-      '--exit-with-parent',
-      '--read-only',
-      '--export-name=MY_SECRET_EXPORT',
-    ],
-    {
-      stdio: ['inherit', 'inherit', 'inherit'],
-    }
-  )
-
-  const client = new NbdClient({
-    address: 'localhost',
-    exportname: 'MY_SECRET_EXPORT',
-    secure: false,
-  })
-
-  await client.connect()
-  const CHUNK_SIZE = 32 * 1024 // non default size
-  const indexes = []
-  for (let i = 0; i < FILE_SIZE / CHUNK_SIZE; i++) {
-    indexes.push(i)
-  }
-  // read mutiple blocks in parallel
-  await asyncEach(
-    indexes,
-    async i => {
-      const block = await client.readBlock(i, CHUNK_SIZE)
-      let blockOk = true
-      let firstFail
-      for (let j = 0; j < CHUNK_SIZE; j += 4) {
-        const wanted = i * CHUNK_SIZE + j
-        const found = block.readUInt32BE(j)
-        blockOk = blockOk && found === wanted
-        if (!blockOk && firstFail === undefined) {
-          firstFail = j
-        }
-      }
-      tap.ok(blockOk, `check block ${i} content`)
-    },
-    { concurrency: 8 }
-  )
-  await client.disconnect()
-  nbdServer.kill()
-  await fs.unlink(path)
-})

@vates/nbd-client/package.json

@@ -1,30 +0,0 @@
-{
-  "private": true,
-  "name": "@vates/nbd-client",
-  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/nbd-client",
-  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
-  "repository": {
-    "directory": "@vates/nbd-client",
-    "type": "git",
-    "url": "https://github.com/vatesfr/xen-orchestra.git"
-  },
-  "author": {
-    "name": "Vates SAS",
-    "url": "https://vates.fr"
-  },
-  "license": "AGPL-3.0-or-later",
-  "version": "0.0.1",
-  "engines": {
-    "node": ">=14.0"
-  },
-  "dependencies": {
-    "@vates/async-each": "^1.0.0",
-    "@xen-orchestra/async-map": "^0.1.2",
-    "promise-toolbox": "^0.21.0",
-    "xen-api": "^1.2.2"
-  },
-  "devDependencies": {
-    "tap": "^16.3.0",
-    "tmp": "^0.2.1"
-  }
-}

@vates/nbd-client/tests/index.mjs

@@ -1,152 +0,0 @@
-import NbdClient from '../index.js'
-import { Xapi } from 'xen-api'
-import readline from 'node:readline'
-import { stdin as input, stdout as output } from 'node:process'
-import { asyncMap } from '@xen-orchestra/async-map'
-import { downloadVhd, getFullBlocks, getChangedNbdBlocks } from './utils.mjs'
-
-const xapi = new Xapi({
-  auth: {
-    user: 'root',
-    password: 'vateslab',
-  },
-  url: '172.16.210.11',
-  allowUnauthorized: true,
-})
-await xapi.connect()
-
-const networks = await xapi.call('network.get_all_records')
-
-const nbdNetworks = Object.values(networks).filter(
-  network => network.purpose.includes('nbd') || network.purpose.includes('insecure_nbd')
-)
-
-let secure = false
-if (!nbdNetworks.length) {
-  console.log(`you don't have any nbd enabled network`)
-  console.log(`please add a purpose of nbd (to use tls) or insecure_nbd to oneof the host network`)
-  process.exit()
-}
-
-const network = nbdNetworks[0]
-secure = network.purpose.includes('nbd')
-console.log(`we will use network **${network.name_label}** ${secure ? 'with' : 'without'} TLS`)
-
-const rl = readline.createInterface({ input, output })
-const question = text => {
-  return new Promise(resolve => {
-    rl.question(text, resolve)
-  })
-}
-
-let vmuuid, vmRef
-do {
-  vmuuid = await question('VM uuid ? ')
-  try {
-    vmRef = xapi.getObject(vmuuid).$ref
-  } catch (e) {
-    // console.log(e)
-    console.log('maybe the objects was not loaded, try again ')
-    await new Promise(resolve => setTimeout(resolve, 1000))
-  }
-} while (!vmRef)
-
-const vdiRefs = (
-  await asyncMap(await xapi.call('VM.get_VBDs', vmRef), async vbd => {
-    const vdi = await xapi.call('VBD.get_VDI', vbd)
-    return vdi
-  })
-).filter(vdiRef => vdiRef !== 'OpaqueRef:NULL')
-
-const vdiRef = vdiRefs[0]
-
-const vdi = xapi.getObject(vdiRef)
-
-console.log('Will work on vdi  [', vdi.name_label, ']')
-const cbt_enabled = vdi.cbt_enabled
-console.log('Change block tracking is [', cbt_enabled ? 'enabled' : 'disabled', ']')
-
-if (!cbt_enabled) {
-  const shouldEnable = await question('would you like to enable it ? Y/n ')
-  if (shouldEnable === 'Y') {
-    await xapi.call('VDI.enable_cbt', vdiRef)
-    console.log('CBT is now enable for this VDI')
-    console.log('You must make a snapshot, write some data and relaunch this script to backup changes')
-  } else {
-    console.warn('did nothing')
-  }
-  process.exit()
-}
-
-console.log('will search for suitable snapshots')
-const snapshots = vdi.snapshots.map(snapshotRef => xapi.getObject(snapshotRef)).filter(({ cbt_enabled }) => cbt_enabled)
-
-if (snapshots.length < 2) {
-  throw new Error(`not enough snapshots with cbt enabled , found ${snapshots.length} and 2 are needed`)
-}
-
-console.log('found snapshots will compare last two snapshots with cbt_enabled')
-const snapshotRef = xapi.getObject(snapshots[snapshots.length - 1].uuid).$ref
-const snapshotTarget = xapi.getObject(snapshots[snapshots.length - 2].uuid).$ref
-console.log('older snapshot is ', xapi.getObject(snapshotRef).snapshot_time)
-console.log('newer one is ', xapi.getObject(snapshotTarget).snapshot_time)
-
-console.log('## will get bitmap of changed blocks')
-const cbt = Buffer.from(await xapi.call('VDI.list_changed_blocks', snapshotRef, snapshotTarget), 'base64')
-
-console.log('got changes')
-console.log('will connect to NBD server')
-
-const nbd = (await xapi.call('VDI.get_nbd_info', snapshotTarget))[0]
-
-if (!nbd) {
-  console.error('Nbd is not enabled on the host')
-  console.error('you should add `insecure_nbd` as the `purpose` of a network of this host')
-  process.exit()
-}
-
-nbd.secure = true
-// console.log(nbd)
-const client = new NbdClient(nbd)
-await client.connect()
-
-// @todo : should also handle last blocks that could be incomplete
-
-const stats = {}
-
-for (const nbBlocksRead of [32, 16, 8, 4, 2, 1]) {
-  const blockSize = nbBlocksRead * 64 * 1024
-  stats[blockSize] = {}
-  const MASK = 0x80
-  const test = (map, bit) => ((map[bit >> 3] << (bit & 7)) & MASK) !== 0
-
-  const changed = []
-  for (let i = 0; i < (cbt.length * 8) / nbBlocksRead; i++) {
-    let blockChanged = false
-    for (let j = 0; j < nbBlocksRead; j++) {
-      blockChanged = blockChanged || test(cbt, i * nbBlocksRead + j)
-    }
-    if (blockChanged) {
-      changed.push(i)
-    }
-  }
-  console.log(changed.length, 'block changed')
-  for (const concurrency of [32, 16, 8, 4, 2]) {
-    const { speed } = await getChangedNbdBlocks(client, changed, concurrency, blockSize)
-    stats[blockSize][concurrency] = speed
-  }
-}
-console.log('speed summary')
-console.table(stats)
-
-console.log('## will check full download of the base vdi ')
-
-await getFullBlocks(client, 16, 512 * 1024) // a good sweet spot
-
-console.log('## will check vhd delta export  size and speed')
-
-console.log('## will check full vhd export  size and speed')
-await downloadVhd(xapi, {
-  format: 'vhd',
-  vdi: snapshotTarget,
-})

@vates/nbd-client/tests/test-coherence.mjs

@@ -1,97 +0,0 @@
-import NbdClient from '../index.js'
-import { Xapi } from 'xen-api'
-import { asyncMap } from '@xen-orchestra/async-map'
-import { downloadVhd, getFullBlocks } from './utils.mjs'
-import fs from 'fs/promises'
-
-const xapi = new Xapi({
-  auth: {
-    user: 'root',
-    password: 'vateslab',
-  },
-  url: '172.16.210.11',
-  allowUnauthorized: true,
-})
-await xapi.connect()
-
-const vmuuid = '123e4f2b-498e-d0af-15ae-f835a1e9f59f'
-let vmRef
-do {
-  try {
-    vmRef = xapi.getObject(vmuuid).$ref
-  } catch (e) {
-    console.log('maybe the objects was not loaded, try again ')
-    await new Promise(resolve => setTimeout(resolve, 1000))
-  }
-} while (!vmRef)
-
-const vdiRefs = (
-  await asyncMap(await xapi.call('VM.get_VBDs', vmRef), async vbd => {
-    const vdi = await xapi.call('VBD.get_VDI', vbd)
-    return vdi
-  })
-).filter(vdiRef => vdiRef !== 'OpaqueRef:NULL')
-
-const vdiRef = vdiRefs[0]
-
-const vdi = xapi.getObject(vdiRef)
-
-console.log('Will work on vdi  [', vdi.name_label, ']')
-
-console.log('will search for suitable snapshots')
-const snapshots = vdi.snapshots.map(snapshotRef => xapi.getObject(snapshotRef))
-
-console.log('found snapshots will use the last one for tests')
-const snapshotRef = xapi.getObject(snapshots[snapshots.length - 1].uuid).$ref
-
-console.log('will connect to NBD server')
-
-const nbd = (await xapi.call('VDI.get_nbd_info', snapshotRef))[0]
-
-if (!nbd) {
-  console.error('Nbd is not enabled on the host')
-  console.error('you should add `insecure_nbd` as the `purpose` of a network of this host')
-  process.exit()
-}
-
-if (!nbd) {
-  console.error('Nbd is not enabled on the host')
-  console.error('you should add `insecure_nbd` as the `purpose` of a network of this host')
-  process.exit()
-}
-
-const nbdClient = new NbdClient(nbd)
-await nbdClient.connect()
-let fd = await fs.open('/tmp/nbd.raw', 'w')
-await getFullBlocks({
-  nbdClient,
-  concurrency: 8,
-  nbBlocksRead: 16 /* 1MB block */,
-  fd,
-})
-console.log(' done nbd ')
-await fd.close()
-
-fd = await fs.open('/tmp/export.raw', 'w')
-await downloadVhd({
-  xapi,
-  query: {
-    format: 'raw',
-    vdi: snapshotRef,
-  },
-  fd,
-})
-
-fd.close()
-
-fd = await fs.open('/tmp/export.vhd', 'w')
-await downloadVhd({
-  xapi,
-  query: {
-    format: 'vhd',
-    vdi: snapshotRef,
-  },
-  fd,
-})
-
-fd.close()

@vates/nbd-client/tests/test-transfer-speed.mjs

@@ -1,117 +0,0 @@
-import NbdClient from '../index.js'
-import { Xapi } from 'xen-api'
-import readline from 'node:readline'
-import { stdin as input, stdout as output } from 'node:process'
-import { asyncMap } from '@xen-orchestra/async-map'
-import { downloadVhd, getFullBlocks } from './utils.mjs'
-
-const xapi = new Xapi({
-  auth: {
-    user: 'root',
-    password: 'vateslab',
-  },
-  url: '172.16.210.11',
-  allowUnauthorized: true,
-})
-await xapi.connect()
-
-const networks = await xapi.call('network.get_all_records')
-console.log({ networks })
-const nbdNetworks = Object.values(networks).filter(
-  network => network.purpose.includes('nbd') || network.purpose.includes('insecure_nbd')
-)
-
-let secure = false
-if (!nbdNetworks.length) {
-  console.log(`you don't have any nbd enabled network`)
-  console.log(`please add a purpose of nbd (to use tls) or insecure_nbd to oneof the host network`)
-  process.exit()
-}
-
-const network = nbdNetworks[0]
-secure = network.purpose.includes('nbd')
-console.log(`we will use network **${network.name_label}** ${secure ? 'with' : 'without'} TLS`)
-
-const rl = readline.createInterface({ input, output })
-const question = text => {
-  return new Promise(resolve => {
-    rl.question(text, resolve)
-  })
-}
-
-let vmuuid, vmRef
-do {
-  vmuuid = '123e4f2b-498e-d0af-15ae-f835a1e9f59f' // await question('VM uuid ? ')
-  try {
-    vmRef = xapi.getObject(vmuuid).$ref
-  } catch (e) {
-    console.log(e)
-    console.log('maybe the objects was not loaded, try again ')
-  }
-} while (!vmRef)
-
-const vdiRefs = (
-  await asyncMap(await xapi.call('VM.get_VBDs', vmRef), async vbd => {
-    const vdi = await xapi.call('VBD.get_VDI', vbd)
-    return vdi
-  })
-).filter(vdiRef => vdiRef !== 'OpaqueRef:NULL')
-
-const vdiRef = vdiRefs[0]
-
-const vdi = xapi.getObject(vdiRef)
-
-console.log('Will work on vdi  [', vdi.name_label, ']')
-
-console.log('will search for suitable snapshots')
-const snapshots = vdi.snapshots.map(snapshotRef => xapi.getObject(snapshotRef))
-
-console.log('found snapshots will use the last one for tests')
-const snapshotRef = xapi.getObject(snapshots[snapshots.length - 1].uuid).$ref
-
-console.log('will connect to NBD server')
-
-const nbd = (await xapi.call('VDI.get_nbd_info', snapshotRef))[0]
-
-if (!nbd) {
-  console.error('Nbd is not enabled on the host')
-  console.error('you should add `insecure_nbd` as the `purpose` of a network of this host')
-  process.exit()
-}
-
-nbd.secure = secure
-const nbdClient = new NbdClient(nbd)
-await nbdClient.connect()
-
-const maxDuration =
-  parseInt(await question('Maximum duration per test in second ? (-1 for unlimited, default 30) '), 10) || 30
-console.log('Will start downloading blocks during ', maxDuration, 'seconds')
-
-console.log('## will check the vhd download speed')
-
-const stats = {}
-
-for (const nbBlocksRead of [32, 16, 8, 4, 2, 1]) {
-  stats[nbBlocksRead * 64 * 1024] = {}
-  for (const concurrency of [32, 16, 8, 4, 2]) {
-    const { speed } = await getFullBlocks({ nbdClient, concurrency, nbBlocksRead })
-
-    stats[concurrency] = speed
-  }
-}
-
-console.log('speed summary')
-console.table(stats)
-
-console.log('## will check full vhd export  size and speed')
-await downloadVhd(xapi, {
-  format: 'vhd',
-  vdi: snapshotRef,
-})
-
-console.log('## will check full raw export  size and speed')
-await downloadVhd(xapi, {
-  format: 'raw',
-  vdi: snapshotRef,
-})
-process.exit()

@vates/nbd-client/tests/utils.mjs

@@ -1,116 +0,0 @@
-import { asyncEach } from '@vates/async-each'
-import { CancelToken } from 'promise-toolbox'
-import zlib from 'node:zlib'
-
-export async function getChangedNbdBlocks(nbdClient, changed, concurrency, blockSize) {
-  let nbModified = 0
-  let size = 0
-  let compressedSize = 0
-  const start = new Date()
-  console.log('### with concurrency ', concurrency, ' blockSize ', blockSize / 1024 / 1024, 'MB')
-  const interval = setInterval(() => {
-    console.log(`${nbModified} block handled in ${new Date() - start} ms`)
-  }, 5000)
-  await asyncEach(
-    changed,
-    async blockIndex => {
-      if (new Date() - start > 30000) {
-        return
-      }
-      const data = await nbdClient.readBlock(blockIndex, blockSize)
-
-      await new Promise(resolve => {
-        zlib.gzip(data, { level: zlib.constants.Z_BEST_SPEED }, (_, compressed) => {
-          compressedSize += compressed.length
-          resolve()
-        })
-      })
-      size += data?.length ?? 0
-      nbModified++
-    },
-    {
-      concurrency,
-    }
-  )
-  clearInterval(interval)
-  console.log('duration :', new Date() - start)
-  console.log('read : ', size, 'octets, compressed: ', compressedSize, 'ratio ', size / compressedSize)
-  console.log('speed : ', Math.round(((size / 1024 / 1024) * 1000) / (new Date() - start)), 'MB/s')
-  return { speed: Math.round(((size / 1024 / 1024) * 1000) / (new Date() - start)) }
-}
-
-export async function getFullBlocks({ nbdClient, concurrency = 1, nbBlocksRead = 1, fd, maxDuration = -1 } = {}) {
-  const blockSize = nbBlocksRead * 64 * 1024
-  let nbModified = 0
-  let size = 0
-  console.log('### with concurrency ', concurrency)
-  const start = new Date()
-  console.log(' max nb blocks ', nbdClient.nbBlocks / nbBlocksRead)
-  function* blockIterator() {
-    for (let i = 0; i < nbdClient.nbBlocks / nbBlocksRead; i++) {
-      yield i
-    }
-  }
-  const interval = setInterval(() => {
-    console.log(`${nbModified} block handled in ${new Date() - start} ms`)
-  }, 5000)
-  await asyncEach(
-    blockIterator(),
-    async blockIndex => {
-      if (maxDuration > 0 && new Date() - start > maxDuration * 1000) {
-        return
-      }
-      const data = await nbdClient.readBlock(blockIndex, blockSize)
-      size += data?.length ?? 0
-      nbModified++
-      if (fd) {
-        await fd.write(data, 0, data.length, blockIndex * blockSize)
-      }
-    },
-    {
-      concurrency,
-    }
-  )
-  clearInterval(interval)
-  if (new Date() - start < 10000) {
-    console.warn(
-      `data set too small or performance to high, result won't be usefull. Please relaunch with bigger snapshot or higher maximum data size `
-    )
-  }
-  console.log('duration :', new Date() - start)
-  console.log('nb blocks : ', nbModified)
-  console.log('read : ', size, 'octets')
-  const speed = Math.round(((size / 1024 / 1024) * 1000 * 100) / (new Date() - start)) / 100
-  console.log('speed : ', speed, 'MB/s')
-  return { speed }
-}
-
-export async function downloadVhd({ xapi, query, fd, maxDuration = -1 } = {}) {
-  const startStream = new Date()
-  let sizeStream = 0
-  let nbChunk = 0
-
-  const interval = setInterval(() => {
-    console.log(`${nbChunk} chunks , ${sizeStream} octets handled in ${new Date() - startStream} ms`)
-  }, 5000)
-  const stream = await xapi.getResource(CancelToken.none, '/export_raw_vdi/', {
-    query,
-  })
-  for await (const chunk of stream) {
-    sizeStream += chunk.length
-
-    if (fd) {
-      await fd.write(chunk)
-    }
-    nbChunk++
-
-    if (maxDuration > 0 && new Date() - startStream > maxDuration * 1000) {
-      break
-    }
-  }
-  clearInterval(interval)
-  console.log('Stream duration :', new Date() - startStream)
-  console.log('Stream read : ', sizeStream, 'octets')
-  const speed = Math.round(((sizeStream / 1024 / 1024) * 1000 * 100) / (new Date() - startStream)) / 100
-  console.log('speed : ', speed, 'MB/s')
-}

@xen-orchestra/backups-cli/commands/clean-vms.js

@@ -26,13 +26,7 @@ module.exports = async function main(args) {
   await asyncMap(_, async vmDir => {
     vmDir = resolve(vmDir)
     try {
-      await adapter.cleanVm(vmDir, {
-        fixMetadata: fix,
-        remove,
-        merge,
-        logInfo: (...args) => console.log(...args),
-        logWarn: (...args) => console.warn(...args),
-      })
+      await adapter.cleanVm(vmDir, { fixMetadata: fix, remove, merge, onLog: (...args) => console.warn(...args) })
     } catch (error) {
       console.error('adapter.cleanVm', vmDir, error)
     }

@xen-orchestra/backups-cli/package.json

@@ -7,8 +7,8 @@
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.28.0",
-    "@xen-orchestra/fs": "^3.1.0",
+    "@xen-orchestra/backups": "^0.26.0",
+    "@xen-orchestra/fs": "^1.1.0",
     "filenamify": "^4.1.0",
     "getopts": "^2.2.5",
     "lodash": "^4.17.15",
@@ -27,7 +27,7 @@
   "scripts": {
     "postversion": "npm publish --access public"
   },
-  "version": "0.7.8",
+  "version": "0.7.4",
   "license": "AGPL-3.0-or-later",
   "author": {
     "name": "Vates SAS",

@xen-orchestra/backups/Backup.js

@@ -245,7 +245,7 @@ exports.Backup = class Backup {
           })
         )
       ),
-      () => (settings.healthCheckSr !== undefined ? this._getRecord('SR', settings.healthCheckSr) : undefined),
+      () => settings.healthCheckSr !== undefined ? this._getRecord('SR', settings.healthCheckSr) : undefined,
       async (srs, remoteAdapters, healthCheckSr) => {
         // remove adapters that failed (already handled)
         remoteAdapters = remoteAdapters.filter(_ => _ !== undefined)

@xen-orchestra/backups/RemoteAdapter.js

@@ -22,15 +22,11 @@ const zlib = require('zlib')
 
 const { BACKUP_DIR } = require('./_getVmBackupDir.js')
 const { cleanVm } = require('./_cleanVm.js')
-const { formatFilenameDate } = require('./_filenameDate.js')
 const { getTmpDir } = require('./_getTmpDir.js')
 const { isMetadataFile } = require('./_backupType.js')
 const { isValidXva } = require('./_isValidXva.js')
 const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js')
 const { lvs, pvs } = require('./_lvm.js')
-// @todo : this import is marked extraneous , sould be fixed when lib is published
-const { mount } = require('@vates/fuse-vhd')
-const { asyncEach } = require('@vates/async-each')
 
 const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
 exports.DIR_XO_CONFIG_BACKUPS = DIR_XO_CONFIG_BACKUPS
@@ -38,7 +34,7 @@ exports.DIR_XO_CONFIG_BACKUPS = DIR_XO_CONFIG_BACKUPS
 const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
 exports.DIR_XO_POOL_METADATA_BACKUPS = DIR_XO_POOL_METADATA_BACKUPS
 
-const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
+const { warn } = createLogger('xo:backups:RemoteAdapter')
 
 const compareTimestamp = (a, b) => a.timestamp - b.timestamp
 
@@ -48,6 +44,8 @@ const resolveRelativeFromFile = (file, path) => resolve('/', dirname(file), path
 
 const resolveSubpath = (root, path) => resolve(root, `.${resolve('/', path)}`)
 
+const RE_VHDI = /^vhdi(\d+)$/
+
 async function addDirectory(files, realPath, metadataPath) {
   const stats = await lstat(realPath)
   if (stats.isDirectory()) {
@@ -76,14 +74,12 @@ const debounceResourceFactory = factory =>
   }
 
 class RemoteAdapter {
-  constructor(handler, { debounceResource = res => res, dirMode, vhdDirectoryCompression, useGetDiskLegacy=false } = {}) {
+  constructor(handler, { debounceResource = res => res, dirMode, vhdDirectoryCompression } = {}) {
     this._debounceResource = debounceResource
     this._dirMode = dirMode
     this._handler = handler
     this._vhdDirectoryCompression = vhdDirectoryCompression
     this._readCacheListVmBackups = synchronized.withKey()(this._readCacheListVmBackups)
-    this._useGetDiskLegacy = useGetDiskLegacy
-
   }
 
   get handler() {
@@ -131,9 +127,7 @@ class RemoteAdapter {
   }
 
   async *_getPartition(devicePath, partition) {
-    // the norecovery option is necessary because if the partition is dirty,
-    // mount will try to fix it which is impossible if because the device is read-only
-    const options = ['loop', 'ro', 'norecovery']
+    const options = ['loop', 'ro']
 
     if (partition !== undefined) {
       const { size, start } = partition
@@ -230,30 +224,11 @@ class RemoteAdapter {
     return promise
   }
 
-  #removeVmBackupsFromCache(backups) {
-    for (const [dir, filenames] of Object.entries(
-      groupBy(
-        backups.map(_ => _._filename),
-        dirname
-      )
-    )) {
-      // detached async action, will not reject
-      this._updateCache(dir + '/cache.json.gz', backups => {
-        for (const filename of filenames) {
-          debug('removing cache entry', { entry: filename })
-          delete backups[filename]
-        }
-      })
-    }
-  }
-
   async deleteDeltaVmBackups(backups) {
     const handler = this._handler
 
     // this will delete the json, unused VHDs will be detected by `cleanVm`
     await asyncMapSettled(backups, ({ _filename }) => handler.unlink(_filename))
-
-    this.#removeVmBackupsFromCache(backups)
   }
 
   async deleteMetadataBackup(backupId) {
@@ -281,8 +256,6 @@ class RemoteAdapter {
     await asyncMapSettled(backups, ({ _filename, xva }) =>
       Promise.all([handler.unlink(_filename), handler.unlink(resolveRelativeFromFile(_filename, xva))])
     )
-
-    this.#removeVmBackupsFromCache(backups)
   }
 
   deleteVmBackup(file) {
@@ -303,13 +276,14 @@ class RemoteAdapter {
       full !== undefined && this.deleteFullVmBackups(full),
     ])
 
-    await asyncMap(new Set(files.map(file => dirname(file))), dir =>
-      // - don't merge in main process, unused VHDs will be merged in the next backup run
-      // - don't error in case this fails:
-      //   - if lock is already being held, a backup is running and cleanVm will be ran at the end
-      //   - otherwise, there is nothing more we can do, orphan file will be cleaned in the future
-      this.cleanVm(dir, { remove: true, logWarn: warn }).catch(noop)
-    )
+    const dirs = new Set(files.map(file => dirname(file)))
+    for (const dir of dirs) {
+      // don't merge in main process, unused VHDs will be merged in the next backup run
+      await this.cleanVm(dir, { remove: true, onLog: warn })
+    }
+
+    const dedupedVmUuid = new Set(metadatas.map(_ => _.vm.uuid))
+    await asyncMap(dedupedVmUuid, vmUuid => this.invalidateVmBackupListCache(vmUuid))
   }
 
   #getCompressionType() {
@@ -324,10 +298,7 @@ class RemoteAdapter {
     return this.#useVhdDirectory()
   }
 
-
-  async *#getDiskLegacy(diskId) {
-
-    const RE_VHDI = /^vhdi(\d+)$/
+  async *getDisk(diskId) {
     const handler = this._handler
 
     const diskPath = handler._getFilePath('/' + diskId)
@@ -357,20 +328,6 @@ class RemoteAdapter {
     }
   }
 
-  async *getDisk(diskId) {
-    if(this._useGetDiskLegacy){
-      yield * this.#getDiskLegacy(diskId)
-      return
-    }
-    const handler = this._handler
-    // this is a disposable
-    const mountDir = yield getTmpDir()
-    // this is also a disposable
-    yield mount(handler, diskId, mountDir)
-    // this will yield disk path to caller
-    yield `${mountDir}/vhd0`
-  }
-
   // partitionId values:
   //
   // - undefined: raw disk
@@ -421,25 +378,22 @@ class RemoteAdapter {
   listPartitionFiles(diskId, partitionId, path) {
     return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => {
       path = resolveSubpath(rootPath, path)
+
       const entriesMap = {}
-      await asyncEach(
-        await readdir(path),
-        async name => {
-          try {
-            const stats = await lstat(`${path}/${name}`)
-            if (stats.isDirectory()) {
-              entriesMap[name + '/'] = {}
-            } else if (stats.isFile()) {
-              entriesMap[name] = {}
-            }
-          } catch (error) {
-            if (error == null || error.code !== 'ENOENT') {
-              throw error
-            }
+      await asyncMap(await readdir(path), async name => {
+        try {
+          const stats = await lstat(`${path}/${name}`)
+          if (stats.isDirectory()) {
+            entriesMap[name + '/'] = {}
+          } else if (stats.isFile()) {
+            entriesMap[name] = {}
           }
-        },
-        { concurrency: 1 }
-      )
+        } catch (error) {
+          if (error == null || error.code !== 'ENOENT') {
+            throw error
+          }
+        }
+      })
 
       return entriesMap
     })
@@ -504,46 +458,11 @@ class RemoteAdapter {
     return backupsByPool
   }
 
-  #getVmBackupsCache(vmUuid) {
-    return `${BACKUP_DIR}/${vmUuid}/cache.json.gz`
-  }
-
-  async #readCache(path) {
-    try {
-      return JSON.parse(await fromCallback(zlib.gunzip, await this.handler.readFile(path)))
-    } catch (error) {
-      if (error.code !== 'ENOENT') {
-        warn('#readCache', { error, path })
-      }
-    }
-  }
-
-  _updateCache = synchronized.withKey()(this._updateCache)
-  // eslint-disable-next-line no-dupe-class-members
-  async _updateCache(path, fn) {
-    const cache = await this.#readCache(path)
-    if (cache !== undefined) {
-      fn(cache)
-
-      await this.#writeCache(path, cache)
-    }
-  }
-
-  async #writeCache(path, data) {
-    try {
-      await this.handler.writeFile(path, await fromCallback(zlib.gzip, JSON.stringify(data)), { flags: 'w' })
-    } catch (error) {
-      warn('#writeCache', { error, path })
-    }
-  }
-
   async invalidateVmBackupListCache(vmUuid) {
-    await this.handler.unlink(this.#getVmBackupsCache(vmUuid))
+    await this.handler.unlink(`${BACKUP_DIR}/${vmUuid}/cache.json.gz`)
   }
 
   async #getCachabledDataListVmBackups(dir) {
-    debug('generating cache', { path: dir })
-
     const handler = this._handler
     const backups = {}
 
@@ -579,26 +498,41 @@ class RemoteAdapter {
   // if cache is missing  or broken  => regenerate it and return
 
   async _readCacheListVmBackups(vmUuid) {
-    const path = this.#getVmBackupsCache(vmUuid)
+    const dir = `${BACKUP_DIR}/${vmUuid}`
+    const path = `${dir}/cache.json.gz`
 
-    const cache = await this.#readCache(path)
-    if (cache !== undefined) {
-      debug('found VM backups cache, using it', { path })
-      return cache
+    try {
+      const gzipped = await this.handler.readFile(path)
+      const text = await fromCallback(zlib.gunzip, gzipped)
+      return JSON.parse(text)
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        warn('Cache file was unreadable', { vmUuid, error })
+      }
     }
 
     // nothing cached, or cache unreadable => regenerate it
-    const backups = await this.#getCachabledDataListVmBackups(`${BACKUP_DIR}/${vmUuid}`)
+    const backups = await this.#getCachabledDataListVmBackups(dir)
     if (backups === undefined) {
       return
     }
 
     // detached async action, will not reject
-    this.#writeCache(path, backups)
+    this.#writeVmBackupsCache(path, backups)
 
     return backups
   }
 
+  async #writeVmBackupsCache(cacheFile, backups) {
+    try {
+      const text = JSON.stringify(backups)
+      const zipped = await fromCallback(zlib.gzip, text)
+      await this.handler.writeFile(cacheFile, zipped, { flags: 'w' })
+    } catch (error) {
+      warn('writeVmBackupsCache', { cacheFile, error })
+    }
+  }
+
   async listVmBackups(vmUuid, predicate) {
     const backups = []
     const cached = await this._readCacheListVmBackups(vmUuid)
@@ -637,41 +571,18 @@ class RemoteAdapter {
     return backups.sort(compareTimestamp)
   }
 
-  async writeVmBackupMetadata(vmUuid, metadata) {
-    const path = `/${BACKUP_DIR}/${vmUuid}/${formatFilenameDate(metadata.timestamp)}.json`
-
-    await this.handler.outputFile(path, JSON.stringify(metadata), {
-      dirMode: this._dirMode,
-    })
-
-    // will not throw
-    this._updateCache(this.#getVmBackupsCache(vmUuid), backups => {
-      debug('adding cache entry', { entry: path })
-      backups[path] = {
-        ...metadata,
-
-        // these values are required in the cache
-        _filename: path,
-        id: path,
-      }
-    })
-
-    return path
-  }
-
-  async writeVhd(path, input, { checksum = true, validator = noop, writeBlockConcurrency, nbdClient } = {}) {
+  async writeVhd(path, input, { checksum = true, validator = noop } = {}) {
     const handler = this._handler
 
     if (this.#useVhdDirectory()) {
       const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
       await createVhdDirectoryFromStream(handler, dataPath, input, {
-        concurrency: writeBlockConcurrency,
+        concurrency: 16,
         compression: this.#getCompressionType(),
         async validator() {
           await input.task
           return validator.apply(this, arguments)
         },
-        nbdClient,
       })
       await VhdAbstract.createAlias(handler, path, dataPath)
     } else {

@xen-orchestra/backups/Task.js

@@ -3,10 +3,8 @@
 const CancelToken = require('promise-toolbox/CancelToken')
 const Zone = require('node-zone')
 
-const logAfterEnd = log => {
-  const error = new Error('task has already ended')
-  error.log = log
-  throw error
+const logAfterEnd = () => {
+  throw new Error('task has already ended')
 }
 
 const noop = Function.prototype

@xen-orchestra/backups/_VmBackup.js

@@ -128,49 +128,42 @@ class VmBackup {
   }
 
   // calls fn for each function, warns of any errors, and throws only if there are no writers left
-  async _callWriters(fn, step, parallel = true) {
+  async _callWriters(fn, warnMessage, parallel = true) {
     const writers = this._writers
     const n = writers.size
     if (n === 0) {
       return
     }
-
-    async function callWriter(writer) {
-      const { name } = writer.constructor
-      try {
-        debug('writer step starting', { step, writer: name })
-        await fn(writer)
-        debug('writer step succeeded', { duration: step, writer: name })
-      } catch (error) {
-        writers.delete(writer)
-
-        warn('writer step failed', { error, step, writer: name })
-
-        // these two steps are the only one that are not already in their own sub tasks
-        if (step === 'writer.checkBaseVdis()' || step === 'writer.beforeBackup()') {
-          Task.warning(
-            `the writer ${name} has failed the step ${step} with error ${error.message}. It won't be used anymore in this job execution.`
-          )
-        }
-
-        throw error
-      }
-    }
     if (n === 1) {
       const [writer] = writers
-      return callWriter(writer)
+      try {
+        await fn(writer)
+      } catch (error) {
+        writers.delete(writer)
+        throw error
+      }
+      return
     }
 
     const errors = []
     await (parallel ? asyncMap : asyncEach)(writers, async function (writer) {
       try {
-        await callWriter(writer)
+        await fn(writer)
       } catch (error) {
         errors.push(error)
+        this.delete(writer)
+        warn(warnMessage, { error, writer: writer.constructor.name })
+
+        // these two steps are the only one that are not already in their own sub tasks
+        if (warnMessage === 'writer.checkBaseVdis()' || warnMessage === 'writer.beforeBackup()') {
+          Task.warning(
+            `the writer ${writer.constructor.name} has failed the step ${warnMessage} with error ${error.message}. It won't be used anymore in this job execution.`
+          )
+        }
       }
     })
     if (writers.size === 0) {
-      throw new AggregateError(errors, 'all targets have failed, step: ' + step)
+      throw new AggregateError(errors, 'all targets have failed, step: ' + warnMessage)
     }
   }
 

@xen-orchestra/backups/_cleanVm.integ.spec.js

@@ -35,7 +35,7 @@ afterEach(async () => {
 })
 
 const uniqueId = () => uuid.v1()
-const uniqueIdBuffer = () => uuid.v1({}, Buffer.alloc(16))
+const uniqueIdBuffer = () => Buffer.from(uniqueId(), 'utf-8')
 
 async function generateVhd(path, opts = {}) {
   let vhd
@@ -78,15 +78,15 @@ test('It remove broken vhd', async () => {
   await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
   expect((await handler.list(basePath)).length).toEqual(1)
   let loggued = ''
-  const logInfo = message => {
+  const onLog = message => {
     loggued += message
   }
-  await adapter.cleanVm('/', { remove: false, logInfo, logWarn: logInfo, lock: false })
-  expect(loggued).toEqual(`VHD check error`)
+  await adapter.cleanVm('/', { remove: false, onLog })
+  expect(loggued).toEqual(`error while checking the VHD with path /${basePath}/notReallyAVhd.vhd`)
   // not removed
   expect((await handler.list(basePath)).length).toEqual(1)
   // really remove it
-  await adapter.cleanVm('/', { remove: true, logInfo, logWarn: () => {}, lock: false })
+  await adapter.cleanVm('/', { remove: true, onLog })
   expect((await handler.list(basePath)).length).toEqual(0)
 })
 
@@ -118,13 +118,15 @@ test('it remove vhd with missing or multiple ancestors', async () => {
   )
   // clean
   let loggued = ''
-  const logInfo = message => {
+  const onLog = message => {
     loggued += message + '\n'
   }
-  await adapter.cleanVm('/', { remove: true, logInfo, logWarn: logInfo, lock: false })
+  await adapter.cleanVm('/', { remove: true, onLog })
 
   const deletedOrphanVhd = loggued.match(/deleting orphan VHD/g) || []
   expect(deletedOrphanVhd.length).toEqual(1) // only one vhd should have been deleted
+  const deletedAbandonnedVhd = loggued.match(/abandonned.vhd is missing/g) || []
+  expect(deletedAbandonnedVhd.length).toEqual(1) // and it must be abandonned.vhd
 
   // we don't test the filew on disk, since they will all be marker as unused and deleted without a metadata.json file
 })
@@ -157,12 +159,14 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy
   })
 
   let loggued = ''
-  const logInfo = message => {
+  const onLog = message => {
     loggued += message + '\n'
   }
-  await adapter.cleanVm('/', { remove: true, logInfo, logWarn: logInfo, lock: false })
-  let matched = loggued.match(/deleting unused VHD/g) || []
+  await adapter.cleanVm('/', { remove: true, onLog })
+  let matched = loggued.match(/deleting unused VHD /g) || []
   expect(matched.length).toEqual(1) // only one vhd should have been deleted
+  matched = loggued.match(/abandonned.vhd is unused/g) || []
+  expect(matched.length).toEqual(1) // and it must be abandonned.vhd
 
   // a missing vhd cause clean to remove all vhds
   await handler.writeFile(
@@ -179,8 +183,8 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy
     { flags: 'w' }
   )
   loggued = ''
-  await adapter.cleanVm('/', { remove: true, logInfo, logWarn: () => {}, lock: false })
-  matched = loggued.match(/deleting unused VHD/g) || []
+  await adapter.cleanVm('/', { remove: true, onLog })
+  matched = loggued.match(/deleting unused VHD /g) || []
   expect(matched.length).toEqual(2) // all vhds (orphan and  child  ) should have been deleted
 })
 
@@ -216,16 +220,16 @@ test('it merges delta of non destroyed chain', async () => {
   })
 
   let loggued = []
-  const logInfo = message => {
+  const onLog = message => {
     loggued.push(message)
   }
-  await adapter.cleanVm('/', { remove: true, logInfo, logWarn: logInfo, lock: false })
-  expect(loggued[0]).toEqual(`incorrect backup size in metadata`)
+  await adapter.cleanVm('/', { remove: true, onLog })
+  expect(loggued[0]).toEqual(`incorrect size in metadata: 12000 instead of 209920`)
 
   loggued = []
-  await adapter.cleanVm('/', { remove: true, merge: true, logInfo, logWarn: () => {}, lock: false })
+  await adapter.cleanVm('/', { remove: true, merge: true, onLog })
   const [merging] = loggued
-  expect(merging).toEqual(`merging VHD chain`)
+  expect(merging).toEqual(`merging 1 children into /${basePath}/orphan.vhd`)
 
   const metadata = JSON.parse(await handler.readFile(`metadata.json`))
   // size should be the size of children + grand children after the merge
@@ -271,7 +275,7 @@ test('it finish unterminated merge ', async () => {
     })
   )
 
-  await adapter.cleanVm('/', { remove: true, merge: true, logWarn: () => {}, lock: false })
+  await adapter.cleanVm('/', { remove: true, merge: true })
   // merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
 
   // only check deletion
@@ -378,7 +382,7 @@ describe('tests multiple combination ', () => {
           })
         )
 
-        await adapter.cleanVm('/', { remove: true, merge: true, logWarn: () => {}, lock: false })
+        await adapter.cleanVm('/', { remove: true, merge: true })
 
         const metadata = JSON.parse(await handler.readFile(`metadata.json`))
         // size should be the size of children + grand children + clean after the merge
@@ -414,7 +418,7 @@ describe('tests multiple combination ', () => {
 test('it cleans orphan merge states ', async () => {
   await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
 
-  await adapter.cleanVm('/', { remove: true, logWarn: () => {}, lock: false })
+  await adapter.cleanVm('/', { remove: true })
 
   expect(await handler.list(basePath)).toEqual([])
 })
@@ -429,11 +433,7 @@ test('check Aliases should work alone', async () => {
 
   await generateVhd(`vhds/data/missingalias.vhd`)
 
-  await checkAliases(['vhds/missingData.alias.vhd', 'vhds/ok.alias.vhd'], 'vhds/data', {
-    remove: true,
-    handler,
-    logWarn: () => {},
-  })
+  await checkAliases(['vhds/missingData.alias.vhd', 'vhds/ok.alias.vhd'], 'vhds/data', { remove: true, handler })
 
   // only ok have suvived
   const alias = (await handler.list('vhds')).filter(f => f.endsWith('.vhd'))

@xen-orchestra/backups/_cleanVm.js

@@ -1,27 +1,22 @@
 'use strict'
 
+const assert = require('assert')
 const sum = require('lodash/sum')
-const UUID = require('uuid')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { Constants, openVhd, VhdAbstract, VhdFile } = require('vhd-lib')
+const { Constants, mergeVhd, openVhd, VhdAbstract, VhdFile } = require('vhd-lib')
 const { isVhdAlias, resolveVhdAlias } = require('vhd-lib/aliases')
 const { dirname, resolve } = require('path')
 const { DISK_TYPES } = Constants
 const { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } = require('./_backupType.js')
 const { limitConcurrency } = require('limit-concurrency-decorator')
-const { mergeVhdChain } = require('vhd-lib/merge')
 
 const { Task } = require('./Task.js')
 const { Disposable } = require('promise-toolbox')
-const handlerPath = require('@xen-orchestra/fs/path')
 
 // checking the size of a vhd directory is costly
 // 1 Http Query per 1000 blocks
 // we only check size of all the vhd are VhdFiles
-function shouldComputeVhdsSize(handler, vhds) {
-  if (handler.isEncrypted) {
-    return false
-  }
+function shouldComputeVhdsSize(vhds) {
   return vhds.every(vhd => vhd instanceof VhdFile)
 }
 
@@ -29,49 +24,63 @@ const computeVhdsSize = (handler, vhdPaths) =>
   Disposable.use(
     vhdPaths.map(vhdPath => openVhd(handler, vhdPath)),
     async vhds => {
-      if (shouldComputeVhdsSize(handler, vhds)) {
+      if (shouldComputeVhdsSize(vhds)) {
         const sizes = await asyncMap(vhds, vhd => vhd.getSize())
         return sum(sizes)
       }
     }
   )
 
-// chain is [ ancestor, child_1, ..., child_n ]
-async function _mergeVhdChain(handler, chain, { logInfo, remove, merge, mergeBlockConcurrency }) {
+// chain is [ ancestor, child1, ..., childn]
+// 1. Create a VhdSynthetic from all children
+// 2. Merge the VhdSynthetic into the ancestor
+// 3. Delete all (now) unused VHDs
+// 4. Rename the ancestor with the merged data to the latest child
+//
+//                  VhdSynthetic
+//                       |
+//              /‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾\
+//  [ ancestor, child1, ...,child n-1,  childn ]
+//         |    \___________________/     ^
+//         |             |                |
+//         |       unused VHDs            |
+//         |                              |
+//         \___________rename_____________/
+
+async function mergeVhdChain(chain, { handler, logInfo, remove, merge }) {
+  assert(chain.length >= 2)
+  const chainCopy = [...chain]
+  const parent = chainCopy.pop()
+  const children = chainCopy
+
   if (merge) {
-    logInfo(`merging VHD chain`, { chain })
+    logInfo(`merging children into parent`, { childrenCount: children.length, parent })
 
     let done, total
     const handle = setInterval(() => {
       if (done !== undefined) {
-        logInfo('merge in progress', {
-          done,
-          parent: chain[0],
-          progress: Math.round((100 * done) / total),
-          total,
-        })
+        logInfo(`merging children in progress`, { children, parent, doneCount: done, totalCount: total })
       }
     }, 10e3)
-    try {
-      return await mergeVhdChain(handler, chain, {
-        logInfo,
-        mergeBlockConcurrency,
-        onProgress({ done: d, total: t }) {
-          done = d
-          total = t
-        },
-        removeUnused: remove,
-      })
-    } finally {
-      clearInterval(handle)
-    }
+
+    const mergedSize = await mergeVhd(handler, parent, handler, children, {
+      logInfo,
+      onProgress({ done: d, total: t }) {
+        done = d
+        total = t
+      },
+      remove,
+    })
+
+    clearInterval(handle)
+    return mergedSize
   }
 }
 
 const noop = Function.prototype
 
 const INTERRUPTED_VHDS_REG = /^\.(.+)\.merge.json$/
-const listVhds = async (handler, vmDir, logWarn) => {
+const listVhds = async (handler, vmDir) => {
   const vhds = new Set()
   const aliases = {}
   const interruptedVhds = new Map()
@@ -91,23 +100,12 @@ const listVhds = async (handler, vmDir, logWarn) => {
             filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file),
           })
           aliases[vdiDir] = list.filter(vhd => isVhdAlias(vhd)).map(file => `${vdiDir}/${file}`)
-
-          await asyncMap(list, async file => {
+          list.forEach(file => {
             const res = INTERRUPTED_VHDS_REG.exec(file)
             if (res === null) {
               vhds.add(`${vdiDir}/${file}`)
             } else {
-              try {
-                const mergeState = JSON.parse(await handler.readFile(`${vdiDir}/${file}`))
-                interruptedVhds.set(`${vdiDir}/${res[1]}`, {
-                  statePath: `${vdiDir}/${file}`,
-                  chain: mergeState.chain,
-                })
-              } catch (error) {
-                // fall back to a non resuming merge
-                vhds.add(`${vdiDir}/${file}`)
-                logWarn('failed to read existing merge state', { path: file, error })
-              }
+              interruptedVhds.set(`${vdiDir}/${res[1]}`, `${vdiDir}/${file}`)
             }
           })
         }
@@ -123,15 +121,15 @@ async function checkAliases(
   { handler, logInfo = noop, logWarn = console.warn, remove = false }
 ) {
   const aliasFound = []
-  for (const alias of aliasPaths) {
-    const target = await resolveVhdAlias(handler, alias)
+  for (const path of aliasPaths) {
+    const target = await resolveVhdAlias(handler, path)
 
     if (!isVhdFile(target)) {
-      logWarn('alias references non VHD target', { alias, target })
+      logWarn('alias references non VHD target', { path, target })
       if (remove) {
-        logInfo('removing alias and non VHD target', { alias, target })
+        logInfo('removing alias and non VHD target', { path, target })
         await handler.unlink(target)
-        await handler.unlink(alias)
+        await handler.unlink(path)
       }
       continue
     }
@@ -144,13 +142,13 @@ async function checkAliases(
         // error during dispose should not trigger a deletion
       }
     } catch (error) {
-      logWarn('missing or broken alias target', { alias, target, error })
+      logWarn('missing or broken alias target', { target, path, error })
       if (remove) {
         try {
-          await VhdAbstract.unlink(handler, alias)
+          await VhdAbstract.unlink(handler, path)
         } catch (error) {
           if (error.code !== 'ENOENT') {
-            logWarn('error deleting alias target', { alias, target, error })
+            logWarn('error deleting alias target', { target, path, error })
           }
         }
       }
@@ -160,17 +158,17 @@ async function checkAliases(
     aliasFound.push(resolve('/', target))
   }
 
-  const vhds = await handler.list(targetDataRepository, {
+  const entries = await handler.list(targetDataRepository, {
     ignoreMissing: true,
     prependDir: true,
   })
 
-  await asyncMap(vhds, async path => {
-    if (!aliasFound.includes(path)) {
-      logWarn('no alias references VHD', { path })
+  entries.forEach(async entry => {
+    if (!aliasFound.includes(entry)) {
+      logWarn('no alias references VHD', { entry })
       if (remove) {
-        logInfo('deleting unused VHD', { path })
-        await VhdAbstract.unlink(handler, path)
+        logInfo('deleting unaliased VHD')
+        await VhdAbstract.unlink(handler, entry)
       }
     }
   })
@@ -182,26 +180,18 @@ const defaultMergeLimiter = limitConcurrency(1)
 
 exports.cleanVm = async function cleanVm(
   vmDir,
-  {
-    fixMetadata,
-    remove,
-    merge,
-    mergeBlockConcurrency,
-    mergeLimiter = defaultMergeLimiter,
-    logInfo = noop,
-    logWarn = console.warn,
-  }
+  { fixMetadata, remove, merge, mergeLimiter = defaultMergeLimiter, logInfo = noop, logWarn = console.warn }
 ) {
-  const limitedMergeVhdChain = mergeLimiter(_mergeVhdChain)
+  const limitedMergeVhdChain = mergeLimiter(mergeVhdChain)
 
   const handler = this._handler
 
   const vhdsToJSons = new Set()
-  const vhdById = new Map()
   const vhdParents = { __proto__: null }
   const vhdChildren = { __proto__: null }
+  const vhdById = new Map()
 
-  const { vhds, interruptedVhds, aliases } = await listVhds(handler, vmDir, logWarn)
+  const { vhds, interruptedVhds, aliases } = await listVhds(handler, vmDir)
 
   // remove broken VHDs
   await asyncMap(vhds, async path => {
@@ -219,40 +209,36 @@ exports.cleanVm = async function cleanVm(
           }
           vhdChildren[parent] = path
         }
-        // Detect VHDs with the same UUIDs
-        //
-        // Due to a bug introduced in a1bcd35e2
-        const duplicate = vhdById.get(UUID.stringify(vhd.footer.uuid))
-        let vhdKept = vhd
+        const duplicate = vhdById.get(vhd.footer.uuid)
         if (duplicate !== undefined) {
-          logWarn('uuid is duplicated', { uuid: UUID.stringify(vhd.footer.uuid) })
+          logWarn('uuid is duplicated', { uuid: vhd.footer.uuid })
           if (duplicate.containsAllDataOf(vhd)) {
             logWarn(`should delete ${path}`)
-            vhdKept = duplicate
-            vhds.delete(path)
           } else if (vhd.containsAllDataOf(duplicate)) {
             logWarn(`should delete ${duplicate._path}`)
-            vhds.delete(duplicate._path)
           } else {
-            logWarn('same ids but different content')
+            logWarn(`same ids but different content`)
           }
         }
-        vhdById.set(UUID.stringify(vhdKept.footer.uuid), vhdKept)
+        vhdById.set(vhd.footer.uuid, vhd)
       })
     } catch (error) {
       vhds.delete(path)
       logWarn('VHD check error', { path, error })
       if (error?.code === 'ERR_ASSERTION' && remove) {
-        logInfo('deleting broken VHD', { path })
+        logInfo('deleting broken path', { path })
         return VhdAbstract.unlink(handler, path)
       }
     }
   })
+  // the vhd are closed at the end of the disposable
+  // it's unsafe to use them later
+  vhdById.clear()
 
   // remove interrupted merge states for missing VHDs
   for (const interruptedVhd of interruptedVhds.keys()) {
     if (!vhds.has(interruptedVhd)) {
-      const { statePath } = interruptedVhds.get(interruptedVhd)
+      const statePath = interruptedVhds.get(interruptedVhd)
       interruptedVhds.delete(interruptedVhd)
 
       logWarn('orphan merge state', {
@@ -291,9 +277,9 @@ exports.cleanVm = async function cleanVm(
       if (!vhds.has(parent)) {
         vhds.delete(vhdPath)
 
-        logWarn('parent VHD is missing', { parent, child: vhdPath })
+        logWarn('parent VHD is missing', { parent, vhdPath })
         if (remove) {
-          logInfo('deleting orphan VHD', { path: vhdPath })
+          logInfo('deleting orphan VHD', { vhdPath })
           deletions.push(VhdAbstract.unlink(handler, vhdPath))
         }
       }
@@ -311,7 +297,6 @@ exports.cleanVm = async function cleanVm(
   }
 
   const jsons = new Set()
-  let mustInvalidateCache = false
   const xvas = new Set()
   const xvaSums = []
   const entries = await handler.list(vmDir, {
@@ -345,7 +330,7 @@ exports.cleanVm = async function cleanVm(
     try {
       metadata = JSON.parse(await handler.readFile(json))
     } catch (error) {
-      logWarn('failed to read backup metadata', { path: json, error })
+      logWarn('failed to read metadata file', { json, error })
       jsons.delete(json)
       return
     }
@@ -356,11 +341,10 @@ exports.cleanVm = async function cleanVm(
       if (xvas.has(linkedXva)) {
         unusedXvas.delete(linkedXva)
       } else {
-        logWarn('the XVA linked to the backup is missing', { backup: json, xva: linkedXva })
+        logWarn('metadata XVA is missing', { json })
         if (remove) {
-          logInfo('deleting incomplete backup', { path: json })
+          logInfo('deleting incomplete backup', { json })
           jsons.delete(json)
-          mustInvalidateCache = true
           await handler.unlink(json)
         }
       }
@@ -380,10 +364,9 @@ exports.cleanVm = async function cleanVm(
           vhdsToJSons[path] = json
         })
       } else {
-        logWarn('some VHDs linked to the backup are missing', { backup: json, missingVhds })
+        logWarn('some metadata VHDs are missing', { json, missingVhds })
         if (remove) {
-          logInfo('deleting incomplete backup', { path: json })
-          mustInvalidateCache = true
+          logInfo('deleting incomplete backup', { json })
           jsons.delete(json)
           await handler.unlink(json)
         }
@@ -395,7 +378,7 @@ exports.cleanVm = async function cleanVm(
   const unusedVhdsDeletion = []
   const toMerge = []
   {
-    // VHD chains (as list from oldest to most recent) to merge indexed by most recent
+    // VHD chains (as list from child to ancestor) to merge indexed by last
     // ancestor
     const vhdChainsToMerge = { __proto__: null }
 
@@ -419,14 +402,14 @@ exports.cleanVm = async function cleanVm(
       if (child !== undefined) {
         const chain = getUsedChildChainOrDelete(child)
         if (chain !== undefined) {
-          chain.unshift(vhd)
+          chain.push(vhd)
           return chain
         }
       }
 
-      logWarn('unused VHD', { path: vhd })
+      logWarn('unused VHD', { vhd })
       if (remove) {
-        logInfo('deleting unused VHD', { path: vhd })
+        logInfo('deleting unused VHD', { vhd })
         unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
       }
     }
@@ -437,13 +420,7 @@ exports.cleanVm = async function cleanVm(
 
     // merge interrupted VHDs
     for (const parent of interruptedVhds.keys()) {
-      // before #6349 the chain wasn't in the mergeState
-      const { chain, statePath } = interruptedVhds.get(parent)
-      if (chain === undefined) {
-        vhdChainsToMerge[parent] = [parent, vhdChildren[parent]]
-      } else {
-        vhdChainsToMerge[parent] = chain.map(vhdPath => handlerPath.resolveFromFile(statePath, vhdPath))
-      }
+      vhdChainsToMerge[parent] = [vhdChildren[parent], parent]
     }
 
     Object.values(vhdChainsToMerge).forEach(chain => {
@@ -456,15 +433,9 @@ exports.cleanVm = async function cleanVm(
   const metadataWithMergedVhd = {}
   const doMerge = async () => {
     await asyncMap(toMerge, async chain => {
-      const merged = await limitedMergeVhdChain(handler, chain, {
-        logInfo,
-        logWarn,
-        remove,
-        merge,
-        mergeBlockConcurrency,
-      })
+      const merged = await limitedMergeVhdChain(chain, { handler, logInfo, logWarn, remove, merge })
       if (merged !== undefined) {
-        const metadataPath = vhdsToJSons[chain[chain.length - 1]] // all the chain should have the same metada file
+        const metadataPath = vhdsToJSons[chain[0]] // all the chain should have the same metada file
         metadataWithMergedVhd[metadataPath] = true
       }
     })
@@ -507,11 +478,7 @@ exports.cleanVm = async function cleanVm(
       if (mode === 'full') {
         // a full backup : check size
         const linkedXva = resolve('/', vmDir, xva)
-        try {
-          fileSystemSize = await handler.getSize(linkedXva)
-        } catch (error) {
-          // can fail with encrypted remote
-        }
+        fileSystemSize = await handler.getSize(linkedXva)
       } else if (mode === 'delta') {
         const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
         fileSystemSize = await computeVhdsSize(handler, linkedVhds)
@@ -523,15 +490,11 @@ exports.cleanVm = async function cleanVm(
 
         // don't warn if the size has changed after a merge
         if (!merged && fileSystemSize !== size) {
-          logWarn('incorrect backup size in metadata', {
-            path: metadataPath,
-            actual: size ?? 'none',
-            expected: fileSystemSize,
-          })
+          logWarn('incorrect size in metadata', { size: size ?? 'none', fileSystemSize })
         }
       }
     } catch (error) {
-      logWarn('failed to get backup size', { backup: metadataPath, error })
+      logWarn('failed to get metadata size', { metadataPath, error })
       return
     }
 
@@ -541,16 +504,11 @@ exports.cleanVm = async function cleanVm(
       try {
         await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })
       } catch (error) {
-        logWarn('failed to update backup size in metadata', { path: metadataPath, error })
+        logWarn('metadata size update failed', { metadataPath, error })
       }
     }
   })
 
-  // purge cache if a metadata file has been deleted
-  if (mustInvalidateCache) {
-    await handler.unlink(vmDir + '/cache.json.gz')
-  }
-
   return {
     // boolean whether some VHDs were merged (or should be merged)
     merge: toMerge.length !== 0,

@xen-orchestra/backups/_deltaVm.js

@@ -1,12 +1,12 @@
 'use strict'
 
+const compareVersions = require('compare-versions')
 const find = require('lodash/find.js')
 const groupBy = require('lodash/groupBy.js')
 const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const omit = require('lodash/omit.js')
 const { asyncMap } = require('@xen-orchestra/async-map')
 const { CancelToken } = require('promise-toolbox')
-const { compareVersions } = require('compare-versions')
 const { createVhdStreamWithLength } = require('vhd-lib')
 const { defer } = require('golike-defer')
 

@xen-orchestra/backups/_forkStreamUnpipe.js

@@ -3,8 +3,6 @@
 const eos = require('end-of-stream')
 const { PassThrough } = require('stream')
 
-const { debug } = require('@xen-orchestra/log').createLogger('xo:backups:forkStreamUnpipe')
-
 // create a new readable stream from an existing one which may be piped later
 //
 // in case of error in the new readable stream, it will simply be unpiped
@@ -13,23 +11,18 @@ exports.forkStreamUnpipe = function forkStreamUnpipe(stream) {
   const { forks = 0 } = stream
   stream.forks = forks + 1
 
-  debug('forking', { forks: stream.forks })
-
   const proxy = new PassThrough()
   stream.pipe(proxy)
   eos(stream, error => {
     if (error !== undefined) {
-      debug('error on original stream, destroying fork', { error })
       proxy.destroy(error)
     }
   })
-  eos(proxy, error => {
-    debug('end of stream, unpiping', { error, forks: --stream.forks })
-
+  eos(proxy, _ => {
+    stream.forks--
     stream.unpipe(proxy)
 
     if (stream.forks === 0) {
-      debug('no more forks, destroying original stream')
       stream.destroy(new Error('no more consumers for this stream'))
     }
   })

@xen-orchestra/backups/_isValidXva.js

@@ -49,11 +49,6 @@ const isValidTar = async (handler, size, fd) => {
 // TODO: find an heuristic for compressed files
 async function isValidXva(path) {
   const handler = this._handler
-
-  // size is longer when encrypted  + reading part of an encrypted file is not implemented
-  if (handler.isEncrypted) {
-    return true
-  }
   try {
     const fd = await handler.openFile(path, 'r')
     try {
@@ -71,6 +66,7 @@ async function isValidXva(path) {
     }
   } catch (error) {
     // never throw, log and report as valid to avoid side effects
+    console.error('isValidXva', path, error)
     return true
   }
 }

@xen-orchestra/backups/docs/VM backups/README.md

@@ -14,14 +14,12 @@
 
 ## File structure on remote
 
-### with vhd files
-
 ```
 <remote>
 └─ xo-vm-backups
   ├─ index.json // TODO
   └─ <VM UUID>
-     ├─ cache.json.gz
+     ├─ index.json // TODO
      ├─ vdis
      │  └─ <job UUID>
      │     └─ <VDI UUID>
@@ -32,31 +30,6 @@
      └─ <YYYYMMDD>T<HHmmss>.xva.checksum
 ```
 
-### with vhd directories
-
-When `useVhdDirectory` is enabled on the remote, the directory containing the VHDs has a slightly different architecture:
-
-```
-<vdis>/<job UUID>/<VDI UUID>
-  ├─ <YYYYMMDD>T<HHmmss>.alias.vhd // contains the relative path to a VHD directory
-  ├─ <YYYYMMDD>T<HHmmss>.alias.vhd
-  └─ data
-    ├─ <uuid>.vhd // VHD directory format is described in vhd-lib/Vhd/VhdDirectory.js
-    └─ <uuid>.vhd
-```
-
-## Cache for a VM
-
-In a VM directory, if the file `cache.json.gz` exists, it contains the metadata for all the backups for this VM.
-
-Add the following file: `xo-vm-backups/<VM UUID>/cache.json.gz`.
-
-This cache is compressed in Gzip and contains an JSON object with the metadata for all the backups of this VM indexed by their absolute path (i.e. `/xo-vm-backups/<VM UUID>/<timestamp>.json`).
-
-This file is generated on demande when listing the backups, and directly updated on backup creation/deletion.
-
-In case any incoherence is detected, the file is deleted so it will be fully generated when required.
-
 ## Attributes
 
 ### Of created snapshots

@xen-orchestra/backups/merge-worker/cli.js

@@ -64,7 +64,7 @@ const main = Disposable.wrap(async function* main(args) {
     try {
       const vmDir = getVmBackupDir(String(await handler.readFile(taskFile)))
       try {
-        await adapter.cleanVm(vmDir, { merge: true, logInfo: info, logWarn: warn, remove: true })
+        await adapter.cleanVm(vmDir, { merge: true, onLog: info, remove: true })
       } catch (error) {
         // consider the clean successful if the VM dir is missing
         if (error.code !== 'ENOENT') {

@xen-orchestra/backups/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.28.0",
+  "version": "0.26.0",
   "engines": {
     "node": ">=14.6"
   },
@@ -16,18 +16,16 @@
     "postversion": "npm publish --access public"
   },
   "dependencies": {
-    "@vates/async-each": "^1.0.0",
     "@vates/cached-dns.lookup": "^1.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.0.0",
     "@vates/disposable": "^0.1.1",
-    "@vates/fuse-vhd": "^1.0.0",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^3.1.0",
+    "@xen-orchestra/fs": "^1.1.0",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/template": "^0.1.0",
-    "compare-versions": "^5.0.1",
+    "compare-versions": "^4.0.1",
     "d3-time-format": "^3.0.0",
     "decorator-synchronized": "^0.6.0",
     "end-of-stream": "^1.4.4",
@@ -39,8 +37,8 @@
     "parse-pairs": "^1.1.0",
     "promise-toolbox": "^0.21.0",
     "proper-lockfile": "^4.1.2",
-    "uuid": "^9.0.0",
-    "vhd-lib": "^4.1.0",
+    "uuid": "^8.3.2",
+    "vhd-lib": "^3.3.1",
     "yazl": "^2.5.1"
   },
   "devDependencies": {
@@ -48,7 +46,7 @@
     "tmp": "^0.2.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^1.5.0"
+    "@xen-orchestra/xapi": "^1.4.0"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

@xen-orchestra/backups/writers/DeltaBackupWriter.js

@@ -19,7 +19,9 @@ const { AbstractDeltaWriter } = require('./_AbstractDeltaWriter.js')
 const { checkVhd } = require('./_checkVhd.js')
 const { packUuid } = require('./_packUuid.js')
 const { Disposable } = require('promise-toolbox')
-const NbdClient = require('nbd-client')
+const { HealthCheckVmBackup } = require('../HealthCheckVmBackup.js')
+const { ImportVmBackup } = require('../ImportVmBackup.js')
+
 const { warn } = createLogger('xo:backups:DeltaBackupWriter')
 
 exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(AbstractDeltaWriter) {
@@ -36,7 +38,6 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
       try {
         const vhds = await handler.list(`${vdisDir}/${srcVdi.uuid}`, {
           filter: _ => _[0] !== '.' && _.endsWith('.vhd'),
-          ignoreMissing: true,
           prependDir: true,
         })
         const packedBaseUuid = packUuid(baseUuid)
@@ -70,6 +71,35 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
     return this._cleanVm({ merge: true })
   }
 
+  healthCheck(sr) {
+    return Task.run(
+      {
+        name: 'health check',
+      },
+      async () => {
+        const xapi = sr.$xapi
+        const srUuid = sr.uuid
+        const adapter = this._adapter
+        const metadata = await adapter.readVmBackupMetadata(this._metadataFileName)
+        const { id: restoredId } = await new ImportVmBackup({
+          adapter,
+          metadata,
+          srUuid,
+          xapi,
+        }).run()
+        const restoredVm = xapi.getObject(restoredId)
+        try {
+          await new HealthCheckVmBackup({
+            restoredVm,
+            xapi,
+          }).run()
+        } finally {
+          await xapi.VM_destroy(restoredVm.$ref)
+        }
+      }
+    )
+  }
+
   prepare({ isFull }) {
     // create the task related to this export and ensure all methods are called in this context
     const task = new Task({
@@ -159,6 +189,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
         }/${adapter.getVhdFileName(basename)}`
     )
 
+    const metadataFilename = (this._metadataFileName = `${backupDir}/${basename}.json`)
     const metadataContent = {
       jobId,
       mode: job.mode,
@@ -199,24 +230,11 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
             await checkVhd(handler, parentPath)
           }
 
-          // get nbd if possible
-          const vdiRef = vm.$xapi.getObject(vdi.uuid).$ref
-          let nbdClient
-          try {
-            const [nbdInfo] = await vm.$xapi.call('VDI.get_nbd_info', vdiRef)
-            nbdClient = new NbdClient(nbdInfo)
-            await nbdClient.connect()
-          } catch (e) {
-            console.log('NBD error', e)
-          }
-
           await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
             // no checksum for VHDs, because they will be invalidated by
             // merges and chainings
             checksum: false,
             validator: tmpPath => checkVhd(handler, tmpPath),
-            writeBlockConcurrency: this._backup.config.writeBlockConcurrency,
-            nbdClient,
           })
 
           if (isDelta) {
@@ -236,7 +254,9 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
       }
     })
     metadataContent.size = size
-    this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadataContent)
+    await handler.outputFile(metadataFilename, JSON.stringify(metadataContent), {
+      dirMode: backup.config.dirMode,
+    })
 
     // TODO: run cleanup?
   }

@xen-orchestra/backups/writers/FullBackupWriter.js

@@ -34,6 +34,7 @@ exports.FullBackupWriter = class FullBackupWriter extends MixinBackupWriter(Abst
     const { job, scheduleId, vm } = backup
 
     const adapter = this._adapter
+    const handler = adapter.handler
     const backupDir = getVmBackupDir(vm.uuid)
 
     // TODO: clean VM backup directory
@@ -73,7 +74,9 @@ exports.FullBackupWriter = class FullBackupWriter extends MixinBackupWriter(Abst
       return { size: sizeContainer.size }
     })
     metadata.size = sizeContainer.size
-    this._metadataFileName = await adapter.writeVmBackupMetadata(vm.uuid, metadata)
+    await handler.outputFile(metadataFilename, JSON.stringify(metadata), {
+      dirMode: backup.config.dirMode,
+    })
 
     if (!deleteFirst) {
       await deleteOldBackups()

@xen-orchestra/backups/writers/_MixinBackupWriter.js

@@ -3,13 +3,10 @@
 const { createLogger } = require('@xen-orchestra/log')
 const { join } = require('path')
 
-const assert = require('assert')
-const { formatFilenameDate } = require('../_filenameDate.js')
 const { getVmBackupDir } = require('../_getVmBackupDir.js')
-const { HealthCheckVmBackup } = require('../HealthCheckVmBackup.js')
-const { ImportVmBackup } = require('../ImportVmBackup.js')
-const { Task } = require('../Task.js')
 const MergeWorker = require('../merge-worker/index.js')
+const { formatFilenameDate } = require('../_filenameDate.js')
+const { Task } = require('../Task.js')
 
 const { info, warn } = createLogger('xo:backups:MixinBackupWriter')
 
@@ -39,7 +36,6 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
               Task.warning(message, data)
             },
             lock: false,
-            mergeBlockConcurrency: this._backup.config.mergeBlockConcurrency,
           })
         })
       } catch (error) {
@@ -75,39 +71,6 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
         const remotePath = handler._getRealPath()
         await MergeWorker.run(remotePath)
       }
-    }
-
-    healthCheck(sr) {
-      assert.notStrictEqual(
-        this._metadataFileName,
-        undefined,
-        'Metadata file name should be defined before making a healthcheck'
-      )
-      return Task.run(
-        {
-          name: 'health check',
-        },
-        async () => {
-          const xapi = sr.$xapi
-          const srUuid = sr.uuid
-          const adapter = this._adapter
-          const metadata = await adapter.readVmBackupMetadata(this._metadataFileName)
-          const { id: restoredId } = await new ImportVmBackup({
-            adapter,
-            metadata,
-            srUuid,
-            xapi,
-          }).run()
-          const restoredVm = xapi.getObject(restoredId)
-          try {
-            await new HealthCheckVmBackup({
-              restoredVm,
-              xapi,
-            }).run()
-          } finally {
-            await xapi.VM_destroy(restoredVm.$ref)
-          }
-        }
-      )
+      await this._adapter.invalidateVmBackupListCache(this._backup.vm.uuid)
     }
   }

@xen-orchestra/cr-seed-cli/package.json

@@ -18,7 +18,7 @@
   "preferGlobal": true,
   "dependencies": {
     "golike-defer": "^0.5.1",
-    "xen-api": "^1.2.2"
+    "xen-api": "^1.2.1"
   },
   "scripts": {
     "postversion": "npm publish"

@xen-orchestra/fs/docs/encryption.md

@@ -1,19 +0,0 @@
-## metadata files
-
-- Older remotes dont have any metadata file
-- Remote used since 5.75 have two files : encryption.json and metadata.json
-
-The metadata files are checked by the sync() method. If the check fails it MUST throw an error and dismount.
-
-If the remote is empty, the `sync` method creates them
-
-### encryption.json
-
-A non encrypted file contain the algorithm and parameters used for this remote.
-This MUST NOT contains the key.
-
-### metadata.json
-
-An encrypted JSON file containing the settings of a remote. Today this is an empty JSON file ( `{random: <randomuuid>}` ), it serves to check if the encryption key set in the remote is valid, but in the future will be able to store some remote settings to ease disaster recovery.
-
-If this file can't be read (decrypted, decompressed, .. ), that means that the remote settings have been updated. If the remote is empty, update the `encryption.json` and `metadata.json` files , else raise an error.

@xen-orchestra/fs/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/fs",
-  "version": "3.1.0",
+  "version": "1.1.0",
   "license": "AGPL-3.0-or-later",
   "description": "The File System for Xen Orchestra backups.",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
@@ -17,18 +17,18 @@
     "xo-fs": "./cli.js"
   },
   "engines": {
-    "node": ">=14.13"
+    "node": ">=14"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.54.0",
     "@aws-sdk/lib-storage": "^3.54.0",
     "@aws-sdk/middleware-apply-body-checksum": "^3.58.0",
     "@aws-sdk/node-http-handler": "^3.54.0",
+    "@marsaud/smb2": "^0.18.0",
     "@sindresorhus/df": "^3.1.1",
-    "@vates/async-each": "^1.0.0",
+    "@vates/async-each": "^0.1.0",
     "@vates/coalesce-calls": "^0.1.0",
     "@vates/decorate-with": "^2.0.0",
-    "@vates/read-chunk": "^1.0.0",
     "@xen-orchestra/async-map": "^0.1.2",
     "@xen-orchestra/log": "^0.3.0",
     "bind-property-descriptor": "^2.0.0",
@@ -40,8 +40,7 @@
     "lodash": "^4.17.4",
     "promise-toolbox": "^0.21.0",
     "proper-lockfile": "^4.1.2",
-    "pumpify": "^2.0.1",
-    "readable-stream": "^4.1.0",
+    "readable-stream": "^3.0.6",
     "through2": "^4.0.2",
     "xo-remote-parser": "^0.9.1"
   },
@@ -51,6 +50,7 @@
     "@babel/plugin-proposal-decorators": "^7.1.6",
     "@babel/plugin-proposal-function-bind": "^7.0.0",
     "@babel/preset-env": "^7.8.0",
+    "async-iterator-to-stream": "^1.1.0",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^7.0.2",
     "dotenv": "^16.0.0",
@@ -68,9 +68,5 @@
   "author": {
     "name": "Vates SAS",
     "url": "https://vates.fr"
-  },
-  "exports": {
-    ".": "./dist/index.js",
-    "./path": "./dist/path.js"
   }
 }

@xen-orchestra/fs/src/_encryptor.js

@@ -1,71 +0,0 @@
-const { readChunk } = require('@vates/read-chunk')
-const crypto = require('crypto')
-const pumpify = require('pumpify')
-
-function getEncryptor(key) {
-  if (key === undefined) {
-    return {
-      id: 'NULL_ENCRYPTOR',
-      algorithm: 'none',
-      key: 'none',
-      ivLength: 0,
-      encryptData: buffer => buffer,
-      encryptStream: stream => stream,
-      decryptData: buffer => buffer,
-      decryptStream: stream => stream,
-    }
-  }
-  const algorithm = 'aes-256-cbc'
-  const ivLength = 16
-
-  function encryptStream(input) {
-    const iv = crypto.randomBytes(ivLength)
-    const cipher = crypto.createCipheriv(algorithm, Buffer.from(key), iv)
-
-    const encrypted = pumpify(input, cipher)
-    encrypted.unshift(iv)
-    return encrypted
-  }
-
-  async function decryptStream(encryptedStream) {
-    const iv = await readChunk(encryptedStream, ivLength)
-    const cipher = crypto.createDecipheriv(algorithm, Buffer.from(key), iv)
-    /**
-     * WARNING
-     *
-     * the crytped size has an initializtion vector + a padding at the end
-     * whe can't predict the decrypted size from the start of the encrypted size
-     * thus, we can't set decrypted.length reliably
-     *
-     */
-    return pumpify(encryptedStream, cipher)
-  }
-
-  function encryptData(buffer) {
-    const iv = crypto.randomBytes(ivLength)
-    const cipher = crypto.createCipheriv(algorithm, Buffer.from(key), iv)
-    const encrypted = cipher.update(buffer)
-    return Buffer.concat([iv, encrypted, cipher.final()])
-  }
-
-  function decryptData(buffer) {
-    const iv = buffer.slice(0, ivLength)
-    const encrypted = buffer.slice(ivLength)
-    const decipher = crypto.createDecipheriv(algorithm, Buffer.from(key), iv)
-    const decrypted = decipher.update(encrypted)
-    return Buffer.concat([decrypted, decipher.final()])
-  }
-
-  return {
-    id: algorithm,
-    algorithm,
-    key,
-    ivLength,
-    encryptData,
-    encryptStream,
-    decryptData,
-    decryptStream,
-  }
-}
-
-exports._getEncryptor = getEncryptor

@xen-orchestra/fs/src/_path.js

@@ -1,6 +1,6 @@
 import path from 'path'
 
-const { basename, dirname, join, resolve, relative, sep } = path.posix
+const { basename, dirname, join, resolve, sep } = path.posix
 
 export { basename, dirname, join }
 
@@ -19,6 +19,3 @@ export function split(path) {
 
   return parts
 }
-
-export const relativeFromFile = (file, path) => relative(dirname(file), path)
-export const resolveFromFile = (file, path) => resolve('/', dirname(file), path).slice(1)

@xen-orchestra/fs/src/abstract.js

@@ -1,5 +1,4 @@
 import asyncMapSettled from '@xen-orchestra/async-map/legacy'
-import assert from 'assert'
 import getStream from 'get-stream'
 import { coalesceCalls } from '@vates/coalesce-calls'
 import { createLogger } from '@xen-orchestra/log'
@@ -7,14 +6,13 @@ import { fromCallback, fromEvent, ignoreErrors, timeout } from 'promise-toolbox'
 import { limitConcurrency } from 'limit-concurrency-decorator'
 import { parse } from 'xo-remote-parser'
 import { pipeline } from 'stream'
-import { randomBytes, randomUUID } from 'crypto'
+import { randomBytes } from 'crypto'
 import { synchronized } from 'decorator-synchronized'
 
-import { basename, dirname, normalize as normalizePath } from './path'
+import { basename, dirname, normalize as normalizePath } from './_path'
 import { createChecksumStream, validChecksumOfReadStream } from './checksum'
-import { _getEncryptor } from './_encryptor'
 
-const { info, warn } = createLogger('@xen-orchestra:fs')
+const { warn } = createLogger('@xen-orchestra:fs')
 
 const checksumFile = file => file + '.checksum'
 const computeRate = (hrtime, size) => {
@@ -25,9 +23,6 @@ const computeRate = (hrtime, size) => {
 const DEFAULT_TIMEOUT = 6e5 // 10 min
 const DEFAULT_MAX_PARALLEL_OPERATIONS = 10
 
-const ENCRYPTION_DESC_FILENAME = 'encryption.json'
-const ENCRYPTION_METADATA_FILENAME = 'metadata.json'
-
 const ignoreEnoent = error => {
   if (error == null || error.code !== 'ENOENT') {
     throw error
@@ -68,7 +63,6 @@ class PrefixWrapper {
 }
 
 export default class RemoteHandlerAbstract {
-  _encryptor
   constructor(remote, options = {}) {
     if (remote.url === 'test://') {
       this._remote = remote
@@ -79,7 +73,6 @@ export default class RemoteHandlerAbstract {
       }
     }
     ;({ highWaterMark: this._highWaterMark, timeout: this._timeout = DEFAULT_TIMEOUT } = options)
-    this._encryptor = _getEncryptor(this._remote.encryptionKey)
 
     const sharedLimit = limitConcurrency(options.maxParallelOperations ?? DEFAULT_MAX_PARALLEL_OPERATIONS)
     this.closeFile = sharedLimit(this.closeFile)
@@ -118,51 +111,90 @@ export default class RemoteHandlerAbstract {
     await this.__closeFile(fd)
   }
 
-  async createReadStream(file, { checksum = false, ignoreMissingChecksum = false, ...options } = {}) {
-    if (options.end !== undefined || options.start !== undefined) {
-      assert.strictEqual(this.isEncrypted, false, `Can't read part of a file when encryption is active ${file}`)
-    }
+  // TODO: remove method
+  async createOutputStream(file, { checksum = false, dirMode, ...options } = {}) {
     if (typeof file === 'string') {
       file = normalizePath(file)
     }
-
-    let stream = await timeout.call(
-      this._createReadStream(file, { ...options, highWaterMark: this._highWaterMark }),
+    const path = typeof file === 'string' ? file : file.path
+    const streamP = timeout.call(
+      this._createOutputStream(file, {
+        dirMode,
+        flags: 'wx',
+        ...options,
+      }),
       this._timeout
     )
 
-    // detect early errors
-    await fromEvent(stream, 'readable')
-
-    if (checksum) {
-      try {
-        const path = typeof file === 'string' ? file : file.path
-        const checksum = await this._readFile(checksumFile(path), { flags: 'r' })
-
-        const { length } = stream
-        stream = validChecksumOfReadStream(stream, String(checksum).trim())
-        stream.length = length
-      } catch (error) {
-        if (!(ignoreMissingChecksum && error.code === 'ENOENT')) {
-          throw error
-        }
-      }
+    if (!checksum) {
+      return streamP
     }
 
-    if (this.isEncrypted) {
-      stream = this._encryptor.decryptStream(stream)
-    } else {
-      // try to add the length prop if missing and not a range stream
-      if (stream.length === undefined && options.end === undefined && options.start === undefined) {
-        try {
-          stream.length = await this._getSize(file)
-        } catch (error) {
-          // ignore errors
-        }
-      }
+    const checksumStream = createChecksumStream()
+    const forwardError = error => {
+      checksumStream.emit('error', error)
     }
 
-    return stream
+    const stream = await streamP
+    stream.on('error', forwardError)
+    checksumStream.pipe(stream)
+
+    checksumStream.checksumWritten = checksumStream.checksum
+      .then(value => this._outputFile(checksumFile(path), value, { flags: 'wx' }))
+      .catch(forwardError)
+
+    return checksumStream
+  }
+
+  createReadStream(file, { checksum = false, ignoreMissingChecksum = false, ...options } = {}) {
+    if (typeof file === 'string') {
+      file = normalizePath(file)
+    }
+    const path = typeof file === 'string' ? file : file.path
+    const streamP = timeout
+      .call(this._createReadStream(file, { ...options, highWaterMark: this._highWaterMark }), this._timeout)
+      .then(stream => {
+        // detect early errors
+        let promise = fromEvent(stream, 'readable')
+
+        // try to add the length prop if missing and not a range stream
+        if (stream.length === undefined && options.end === undefined && options.start === undefined) {
+          promise = Promise.all([
+            promise,
+            ignoreErrors.call(
+              this._getSize(file).then(size => {
+                stream.length = size
+              })
+            ),
+          ])
+        }
+
+        return promise.then(() => stream)
+      })
+
+    if (!checksum) {
+      return streamP
+    }
+
+    // avoid a unhandled rejection warning
+    ignoreErrors.call(streamP)
+
+    return this._readFile(checksumFile(path), { flags: 'r' }).then(
+      checksum =>
+        streamP.then(stream => {
+          const { length } = stream
+          stream = validChecksumOfReadStream(stream, String(checksum).trim())
+          stream.length = length
+
+          return stream
+        }),
+      error => {
+        if (ignoreMissingChecksum && error && error.code === 'ENOENT') {
+          return streamP
+        }
+        throw error
+      }
+    )
   }
 
   /**
@@ -178,8 +210,6 @@ export default class RemoteHandlerAbstract {
   async outputStream(path, input, { checksum = true, dirMode, validator } = {}) {
     path = normalizePath(path)
     let checksumStream
-
-    input = this._encryptor.encryptStream(input)
     if (checksum) {
       checksumStream = createChecksumStream()
       pipeline(input, checksumStream, noop)
@@ -190,8 +220,6 @@ export default class RemoteHandlerAbstract {
       validator,
     })
     if (checksum) {
-      // using _outpuFile means the checksum will NOT be encrypted
-      // it is by design to allow checking of encrypted files without the key
       await this._outputFile(checksumFile(path), await checksumStream.checksum, { dirMode, flags: 'wx' })
     }
   }
@@ -211,13 +239,8 @@ export default class RemoteHandlerAbstract {
     return timeout.call(this._getInfo(), this._timeout)
   }
 
-  // when using encryption, the file size is aligned with the encryption block size ( 16 bytes )
-  // that means that the size will be 1 to 16 bytes more than the content size + the initialized vector length (16 bytes)
   async getSize(file) {
-    assert.strictEqual(this.isEncrypted, false, `Can't compute size of an encrypted file ${file}`)
-
-    const size = await timeout.call(this._getSize(typeof file === 'string' ? normalizePath(file) : file), this._timeout)
-    return size - this._encryptor.ivLength
+    return timeout.call(this._getSize(typeof file === 'string' ? normalizePath(file) : file), this._timeout)
   }
 
   async list(dir, { filter, ignoreMissing = false, prependDir = false } = {}) {
@@ -263,18 +286,15 @@ export default class RemoteHandlerAbstract {
   }
 
   async outputFile(file, data, { dirMode, flags = 'wx' } = {}) {
-    const encryptedData = this._encryptor.encryptData(data)
-    await this._outputFile(normalizePath(file), encryptedData, { dirMode, flags })
+    await this._outputFile(normalizePath(file), data, { dirMode, flags })
   }
 
   async read(file, buffer, position) {
-    assert.strictEqual(this.isEncrypted, false, `Can't read part of an encrypted file ${file}`)
     return this._read(typeof file === 'string' ? normalizePath(file) : file, buffer, position)
   }
 
   async readFile(file, { flags = 'r' } = {}) {
-    const data = await this._readFile(normalizePath(file), { flags })
-    return this._encryptor.decryptData(data)
+    return this._readFile(normalizePath(file), { flags })
   }
 
   async rename(oldPath, newPath, { checksum = false } = {}) {
@@ -314,61 +334,6 @@ export default class RemoteHandlerAbstract {
   @synchronized()
   async sync() {
     await this._sync()
-    try {
-      await this._checkMetadata()
-    } catch (error) {
-      await this._forget()
-      throw error
-    }
-  }
-
-  async _canWriteMetadata() {
-    const list = await this.list('/', {
-      filter: e => !e.startsWith('.') && e !== ENCRYPTION_DESC_FILENAME && e !== ENCRYPTION_METADATA_FILENAME,
-    })
-    return list.length === 0
-  }
-
-  async _createMetadata() {
-    await Promise.all([
-      this._writeFile(
-        normalizePath(ENCRYPTION_DESC_FILENAME),
-        JSON.stringify({ algorithm: this._encryptor.algorithm }),
-        {
-          flags: 'w',
-        }
-      ), // not encrypted
-      this.writeFile(ENCRYPTION_METADATA_FILENAME, `{"random":"${randomUUID()}"}`, { flags: 'w' }), // encrypted
-    ])
-  }
-
-  async _checkMetadata() {
-    try {
-      // this file is not encrypted
-      const data = await this._readFile(normalizePath(ENCRYPTION_DESC_FILENAME))
-      JSON.parse(data)
-    } catch (error) {
-      if (error.code !== 'ENOENT') {
-        throw error
-      }
-    }
-
-    try {
-      // this file is encrypted
-      const data = await this.readFile(ENCRYPTION_METADATA_FILENAME)
-      JSON.parse(data)
-    } catch (error) {
-      if (error.code === 'ENOENT' || (await this._canWriteMetadata())) {
-        info('will update metadata of this remote')
-        return this._createMetadata()
-      }
-      warn(
-        `The encryptionKey settings of this remote does not match the key used to create it. You won't be able to read any data from this remote`,
-        { error }
-      )
-      // will probably send a ERR_OSSL_EVP_BAD_DECRYPT if key is incorrect
-      throw error
-    }
   }
 
   async test() {
@@ -422,13 +387,11 @@ export default class RemoteHandlerAbstract {
   }
 
   async write(file, buffer, position) {
-    assert.strictEqual(this.isEncrypted, false, `Can't write part of a file with encryption ${file}`)
     await this._write(typeof file === 'string' ? normalizePath(file) : file, buffer, position)
   }
 
   async writeFile(file, data, { flags = 'wx' } = {}) {
-    const encryptedData = this._encryptor.encryptData(data)
-    await this._writeFile(normalizePath(file), encryptedData, { flags })
+    await this._writeFile(normalizePath(file), data, { flags })
   }
 
   // Methods that can be called by private methods to avoid parallel limit on public methods
@@ -547,13 +510,9 @@ export default class RemoteHandlerAbstract {
 
   async _outputStream(path, input, { dirMode, validator }) {
     const tmpPath = `${dirname(path)}/.${basename(path)}`
-    const output = await timeout.call(
-      this._createOutputStream(tmpPath, {
-        dirMode,
-        flags: 'wx',
-      }),
-      this._timeout
-    )
+    const output = await this.createOutputStream(tmpPath, {
+      dirMode,
+    })
     try {
       await fromCallback(pipeline, input, output)
       if (validator !== undefined) {
@@ -636,10 +595,6 @@ export default class RemoteHandlerAbstract {
   async _writeFile(file, data, options) {
     throw new Error('Not implemented')
   }
-
-  get isEncrypted() {
-    return this._encryptor.id !== 'NULL_ENCRYPTOR'
-  }
 }
 
 function createPrefixWrapperMethods() {

@xen-orchestra/fs/src/abstract.spec.js

@@ -30,6 +30,18 @@ describe('closeFile()', () => {
   })
 })
 
+describe('createOutputStream()', () => {
+  it(`throws in case of timeout`, async () => {
+    const testHandler = new TestHandler({
+      createOutputStream: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.createOutputStream('File')
+    jest.advanceTimersByTime(TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
 describe('getInfo()', () => {
   it('throws in case of timeout', async () => {
     const testHandler = new TestHandler({

@xen-orchestra/fs/src/fs.spec.js

@@ -1,7 +1,10 @@
 /* eslint-env jest */
 
 import 'dotenv/config'
+import asyncIteratorToStream from 'async-iterator-to-stream'
 import { forOwn, random } from 'lodash'
+import { fromCallback } from 'promise-toolbox'
+import { pipeline } from 'readable-stream'
 import { tmpdir } from 'os'
 
 import { getHandler } from '.'
@@ -24,6 +27,9 @@ const unsecureRandomBytes = n => {
 
 const TEST_DATA_LEN = 1024
 const TEST_DATA = unsecureRandomBytes(TEST_DATA_LEN)
+const createTestDataStream = asyncIteratorToStream(function* () {
+  yield TEST_DATA
+})
 
 const rejectionOf = p =>
   p.then(
@@ -76,6 +82,14 @@ handlers.forEach(url => {
       })
     })
 
+    describe('#createOutputStream()', () => {
+      it('creates parent dir if missing', async () => {
+        const stream = await handler.createOutputStream('dir/file')
+        await fromCallback(pipeline, createTestDataStream(), stream)
+        await expect(await handler.readFile('dir/file')).toEqual(TEST_DATA)
+      })
+    })
+
     describe('#getInfo()', () => {
       let info
       beforeAll(async () => {

@xen-orchestra/fs/src/index.js

@@ -5,6 +5,7 @@ import RemoteHandlerLocal from './local'
 import RemoteHandlerNfs from './nfs'
 import RemoteHandlerS3 from './s3'
 import RemoteHandlerSmb from './smb'
+import RemoteHandlerSmbMount from './smb-mount'
 
 const HANDLERS = {
   file: RemoteHandlerLocal,
@@ -14,8 +15,10 @@ const HANDLERS = {
 
 try {
   execa.sync('mount.cifs', ['-V'])
+  HANDLERS.smb = RemoteHandlerSmbMount
+} catch (_) {
   HANDLERS.smb = RemoteHandlerSmb
-} catch (_) {}
+}
 
 export const getHandler = (remote, ...rest) => {
   const Handler = HANDLERS[parse(remote.url).type]

@xen-orchestra/fs/src/local.js

@@ -1,38 +1,13 @@
 import df from '@sindresorhus/df'
 import fs from 'fs-extra'
 import lockfile from 'proper-lockfile'
-import { createLogger } from '@xen-orchestra/log'
 import { fromEvent, retry } from 'promise-toolbox'
 
 import RemoteHandlerAbstract from './abstract'
 
-const { info, warn } = createLogger('xo:fs:local')
-
-// save current stack trace and add it to any rejected error
-//
-// This is especially useful when the resolution is separate from the initial
-// call, which is often the case with RPC libs.
-//
-// There is a perf impact and it should be avoided in production.
-async function addSyncStackTrace(fn, ...args) {
-  const stackContainer = new Error()
-  try {
-    return await fn.apply(this, args)
-  } catch (error) {
-    error.syncStack = stackContainer.stack
-    throw error
-  }
-}
-
-function dontAddSyncStackTrace(fn, ...args) {
-  return fn.apply(this, args)
-}
-
 export default class LocalHandler extends RemoteHandlerAbstract {
   constructor(remote, opts = {}) {
     super(remote)
-
-    this._addSyncStackTrace = opts.syncStackTraces ?? true ? addSyncStackTrace : dontAddSyncStackTrace
     this._retriesOnEagain = {
       delay: 1e3,
       retries: 9,
@@ -55,17 +30,17 @@ export default class LocalHandler extends RemoteHandlerAbstract {
   }
 
   async _closeFile(fd) {
-    return this._addSyncStackTrace(fs.close, fd)
+    return fs.close(fd)
   }
 
   async _copy(oldPath, newPath) {
-    return this._addSyncStackTrace(fs.copy, this._getFilePath(oldPath), this._getFilePath(newPath))
+    return fs.copy(this._getFilePath(oldPath), this._getFilePath(newPath))
   }
 
   async _createReadStream(file, options) {
     if (typeof file === 'string') {
       const stream = fs.createReadStream(this._getFilePath(file), options)
-      await this._addSyncStackTrace(fromEvent, stream, 'open')
+      await fromEvent(stream, 'open')
       return stream
     }
     return fs.createReadStream('', {
@@ -78,7 +53,7 @@ export default class LocalHandler extends RemoteHandlerAbstract {
   async _createWriteStream(file, options) {
     if (typeof file === 'string') {
       const stream = fs.createWriteStream(this._getFilePath(file), options)
-      await this._addSyncStackTrace(fromEvent, stream, 'open')
+      await fromEvent(stream, 'open')
       return stream
     }
     return fs.createWriteStream('', {
@@ -104,98 +79,71 @@ export default class LocalHandler extends RemoteHandlerAbstract {
   }
 
   async _getSize(file) {
-    const stats = await this._addSyncStackTrace(fs.stat, this._getFilePath(typeof file === 'string' ? file : file.path))
+    const stats = await fs.stat(this._getFilePath(typeof file === 'string' ? file : file.path))
     return stats.size
   }
 
   async _list(dir) {
-    return this._addSyncStackTrace(fs.readdir, this._getFilePath(dir))
+    return fs.readdir(this._getFilePath(dir))
   }
 
-  async _lock(path) {
-    const acquire = lockfile.lock.bind(undefined, this._getFilePath(path), {
-      async onCompromised(error) {
-        warn('lock compromised', { error })
-        try {
-          release = await acquire()
-          info('compromised lock was reacquired')
-        } catch (error) {
-          warn('compromised lock could not be reacquired', { error })
-        }
-      },
-    })
-
-    let release = await this._addSyncStackTrace(acquire)
-
-    return async () => {
-      try {
-        await this._addSyncStackTrace(release)
-      } catch (error) {
-        warn('lock could not be released', { error })
-      }
-    }
+  _lock(path) {
+    return lockfile.lock(this._getFilePath(path))
   }
 
   _mkdir(dir, { mode }) {
-    return this._addSyncStackTrace(fs.mkdir, this._getFilePath(dir), { mode })
+    return fs.mkdir(this._getFilePath(dir), { mode })
   }
 
   async _openFile(path, flags) {
-    return this._addSyncStackTrace(fs.open, this._getFilePath(path), flags)
+    return fs.open(this._getFilePath(path), flags)
   }
 
   async _read(file, buffer, position) {
     const needsClose = typeof file === 'string'
-    file = needsClose ? await this._addSyncStackTrace(fs.open, this._getFilePath(file), 'r') : file.fd
+    file = needsClose ? await fs.open(this._getFilePath(file), 'r') : file.fd
     try {
-      return await this._addSyncStackTrace(
-        fs.read,
-        file,
-        buffer,
-        0,
-        buffer.length,
-        position === undefined ? null : position
-      )
+      return await fs.read(file, buffer, 0, buffer.length, position === undefined ? null : position)
     } finally {
       if (needsClose) {
-        await this._addSyncStackTrace(fs.close, file)
+        await fs.close(file)
       }
     }
   }
 
   async _readFile(file, options) {
     const filePath = this._getFilePath(file)
-    return await this._addSyncStackTrace(retry, () => fs.readFile(filePath, options), this._retriesOnEagain)
+    return await retry(() => fs.readFile(filePath, options), this._retriesOnEagain)
   }
 
   async _rename(oldPath, newPath) {
-    return this._addSyncStackTrace(fs.rename, this._getFilePath(oldPath), this._getFilePath(newPath))
+    return fs.rename(this._getFilePath(oldPath), this._getFilePath(newPath))
   }
 
   async _rmdir(dir) {
-    return this._addSyncStackTrace(fs.rmdir, this._getFilePath(dir))
+    return fs.rmdir(this._getFilePath(dir))
   }
 
   async _sync() {
     const path = this._getRealPath('/')
-    await this._addSyncStackTrace(fs.ensureDir, path)
-    await this._addSyncStackTrace(fs.access, path, fs.R_OK | fs.W_OK)
+    await fs.ensureDir(path)
+    await fs.access(path, fs.R_OK | fs.W_OK)
   }
 
   _truncate(file, len) {
-    return this._addSyncStackTrace(fs.truncate, this._getFilePath(file), len)
+    return fs.truncate(this._getFilePath(file), len)
   }
 
   async _unlink(file) {
     const filePath = this._getFilePath(file)
-    return await this._addSyncStackTrace(retry, () => fs.unlink(filePath), this._retriesOnEagain)
+    return await retry(() => fs.unlink(filePath), this._retriesOnEagain)
   }
 
   _writeFd(file, buffer, position) {
-    return this._addSyncStackTrace(fs.write, file.fd, buffer, 0, buffer.length, position)
+    return fs.write(file.fd, buffer, 0, buffer.length, position)
   }
 
   _writeFile(file, data, { flags }) {
-    return this._addSyncStackTrace(fs.writeFile, this._getFilePath(file), data, { flag: flags })
+    return fs.writeFile(this._getFilePath(file), data, { flag: flags })
   }
 }

@xen-orchestra/fs/src/s3.js

@@ -27,7 +27,7 @@ import copyStreamToBuffer from './_copyStreamToBuffer.js'
 import createBufferFromStream from './_createBufferFromStream.js'
 import guessAwsRegion from './_guessAwsRegion.js'
 import RemoteHandlerAbstract from './abstract'
-import { basename, join, split } from './path'
+import { basename, join, split } from './_path'
 import { asyncEach } from '@vates/async-each'
 
 // endpoints https://docs.aws.amazon.com/general/latest/gr/s3.html
@@ -155,14 +155,6 @@ export default class S3Handler extends RemoteHandlerAbstract {
       if (e.name === 'EntityTooLarge') {
         return this._multipartCopy(oldPath, newPath)
       }
-      // normalize this error code
-      if (e.name === 'NoSuchKey') {
-        const error = new Error(`ENOENT: no such file or directory '${oldPath}'`)
-        error.cause = e
-        error.code = 'ENOENT'
-        error.path = oldPath
-        throw error
-      }
       throw e
     }
   }

@xen-orchestra/fs/src/smb-mount.js

@@ -0,0 +1,23 @@
+import { parse } from 'xo-remote-parser'
+
+import MountHandler from './_mount'
+import { normalize } from './_path'
+
+export default class SmbMountHandler extends MountHandler {
+  constructor(remote, opts) {
+    const { domain = 'WORKGROUP', host, password, path, username } = parse(remote.url)
+    super(remote, opts, {
+      type: 'cifs',
+      device: '//' + host + normalize(path),
+      options: `domain=${domain}`,
+      env: {
+        USER: username,
+        PASSWD: password,
+      },
+    })
+  }
+
+  get type() {
+    return 'smb'
+  }
+}

@xen-orchestra/fs/src/smb.js

@@ -1,23 +1,163 @@
-import { parse } from 'xo-remote-parser'
+import Smb2 from '@marsaud/smb2'
 
-import MountHandler from './_mount'
-import { normalize } from './path'
+import RemoteHandlerAbstract from './abstract'
 
-export default class SmbHandler extends MountHandler {
+// Normalize the error code for file not found.
+const wrapError = (error, code) => ({
+  __proto__: error,
+  cause: error,
+  code,
+})
+const normalizeError = (error, shouldBeDirectory) => {
+  const { code } = error
+
+  throw code === 'STATUS_DIRECTORY_NOT_EMPTY'
+    ? wrapError(error, 'ENOTEMPTY')
+    : code === 'STATUS_FILE_IS_A_DIRECTORY'
+    ? wrapError(error, 'EISDIR')
+    : code === 'STATUS_NOT_A_DIRECTORY'
+    ? wrapError(error, 'ENOTDIR')
+    : code === 'STATUS_OBJECT_NAME_NOT_FOUND' || code === 'STATUS_OBJECT_PATH_NOT_FOUND'
+    ? wrapError(error, 'ENOENT')
+    : code === 'STATUS_OBJECT_NAME_COLLISION'
+    ? wrapError(error, 'EEXIST')
+    : code === 'STATUS_NOT_SUPPORTED' || code === 'STATUS_INVALID_PARAMETER'
+    ? wrapError(error, shouldBeDirectory ? 'ENOTDIR' : 'EISDIR')
+    : error
+}
+const normalizeDirError = error => normalizeError(error, true)
+
+export default class SmbHandler extends RemoteHandlerAbstract {
   constructor(remote, opts) {
-    const { domain = 'WORKGROUP', host, password, path, username } = parse(remote.url)
-    super(remote, opts, {
-      type: 'cifs',
-      device: '//' + host + normalize(path),
-      options: `domain=${domain}`,
-      env: {
-        USER: username,
-        PASSWD: password,
-      },
-    })
+    super(remote, opts)
+
+    // defined in _sync()
+    this._client = undefined
+
+    const prefix = this._remote.path
+    this._prefix = prefix !== '' ? prefix + '\\' : prefix
   }
 
   get type() {
     return 'smb'
   }
+
+  _getFilePath(file) {
+    return this._prefix + (typeof file === 'string' ? file : file.path).slice(1).replace(/\//g, '\\')
+  }
+
+  _dirname(file) {
+    const parts = file.split('\\')
+    parts.pop()
+    return parts.join('\\')
+  }
+
+  _closeFile(file) {
+    return this._client.close(file).catch(normalizeError)
+  }
+
+  _createReadStream(file, options) {
+    if (typeof file === 'string') {
+      file = this._getFilePath(file)
+    } else {
+      options = { autoClose: false, ...options, fd: file.fd }
+      file = ''
+    }
+    return this._client.createReadStream(file, options).catch(normalizeError)
+  }
+
+  _createWriteStream(file, options) {
+    if (typeof file === 'string') {
+      file = this._getFilePath(file)
+    } else {
+      options = { autoClose: false, ...options, fd: file.fd }
+      file = ''
+    }
+    return this._client.createWriteStream(file, options).catch(normalizeError)
+  }
+
+  _forget() {
+    const client = this._client
+    this._client = undefined
+    return client.disconnect()
+  }
+
+  _getSize(file) {
+    return this._client.getSize(this._getFilePath(file)).catch(normalizeError)
+  }
+
+  _list(dir) {
+    return this._client.readdir(this._getFilePath(dir)).catch(normalizeDirError)
+  }
+
+  _mkdir(dir, { mode }) {
+    return this._client.mkdir(this._getFilePath(dir), mode).catch(normalizeDirError)
+  }
+
+  // TODO: add flags
+  _openFile(path, flags) {
+    return this._client.open(this._getFilePath(path), flags).catch(normalizeError)
+  }
+
+  async _read(file, buffer, position) {
+    const client = this._client
+    const needsClose = typeof file === 'string'
+    file = needsClose ? await client.open(this._getFilePath(file)) : file.fd
+    try {
+      return await client.read(file, buffer, 0, buffer.length, position)
+    } catch (error) {
+      normalizeError(error)
+    } finally {
+      if (needsClose) {
+        await client.close(file)
+      }
+    }
+  }
+
+  _readFile(file, options) {
+    return this._client.readFile(this._getFilePath(file), options).catch(normalizeError)
+  }
+
+  _rename(oldPath, newPath) {
+    return this._client
+      .rename(this._getFilePath(oldPath), this._getFilePath(newPath), {
+        replace: true,
+      })
+      .catch(normalizeError)
+  }
+
+  _rmdir(dir) {
+    return this._client.rmdir(this._getFilePath(dir)).catch(normalizeDirError)
+  }
+
+  _sync() {
+    const remote = this._remote
+
+    this._client = new Smb2({
+      share: `\\\\${remote.host}`,
+      domain: remote.domain,
+      username: remote.username,
+      password: remote.password,
+      autoCloseTimeout: 0,
+    })
+
+    // Check access (smb2 does not expose connect in public so far...)
+    return this.list('.')
+  }
+
+  _truncate(file, len) {
+    return this._client.truncate(this._getFilePath(file), len).catch(normalizeError)
+  }
+
+  _unlink(file) {
+    return this._client.unlink(this._getFilePath(file)).catch(normalizeError)
+  }
+
+  _writeFd(file, buffer, position) {
+    return this._client.write(file.fd, buffer, 0, buffer.length, position)
+  }
+
+  _writeFile(file, data, options) {
+    return this._client.writeFile(this._getFilePath(file), data, options).catch(normalizeError)
+  }
 }

@xen-orchestra/mixins/Hooks.mjs

@@ -30,41 +30,20 @@ export default class Hooks extends EventEmitter {
   // Run *start* async listeners.
   //
   // They initialize the application.
-  //
-  // *startCore* is automatically called if necessary.
   async start() {
-    if (this._status === 'stopped') {
-      await this.startCore()
-    } else {
-      assert.strictEqual(this._status, 'core started')
-    }
+    assert.strictEqual(this._status, 'stopped')
     this._status = 'starting'
     await runHook(this, 'start')
     this.emit((this._status = 'started'))
   }
 
-  // Run *start core* async listeners.
-  //
-  // They initialize core features of the application (connect to databases,
-  // etc.) and should be fast and side-effects free.
-  async startCore() {
-    assert.strictEqual(this._status, 'stopped')
-    this._status = 'starting core'
-    await runHook(this, 'start core')
-    this.emit((this._status = 'core started'))
-  }
-
-  // Run *stop*  async listeners if necessary and *stop core* listeners.
+  // Run *stop* async listeners.
   //
   // They close connections, unmount file systems, save states, etc.
   async stop() {
-    if (this._status !== 'core started') {
-      assert.strictEqual(this._status, 'started')
-      this._status = 'stopping'
-      await runHook(this, 'stop')
-      this._status = 'core started'
-    }
-    await runHook(this, 'stop core')
+    assert.strictEqual(this._status, 'started')
+    this._status = 'stopping'
+    await runHook(this, 'stop')
     this.emit((this._status = 'stopped'))
   }
 }

@xen-orchestra/mixins/HttpProxy.mjs

@@ -40,7 +40,7 @@ export default class HttpProxy {
     this.#app = app
 
     const events = new EventListenersManager(httpServer)
-    app.config.watch('http.proxy.enabled', (enabled = true) => {
+    app.config.watch('http.proxy.enabled', (enabled = false) => {
       events.removeAll()
       if (enabled) {
         events.add('connect', this.#handleConnect.bind(this)).add('request', this.#handleRequest.bind(this))

@xen-orchestra/mixins/SslCertificate.mjs

@@ -1,214 +0,0 @@
-import { createLogger } from '@xen-orchestra/log'
-import { createSecureContext } from 'tls'
-import { dirname } from 'node:path'
-import { X509Certificate } from 'node:crypto'
-import acme from 'acme-client'
-import fs from 'node:fs/promises'
-import get from 'lodash/get.js'
-
-const { debug, info, warn } = createLogger('xo:mixins:sslCertificate')
-
-acme.setLogger(message => {
-  debug(message)
-})
-
-// - create any missing parent directories
-// - replace existing files
-// - secure permissions (read-only for the owner)
-async function outputFile(path, content) {
-  await fs.mkdir(dirname(path), { recursive: true })
-  try {
-    await fs.unlink(path)
-  } catch (error) {
-    if (error.code !== 'ENOENT') {
-      throw error
-    }
-  }
-  await fs.writeFile(path, content, { flag: 'wx', mode: 0o400 })
-}
-
-// from https://github.com/publishlab/node-acme-client/blob/master/examples/auto.js
-class SslCertificate {
-  #cert
-  #challengeCreateFn
-  #challengeRemoveFn
-  #delayBeforeRenewal = 30 * 24 * 60 * 60 * 1000 // 30 days
-  #secureContext
-  #updateSslCertificatePromise
-
-  constructor({ challengeCreateFn, challengeRemoveFn }, cert, key) {
-    this.#challengeCreateFn = challengeCreateFn
-    this.#challengeRemoveFn = challengeRemoveFn
-
-    this.#set(cert, key)
-  }
-
-  get #isValid() {
-    const cert = this.#cert
-    return cert !== undefined && Date.parse(cert.validTo) > Date.now() && cert.issuer !== cert.subject
-  }
-
-  get #shouldBeRenewed() {
-    return !(this.#isValid && Date.parse(this.#cert.validTo) > Date.now() + this.#delayBeforeRenewal)
-  }
-
-  #set(cert, key) {
-    this.#cert = new X509Certificate(cert)
-    this.#secureContext = createSecureContext({ cert, key })
-  }
-
-  async getSecureContext(config) {
-    if (!this.#shouldBeRenewed) {
-      return this.#secureContext
-    }
-
-    if (this.#updateSslCertificatePromise === undefined) {
-      // not currently updating certificate
-      //
-      // ensure we only refresh certificate once at a time
-      //
-      // promise is cleaned by #updateSslCertificate itself
-      this.#updateSslCertificatePromise = this.#updateSslCertificate(config)
-    }
-
-    // old certificate is still here, return it while updating
-    if (this.#isValid) {
-      return this.#secureContext
-    }
-
-    return this.#updateSslCertificatePromise
-  }
-
-  async #save(certPath, cert, keyPath, key) {
-    try {
-      await Promise.all([outputFile(keyPath, key), outputFile(certPath, cert)])
-      info('new certificate generated', { cert: certPath, key: keyPath })
-    } catch (error) {
-      warn(`couldn't write let's encrypt certificates to disk `, { error })
-    }
-  }
-
-  async #updateSslCertificate(config) {
-    const { cert: certPath, key: keyPath, acmeEmail, acmeDomain } = config
-    try {
-      let { acmeCa = 'letsencrypt/production' } = config
-      if (!(acmeCa.startsWith('http:') || acmeCa.startsWith('https:'))) {
-        acmeCa = get(acme.directory, acmeCa.split('/'))
-      }
-
-      /* Init client */
-      const client = new acme.Client({
-        directoryUrl: acmeCa,
-        accountKey: await acme.crypto.createPrivateKey(),
-      })
-
-      /* Create CSR */
-      let [key, csr] = await acme.crypto.createCsr({
-        commonName: acmeDomain,
-      })
-      csr = csr.toString()
-      key = key.toString()
-      debug('Successfully generated key and csr')
-
-      /* Certificate */
-      const cert = await client.auto({
-        challengeCreateFn: this.#challengeCreateFn,
-        challengePriority: ['http-01'],
-        challengeRemoveFn: this.#challengeRemoveFn,
-        csr,
-        email: acmeEmail,
-        skipChallengeVerification: true,
-        termsOfServiceAgreed: true,
-      })
-      debug('Successfully generated certificate')
-
-      this.#set(cert, key)
-
-      // don't wait for this
-      this.#save(certPath, cert, keyPath, key)
-
-      return this.#secureContext
-    } catch (error) {
-      warn(`couldn't renew ssl certificate`, { acmeDomain, error })
-    } finally {
-      this.#updateSslCertificatePromise = undefined
-    }
-  }
-}
-
-export default class SslCertificates {
-  #app
-  #challenges = new Map()
-  #challengeHandlers = {
-    challengeCreateFn: (authz, challenge, keyAuthorization) => {
-      this.#challenges.set(challenge.token, keyAuthorization)
-    },
-    challengeRemoveFn: (authz, challenge, keyAuthorization) => {
-      this.#challenges.delete(challenge.token)
-    },
-  }
-  #handlers = new Map()
-
-  constructor(app, { httpServer }) {
-    // don't setup the proxy if httpServer is not present
-    //
-    // that can happen when the app is instanciated in another context like xo-server-recover-account
-    if (httpServer === undefined) {
-      return
-    }
-    const prefix = '/.well-known/acme-challenge/'
-    httpServer.on('request', (req, res) => {
-      const { url } = req
-      if (url.startsWith(prefix)) {
-        const token = url.slice(prefix.length)
-        this.#acmeChallendMiddleware(req, res, token)
-      }
-    })
-
-    this.#app = app
-
-    httpServer.getSecureContext = this.getSecureContext.bind(this)
-  }
-
-  async getSecureContext(httpsDomainName, configKey, initialCert, initialKey) {
-    const config = this.#app.config.get(['http', 'listen', configKey])
-    const handlers = this.#handlers
-
-    const { acmeDomain } = config
-
-    // not a let's encrypt protected end point, sommething changed in the configuration
-    if (acmeDomain === undefined) {
-      handlers.delete(configKey)
-      return
-    }
-
-    // server has been access with another domain, don't use the certificate
-    if (acmeDomain !== httpsDomainName) {
-      return
-    }
-
-    let handler = handlers.get(configKey)
-    if (handler === undefined) {
-      // register the handler for this domain
-      handler = new SslCertificate(this.#challengeHandlers, initialCert, initialKey)
-      handlers.set(configKey, handler)
-    }
-    return handler.getSecureContext(config)
-  }
-
-  // middleware that will serve the http challenge to let's encrypt servers
-  #acmeChallendMiddleware(req, res, token) {
-    debug('fetching challenge for token ', token)
-    const challenge = this.#challenges.get(token)
-    debug('challenge content is ', challenge)
-    if (challenge === undefined) {
-      res.statusCode = 404
-      res.end()
-      return
-    }
-
-    res.write(challenge)
-    res.end()
-    debug('successfully answered challenge ')
-  }
-}

@xen-orchestra/mixins/docs/HttpProxy.md

@@ -10,11 +10,11 @@
 
 ## Set up
 
-The proxy is enabled by default, to disable it, add the following lines to your config:
+The proxy is disabled by default, to enable it, add the following lines to your config:
 
 ```toml
 [http.proxy]
-enabled = false
+enabled = true
 ```
 
 ## Usage

@xen-orchestra/mixins/docs/SslCertificate.md

@@ -1,49 +0,0 @@
-> This module provides [Let's Encrypt](https://letsencrypt.org/) integration to `xo-proxy` and `xo-server`.
-
-First of all, make sure your server is listening on HTTP on port 80 and on HTTPS 443.
-
-In `xo-server`, to avoid HTTP access, enable the redirection to HTTPs:
-
-```toml
-[http]
-redirectToHttps = true
-```
-
-Your server must be reachable with the configured domain to the certificate provider (e.g. Let's Encrypt), it usually means publicly reachable.
-
-Finally, add the following entries to your HTTPS configuration.
-
-```toml
-# Must be set to true for this feature
-autoCert = true
-
-# These entries are required and indicates where the certificate and the
-# private key will be saved.
-cert = 'path/to/cert.pem'
-key = 'path/to/key.pem'
-
-# ACME (e.g. Let's Encrypt, ZeroSSL) CA directory
-#
-# Specifies the URL to the ACME CA's directory.
-#
-# A identifier `provider/directory` can be passed instead of a URL, see the
-# list of supported directories here: https://www.npmjs.com/package/acme-client#directory-urls
-#
-# Note that the application cannot detect that this value has changed.
-#
-# In that case delete the certificate and the key files, and restart the
-# application to generate new ones.
-#
-# Default is 'letsencrypt/production'
-acmeCa = 'zerossl/production'
-
-# Domain for which the certificate should be created.
-#
-# This entry is required.
-acmeDomain = 'my.domain.net'
-
-# Optional email address which will be used for the certificate creation.
-#
-# It will be notified of any issues.
-acmeEmail = 'admin@my.domain.net'
-```

@xen-orchestra/mixins/package.json

@@ -14,17 +14,16 @@
     "url": "https://vates.fr"
   },
   "license": "AGPL-3.0-or-later",
-  "version": "0.8.0",
+  "version": "0.5.0",
   "engines": {
-    "node": ">=15.6"
+    "node": ">=12"
   },
   "dependencies": {
     "@vates/event-listeners-manager": "^1.0.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/emit-async": "^1.0.0",
     "@xen-orchestra/log": "^0.3.0",
-    "acme-client": "^5.0.0",
-    "app-conf": "^2.3.0",
+    "app-conf": "^2.1.0",
     "lodash": "^4.17.21",
     "promise-toolbox": "^0.21.0"
   },

@xen-orchestra/proxy-cli/package.json

@@ -28,7 +28,7 @@
     "@iarna/toml": "^2.2.0",
     "@vates/read-chunk": "^1.0.0",
     "ansi-colors": "^4.1.1",
-    "app-conf": "^2.3.0",
+    "app-conf": "^2.1.0",
     "content-type": "^1.0.4",
     "cson-parser": "^4.0.7",
     "getopts": "^2.2.3",

@xen-orchestra/proxy/app/index.mjs

@@ -1,7 +1,6 @@
 import Config from '@xen-orchestra/mixins/Config.mjs'
 import Hooks from '@xen-orchestra/mixins/Hooks.mjs'
 import HttpProxy from '@xen-orchestra/mixins/HttpProxy.mjs'
-import SslCertificate from '@xen-orchestra/mixins/SslCertificate.mjs'
 import mixin from '@xen-orchestra/mixin'
 import { createDebounceResource } from '@vates/disposable/debounceResource.js'
 
@@ -15,23 +14,9 @@ import ReverseProxy from './mixins/reverseProxy.mjs'
 
 export default class App {
   constructor(opts) {
-    mixin(
-      this,
-      {
-        Api,
-        Appliance,
-        Authentication,
-        Backups,
-        Config,
-        Hooks,
-        HttpProxy,
-        Logs,
-        Remotes,
-        ReverseProxy,
-        SslCertificate,
-      },
-      [opts]
-    )
+    mixin(this, { Api, Appliance, Authentication, Backups, Config, Hooks, HttpProxy, Logs, Remotes, ReverseProxy }, [
+      opts,
+    ])
 
     const debounceResource = createDebounceResource()
     this.config.watchDuration('resourceCacheDelay', delay => {

@xen-orchestra/proxy/app/mixins/api.mjs

@@ -1,4 +1,4 @@
-import { format, parse, MethodNotFound, JsonRpcError } from 'json-rpc-protocol'
+import { format, parse, MethodNotFound } from 'json-rpc-protocol'
 import * as errors from 'xo-common/api-errors.js'
 import Ajv from 'ajv'
 import asyncIteratorToStream from 'async-iterator-to-stream'
@@ -9,26 +9,11 @@ import helmet from 'koa-helmet'
 import Koa from 'koa'
 import once from 'lodash/once.js'
 import Router from '@koa/router'
-import stubTrue from 'lodash/stubTrue.js'
 import Zone from 'node-zone'
 import { createLogger } from '@xen-orchestra/log'
 
 const { debug, warn } = createLogger('xo:proxy:api')
 
-// format an error to JSON-RPC but do not hide non JSON-RPC errors
-function formatError(responseId, error) {
-  if (error != null && typeof error.toJsonRpcError !== 'function') {
-    const { message, ...data } = error
-
-    // force these entries even if they are not enumerable
-    data.code = error.code
-    data.stack = error.stack
-
-    error = new JsonRpcError(error.message, undefined, data)
-  }
-  return format.error(responseId, error)
-}
-
 const ndJsonStream = asyncIteratorToStream(async function* (responseId, iterable) {
   try {
     let cursor, iterator
@@ -39,7 +24,7 @@ const ndJsonStream = asyncIteratorToStream(async function* (responseId, iterable
       cursor = await iterator.next()
       yield format.response(responseId, { $responseType: 'ndjson' }) + '\n'
     } catch (error) {
-      yield formatError(responseId, error)
+      yield format.error(responseId, error)
       throw error
     }
 
@@ -78,7 +63,7 @@ export default class Api {
       try {
         body = parse(body)
       } catch (error) {
-        ctx.body = formatError(null, error)
+        ctx.body = format.error(null, error)
         return
       }
 
@@ -92,7 +77,7 @@ export default class Api {
         const { method, params } = body
         warn('call error', { method, params, error })
         ctx.set('Content-Type', 'application/json')
-        ctx.body = formatError(body.id, error)
+        ctx.body = format.error(body.id, error)
         return
       }
 
@@ -181,20 +166,14 @@ export default class Api {
               throw errors.noSuchObject('method', name)
             }
 
-            const { description, params = {}, result = {} } = method
-            return { description, name, params, result }
+            const { description, params = {} } = method
+            return { description, name, params }
           },
           {
             description: 'returns the signature of an API method',
             params: {
               method: { type: 'string' },
             },
-            result: {
-              description: { type: 'string' },
-              name: { type: 'string' },
-              params: { type: 'object' },
-              result: { type: 'object' },
-            },
           },
         ],
       },
@@ -226,29 +205,40 @@ export default class Api {
     })
   }
 
-  addMethod(name, method, { description, params = {}, result: resultSchema } = {}) {
+  addMethod(name, method, { description, params = {} } = {}) {
     const methods = this._methods
 
     if (name in methods) {
       throw new Error(`API method ${name} already exists`)
     }
 
-    const validateParams = this.#compileSchema(params)
-    const validateResult = this.#compileSchema(resultSchema)
+    const ajv = this._ajv
+    const validate = ajv.compile({
+      // we want additional properties to be disabled by default
+      additionalProperties: params['*'] || false,
 
-    const m = async params => {
-      if (!validateParams(params)) {
-        throw errors.invalidParameters(validateParams.errors)
+      properties: params,
+
+      // we want params to be required by default unless explicitly marked so
+      // we use property `optional` instead of object `required`
+      required: Object.keys(params).filter(name => {
+        const param = params[name]
+        const required = !param.optional
+        delete param.optional
+        return required
+      }),
+
+      type: 'object',
+    })
+
+    const m = params => {
+      if (!validate(params)) {
+        throw errors.invalidParameters(validate.errors)
       }
-      const result = await method(params)
-      if (!validateResult(result)) {
-        warn('invalid API method result', { errors: validateResult.error, result })
-      }
-      return result
+      return method(params)
     }
     m.description = description
     m.params = params
-    m.result = resultSchema
 
     methods[name] = m
 
@@ -299,43 +289,4 @@ export default class Api {
     }
     return fn(params)
   }
-
-  #compileSchema(schema) {
-    if (schema === undefined) {
-      return stubTrue
-    }
-
-    if (schema.type === undefined) {
-      schema = { type: 'object', properties: schema }
-    }
-
-    const { type } = schema
-    if (Array.isArray(type) ? type.includes('object') : type === 'object') {
-      const { properties = {} } = schema
-
-      if (schema.additionalProperties === undefined) {
-        const wildCard = properties['*']
-        if (wildCard === undefined) {
-          // we want additional properties to be disabled by default
-          schema.additionalProperties = false
-        } else {
-          delete properties['*']
-          schema.additionalProperties = wildCard
-        }
-      }
-
-      // we want properties to be required by default unless explicitly marked so
-      // we use property `optional` instead of object `required`
-      if (schema.required === undefined) {
-        schema.required = Object.keys(properties).filter(name => {
-          const param = properties[name]
-          const required = !param.optional
-          delete param.optional
-          return required
-        })
-      }
-    }
-
-    return this._ajv.compile(schema)
-  }
 }

@xen-orchestra/proxy/app/mixins/backups.mjs

@@ -407,7 +407,6 @@ export default class Backups {
       debounceResource: app.debounceResource.bind(app),
       dirMode: app.config.get('backups.dirMode'),
       vhdDirectoryCompression: app.config.get('backups.vhdDirectoryCompression'),
-      useGetDiskLegacy: app.config.getOptional('backups.useGetDiskLegacy'),
     })
   }
 

@xen-orchestra/proxy/index.mjs

@@ -56,32 +56,11 @@ ${APP_NAME} v${APP_VERSION}
     createSecureServer: opts => createSecureServer({ ...opts, allowHTTP1: true }),
   })
 
-  forOwn(config.http.listen, async ({ autoCert, cert, key, ...opts }, configKey) => {
-    const useAcme = autoCert && opts.acmeDomain !== undefined
-
-    // don't pass these entries to httpServer.listen(opts)
-    for (const key of Object.keys(opts).filter(_ => _.startsWith('acme'))) {
-      delete opts[key]
-    }
-
+  forOwn(config.http.listen, async ({ autoCert, cert, key, ...opts }) => {
     try {
-      let niceAddress
-      if (cert !== undefined && key !== undefined) {
-        if (useAcme) {
-          opts.SNICallback = async (serverName, callback) => {
-            try {
-              // injected by mixins/SslCertificate
-              const secureContext = await httpServer.getSecureContext(serverName, configKey, opts.cert, opts.key)
-              callback(null, secureContext)
-            } catch (error) {
-              warn(error)
-              callback(error, null)
-            }
-          }
-        }
-
-        niceAddress = await pRetry(
-          async () => {
+      const niceAddress = await pRetry(
+        async () => {
+          if (cert !== undefined && key !== undefined) {
             try {
               opts.cert = fse.readFileSync(cert)
               opts.key = fse.readFileSync(key)
@@ -97,22 +76,20 @@ ${APP_NAME} v${APP_VERSION}
               opts.cert = pems.cert
               opts.key = pems.key
             }
-
-            return httpServer.listen(opts)
-          },
-          {
-            tries: 2,
-            when: e => autoCert && e.code === 'ERR_SSL_EE_KEY_TOO_SMALL',
-            onRetry: () => {
-              warn('deleting invalid certificate')
-              fse.unlinkSync(cert)
-              fse.unlinkSync(key)
-            },
           }
-        )
-      } else {
-        niceAddress = await httpServer.listen(opts)
-      }
+
+          return httpServer.listen(opts)
+        },
+        {
+          tries: 2,
+          when: e => autoCert && e.code === 'ERR_SSL_EE_KEY_TOO_SMALL',
+          onRetry: () => {
+            warn('deleting invalid certificate')
+            fse.unlinkSync(cert)
+            fse.unlinkSync(key)
+          },
+        }
+      )
 
       info(`Web server listening on ${niceAddress}`)
     } catch (error) {
@@ -169,7 +146,6 @@ ${APP_NAME} v${APP_VERSION}
     process.on(signal, () => {
       if (alreadyCalled) {
         warn('forced exit')
-        // eslint-disable-next-line n/no-process-exit
         process.exit(1)
       }
       alreadyCalled = true
@@ -188,7 +164,6 @@ main(process.argv.slice(2)).then(
   error => {
     fatal(error)
 
-    // eslint-disable-next-line n/no-process-exit
     process.exit(1)
   }
 )

@xen-orchestra/proxy/package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "@xen-orchestra/proxy",
-  "version": "0.26.2",
+  "version": "0.23.4",
   "license": "AGPL-3.0-or-later",
   "description": "XO Proxy used to remotely execute backup jobs",
   "keywords": [
@@ -26,33 +26,33 @@
   },
   "dependencies": {
     "@iarna/toml": "^2.2.0",
-    "@koa/router": "^12.0.0",
+    "@koa/router": "^10.0.0",
     "@vates/cached-dns.lookup": "^1.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.0.0",
     "@vates/disposable": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.28.0",
-    "@xen-orchestra/fs": "^3.1.0",
+    "@xen-orchestra/backups": "^0.26.0",
+    "@xen-orchestra/fs": "^1.1.0",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/mixin": "^0.1.0",
-    "@xen-orchestra/mixins": "^0.8.0",
+    "@xen-orchestra/mixins": "^0.5.0",
     "@xen-orchestra/self-signed": "^0.1.3",
-    "@xen-orchestra/xapi": "^1.5.0",
+    "@xen-orchestra/xapi": "^1.4.0",
     "ajv": "^8.0.3",
-    "app-conf": "^2.3.0",
+    "app-conf": "^2.1.0",
     "async-iterator-to-stream": "^1.1.0",
     "fs-extra": "^10.0.0",
     "get-stream": "^6.0.0",
     "getopts": "^2.2.3",
     "golike-defer": "^0.5.1",
-    "http-server-plus": "^0.12.0",
+    "http-server-plus": "^0.11.1",
     "http2-proxy": "^5.0.53",
     "json-rpc-protocol": "^0.13.1",
     "jsonrpc-websocket-client": "^0.7.2",
     "koa": "^2.5.1",
     "koa-compress": "^5.0.1",
-    "koa-helmet": "^6.1.0",
+    "koa-helmet": "^5.1.0",
     "lodash": "^4.17.10",
     "node-zone": "^0.4.0",
     "parse-pairs": "^1.0.0",
@@ -60,7 +60,7 @@
     "source-map-support": "^0.5.16",
     "stoppable": "^1.0.6",
     "xdg-basedir": "^5.1.0",
-    "xen-api": "^1.2.2",
+    "xen-api": "^1.2.1",
     "xo-common": "^0.8.0"
   },
   "devDependencies": {

@xen-orchestra/upload-ova/package.json

@@ -43,7 +43,7 @@
     "pw": "^0.0.4",
     "xdg-basedir": "^4.0.0",
     "xo-lib": "^0.11.1",
-    "xo-vmdk-to-vhd": "^2.4.3"
+    "xo-vmdk-to-vhd": "^2.4.2"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

@xen-orchestra/xapi/docs/vm-sync-hook.md

@@ -1,131 +0,0 @@
-# VM Sync Hook
-
-> This feature is currently _unstable_ and might change or be removed in the future.
->
-> Feedbacks are very welcome on the [project bugtracker](https://github.com/vatesfr/xen-orchestra/issues).
-
-> This feature is not currently supported for backups done with XO Proxy.
-
-Before snapshotting (with or without memory, ie checkpoint), XO can notify the VM via an HTTP request.
-
-A typical use case is to make sure the VM is in a consistent state during the snapshot process, for instance by making sure database writes are flushed to the disk.
-
-> This request will only be sent if the VM is in a running state.
-
-## Configuration
-
-The feature is opt-in via a tag on the VM: `xo:notify-on-snapshot`.
-
-By default, it will be an HTTPS request on the port `1727`, on the first IP address reported by the VM.
-
-If the _VM Tools_ (i.e. management agent) are not installed on the VM or if you which to use another URL, you can specify this in the tag: `xo:notify-on-snapshot=<URL>`.
-
-To guarantee the request comes from XO, a secret must be provided in the `xo-server`'s (and `xo-proxy` if relevant) configuration:
-
-```toml
-[xapiOptions]
-syncHookSecret = 'unique long string to ensure the request comes from XO'
-```
-
-## Specification
-
-XO will waits for the request to be answered before starting the snapshot, but will not wait longer than _1 minute_.
-
-If the request fails for any reason, XO will go ahead with snapshot immediately.
-
-```http
-GET /sync HTTP/1.1
-Authorization: Bearer dW5pcXVlIGxvbmcgc3RyaW5nIHRvIGVuc3VyZSB0aGUgcmVxdWVzdCBjb21lcyBmcm9tIFhP
-```
-
-When the snapshot is finished, another request will be sent:
-
-```http
-GET /post-sync HTTP/1.1
-Authorization: Bearer dW5pcXVlIGxvbmcgc3RyaW5nIHRvIGVuc3VyZSB0aGUgcmVxdWVzdCBjb21lcyBmcm9tIFhP
-```
-
-The created snapshot will have the special `xo:synced` tag set to make it identifiable.
-
-## Example server in Node
-
-`index.cjs`:
-
-```js
-const exec = require('node:util').promisify(require('node:child_process').execFile)
-
-const SECRET = 'unique long string to ensure the request comes from XO'
-
-const HANDLERS = {
-  __proto__: null,
-
-  async '/sync'() {
-    // actions to do before the VM is snapshotted
-
-    // in this example, the Linux command `sync` is called:
-    await exec('sync')
-  },
-
-  async '/post-sync'() {
-    // actions to do after the VM is snapshotted
-  },
-}
-
-function checkAuthorization(req) {
-  try {
-    const { authorization } = req.headers
-    if (authorization !== undefined) {
-      const parts = authorization.split('  ')
-      if (parts.length >= 1 && parts[0].toLowerCase() === 'bearer') {
-        return Buffer.from(parts[1], 'base64').toString() === SECRET
-      }
-    }
-  } catch (error) {
-    console.warn('checkAuthorization', error)
-  }
-  return false
-}
-
-async function main() {
-  // generate a self-signed certificate
-  const [, key, cert] =
-    /^(-----BEGIN PRIVATE KEY-----.+-----END PRIVATE KEY-----\n)(-----BEGIN CERTIFICATE-----.+-----END CERTIFICATE-----\n)$/s.exec(
-      (await exec('openssl', ['req', '-batch', '-new', '-x509', '-nodes', '-newkey', 'rsa:2048', '-keyout', '-']))
-        .stdout
-    )
-
-  const server = require('node:https').createServer({ cert, key }, async function onRequest(req, res) {
-    if (!checkAuthorization(req)) {
-      res.statusCode = 403
-      return res.end('Forbidden')
-    }
-
-    const handler = HANDLERS[req.url.split('?')[0]]
-    if (handler === undefined || req.method !== 'GET') {
-      res.statusCode = 404
-      return res.end('Not Found')
-    }
-
-    try {
-      await handler()
-
-      res.statusCode = 200
-      res.end('Ok')
-    } catch (error) {
-      console.warn(error)
-
-      if (!res.headersSent) {
-        res.statusCode = 500
-        res.write('Internal Error')
-      }
-      res.end()
-    }
-  })
-
-  await new Promise((resolve, reject) => {
-    server.on('close', resolve).on('error', reject).listen(1727)
-  })
-}
-
-main().catch(console.warn)
-```

@xen-orchestra/xapi/index.js

@@ -102,8 +102,6 @@ class Xapi extends Base {
   constructor({
     callRetryWhenTooManyPendingTasks = { delay: 5e3, tries: 10 },
     maxUncoalescedVdis,
-    syncHookSecret,
-    syncHookTimeout,
     vdiDestroyRetryWhenInUse = { delay: 5e3, tries: 10 },
     ...opts
   }) {
@@ -114,8 +112,6 @@ class Xapi extends Base {
       when: { code: 'TOO_MANY_PENDING_TASKS' },
     }
     this._maxUncoalescedVdis = maxUncoalescedVdis
-    this._syncHookSecret = syncHookSecret
-    this._syncHookTimeout = syncHookTimeout
     this._vdiDestroyRetryWhenInUse = {
       ...vdiDestroyRetryWhenInUse,
       onRetry,

@xen-orchestra/xapi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/xapi",
-  "version": "1.5.0",
+  "version": "1.4.0",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/xapi",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
@@ -15,7 +15,7 @@
     "node": ">=14"
   },
   "peerDependencies": {
-    "xen-api": "^1.2.2"
+    "xen-api": "^1.2.1"
   },
   "scripts": {
     "postversion": "npm publish --access public"
@@ -26,11 +26,10 @@
     "@xen-orchestra/log": "^0.3.0",
     "d3-time-format": "^3.0.0",
     "golike-defer": "^0.5.1",
-    "http-request-plus": "^0.14.0",
     "json-rpc-protocol": "^0.13.2",
     "lodash": "^4.17.15",
     "promise-toolbox": "^0.21.0",
-    "vhd-lib": "^4.1.0",
+    "vhd-lib": "^3.3.1",
     "xo-common": "^0.8.0"
   },
   "private": false,

@xen-orchestra/xapi/sr.js

@@ -4,8 +4,7 @@ const { asyncMap, asyncMapSettled } = require('@xen-orchestra/async-map')
 const { decorateClass } = require('@vates/decorate-with')
 const { defer } = require('golike-defer')
 const { incorrectState } = require('xo-common/api-errors')
-const { VDI_FORMAT_VHD } = require('./index.js')
-const assert = require('node:assert').strict
+const { VDI_FORMAT_RAW } = require('./index.js')
 const peekFooterFromStream = require('vhd-lib/peekFooterFromVhdStream')
 
 const AggregateError = require('./_AggregateError.js')
@@ -151,26 +150,12 @@ class Sr {
     $defer,
     ref,
     stream,
-    {
-      format = VDI_FORMAT_VHD,
-      name_label = '[XO] Imported disk - ' + new Date().toISOString(),
-      virtual_size,
-      ...vdiCreateOpts
-    } = {}
+    { name_label = '[XO] Imported disk - ' + new Date().toISOString(), ...vdiCreateOpts } = {}
   ) {
-    if (virtual_size === undefined) {
-      if (format === VDI_FORMAT_VHD) {
-        const footer = await peekFooterFromStream(stream)
-        virtual_size = footer.currentSize
-      } else {
-        virtual_size = stream.length
-        assert.notEqual(virtual_size, undefined)
-      }
-    }
-
-    const vdiRef = await this.VDI_create({ ...vdiCreateOpts, name_label, SR: ref, virtual_size })
+    const footer = await peekFooterFromStream(stream)
+    const vdiRef = await this.VDI_create({ ...vdiCreateOpts, name_label, SR: ref, virtual_size: footer.currentSize })
     $defer.onFailure.call(this, 'callAsync', 'VDI.destroy', vdiRef)
-    await this.VDI_importContent(vdiRef, stream, { format })
+    await this.VDI_importContent(vdiRef, stream, { format: VDI_FORMAT_RAW })
     return vdiRef
   }
 }

@xen-orchestra/xapi/vdi.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const assert = require('node:assert').strict
 const CancelToken = require('promise-toolbox/CancelToken')
 const pCatch = require('promise-toolbox/catch')
 const pRetry = require('promise-toolbox/retry')
@@ -87,8 +86,6 @@ class Vdi {
   }
 
   async importContent(ref, stream, { cancelToken = CancelToken.none, format }) {
-    assert.notEqual(format, undefined)
-
     if (stream.length === undefined) {
       throw new Error('Trying to import a VDI without a length field. Please report this error to Xen Orchestra.')
     }

@xen-orchestra/xapi/vm.js

@@ -2,7 +2,6 @@
 
 const CancelToken = require('promise-toolbox/CancelToken')
 const groupBy = require('lodash/groupBy.js')
-const hrp = require('http-request-plus')
 const ignoreErrors = require('promise-toolbox/ignoreErrors')
 const pickBy = require('lodash/pickBy.js')
 const omit = require('lodash/omit.js')
@@ -47,31 +46,6 @@ const cleanBiosStrings = biosStrings => {
   }
 }
 
-// See: https://github.com/xapi-project/xen-api/blob/324bc6ee6664dd915c0bbe57185f1d6243d9ed7e/ocaml/xapi/xapi_guest_agent.ml#L59-L81
-//
-// Returns <min(n)>/ip || <min(n)>/ipv4/<min(m)> || <min(n)>/ipv6/<min(m)> || undefined
-// where n corresponds to the network interface and m to its IP
-const IPV4_KEY_RE = /^\d+\/ip(?:v4\/\d+)?$/
-const IPV6_KEY_RE = /^\d+\/ipv6\/\d+$/
-function getVmAddress(networks) {
-  if (networks !== undefined) {
-    let ipv6
-    for (const key of Object.keys(networks).sort()) {
-      if (IPV4_KEY_RE.test(key)) {
-        return networks[key]
-      }
-
-      if (ipv6 === undefined && IPV6_KEY_RE.test(key)) {
-        ipv6 = networks[key]
-      }
-    }
-    if (ipv6 !== undefined) {
-      return ipv6
-    }
-  }
-  throw new Error('no VM address found')
-}
-
 async function listNobakVbds(xapi, vbdRefs) {
   const vbds = []
   await asyncMap(vbdRefs, async vbdRef => {
@@ -158,51 +132,6 @@ class Vm {
     }
   }
 
-  async _httpHook({ guest_metrics, power_state, tags, uuid }, pathname) {
-    if (power_state !== 'Running') {
-      return
-    }
-
-    let url
-    let i = tags.length
-    do {
-      if (i === 0) {
-        return
-      }
-      const tag = tags[--i]
-      if (tag === 'xo:notify-on-snapshot') {
-        const { networks } = await this.getRecord('VM_guest_metrics', guest_metrics)
-        url = Object.assign(new URL('https://locahost'), {
-          hostname: getVmAddress(networks),
-          port: 1727,
-        })
-      } else {
-        const prefix = 'xo:notify-on-snapshot='
-        if (tag.startsWith(prefix)) {
-          url = new URL(tag.slice(prefix.length))
-        }
-      }
-    } while (url === undefined)
-
-    url.pathname = pathname
-
-    const headers = {}
-    const secret = this._asyncHookSecret
-    if (secret !== undefined) {
-      headers.authorization = 'Bearer ' + Buffer.from(secret).toString('base64')
-    }
-
-    try {
-      await hrp(url, {
-        headers,
-        rejectUnauthorized: false,
-        timeout: this._syncHookTimeout ?? 60e3,
-      })
-    } catch (error) {
-      warn('HTTP hook failed', { error, url, vm: uuid })
-    }
-  }
-
   async assertHealthyVdiChains(vmRef, tolerance = this._maxUncoalescedVdis) {
     const vdiRefs = {}
     ;(await this.getRecords('VBD', await this.getField('VM', vmRef, 'VBDs'))).forEach(({ VDI: ref }) => {
@@ -219,8 +148,6 @@ class Vm {
   async checkpoint($defer, vmRef, { cancelToken = CancelToken.none, ignoreNobakVdis = false, name_label } = {}) {
     const vm = await this.getRecord('VM', vmRef)
 
-    await this._httpHook(vm, '/sync')
-
     let destroyNobakVdis = false
 
     if (ignoreNobakVdis) {
@@ -241,9 +168,6 @@ class Vm {
     try {
       const ref = await this.callAsync(cancelToken, 'VM.checkpoint', vmRef, name_label).then(extractOpaqueRef)
 
-      // detached async
-      this._httpHook(vm, '/post-sync').catch(noop)
-
       // VM checkpoints are marked as templates, unfortunately it does not play well with XVA export/import
       // which will import them as templates and not VM checkpoints or plain VMs
       await pCatch.call(
@@ -620,8 +544,6 @@ class Vm {
   ) {
     const vm = await this.getRecord('VM', vmRef)
 
-    await this._httpHook(vm, '/sync')
-
     const isHalted = vm.power_state === 'Halted'
 
     // requires the VM to be halted because it's not possible to re-plug VUSB on a live VM
@@ -724,9 +646,6 @@ class Vm {
       ref = await this.callAsync(cancelToken, 'VM.snapshot', vmRef, name_label).then(extractOpaqueRef)
     } while (false)
 
-    // detached async
-    this._httpHook(vm, '/post-sync').catch(noop)
-
     // VM snapshots are marked as templates, unfortunately it does not play well with XVA export/import
     // which will import them as templates and not VM snapshots or plain VMs
     await pCatch.call(

CHANGELOG.unreleased.md

@@ -5,8 +5,14 @@
 
 ### Enhancements
 
+> Users must be able to say: “Nice enhancement, I'm eager to test it”
+
+- [SR] When SR is in maintenance, add "Maintenance mode" badge next to its name (PR [#6313](https://github.com/vatesfr/xen-orchestra/pull/6313))
+
 ### Bug fixes
 
+> Users must be able to say: “I had this issue, happy to know it's fixed”
+
 ### Packages to release
 
 > When modifying a package, add it here with its release type.
@@ -23,4 +29,8 @@
 
 <!--packages-start-->
 
+- @xen-orchestra/backups minor
+- xo-web minor
+- vhd-lib minor
+
 <!--packages-end-->

SECURITY.md

@@ -4,11 +4,11 @@
 
 We apply patches and fix security issues for the following versions:
 
-| Version         | Supported          |
-| --------------- | ------------------ |
-| XOA `latest`    | :white_check_mark: |
-| XOA `stable`    | :white_check_mark: |
-| `master` branch | :white_check_mark: |
+| Version | Supported          |
+| ------- | ------------------ |
+| XOA `latest`     | :white_check_mark: |
+| XOA `stable`     | :white_check_mark: |
+| `master` branch  | :white_check_mark: |
 | anything else   | :x:                |
 
 ## Reporting a Vulnerability

docker/Dockerfile

@@ -4,12 +4,12 @@ FROM ubuntu:xenial
 # https://qastack.fr/programming/25899912/how-to-install-nvm-in-docker
 
 RUN apt-get update
-RUN apt-get install -y curl qemu-utils blktap-utils vmdk-stream-converter git libxml2-utils nbdkit g++
+RUN apt-get install -y curl qemu-utils blktap-utils vmdk-stream-converter git libxml2-utils
 ENV NVM_DIR /usr/local/nvm
 RUN mkdir -p /usr/local/nvm
 RUN cd /usr/local/nvm
 RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
-ENV NODE_VERSION v16.10.0
+ENV NODE_VERSION v17.0.1
 RUN /bin/bash -c "source $NVM_DIR/nvm.sh && nvm install $NODE_VERSION && nvm use --delete-prefix $NODE_VERSION"
 
 ENV NODE_PATH $NVM_DIR/versions/node/$NODE_VERSION/lib/node_modules

docs/backup_troubleshooting.md

@@ -109,13 +109,10 @@ As a temporary workaround you can increase the timeout higher than the default v
 :::
 
 Create the following file:
-
 ```
 /etc/xo-server/config.httpInactivityTimeout.toml
 ```
-
 Add the following lines:
-
 ```
 # XOA Support - Work-around HTTP timeout issue during backups
 [xapiOptions]

docs/configuration.md

@@ -28,7 +28,7 @@ hostname = '0.0.0.0'
 port = 80
 ```
 
-## HTTPS and certificates
+## HTTPS
 
 XO-server can also run in HTTPS (you can run HTTP and HTTPS at the same time) - just modify what's needed in the `# Basic HTTPS` section, this time with the certificates/keys you need and their path:
 

docs/contributing.md

@@ -24,15 +24,16 @@ Please, do explain:
 The best way to propose a change to the documentation or code is
 to create a [GitHub pull request](https://help.github.com/articles/using-pull-requests/).
 
-1. Fork the [Xen Orchestra repository](https://github.com/vatesfr/xen-orchestra) using the Fork button
-2. Follow [the documentation](installation.md#from-the-sources) to install and run Xen Orchestra from the sources
-3. Create a branch for your work
-4. Edit the source files
-5. Add a summary of your changes to `CHANGELOG.unreleased.md`, if your changes do not relate to an existing changelog item and update the list of packages that must be released to take your changes into account
-6. [Create a pull request](https://github.com/vatesfr/xen-orchestra/compare) for this branch against the `master` branch
-7. Push into the branch until the pull request is ready to merge
-8. Avoid unnecessary merges: keep you branch up to date by regularly rebasing `git rebase origin/master`
-9. When ready to merge, clean up the history (reorder commits, squash some of them together, rephrase messages): `git rebase -i origin/master`
+:::tip
+Your pull request should always be against the `master` branch and not against `stable` which is the stable branch!
+:::
+
+1. Create a branch for your work
+2. Add a summary of your changes to `CHANGELOG.md` under the `next` section, if your changes do not relate to an existing changelog item
+3. Create a pull request for this branch against the `master` branch
+4. Push into the branch until the pull request is ready to merge
+5. Avoid unnecessary merges: keep you branch up to date by regularly rebasing `git rebase origin/master`
+6. When ready to merge, clean up the history (reorder commits, squash some of them together, rephrase messages): `git rebase -i origin/master`
 
 ### Issue triage
 

docs/restapi.md

@@ -143,14 +143,14 @@ curl \
 
 ## VDI Import
 
-A VHD or a raw export can be imported on an SR to create a new VDI at `/rest/v0/srs/<sr uuid>/vdis`.
+A VHD can be imported on an SR to create a VDI at `/rest/v0/srs/<sr uuid>/vdis`.
 
 ```bash
 curl \
   -X POST \
   -b authenticationToken=KQxQdm2vMiv7jBIK0hgkmgxKzemd8wSJ7ugFGKFkTbs \
-  -T myDisk.raw \
-  'https://xo.example.org/rest/v0/srs/357bd56c-71f9-4b2a-83b8-3451dec04b8f/vdis?raw&name_label=my_imported_VDI' \
+  -T myDisk.vhd \
+  'https://xo.example.org/rest/v0/srs/357bd56c-71f9-4b2a-83b8-3451dec04b8f/vdis?name_label=my_imported_VDI' \
   | cat
 ```
 
@@ -162,7 +162,6 @@ The following query parameters are supported to customize the created VDI:
 
 - `name_label`
 - `name_description`
-- `raw`: this parameter must be used if importing a raw export instead of a VHD
 
 ## The future
 

docs/sdn_controller.md

@@ -15,7 +15,7 @@ Please read the [dedicated devblog on the SDN Controller](https://xen-orchestra.
 :::warning
 As VxLAN and GRE are protocols using extra encapsulation, they require extra bits on a network packet. If you create a Global Private Network with a default MTU at `1500`, you won't be able to use it "as is" in your VMs, unless you configure a smaller MTU for each virtual interface, in your VM operating system (eg: `1400`).
 
-If you want something entirely transparent for your VMs, then you'll need to create a network with a MTU of `1546` for GRE or `1550` for VxLAN. However, larger MTU will require capable network equipments.
+If you want something entirely transparent for your VMs, then you'll need to create a network with a MTU of `1546` for GRE or `1550` for VxLAN. However, larger MTU will require capable network equipements.
 :::
 
 ### Network creation

docs/users.md

@@ -109,6 +109,10 @@ In the "Settings" then "Plugins" view, expand the SAML plugin configuration. The
 
 Save the configuration and then activate the plugin (button on top).
 
+:::warning
+When registering your instance to your identity provider, you must configure its callback URL to `http://xo.example.net/signin/saml/callback`!
+:::
+
 ### GitHub
 
 This plugin allows GitHub users to authenticate to Xen-Orchestra.

docs/xoa.md

@@ -206,23 +206,6 @@ In any case, if you lose your password, you can reset the database and get the d
 
 You can verify that your time is correctly set with the `date` command. To set XOA to your current timezone, use `sudo dpkg-reconfigure tzdata`.
 
-## Setting a custom NTP server
-
-By default, XOA is configured to use the standard Debian NTP servers:
-
-```
-pool 0.debian.pool.ntp.org iburst
-pool 1.debian.pool.ntp.org iburst
-pool 2.debian.pool.ntp.org iburst
-pool 3.debian.pool.ntp.org iburst
-```
-
-If you'd like to use your own NTP server or another pool, you can make the changes directly in `/etc/ntp.conf`.
-
-You will need to be root to edit this file (or use `sudo`). We recommend adding your custom server to the top of the list, leaving the debian server entries if possible.
-
-For changes to take effect, you will need to restart NTP: `sudo systemctl restart ntp.service`.
-
 ## Restart the service
 
 You can restart Xen Orchestra by accessing XOA via SSH (or console) and running `systemctl restart xo-server.service`.

package.json

@@ -3,7 +3,7 @@
     "@babel/core": "^7.0.0",
     "@babel/eslint-parser": "^7.13.8",
     "@babel/register": "^7.0.0",
-    "babel-jest": "^29.0.3",
+    "babel-jest": "^27.3.1",
     "benchmark": "^2.1.4",
     "deptree": "^1.0.0",
     "eslint": "^8.7.0",
@@ -19,8 +19,8 @@
     "globby": "^13.1.1",
     "handlebars": "^4.7.6",
     "husky": "^4.2.5",
-    "jest": "^29.0.3",
-    "lint-staged": "^13.0.3",
+    "jest": "^27.3.1",
+    "lint-staged": "^12.0.3",
     "lodash": "^4.17.4",
     "prettier": "^2.0.5",
     "promise-toolbox": "^0.21.0",
@@ -77,7 +77,7 @@
   "private": true,
   "scripts": {
     "build": "scripts/run-script.js --parallel build",
-    "ci": "yarn && yarn build &&  yarn test-integration",
+    "ci": "yarn && yarn test-integration",
     "clean": "scripts/run-script.js --parallel clean",
     "dev": "scripts/run-script.js --parallel dev",
     "dev-test": "jest --bail --watch \"^(?!.*\\.integ\\.spec\\.js$)\"",

packages/complex-matcher/index.js

@@ -253,7 +253,7 @@ class RegExpNode extends Node {
     return this.re.toString()
   }
 }
-exports.RegExp = exports.RegExpNode = RegExpNode
+exports.RegExp = RegExpNode
 
 class StringNode extends Node {
   constructor(value) {

packages/vhd-cli/commands/info.js

@@ -47,7 +47,6 @@ async function showDetails(handler, path) {
   }
 
   console.log(
-    'footer:',
     mapProperties(vhd.footer, {
       currentSize: 'bytes',
       diskType: 'diskType',
@@ -58,7 +57,6 @@ async function showDetails(handler, path) {
   )
 
   console.log(
-    'header:',
     mapProperties(vhd.header, {
       blockSize: 'bytes',
       parentTimestamp: 'date',

packages/vhd-cli/commands/merge.js

@@ -1,8 +1,8 @@
 'use strict'
 
 const { Bar } = require('cli-progress')
+const { mergeVhd } = require('vhd-lib')
 const { getHandler } = require('@xen-orchestra/fs')
-const { mergeVhdChain } = require('vhd-lib/merge')
 const { resolve } = require('path')
 
 module.exports = async function merge(args) {
@@ -12,7 +12,7 @@ module.exports = async function merge(args) {
 
   const handler = getHandler({ url: 'file:///' })
   let bar
-  await mergeVhdChain(handler, [resolve(args[1]), resolve(args[0])], {
+  await mergeVhd(handler, resolve(args[1]), handler, resolve(args[0]), {
     onProgress({ done, total }) {
       if (bar === undefined) {
         bar = new Bar({

packages/vhd-cli/info.integ.spec.js

@@ -1,13 +1,11 @@
-'use strict'
-
 /* eslint-env jest */
 
-const execa = require('execa')
-const rimraf = require('rimraf')
-const tmp = require('tmp')
-const { pFromCallback } = require('promise-toolbox')
+import execa from 'execa'
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { pFromCallback } from 'promise-toolbox'
 
-const command = require('./commands/info')
+import command from './commands/info'
 
 const initialDir = process.cwd()
 

packages/vhd-cli/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "vhd-cli",
-  "version": "0.9.1",
+  "version": "0.8.0",
   "license": "ISC",
   "description": "Tools to read/create and merge VHD files",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-cli",
@@ -23,22 +23,17 @@
     "node": ">=10"
   },
   "dependencies": {
-    "@xen-orchestra/fs": "^3.1.0",
+    "@xen-orchestra/fs": "^1.1.0",
     "cli-progress": "^3.1.0",
     "exec-promise": "^0.7.0",
     "getopts": "^2.2.3",
     "human-format": "^1.0.0",
     "lodash": "^4.17.21",
     "promise-toolbox": "^0.21.0",
-    "uuid": "^9.0.0",
-    "vhd-lib": "^4.1.0"
+    "uuid": "^8.3.2",
+    "vhd-lib": "^3.3.1"
   },
   "scripts": {
     "postversion": "npm publish"
-  },
-  "devDependencies": {
-    "execa": "^4.0.0",
-    "rimraf": "^3.0.2",
-    "tmp": "^0.2.1"
   }
 }

packages/vhd-lib/Vhd/VhdAbstract.integ.spec.js

@@ -275,3 +275,26 @@ it('can stream content', async () => {
     }
   })
 })
+
+it('can check vhd contained in on another', async () => {
+  const rawFile = `${tempDir}/contained`
+  await createRandomFile(rawFile, 4)
+  const containedVhdFileName = `${tempDir}/contained.vhd`
+  await convertFromRawToVhd(rawFile, containedVhdFileName)
+
+  const after = `${tempDir}/after`
+  await createRandomFile(after, 4)
+
+  fs.appendFile(rawFile, await fs.readFile(after))
+
+  const cnotainerVhdFileName = `${tempDir}/container.vhd`
+  await convertFromRawToVhd(rawFile, cnotainerVhdFileName)
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file://' + tempDir })
+    const contained = yield openVhd(handler, 'contained.vhd')
+    const container = yield openVhd(handler, 'container.vhd')
+    expect(await contained.contains(container)).toEqual(false)
+    expect(await container.contains(contained)).toEqual(true)
+  })
+})

packages/vhd-lib/Vhd/VhdAbstract.js

@@ -86,19 +86,10 @@ exports.VhdAbstract = class VhdAbstract {
   }
 
   /**
-   * @typedef {Object} BitmapBlock
-   * @property {number} id
-   * @property {Buffer} bitmap
-   *
-   * @typedef {Object} FullBlock
-   * @property {number} id
-   * @property {Buffer} bitmap
-   * @property {Buffer} data
-   * @property {Buffer} buffer - bitmap + data
    *
    * @param {number} blockId
    * @param {boolean} onlyBitmap
-   * @returns {Promise<BitmapBlock | FullBlock>}
+   * @returns {Buffer}
    */
   readBlock(blockId, onlyBitmap = false) {
     throw new Error(`reading  ${onlyBitmap ? 'bitmap of block' : 'block'} ${blockId} is not implemented`)
@@ -344,6 +335,10 @@ exports.VhdAbstract = class VhdAbstract {
     return stream
   }
 
+  /*
+   * check if all the data of a child are already contained in this vhd
+   */
+
   async containsAllDataOf(child) {
     await this.readBlockAllocationTable()
     await child.readBlockAllocationTable()
@@ -360,38 +355,4 @@ exports.VhdAbstract = class VhdAbstract {
     }
     return true
   }
-
-  async readRawData(start, length, cache, buf) {
-    const header = this.header
-    const blockSize = header.blockSize
-    const startBlockId = Math.floor(start / blockSize)
-    const endBlockId = Math.floor((start + length) / blockSize)
-
-    const startOffset = start % blockSize
-    let copied = 0
-    for (let blockId = startBlockId; blockId <= endBlockId; blockId++) {
-      let data
-      if (this.containsBlock(blockId)) {
-        if (!cache.has(blockId)) {
-          cache.set(
-            blockId,
-            // promise is awaited later, so it won't generate unbounded error
-            this.readBlock(blockId).then(block => {
-              return block.data
-            })
-          )
-        }
-        // the cache contains a promise
-        data = await cache.get(blockId)
-      } else {
-        data = Buffer.alloc(blockSize, 0)
-      }
-      const offsetStart = blockId === startBlockId ? startOffset : 0
-      const offsetEnd = blockId === endBlockId ? (start + length) % blockSize : blockSize
-      data.copy(buf, copied, offsetStart, offsetEnd)
-      copied += offsetEnd - offsetStart
-    }
-    assert.strictEqual(copied, length, 'invalid length')
-    return copied
-  }
 }

packages/vhd-lib/Vhd/VhdDirectory.js

@@ -6,7 +6,6 @@ const { fuFooter, fuHeader, checksumStruct } = require('../_structs')
 const { test, set: setBitmap } = require('../_bitmap')
 const { VhdAbstract } = require('./VhdAbstract')
 const assert = require('assert')
-const { synchronized } = require('decorator-synchronized')
 const promisify = require('promise-toolbox/promisify')
 const zlib = require('zlib')
 
@@ -132,7 +131,6 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
     this._path = path
     this._opts = opts
     this.#compressor = getCompressor(opts?.compression)
-    this.writeBlockAllocationTable = synchronized()(this.writeBlockAllocationTable)
   }
 
   async readBlockAllocationTable() {
@@ -253,7 +251,7 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
   async mergeBlock(child, blockId, isResumingMerge = false) {
     const childBlockPath = child._getFullBlockPath?.(blockId)
     if (
-      childBlockPath === undefined ||
+      childBlockPath !== undefined ||
       this._handler !== child._handler ||
       child.compressionType !== this.compressionType ||
       child.compressionType === 'MIXED'
@@ -261,12 +259,7 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
       return super.mergeBlock(child, blockId)
     }
     try {
-      const blockExists = this.containsBlock(blockId)
       await this._handler.rename(childBlockPath, this._getFullBlockPath(blockId))
-      if (!blockExists) {
-        setBitmap(this.#blockTable, blockId)
-        await this.writeBlockAllocationTable()
-      }
     } catch (error) {
       if (error.code === 'ENOENT' && isResumingMerge === true) {
         // when resuming, the blocks moved since the last merge state write are

packages/vhd-lib/Vhd/VhdFile.integ.spec.js

@@ -72,7 +72,7 @@ test('blocks can be moved', async () => {
   await newVhd._freeFirstBlockSpace(8000000)
   const recoveredFileName = `${tempDir}/recovered`
   await recoverRawContent(vhdFileName, recoveredFileName, originalSize)
-  expect((await fs.readFile(recoveredFileName)).equals(await fs.readFile(rawFileName))).toEqual(true)
+  expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(rawFileName))
 })
 
 test('the BAT MSB is not used for sign', async () => {
@@ -116,7 +116,7 @@ test('the BAT MSB is not used for sign', async () => {
       end: hugePositionBytes + randomBuffer.length - 1,
     })
   )
-  expect(recovered.equals(randomBuffer)).toEqual(true)
+  expect(recovered).toEqual(randomBuffer)
 })
 
 test('writeData on empty file', async () => {
@@ -134,7 +134,7 @@ test('writeData on empty file', async () => {
   await newVhd.writeData(0, randomData)
   const recoveredFileName = `${tempDir}/recovered`
   await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
-  expect((await fs.readFile(recoveredFileName)).equals(randomData)).toEqual(true)
+  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
 })
 
 test('writeData in 2 non-overlaping operations', async () => {
@@ -154,7 +154,7 @@ test('writeData in 2 non-overlaping operations', async () => {
   await newVhd.writeData(0, randomData.slice(0, splitPointSectors * 512))
   await newVhd.writeData(splitPointSectors, randomData.slice(splitPointSectors * 512))
   await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
-  expect((await fs.readFile(recoveredFileName)).equals(randomData)).toEqual(true)
+  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
 })
 
 test('writeData in 2 overlaping operations', async () => {
@@ -175,7 +175,7 @@ test('writeData in 2 overlaping operations', async () => {
   await newVhd.writeData(0, randomData.slice(0, endFirstWrite * 512))
   await newVhd.writeData(startSecondWrite, randomData.slice(startSecondWrite * 512))
   await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
-  expect((await fs.readFile(recoveredFileName)).equals(randomData)).toEqual(true)
+  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
 })
 
 test('BAT can be extended and blocks moved', async () => {
@@ -193,7 +193,7 @@ test('BAT can be extended and blocks moved', async () => {
   await newVhd.ensureBatSize(2000)
   await newVhd.writeBlockAllocationTable()
   await recoverRawContent(vhdFileName, recoveredFileName, originalSize)
-  expect((await fs.readFile(recoveredFileName)).equals(await fs.readFile(rawFileName))).toEqual(true)
+  expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(rawFileName))
 })
 
 test('Can coalesce block', async () => {
@@ -227,13 +227,13 @@ test('Can coalesce block', async () => {
     await parentVhd.writeBlockAllocationTable()
     let parentBlockData = (await parentVhd.readBlock(0)).data
     let childBlockData = (await childFileVhd.readBlock(0)).data
-    expect(parentBlockData.equals(childBlockData)).toEqual(true)
+    expect(parentBlockData).toEqual(childBlockData)
 
     await parentVhd.mergeBlock(childDirectoryVhd, 0)
     await parentVhd.writeFooter()
     await parentVhd.writeBlockAllocationTable()
     parentBlockData = (await parentVhd.readBlock(0)).data
     childBlockData = (await childDirectoryVhd.readBlock(0)).data
-    expect(parentBlockData.equals(childBlockData)).toEqual(true)
+    expect(parentBlockData).toEqual(childBlockData)
   })
 })

packages/vhd-lib/Vhd/VhdFile.js

@@ -83,7 +83,6 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
   }
 
   static async open(handler, path, { flags, checkSecondFooter = true } = {}) {
-    assert(!handler.isEncrypted, `VHDFile implementation is not compatible with encrypted remote`)
     const fd = await handler.openFile(path, flags ?? 'r+')
     const vhd = new VhdFile(handler, fd)
     // openning a file for reading does not trigger EISDIR as long as we don't really read from it :

packages/vhd-lib/Vhd/VhdSynthetic.integ.spec.js

@@ -55,7 +55,7 @@ test('It can read block and parent locator from a synthetic vhd', async () => {
 
     await bigVhd.readHeaderAndFooter()
 
-    const syntheticVhd = yield VhdSynthetic.open(handler, [bigVhdFileName, smallVhdFileName])
+    const syntheticVhd = yield VhdSynthetic.open(handler, [smallVhdFileName, bigVhdFileName])
     await syntheticVhd.readBlockAllocationTable()
 
     expect(syntheticVhd.header.diskType).toEqual(bigVhd.header.diskType)
@@ -65,24 +65,25 @@ test('It can read block and parent locator from a synthetic vhd', async () => {
     const buf = Buffer.alloc(syntheticVhd.sectorsPerBlock * SECTOR_SIZE, 0)
     let content = (await syntheticVhd.readBlock(0)).data
     await handler.read(smallRawFileName, buf, 0)
-    expect(content.equals(buf)).toEqual(true)
+    expect(content).toEqual(buf)
 
     content = (await syntheticVhd.readBlock(1)).data
     await handler.read(smallRawFileName, buf, buf.length)
-    expect(content.equals(buf)).toEqual(true)
+    expect(content).toEqual(buf)
 
     // the next one from big
 
     content = (await syntheticVhd.readBlock(2)).data
     await handler.read(bigRawFileName, buf, buf.length * 2)
-    expect(content.equals(buf)).toEqual(true)
+    expect(content).toEqual(buf)
 
     content = (await syntheticVhd.readBlock(3)).data
     await handler.read(bigRawFileName, buf, buf.length * 3)
-    expect(content.equals(buf)).toEqual(true)
+    expect(content).toEqual(buf)
 
     // the parent locator should the one of the root vhd
     const parentLocator = await syntheticVhd.readParentLocator(0)
     expect(parentLocator.platformCode).toEqual(PLATFORMS.W2KU)
+    expect(Buffer.from(parentLocator.data, 'utf-8').toString()).toEqual('I am in the big one')
   })
 })

packages/vhd-lib/Vhd/VhdSynthetic.js

@@ -15,13 +15,14 @@ const VhdSynthetic = class VhdSynthetic extends VhdAbstract {
   #vhds = []
 
   get header() {
-    // this the most recent vhd
-    const vhd = this.#vhds[this.#vhds.length - 1]
+    // this the VHD we want to synthetize
+    const vhd = this.#vhds[0]
 
     // this is the root VHD
-    const rootVhd = this.#vhds[0]
+    const rootVhd = this.#vhds[this.#vhds.length - 1]
 
     // data of our synthetic VHD
+    // TODO: set parentLocatorEntry-s in header
     return {
       ...vhd.header,
       parentLocatorEntry: cloneDeep(rootVhd.header.parentLocatorEntry),
@@ -33,13 +34,10 @@ const VhdSynthetic = class VhdSynthetic extends VhdAbstract {
   }
 
   get footer() {
-    // this the most recent vhd
-    const vhd = this.#vhds[this.#vhds.length - 1]
-
-    // this is the oldest VHD
-    const rootVhd = this.#vhds[0]
+    // this is the root VHD
+    const rootVhd = this.#vhds[this.#vhds.length - 1]
     return {
-      ...vhd.footer,
+      ...this.#vhds[0].footer,
       dataOffset: FOOTER_SIZE,
       diskType: rootVhd.footer.diskType,
     }
@@ -79,21 +77,17 @@ const VhdSynthetic = class VhdSynthetic extends VhdAbstract {
     await asyncMap(vhds, vhd => vhd.readHeaderAndFooter())
 
     for (let i = 0, n = vhds.length - 1; i < n; ++i) {
-      const parent = vhds[i]
-      const child = vhds[i + 1]
+      const child = vhds[i]
+      const parent = vhds[i + 1]
       assert.strictEqual(child.footer.diskType, DISK_TYPES.DIFFERENCING)
       assert.strictEqual(UUID.stringify(child.header.parentUuid), UUID.stringify(parent.footer.uuid))
     }
   }
 
   #getVhdWithBlock(blockId) {
-    for (let i = this.#vhds.length - 1; i >= 0; i--) {
-      const vhd = this.#vhds[i]
-      if (vhd.containsBlock(blockId)) {
-        return vhd
-      }
-    }
-    assert(false, `no such block ${blockId}`)
+    const index = this.#vhds.findIndex(vhd => vhd.containsBlock(blockId))
+    assert(index !== -1, `no such block ${blockId}`)
+    return this.#vhds[index]
   }
 
   async readBlock(blockId, onlyBitmap = false) {
@@ -126,7 +120,7 @@ VhdSynthetic.fromVhdChain = Disposable.factory(async function* fromVhdChain(hand
   const vhds = []
   do {
     vhd = yield openVhd(handler, vhdPath)
-    vhds.unshift(vhd) // from oldest to most recent
+    vhds.push(vhd)
     vhdPath = resolveRelativeFromFile(vhdPath, vhd.header.parentUnicodeName)
   } while (vhd.footer.diskType !== DISK_TYPES.DYNAMIC)
 

packages/vhd-lib/aliases.js

@@ -12,14 +12,11 @@ exports.resolveVhdAlias = async function resolveVhdAlias(handler, filename) {
   if (!isVhdAlias(filename)) {
     return filename
   }
-  if (!handler.isEncrypted) {
-    const size = await handler.getSize(filename)
-    if (size > ALIAS_MAX_PATH_LENGTH) {
-      // seems reasonnable for a relative path
-      throw new Error(`The alias file ${filename} is too big (${size} bytes)`)
-    }
+  const size = await handler.getSize(filename)
+  if (size > ALIAS_MAX_PATH_LENGTH) {
+    // seems reasonnable for a relative path
+    throw new Error(`The alias file ${filename} is too big (${size} bytes)`)
   }
-
   const aliasContent = (await handler.readFile(filename)).toString().trim()
   // also handle circular references and unreasonnably long chains
   if (isVhdAlias(aliasContent)) {

packages/vhd-lib/createVhdDirectoryFromStream.js

@@ -8,10 +8,10 @@ const { asyncEach } = require('@vates/async-each')
 
 const { warn } = createLogger('vhd-lib:createVhdDirectoryFromStream')
 
-const buildVhd = Disposable.wrap(async function* (handler, path, inputStream, { concurrency, compression, nbdClient }) {
+const buildVhd = Disposable.wrap(async function* (handler, path, inputStream, { concurrency, compression }) {
   const vhd = yield VhdDirectory.create(handler, path, { compression })
   await asyncEach(
-    parseVhdStream(inputStream, nbdClient),
+    parseVhdStream(inputStream),
     async function (item) {
       switch (item.type) {
         case 'footer':
@@ -44,10 +44,10 @@ exports.createVhdDirectoryFromStream = async function createVhdDirectoryFromStre
   handler,
   path,
   inputStream,
-  { validator, concurrency = 16, compression, nbdClient } = {}
+  { validator, concurrency = 16, compression } = {}
 ) {
   try {
-    await buildVhd(handler, path, inputStream, { concurrency, compression, nbdClient })
+    await buildVhd(handler, path, inputStream, { concurrency, compression })
     if (validator !== undefined) {
       await validator.call(this, path)
     }

packages/vhd-lib/index.js

@@ -6,6 +6,8 @@ exports.checkVhdChain = require('./checkChain')
 exports.createReadableSparseStream = require('./createReadableSparseStream')
 exports.createVhdStreamWithLength = require('./createVhdStreamWithLength')
 exports.createVhdDirectoryFromStream = require('./createVhdDirectoryFromStream').createVhdDirectoryFromStream
+const { mergeVhd } = require('./merge')
+exports.mergeVhd = mergeVhd
 exports.peekFooterFromVhdStream = require('./peekFooterFromVhdStream')
 exports.openVhd = require('./openVhd').openVhd
 exports.VhdAbstract = require('./Vhd/VhdAbstract').VhdAbstract

packages/vhd-lib/merge.integ.spec.js

@@ -8,8 +8,8 @@ const tmp = require('tmp')
 const { getHandler } = require('@xen-orchestra/fs')
 const { pFromCallback } = require('promise-toolbox')
 
-const { VhdFile, chainVhd } = require('./index')
-const { _cleanupVhds: cleanupVhds, mergeVhdChain } = require('./merge')
+const { VhdFile, chainVhd, mergeVhd } = require('./index')
+const { _cleanupVhds: cleanupVhds } = require('./merge')
 
 const { checkFile, createRandomFile, convertFromRawToVhd } = require('./tests/utils')
 
@@ -42,7 +42,7 @@ test('merge works in normal cases', async () => {
   await checkFile(`${tempDir}/${parentFileName}`)
 
   // merge
-  await mergeVhdChain(handler, [parentFileName, child1FileName])
+  await mergeVhd(handler, parentFileName, handler, child1FileName)
 
   // check that the merged vhd is still valid
   await checkFile(`${tempDir}/${child1FileName}`)
@@ -65,7 +65,7 @@ test('merge works in normal cases', async () => {
   }
 })
 
-test('it can resume a simple merge ', async () => {
+test('it can resume a merge ', async () => {
   const mbOfFather = 8
   const mbOfChildren = 4
   const parentRandomFileName = `${tempDir}/randomfile`
@@ -95,7 +95,7 @@ test('it can resume a simple merge ', async () => {
     })
   )
   // expect merge to fail since child header is not ok
-  await expect(async () => await mergeVhdChain(handler, ['parent.vhd', 'child1.vhd'])).rejects.toThrow()
+  await expect(async () => await mergeVhd(handler, 'parent.vhd', handler, 'child1.vhd')).rejects.toThrow()
 
   await handler.unlink('.parent.vhd.merge.json')
   await handler.writeFile(
@@ -110,7 +110,7 @@ test('it can resume a simple merge ', async () => {
     })
   )
   // expect merge to fail since parent header is not ok
-  await expect(async () => await mergeVhdChain(handler, ['parent.vhd', 'child1.vhd'])).rejects.toThrow()
+  await expect(async () => await mergeVhd(handler, 'parent.vhd', handler, ['child1.vhd'])).rejects.toThrow()
 
   // break the end footer of parent
   const size = await handler.getSize('parent.vhd')
@@ -137,7 +137,7 @@ test('it can resume a simple merge ', async () => {
   )
 
   // really merge
-  await mergeVhdChain(handler, ['parent.vhd', 'child1.vhd'])
+  await mergeVhd(handler, 'parent.vhd', handler, 'child1.vhd')
 
   // reload header footer and block allocation table , they should succed
   await childVhd.readHeaderAndFooter()
@@ -159,73 +159,6 @@ test('it can resume a simple merge ', async () => {
   }
 })
 
-test('it can resume a multiple merge ', async () => {
-  const mbOfFather = 8
-  const mbOfChildren = 6
-  const mbOfGrandChildren = 4
-  const parentRandomFileName = `${tempDir}/randomfile`
-  const childRandomFileName = `${tempDir}/small_randomfile`
-  const grandChildRandomFileName = `${tempDir}/another_small_randomfile`
-  const parentFileName = `${tempDir}/parent.vhd`
-  const childFileName = `${tempDir}/child.vhd`
-  const grandChildFileName = `${tempDir}/grandchild.vhd`
-  const handler = getHandler({ url: 'file://' })
-  await createRandomFile(parentRandomFileName, mbOfFather)
-  await convertFromRawToVhd(parentRandomFileName, parentFileName)
-
-  await createRandomFile(childRandomFileName, mbOfChildren)
-  await convertFromRawToVhd(childRandomFileName, childFileName)
-  await chainVhd(handler, parentFileName, handler, childFileName, true)
-
-  await createRandomFile(grandChildRandomFileName, mbOfGrandChildren)
-  await convertFromRawToVhd(grandChildRandomFileName, grandChildFileName)
-  await chainVhd(handler, childFileName, handler, grandChildFileName, true)
-
-  const parentVhd = new VhdFile(handler, parentFileName)
-  await parentVhd.readHeaderAndFooter()
-
-  const childVhd = new VhdFile(handler, childFileName)
-  await childVhd.readHeaderAndFooter()
-
-  const grandChildVhd = new VhdFile(handler, grandChildFileName)
-  await grandChildVhd.readHeaderAndFooter()
-
-  await handler.writeFile(
-    `${tempDir}/.parent.vhd.merge.json`,
-    JSON.stringify({
-      parent: {
-        header: parentVhd.header.checksum,
-      },
-      child: {
-        header: childVhd.header.checksum,
-      },
-      currentBlock: 1,
-    })
-  )
-
-  // should fail since the merge state file has only data of parent and child
-  await expect(
-    async () => await mergeVhdChain(handler, [parentFileName, childFileName, grandChildFileName])
-  ).rejects.toThrow()
-  // merge
-  await handler.unlink(`${tempDir}/.parent.vhd.merge.json`)
-  await handler.writeFile(
-    `${tempDir}/.parent.vhd.merge.json`,
-    JSON.stringify({
-      parent: {
-        header: parentVhd.header.checksum,
-      },
-      child: {
-        header: grandChildVhd.header.checksum,
-      },
-      currentBlock: 1,
-      childPath: [childVhd, grandChildVhd],
-    })
-  )
-  // it should succeed
-  await mergeVhdChain(handler, [parentFileName, childFileName, grandChildFileName])
-})
-
 test('it merge multiple child in one pass ', async () => {
   const mbOfFather = 8
   const mbOfChildren = 6
@@ -249,7 +182,7 @@ test('it merge multiple child in one pass ', async () => {
   await chainVhd(handler, childFileName, handler, grandChildFileName, true)
 
   // merge
-  await mergeVhdChain(handler, [parentFileName, childFileName, grandChildFileName])
+  await mergeVhd(handler, parentFileName, handler, [grandChildFileName, childFileName])
 
   // check that vhd is still valid
   await checkFile(grandChildFileName)
@@ -284,7 +217,8 @@ test('it cleans vhd mergedfiles', async () => {
   await handler.writeFile('child2', 'child2Data')
   await handler.writeFile('child3', 'child3Data')
 
-  await cleanupVhds(handler, ['parent', 'child1', 'child2', 'child3'], { merge: true, removeUnused: true })
+  // childPath  is from the grand children to the children
+  await cleanupVhds(handler, 'parent', ['child3', 'child2', 'child1'], { remove: true })
 
   // only child3 should stay, with the data of parent
   const [child3, ...other] = await handler.list('.')

packages/vhd-lib/merge.js

@@ -5,7 +5,6 @@
 const assert = require('assert')
 const noop = require('./_noop')
 const UUID = require('uuid')
-const handlerPath = require('@xen-orchestra/fs/path')
 const { createLogger } = require('@xen-orchestra/log')
 const { limitConcurrency } = require('limit-concurrency-decorator')
 
@@ -56,50 +55,45 @@ function makeThrottledWriter(handler, path, delay) {
 // make the rename / delete part of the merge process
 // will fail if parent and children are in different remote
 
-async function cleanupVhds(handler, chain, { logInfo = noop, removeUnused = false } = {}) {
-  const parent = chain[0]
-  const children = chain.slice(1, -1)
-  const mergeTargetChild = chain[chain.length - 1]
+function cleanupVhds(handler, parent, children, { logInfo = noop, remove = false } = {}) {
+  if (!Array.isArray(children)) {
+    children = [children]
+  }
+  const mergeTargetChild = children.shift()
 
-  await VhdAbstract.rename(handler, parent, mergeTargetChild)
-
-  return asyncMap(children, child => {
-    logInfo(`the VHD child is already merged`, { child })
-    if (removeUnused) {
-      logInfo(`deleting merged VHD child`, { child })
-      return VhdAbstract.unlink(handler, child)
-    }
-  })
+  return Promise.all([
+    VhdAbstract.rename(handler, parent, mergeTargetChild),
+    asyncMap(children, child => {
+      logInfo(`the VHD child is already merged`, { child })
+      if (remove) {
+        logInfo(`deleting merged VHD child`, { child })
+        return VhdAbstract.unlink(handler, child)
+      }
+    }),
+  ])
 }
-
 module.exports._cleanupVhds = cleanupVhds
 
-// Merge a chain of VHDs into a single VHD
-module.exports.mergeVhdChain = limitConcurrency(2)(async function mergeVhdChain(
-  handler,
-  chain,
-  { onProgress = noop, logInfo = noop, removeUnused = false, mergeBlockConcurrency = 2 } = {}
+// Merge one or multiple vhd child into vhd parent.
+// childPath can be array to create a synthetic VHD from multiple VHDs
+// childPath  is from the grand children to the children
+//
+// TODO: rename the VHD file during the merge
+module.exports.mergeVhd = limitConcurrency(2)(async function merge(
+  parentHandler,
+  parentPath,
+  childHandler,
+  childPath,
+  { onProgress = noop, logInfo = noop, remove } = {}
 ) {
-  assert(chain.length >= 2)
-
-  const parentPath = chain[0]
-  const childrenPaths = chain.slice(1)
-
-  const mergeStatePath = dirname(parentPath) + '/.' + basename(parentPath) + '.merge.json'
+  const mergeStatePath = dirname(parentPath) + '/' + '.' + basename(parentPath) + '.merge.json'
 
   return await Disposable.use(async function* () {
     let mergeState
     let isResuming = false
     try {
-      const mergeStateContent = await handler.readFile(mergeStatePath)
+      const mergeStateContent = await parentHandler.readFile(mergeStatePath)
       mergeState = JSON.parse(mergeStateContent)
-
-      // work-around a bug introduce in 97d94b795
-      //
-      // currentBlock could be `null` due to the JSON.stringify of a `NaN` value
-      if (mergeState.currentBlock === null) {
-        mergeState.currentBlock = 0
-      }
     } catch (error) {
       if (error.code !== 'ENOENT') {
         warn('problem while checking the merge state', { error })
@@ -108,23 +102,22 @@ module.exports.mergeVhdChain = limitConcurrency(2)(async function mergeVhdChain(
 
     // during merging, the end footer of the parent can be overwritten by new blocks
     // we should use it as a way to check vhd health
-    const parentVhd = yield openVhd(handler, parentPath, {
+    const parentVhd = yield openVhd(parentHandler, parentPath, {
       flags: 'r+',
       checkSecondFooter: mergeState === undefined,
     })
     let childVhd
     const parentIsVhdDirectory = parentVhd instanceof VhdDirectory
     let childIsVhdDirectory
-    if (childrenPaths.length !== 1) {
-      childVhd = yield VhdSynthetic.open(handler, childrenPaths)
+    if (Array.isArray(childPath)) {
+      childVhd = yield VhdSynthetic.open(childHandler, childPath)
       childIsVhdDirectory = childVhd.checkVhdsClass(VhdDirectory)
     } else {
-      childVhd = yield openVhd(handler, childrenPaths[0])
+      childVhd = yield openVhd(childHandler, childPath)
       childIsVhdDirectory = childVhd instanceof VhdDirectory
     }
 
-    // merging vhdFile must not be concurrently with the potential block reordering after a change
-    const concurrency = parentIsVhdDirectory && childIsVhdDirectory ? mergeBlockConcurrency : 1
+    const concurrency = parentIsVhdDirectory && childIsVhdDirectory ? 16 : 1
     if (mergeState === undefined) {
       // merge should be along a vhd chain
       assert.strictEqual(UUID.stringify(childVhd.header.parentUuid), UUID.stringify(parentVhd.footer.uuid))
@@ -152,7 +145,6 @@ module.exports.mergeVhdChain = limitConcurrency(2)(async function mergeVhdChain(
         parent: { header: parentVhd.header.checksum },
         currentBlock: 0,
         mergedDataSize: 0,
-        chain: chain.map(vhdPath => handlerPath.relativeFromFile(mergeStatePath, vhdPath)),
       }
 
       // finds first allocated block for the 2 following loops
@@ -174,14 +166,13 @@ module.exports.mergeVhdChain = limitConcurrency(2)(async function mergeVhdChain(
     const merging = new Set()
     let counter = 0
 
-    const mergeStateWriter = makeThrottledWriter(handler, mergeStatePath, 10e3)
+    const mergeStateWriter = makeThrottledWriter(parentHandler, mergeStatePath, 10e3)
     await asyncEach(
       toMerge,
       async blockId => {
         merging.add(blockId)
         mergeState.mergedDataSize += await parentVhd.mergeBlock(childVhd, blockId, isResuming)
 
-        mergeState.currentBlock = Math.min(...merging)
         merging.delete(blockId)
 
         onProgress({
@@ -189,6 +180,7 @@ module.exports.mergeVhdChain = limitConcurrency(2)(async function mergeVhdChain(
           done: counter + 1,
         })
         counter++
+        mergeState.currentBlock = Math.min(...merging)
         mergeStateWriter(mergeState)
       },
       {
@@ -212,18 +204,11 @@ module.exports.mergeVhdChain = limitConcurrency(2)(async function mergeVhdChain(
     // creation
     await parentVhd.writeFooter()
 
-    await cleanupVhds(handler, chain, { logInfo, removeUnused })
-
     // should be a disposable
-    handler.unlink(mergeStatePath).catch(warn)
+    parentHandler.unlink(mergeStatePath).catch(warn)
+
+    await cleanupVhds(parentHandler, parentPath, childPath, { logInfo, remove })
 
     return mergeState.mergedDataSize
-  }).catch(error => {
-    try {
-      error.chain = chain
-    } finally {
-      // eslint-disable-next-line no-unsafe-finally
-      throw error
-    }
   })
 })

packages/vhd-lib/openVhd.js

@@ -9,8 +9,7 @@ exports.openVhd = async function openVhd(handler, path, opts) {
   try {
     return await VhdFile.open(handler, resolved, opts)
   } catch (e) {
-    // if the remote is encrypted, trying to open a VhdFile will throw an assertion error before checking if the path is a directory, therefore we should try to open a VhdDirectory anyway.
-    if (e.code !== 'EISDIR' && e.code !== 'ERR_ASSERTION') {
+    if (e.code !== 'EISDIR') {
       throw e
     }
     return await VhdDirectory.open(handler, resolved, opts)

packages/vhd-lib/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "vhd-lib",
-  "version": "4.1.0",
+  "version": "3.3.1",
   "license": "AGPL-3.0-or-later",
   "description": "Primitives for VHD file handling",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-lib",
@@ -15,23 +15,21 @@
     "node": ">=14"
   },
   "dependencies": {
-    "@vates/async-each": "^1.0.0",
+    "@vates/async-each": "^0.1.0",
     "@vates/read-chunk": "^1.0.0",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^3.1.0",
     "@xen-orchestra/log": "^0.3.0",
     "async-iterator-to-stream": "^1.0.2",
-    "decorator-synchronized": "^0.6.0",
     "fs-extra": "^10.0.0",
     "limit-concurrency-decorator": "^0.5.0",
     "lodash": "^4.17.4",
     "promise-toolbox": "^0.21.0",
-    "readable-stream": "^4.1.0",
+    "readable-stream": "^3.0.6",
     "struct-fu": "^1.2.0",
-    "uuid": "^9.0.0"
+    "uuid": "^8.3.1"
   },
   "devDependencies": {
-    "@xen-orchestra/fs": "^3.1.0",
+    "@xen-orchestra/fs": "^1.1.0",
     "execa": "^5.0.0",
     "get-stream": "^6.0.0",
     "rimraf": "^3.0.2",

packages/vhd-lib/parseVhdStream.js

@@ -4,7 +4,7 @@ const { BLOCK_UNUSED, FOOTER_SIZE, HEADER_SIZE, SECTOR_SIZE } = require('./_cons
 const { readChunk } = require('@vates/read-chunk')
 const assert = require('assert')
 const { unpackFooter, unpackHeader, computeFullBlockSize } = require('./Vhd/_utils')
-/*
+
 const cappedBufferConcat = (buffers, maxSize) => {
   let buffer = Buffer.concat(buffers)
   if (buffer.length > maxSize) {
@@ -13,6 +13,114 @@ const cappedBufferConcat = (buffers, maxSize) => {
   return buffer
 }
 
+exports.parseVhdStream = async function* parseVhdStream(stream) {
+  let bytesRead = 0
+
+  // handle empty space between elements
+  // ensure we read stream in order
+  async function read(offset, size) {
+    assert(bytesRead <= offset, `offset is ${offset} but we already read ${bytesRead} bytes`)
+    if (bytesRead < offset) {
+      // empty spaces
+      await read(bytesRead, offset - bytesRead)
+    }
+    const buf = await readChunk(stream, size)
+    assert.strictEqual(buf.length, size, `read ${buf.length} instead of ${size}`)
+    bytesRead += size
+    return buf
+  }
+
+  const bufFooter = await read(0, FOOTER_SIZE)
+
+  const footer = unpackFooter(bufFooter)
+  yield { type: 'footer', footer, offset: 0 }
+
+  const bufHeader = await read(FOOTER_SIZE, HEADER_SIZE)
+  const header = unpackHeader(bufHeader, footer)
+
+  yield { type: 'header', header, offset: SECTOR_SIZE }
+  const blockSize = header.blockSize
+  assert.strictEqual(blockSize % SECTOR_SIZE, 0)
+
+  const fullBlockSize = computeFullBlockSize(blockSize)
+
+  const bitmapSize = fullBlockSize - blockSize
+
+  const index = []
+
+  for (const parentLocatorId in header.parentLocatorEntry) {
+    const parentLocatorEntry = header.parentLocatorEntry[parentLocatorId]
+    // empty parent locator entry, does not exist in the content
+    if (parentLocatorEntry.platformDataSpace === 0) {
+      continue
+    }
+    index.push({
+      ...parentLocatorEntry,
+      type: 'parentLocator',
+      offset: parentLocatorEntry.platformDataOffset,
+      size: parentLocatorEntry.platformDataLength,
+      id: parentLocatorId,
+    })
+  }
+
+  const batOffset = header.tableOffset
+  const batSize = Math.max(1, Math.ceil((header.maxTableEntries * 4) / SECTOR_SIZE)) * SECTOR_SIZE
+
+  index.push({
+    type: 'bat',
+    offset: batOffset,
+    size: batSize,
+  })
+
+  // sometimes some parent locator are before the BAT
+  index.sort((a, b) => a.offset - b.offset)
+  while (index.length > 0) {
+    const item = index.shift()
+    const buffer = await read(item.offset, item.size)
+    item.buffer = buffer
+
+    const { type } = item
+    if (type === 'bat') {
+      // found the BAT : read it and add block to index
+
+      let blockCount = 0
+      for (let blockCounter = 0; blockCounter < header.maxTableEntries; blockCounter++) {
+        const batEntrySector = buffer.readUInt32BE(blockCounter * 4)
+        // unallocated block, no need to export it
+        if (batEntrySector !== BLOCK_UNUSED) {
+          const batEntryBytes = batEntrySector * SECTOR_SIZE
+          // ensure the block is not before the bat
+          assert.ok(batEntryBytes >= batOffset + batSize)
+          index.push({
+            type: 'block',
+            id: blockCounter,
+            offset: batEntryBytes,
+            size: fullBlockSize,
+          })
+          blockCount++
+        }
+      }
+      // sort again index to ensure block and parent locator are in the right order
+      index.sort((a, b) => a.offset - b.offset)
+      item.blockCount = blockCount
+    } else if (type === 'block') {
+      item.bitmap = buffer.slice(0, bitmapSize)
+      item.data = buffer.slice(bitmapSize)
+    }
+
+    yield item
+  }
+
+  /**
+   * the second footer is at filesize - 512 , there can be empty spaces between last block
+   * and the start of the footer
+   *
+   * we read till the end of the stream, and use the last 512 bytes as the footer
+   */
+  const bufFooterEnd = await readLastSector(stream)
+  assert(bufFooter.equals(bufFooterEnd), 'footer1 !== footer2')
+}
+
 function readLastSector(stream) {
   return new Promise((resolve, reject) => {
     let bufFooterEnd = Buffer.alloc(0)
@@ -25,227 +133,4 @@ function readLastSector(stream) {
     stream.on('end', () => resolve(bufFooterEnd))
     stream.on('error', reject)
   })
-} */
-
-class StreamParser {
-  _bitmapSize = 0
-  _bytesRead = 0
-  _stream = null
-  _index = []
-  constructor(stream) {
-    this._stream = stream
-  }
-
-  async _read(offset, size) {
-    assert(this._bytesRead <= offset, `offset is ${offset} but we already read ${this._bytesRead} bytes`)
-    if (this._bytesRead < offset) {
-      // empty spaces
-      await this._read(this._bytesRead, offset - this._bytesRead)
-    }
-    const buf = await readChunk(this._stream, size)
-    assert.strictEqual(buf.length, size, `read ${buf.length} instead of ${size}`)
-    this._bytesRead += size
-    return buf
-  }
-
-  async *headers() {
-    const bufFooter = await this._read(0, FOOTER_SIZE)
-
-    const footer = unpackFooter(bufFooter)
-
-    yield { type: 'footer', footer, offset: 0 }
-    const bufHeader = await this._read(FOOTER_SIZE, HEADER_SIZE)
-    const header = unpackHeader(bufHeader, footer)
-
-    yield { type: 'header', header, offset: SECTOR_SIZE }
-    const blockSize = header.blockSize
-    assert.strictEqual(blockSize % SECTOR_SIZE, 0)
-    const fullBlockSize = computeFullBlockSize(blockSize)
-    this._bitmapSize = fullBlockSize - blockSize
-
-    let batFound = false
-
-    for (const parentLocatorId in header.parentLocatorEntry) {
-      const parentLocatorEntry = header.parentLocatorEntry[parentLocatorId]
-      // empty parent locator entry, does not exist in the content
-      if (parentLocatorEntry.platformDataSpace === 0) {
-        continue
-      }
-      this._index.push({
-        ...parentLocatorEntry,
-        type: 'parentLocator',
-        offset: parentLocatorEntry.platformDataOffset,
-        size: parentLocatorEntry.platformDataLength,
-        id: parentLocatorId,
-      })
-    }
-
-    const batOffset = header.tableOffset
-    const batSize = Math.max(1, Math.ceil((header.maxTableEntries * 4) / SECTOR_SIZE)) * SECTOR_SIZE
-
-    this._index.push({
-      type: 'bat',
-      offset: batOffset,
-      size: batSize,
-    })
-
-    // sometimes some parent locator are before the BAT
-    this._index.sort((a, b) => a.offset - b.offset)
-
-    while (!batFound) {
-      const item = this._index.shift()
-      const buffer = await this._read(item.offset, item.size)
-      item.buffer = buffer
-
-      const { type } = item
-      if (type === 'bat') {
-        // found the BAT : read it and add block to index
-
-        let blockCount = 0
-        for (let blockCounter = 0; blockCounter < header.maxTableEntries; blockCounter++) {
-          const batEntrySector = buffer.readUInt32BE(blockCounter * 4)
-          // unallocated block, no need to export it
-          if (batEntrySector !== BLOCK_UNUSED) {
-            const batEntryBytes = batEntrySector * SECTOR_SIZE
-            // ensure the block is not before the bat
-            assert.ok(batEntryBytes >= batOffset + batSize)
-            this._index.push({
-              type: 'block',
-              id: blockCounter,
-              offset: batEntryBytes,
-              size: fullBlockSize,
-            })
-            blockCount++
-          }
-        }
-        // sort again index to ensure block and parent locator are in the right order
-        this._index.sort((a, b) => a.offset - b.offset)
-        item.blockCount = blockCount
-        batFound = true
-      }
-      yield item
-    }
-  }
-
-  async *blocks() {
-    while (this._index.length > 0) {
-      const item = this._index.shift()
-      const buffer = await this._read(item.offset, item.size)
-
-      item.bitmap = buffer.slice(0, this._bitmapSize)
-      item.data = buffer.slice(this._bitmapSize)
-      item.buffer = buffer
-      yield item
-    }
-  }
-
-  async *parse() {
-    yield* this.headers()
-    yield* this.blocks()
-
-    /**
-     * the second footer is at filesize - 512 , there can be empty spaces between last block
-     * and the start of the footer
-     *
-     * we read till the end of the stream, and use the last 512 bytes as the footer
-     */
-    // const bufFooterEnd = await readLastSector(this._stream)
-    // assert(bufFooter.equals(bufFooterEnd), 'footer1 !== footer2')
-  }
-}
-
-// hybrid mode : read the headers from the vhd stream, and read the blocks from nbd
-class StreamNbdParser extends StreamParser {
-  #nbdClient = null
-  #generateBlockSequentially = true
-  #concurrency = 16
-
-  constructor(stream, nbdClient, { generateBlockSequentially = true } = {}) {
-    super(stream)
-    this.#nbdClient = nbdClient
-    this.#generateBlockSequentially = generateBlockSequentially
-  }
-
-  async readBlockData(item) {
-    const SECTOR_BITMAP = Buffer.alloc(512, 255)
-    const client = this.#nbdClient
-    // we read in a raw file, so the block position is id x length, and have nothing to do with the offset
-    // in the vhd stream
-    const rawDataLength = item.size - SECTOR_BITMAP.length
-    let data = await client.readBlock(item.id, rawDataLength)
-
-    // end of file , non aligned vhd block
-    if (data.length < rawDataLength) {
-      data = Buffer.concat([data, Buffer.alloc(rawDataLength - data.length)])
-    }
-    const buffer = Buffer.concat([SECTOR_BITMAP, data])
-    const block = {
-      ...item,
-      size: rawDataLength,
-      bitmap: SECTOR_BITMAP,
-      data,
-      buffer,
-    }
-    return block
-  }
-
-  async *blocks() {
-    const index = this._index
-
-    const promiseSlots = []
-
-    // parallel yielding
-    function next(position) {
-      const item = index.shift()
-      if (item) {
-        // update this ended
-        promiseSlots[position] = this.readBlockData(item)
-          .then(result => {
-            return { result, position }
-          })
-          .catch(error => {
-            console.error(error)
-            throw error
-          })
-      } else {
-        // no more block to handle
-        promiseSlots[position] = undefined
-      }
-    }
-
-    for (let i = 0; i < this.#concurrency; i++) {
-      next(i)
-    }
-
-    let runningPromises = []
-    while (true) {
-      runningPromises = promiseSlots.filter(p => p !== undefined)
-      if (runningPromises.length === 0) {
-        // done
-        break
-      }
-      // the failing promises will only be seen when ALL the running promises
-      // fails
-      const { result, position } = await Promise.any(runningPromises)
-      next(position)
-      yield result
-    }
-  }
-
-  async *parse() {
-    yield* this.headers()
-
-    yield* this.blocks()
-    this._stream.destroy()
-  }
-}
-
-exports.parseVhdStream = async function* parseVhdStream(stream, nbdClient) {
-  let parser
-  if (nbdClient) {
-    parser = new StreamNbdParser(stream, nbdClient)
-  } else {
-    parser = new StreamParser(stream)
-  }
-  yield* parser.parse()
 }