fix(vhd-lib): don't throw error when a parent locator is missing

See https://xcp-ng.org/forum/topic/5257/problems-building-from-source/20

@vates/async-each/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/async-each/README.md

@@ -0,0 +1,68 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @vates/async-each
+
+[![Package Version](https://badgen.net/npm/v/@vates/async-each)](https://npmjs.org/package/@vates/async-each) ![License](https://badgen.net/npm/license/@vates/async-each) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/async-each)](https://bundlephobia.com/result?p=@vates/async-each) [![Node compatibility](https://badgen.net/npm/node/@vates/async-each)](https://npmjs.org/package/@vates/async-each)
+
+> Run async fn for each item in (async) iterable
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@vates/async-each):
+
+```
+> npm install --save @vates/async-each
+```
+
+## Usage
+
+### `asyncEach(iterable, iteratee, [opts])`
+
+Executes `iteratee` in order for each value yielded by `iterable`.
+
+Returns a promise wich rejects as soon as a call to `iteratee` throws or a promise returned by it rejects, and which resolves when all promises returned by `iteratee` have resolved.
+
+`iterable` must be an iterable or async iterable.
+
+`iteratee` is called with the same `this` value as `asyncEach`, and with the following arguments:
+
+- `value`: the value yielded by `iterable`
+- `index`: the 0-based index for this value
+- `iterable`: the iterable itself
+
+`opts` is an object that can contains the following options:
+
+- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
+- `signal`: an abort signal to stop the iteration
+- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
+
+```js
+import { asyncEach } from '@vates/async-each'
+
+const contents = []
+await asyncEach(
+  ['foo.txt', 'bar.txt', 'baz.txt'],
+  async function (filename, i) {
+    contents[i] = await readFile(filename)
+  },
+  {
+    // reads two files at a time
+    concurrency: 2,
+  }
+)
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

@vates/async-each/USAGE.md

@@ -0,0 +1,35 @@
+### `asyncEach(iterable, iteratee, [opts])`
+
+Executes `iteratee` in order for each value yielded by `iterable`.
+
+Returns a promise wich rejects as soon as a call to `iteratee` throws or a promise returned by it rejects, and which resolves when all promises returned by `iteratee` have resolved.
+
+`iterable` must be an iterable or async iterable.
+
+`iteratee` is called with the same `this` value as `asyncEach`, and with the following arguments:
+
+- `value`: the value yielded by `iterable`
+- `index`: the 0-based index for this value
+- `iterable`: the iterable itself
+
+`opts` is an object that can contains the following options:
+
+- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
+- `signal`: an abort signal to stop the iteration
+- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
+
+```js
+import { asyncEach } from '@vates/async-each'
+
+const contents = []
+await asyncEach(
+  ['foo.txt', 'bar.txt', 'baz.txt'],
+  async function (filename, i) {
+    contents[i] = await readFile(filename)
+  },
+  {
+    // reads two files at a time
+    concurrency: 2,
+  }
+)
+```

@vates/async-each/index.js

@@ -0,0 +1,99 @@
+'use strict'
+
+const noop = Function.prototype
+
+class AggregateError extends Error {
+  constructor(errors, message) {
+    super(message)
+    this.errors = errors
+  }
+}
+
+exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 1, signal, stopOnError = true } = {}) {
+  return new Promise((resolve, reject) => {
+    const it = (iterable[Symbol.iterator] || iterable[Symbol.asyncIterator]).call(iterable)
+    const errors = []
+    let running = 0
+    let index = 0
+
+    let onAbort
+    if (signal !== undefined) {
+      onAbort = () => {
+        onRejectedWrapper(new Error('asyncEach aborted'))
+      }
+      signal.addEventListener('abort', onAbort)
+    }
+
+    const clean = () => {
+      onFulfilled = onRejected = noop
+      if (onAbort !== undefined) {
+        signal.removeEventListener('abort', onAbort)
+      }
+    }
+
+    resolve = (resolve =>
+      function resolveAndClean(value) {
+        resolve(value)
+        clean()
+      })(resolve)
+    reject = (reject =>
+      function rejectAndClean(reason) {
+        reject(reason)
+        clean()
+      })(reject)
+
+    let onFulfilled = value => {
+      --running
+      next()
+    }
+    const onFulfilledWrapper = value => onFulfilled(value)
+
+    let onRejected = stopOnError
+      ? reject
+      : error => {
+          --running
+          errors.push(error)
+          next()
+        }
+    const onRejectedWrapper = reason => onRejected(reason)
+
+    let nextIsRunning = false
+    let next = async () => {
+      if (nextIsRunning) {
+        return
+      }
+      nextIsRunning = true
+      if (running < concurrency) {
+        const cursor = await it.next()
+        if (cursor.done) {
+          next = () => {
+            if (running === 0) {
+              if (errors.length === 0) {
+                resolve()
+              } else {
+                reject(new AggregateError(errors))
+              }
+            }
+          }
+        } else {
+          ++running
+          try {
+            const result = iteratee.call(this, cursor.value, index++, iterable)
+            let then
+            if (result != null && typeof result === 'object' && typeof (then = result.then) === 'function') {
+              then.call(result, onFulfilledWrapper, onRejectedWrapper)
+            } else {
+              onFulfilled(result)
+            }
+          } catch (error) {
+            onRejected(error)
+          }
+        }
+        nextIsRunning = false
+        return next()
+      }
+      nextIsRunning = false
+    }
+    next()
+  })
+}

@vates/async-each/index.spec.js

@@ -0,0 +1,99 @@
+'use strict'
+
+/* eslint-env jest */
+
+const { asyncEach } = require('./')
+
+const randomDelay = (max = 10) =>
+  new Promise(resolve => {
+    setTimeout(resolve, Math.floor(Math.random() * max + 1))
+  })
+
+const rejectionOf = p =>
+  new Promise((resolve, reject) => {
+    p.then(reject, resolve)
+  })
+
+describe('asyncEach', () => {
+  const thisArg = 'qux'
+  const values = ['foo', 'bar', 'baz']
+
+  Object.entries({
+    'sync iterable': () => values,
+    'async iterable': async function* () {
+      for (const value of values) {
+        await randomDelay()
+        yield value
+      }
+    },
+  }).forEach(([what, getIterable]) =>
+    describe('with ' + what, () => {
+      let iterable
+      beforeEach(() => {
+        iterable = getIterable()
+      })
+
+      it('works', async () => {
+        const iteratee = jest.fn(async () => {})
+
+        await asyncEach.call(thisArg, iterable, iteratee)
+
+        expect(iteratee.mock.instances).toEqual(Array.from(values, () => thisArg))
+        expect(iteratee.mock.calls).toEqual(Array.from(values, (value, index) => [value, index, iterable]))
+      })
+      ;[1, 2, 4].forEach(concurrency => {
+        it('respects a concurrency of ' + concurrency, async () => {
+          let running = 0
+
+          await asyncEach(
+            values,
+            async () => {
+              ++running
+              expect(running).toBeLessThanOrEqual(concurrency)
+              await randomDelay()
+              --running
+            },
+            { concurrency }
+          )
+        })
+      })
+
+      it('stops on first error when stopOnError is true', async () => {
+        const error = new Error()
+        const iteratee = jest.fn((_, i) => {
+          if (i === 1) {
+            throw error
+          }
+        })
+
+        expect(await rejectionOf(asyncEach(iterable, iteratee, { stopOnError: true }))).toBe(error)
+        expect(iteratee).toHaveBeenCalledTimes(2)
+      })
+
+      it('rejects AggregateError when stopOnError is false', async () => {
+        const errors = []
+        const iteratee = jest.fn(() => {
+          const error = new Error()
+          errors.push(error)
+          throw error
+        })
+
+        const error = await rejectionOf(asyncEach(iterable, iteratee, { stopOnError: false }))
+        expect(error.errors).toEqual(errors)
+        expect(iteratee.mock.calls).toEqual(Array.from(values, (value, index) => [value, index, iterable]))
+      })
+
+      it('can be interrupted with an AbortSignal', async () => {
+        const ac = new AbortController()
+        const iteratee = jest.fn((_, i) => {
+          if (i === 1) {
+            ac.abort()
+          }
+        })
+
+        await expect(asyncEach(iterable, iteratee, { signal: ac.signal })).rejects.toThrow('asyncEach aborted')
+        expect(iteratee).toHaveBeenCalledTimes(2)
+      })
+    })
+  )
+})

@vates/async-each/package.json

@@ -0,0 +1,34 @@
+{
+  "private": false,
+  "name": "@vates/async-each",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/async-each",
+  "description": "Run async fn for each item in (async) iterable",
+  "keywords": [
+    "array",
+    "async",
+    "collection",
+    "each",
+    "for",
+    "foreach",
+    "iterable",
+    "iterator"
+  ],
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/async-each",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "license": "ISC",
+  "version": "0.1.0",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@xen-orchestra/backups-cli/package.json

@@ -7,8 +7,8 @@
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "dependencies": {
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.15.1",
-    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/backups": "^0.16.0",
+    "@xen-orchestra/fs": "^0.19.1",
     "filenamify": "^4.1.0",
     "getopts": "^2.2.5",
     "lodash": "^4.17.15",
@@ -27,7 +27,7 @@
   "scripts": {
     "postversion": "npm publish --access public"
   },
-  "version": "0.6.0",
+  "version": "0.6.1",
   "license": "AGPL-3.0-or-later",
   "author": {
     "name": "Vates SAS",

@xen-orchestra/backups/RemoteAdapter.js

@@ -3,19 +3,19 @@ const Disposable = require('promise-toolbox/Disposable.js')
 const fromCallback = require('promise-toolbox/fromCallback.js')
 const fromEvent = require('promise-toolbox/fromEvent.js')
 const pDefer = require('promise-toolbox/defer.js')
-const pump = require('pump')
-const { basename, dirname, join, normalize, resolve } = require('path')
+const { dirname, join, normalize, resolve } = require('path')
 const { createLogger } = require('@xen-orchestra/log')
-const { createSyntheticStream, mergeVhd, VhdFile } = require('vhd-lib')
+const { Constants, createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdSynthetic } = require('vhd-lib')
 const { deduped } = require('@vates/disposable/deduped.js')
 const { execFile } = require('child_process')
 const { readdir, stat } = require('fs-extra')
+const { v4: uuidv4 } = require('uuid')
 const { ZipFile } = require('yazl')
 
 const { BACKUP_DIR } = require('./_getVmBackupDir.js')
 const { cleanVm } = require('./_cleanVm.js')
 const { getTmpDir } = require('./_getTmpDir.js')
-const { isMetadataFile, isVhdFile } = require('./_backupType.js')
+const { isMetadataFile } = require('./_backupType.js')
 const { isValidXva } = require('./_isValidXva.js')
 const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js')
 const { lvs, pvs } = require('./_lvm.js')
@@ -77,48 +77,6 @@ class RemoteAdapter {
     return this._handler
   }
 
-  async _deleteVhd(path) {
-    const handler = this._handler
-    const vhds = await asyncMapSettled(
-      await handler.list(dirname(path), {
-        filter: isVhdFile,
-        prependDir: true,
-      }),
-      async path => {
-        try {
-          const vhd = new VhdFile(handler, path)
-          await vhd.readHeaderAndFooter()
-          return {
-            footer: vhd.footer,
-            header: vhd.header,
-            path,
-          }
-        } catch (error) {
-          // Do not fail on corrupted VHDs (usually uncleaned temporary files),
-          // they are probably inconsequent to the backup process and should not
-          // fail it.
-          warn(`BackupNg#_deleteVhd ${path}`, { error })
-        }
-      }
-    )
-    const base = basename(path)
-    const child = vhds.find(_ => _ !== undefined && _.header.parentUnicodeName === base)
-    if (child === undefined) {
-      await handler.unlink(path)
-      return 0
-    }
-
-    try {
-      const childPath = child.path
-      const mergedDataSize = await mergeVhd(handler, path, handler, childPath)
-      await handler.rename(path, childPath)
-      return mergedDataSize
-    } catch (error) {
-      handler.unlink(path).catch(warn)
-      throw error
-    }
-  }
-
   async _findPartition(devicePath, partitionId) {
     const partitions = await listPartitions(devicePath)
     const partition = partitions.find(_ => _.id === partitionId)
@@ -255,7 +213,7 @@ class RemoteAdapter {
     const handler = this._handler
 
     // unused VHDs will be detected by `cleanVm`
-    await asyncMapSettled(backups, ({ _filename }) => handler.unlink(_filename))
+    await asyncMapSettled(backups, ({ _filename }) => VhdAbstract.unlink(handler, _filename))
   }
 
   async deleteMetadataBackup(backupId) {
@@ -354,6 +312,17 @@ class RemoteAdapter {
     return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
   }
 
+  // this function will be the one where we plug the logic of the storage format by fs type/user settings
+
+  // if the file is named .vhd => vhd
+  // if the file is named alias.vhd => alias to a vhd
+  getVhdFileName(baseName) {
+    if (this._handler.type === 's3') {
+      return `${baseName}.alias.vhd` // we want an alias to a vhddirectory
+    }
+    return `${baseName}.vhd`
+  }
+
   async listAllVmBackups() {
     const handler = this._handler
 
@@ -498,6 +467,24 @@ class RemoteAdapter {
     return backups.sort(compareTimestamp)
   }
 
+  async writeVhd(path, input, { checksum = true, validator = noop } = {}) {
+    const handler = this._handler
+    let dataPath = path
+
+    if (path.endsWith('.alias.vhd')) {
+      await createVhdDirectoryFromStream(handler, `${dirname(path)}/data/${uuidv4()}.vhd`, input, {
+        concurrency: 16,
+        async validator() {
+          await input.task
+          return validator.apply(this, arguments)
+        },
+      })
+      await VhdAbstract.createAlias(handler, path, dataPath)
+    } else {
+      await this.outputStream(dataPath, input, { checksum, validator })
+    }
+  }
+
   async outputStream(path, input, { checksum = true, validator = noop } = {}) {
     await this._handler.outputStream(path, input, {
       checksum,
@@ -509,6 +496,52 @@ class RemoteAdapter {
     })
   }
 
+  async _createSyntheticStream(handler, paths) {
+    let disposableVhds = []
+
+    // if it's a path : open all hierarchy of parent
+    if (typeof paths === 'string') {
+      let vhd,
+        vhdPath = paths
+      do {
+        const disposable = await openVhd(handler, vhdPath)
+        vhd = disposable.value
+        disposableVhds.push(disposable)
+        vhdPath = resolveRelativeFromFile(vhdPath, vhd.header.parentUnicodeName)
+      } while (vhd.footer.diskType !== Constants.DISK_TYPES.DYNAMIC)
+    } else {
+      // only open the list of path given
+      disposableVhds = paths.map(path => openVhd(handler, path))
+    }
+
+    // I don't want the vhds to be disposed on return
+    // but only when the stream is done ( or failed )
+    const disposables = await Disposable.all(disposableVhds)
+    const vhds = disposables.value
+
+    let disposed = false
+    const disposeOnce = async () => {
+      if (!disposed) {
+        disposed = true
+
+        try {
+          await disposables.dispose()
+        } catch (error) {
+          warn('_createSyntheticStream: failed to dispose VHDs', { error })
+        }
+      }
+    }
+
+    const synthetic = new VhdSynthetic(vhds)
+    await synthetic.readHeaderAndFooter()
+    await synthetic.readBlockAllocationTable()
+    const stream = await synthetic.stream()
+    stream.on('end', disposeOnce)
+    stream.on('close', disposeOnce)
+    stream.on('error', disposeOnce)
+    return stream
+  }
+
   async readDeltaVmBackup(metadata) {
     const handler = this._handler
     const { vbds, vdis, vhds, vifs, vm } = metadata
@@ -516,7 +549,7 @@ class RemoteAdapter {
 
     const streams = {}
     await asyncMapSettled(Object.keys(vdis), async id => {
-      streams[`${id}.vhd`] = await createSyntheticStream(handler, join(dir, vhds[id]))
+      streams[`${id}.vhd`] = await this._createSyntheticStream(handler, join(dir, vhds[id]))
     })
 
     return {

@xen-orchestra/backups/_cleanVm.integ.spec.js

@@ -0,0 +1,390 @@
+/* eslint-env jest */
+
+const rimraf = require('rimraf')
+const tmp = require('tmp')
+const fs = require('fs-extra')
+const { getHandler } = require('@xen-orchestra/fs')
+const { pFromCallback } = require('promise-toolbox')
+const crypto = require('crypto')
+const { RemoteAdapter } = require('./RemoteAdapter')
+const { VHDFOOTER, VHDHEADER } = require('./tests.fixtures.js')
+const { VhdFile, Constants, VhdDirectory, VhdAbstract } = require('vhd-lib')
+
+let tempDir, adapter, handler, jobId, vdiId, basePath
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+  handler = getHandler({ url: `file://${tempDir}` })
+  await handler.sync()
+  adapter = new RemoteAdapter(handler)
+  jobId = uniqueId()
+  vdiId = uniqueId()
+  basePath = `vdis/${jobId}/${vdiId}`
+  await fs.mkdirp(`${tempDir}/${basePath}`)
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+  await handler.forget()
+})
+
+const uniqueId = () => crypto.randomBytes(16).toString('hex')
+
+async function generateVhd(path, opts = {}) {
+  let vhd
+
+  const dataPath = opts.useAlias ? path + '.data' : path
+  if (opts.mode === 'directory') {
+    await handler.mkdir(dataPath)
+    vhd = new VhdDirectory(handler, dataPath)
+  } else {
+    const fd = await handler.openFile(dataPath, 'wx')
+    vhd = new VhdFile(handler, fd)
+  }
+
+  vhd.header = { ...VHDHEADER, ...opts.header }
+  vhd.footer = { ...VHDFOOTER, ...opts.footer }
+  vhd.footer.uuid = Buffer.from(crypto.randomBytes(16))
+
+  if (vhd.header.parentUnicodeName) {
+    vhd.footer.diskType = Constants.DISK_TYPES.DIFFERENCING
+  } else {
+    vhd.footer.diskType = Constants.DISK_TYPES.DYNAMIC
+  }
+
+  if (opts.useAlias === true) {
+    await VhdAbstract.createAlias(handler, path + '.alias.vhd', dataPath)
+  }
+
+  await vhd.writeBlockAllocationTable()
+  await vhd.writeHeader()
+  await vhd.writeFooter()
+  return vhd
+}
+
+test('It remove broken vhd', async () => {
+  // todo also tests a directory and an alias
+
+  await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
+  expect((await handler.list(basePath)).length).toEqual(1)
+  let loggued = ''
+  const onLog = message => {
+    loggued += message
+  }
+  await adapter.cleanVm('/', { remove: false, onLog })
+  expect(loggued).toEqual(`error while checking the VHD with path /${basePath}/notReallyAVhd.vhd`)
+  // not removed
+  expect((await handler.list(basePath)).length).toEqual(1)
+  // really remove it
+  await adapter.cleanVm('/', { remove: true, onLog })
+  expect((await handler.list(basePath)).length).toEqual(0)
+})
+
+test('it remove vhd with missing or multiple ancestors', async () => {
+  // one with a broken parent
+  await generateVhd(`${basePath}/abandonned.vhd`, {
+    header: {
+      parentUnicodeName: 'gone.vhd',
+      parentUid: Buffer.from(crypto.randomBytes(16)),
+    },
+  })
+
+  // one orphan, which is a full vhd, no parent
+  const orphan = await generateVhd(`${basePath}/orphan.vhd`)
+  // a child to the orphan
+  await generateVhd(`${basePath}/child.vhd`, {
+    header: {
+      parentUnicodeName: 'orphan.vhd',
+      parentUid: orphan.footer.uuid,
+    },
+  })
+
+  // clean
+  let loggued = ''
+  const onLog = message => {
+    loggued += message + '\n'
+  }
+  await adapter.cleanVm('/', { remove: true, onLog })
+
+  const deletedOrphanVhd = loggued.match(/deleting orphan VHD/g) || []
+  expect(deletedOrphanVhd.length).toEqual(1) // only one vhd should have been deleted
+  const deletedAbandonnedVhd = loggued.match(/abandonned.vhd is missing/g) || []
+  expect(deletedAbandonnedVhd.length).toEqual(1) // and it must be abandonned.vhd
+
+  // we don't test the filew on disk, since they will all be marker as unused and deleted without a metadata.json file
+})
+
+test('it remove backup meta data referencing a missing vhd in delta backup', async () => {
+  // create a metadata file marking child and orphan as ok
+  await handler.writeFile(
+    `metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      vhds: [
+        `${basePath}/orphan.vhd`,
+        `${basePath}/child.vhd`,
+        // abandonned.json is not here
+      ],
+    })
+  )
+
+  await generateVhd(`${basePath}/abandonned.vhd`)
+
+  // one orphan, which is a full vhd, no parent
+  const orphan = await generateVhd(`${basePath}/orphan.vhd`)
+
+  // a child to the orphan
+  await generateVhd(`${basePath}/child.vhd`, {
+    header: {
+      parentUnicodeName: 'orphan.vhd',
+      parentUid: orphan.footer.uuid,
+    },
+  })
+
+  let loggued = ''
+  const onLog = message => {
+    loggued += message + '\n'
+  }
+  await adapter.cleanVm('/', { remove: true, onLog })
+  let matched = loggued.match(/deleting unused VHD /g) || []
+  expect(matched.length).toEqual(1) // only one vhd should have been deleted
+  matched = loggued.match(/abandonned.vhd is unused/g) || []
+  expect(matched.length).toEqual(1) // and it must be abandonned.vhd
+
+  // a missing vhd cause clean to remove all vhds
+  await handler.writeFile(
+    `metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      vhds: [
+        `${basePath}/deleted.vhd`, // in metadata but not in vhds
+        `${basePath}/orphan.vhd`,
+        `${basePath}/child.vhd`,
+        // abandonned.json is not here
+      ],
+    }),
+    { flags: 'w' }
+  )
+  loggued = ''
+  await adapter.cleanVm('/', { remove: true, onLog })
+  matched = loggued.match(/deleting unused VHD /g) || []
+  expect(matched.length).toEqual(2) // all vhds (orphan and  child  ) should have been deleted
+})
+
+test('it merges delta of non destroyed chain', async () => {
+  await handler.writeFile(
+    `metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      size: 209920,
+      vhds: [
+        `${basePath}/grandchild.vhd`, // grand child should not be merged
+        `${basePath}/child.vhd`,
+        // orphan is not here, he should be merged in child
+      ],
+    })
+  )
+
+  // one orphan, which is a full vhd, no parent
+  const orphan = await generateVhd(`${basePath}/orphan.vhd`)
+  // a child to the orphan
+  const child = await generateVhd(`${basePath}/child.vhd`, {
+    header: {
+      parentUnicodeName: 'orphan.vhd',
+      parentUid: orphan.footer.uuid,
+    },
+  })
+  // a grand child
+  await generateVhd(`${basePath}/grandchild.vhd`, {
+    header: {
+      parentUnicodeName: 'child.vhd',
+      parentUid: child.footer.uuid,
+    },
+  })
+
+  let loggued = ''
+  const onLog = message => {
+    loggued += message + '\n'
+  }
+  await adapter.cleanVm('/', { remove: true, onLog })
+  expect(loggued).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused\n`)
+  loggued = ''
+  await adapter.cleanVm('/', { remove: true, merge: true, onLog })
+  const [unused, merging] = loggued.split('\n')
+  expect(unused).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused`)
+  expect(merging).toEqual(`merging /${basePath}/child.vhd into /${basePath}/orphan.vhd`)
+
+  // merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
+
+  // only check deletion
+  const remainingVhds = await handler.list(basePath)
+  expect(remainingVhds.length).toEqual(2)
+  expect(remainingVhds.includes('child.vhd')).toEqual(true)
+  expect(remainingVhds.includes('grandchild.vhd')).toEqual(true)
+})
+
+test('it finish unterminated merge ', async () => {
+  await handler.writeFile(
+    `metadata.json`,
+    JSON.stringify({
+      mode: 'delta',
+      size: 209920,
+      vhds: [
+        `${basePath}/orphan.vhd`, // grand child should not be merged
+        `${basePath}/child.vhd`,
+        // orphan is not here, he should be merged in child
+      ],
+    })
+  )
+
+  // one orphan, which is a full vhd, no parent
+  const orphan = await generateVhd(`${basePath}/orphan.vhd`)
+  // a child to the orphan
+  const child = await generateVhd(`${basePath}/child.vhd`, {
+    header: {
+      parentUnicodeName: 'orphan.vhd',
+      parentUid: orphan.footer.uuid,
+    },
+  })
+  // a merge in progress file
+  await handler.writeFile(
+    `${basePath}/.orphan.vhd.merge.json`,
+    JSON.stringify({
+      parent: {
+        header: orphan.header.checksum,
+      },
+      child: {
+        header: child.header.checksum,
+      },
+    })
+  )
+
+  // a unfinished merging
+  await adapter.cleanVm('/', { remove: true, merge: true })
+  // merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
+
+  // only check deletion
+  const remainingVhds = await handler.list(basePath)
+  expect(remainingVhds.length).toEqual(1)
+  expect(remainingVhds.includes('child.vhd')).toEqual(true)
+})
+
+// each of the vhd can be a file, a directory, an alias to a file or an alias to a directory
+// the message an resulting files should be identical to the output with vhd files which is tested independantly
+
+describe('tests mulitple combination ', () => {
+  for (const useAlias of [true, false]) {
+    for (const vhdMode of ['file', 'directory']) {
+      test(`alias : ${useAlias}, mode: ${vhdMode}`, async () => {
+        // a broken VHD
+        const brokenVhdDataPath = basePath + useAlias ? 'broken.data' : 'broken.vhd'
+        if (vhdMode === 'directory') {
+          await handler.mkdir(brokenVhdDataPath)
+        } else {
+          await handler.writeFile(brokenVhdDataPath, 'notreallyavhd')
+        }
+        if (useAlias) {
+          await VhdAbstract.createAlias(handler, 'broken.alias.vhd', brokenVhdDataPath)
+        }
+
+        // a vhd non referenced in metada
+        await generateVhd(`${basePath}/nonreference.vhd`, { useAlias, mode: vhdMode })
+        // an abandonded delta vhd without its parent
+        await generateVhd(`${basePath}/abandonned.vhd`, {
+          useAlias,
+          mode: vhdMode,
+          header: {
+            parentUnicodeName: 'gone.vhd',
+            parentUid: crypto.randomBytes(16),
+          },
+        })
+        // an ancestor of a vhd present in metadata
+        const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, {
+          useAlias,
+          mode: vhdMode,
+        })
+        const child = await generateVhd(`${basePath}/child.vhd`, {
+          useAlias,
+          mode: vhdMode,
+          header: {
+            parentUnicodeName: 'ancestor.vhd' + (useAlias ? '.alias.vhd' : ''),
+            parentUid: ancestor.footer.uuid,
+          },
+        })
+        // a grand child  vhd in metadata
+        await generateVhd(`${basePath}/grandchild.vhd`, {
+          useAlias,
+          mode: vhdMode,
+          header: {
+            parentUnicodeName: 'child.vhd' + (useAlias ? '.alias.vhd' : ''),
+            parentUid: child.footer.uuid,
+          },
+        })
+
+        // an older parent that was merging in clean
+        const cleanAncestor = await generateVhd(`${basePath}/cleanAncestor.vhd`, {
+          useAlias,
+          mode: vhdMode,
+        })
+        // a clean  vhd in metadata
+        const clean = await generateVhd(`${basePath}/clean.vhd`, {
+          useAlias,
+          mode: vhdMode,
+          header: {
+            parentUnicodeName: 'cleanAncestor.vhd' + (useAlias ? '.alias.vhd' : ''),
+            parentUid: cleanAncestor.footer.uuid,
+          },
+        })
+
+        await handler.writeFile(
+          `${basePath}/.cleanAncestor.vhd${useAlias ? '.alias.vhd' : ''}.merge.json`,
+          JSON.stringify({
+            parent: {
+              header: cleanAncestor.header.checksum,
+            },
+            child: {
+              header: clean.header.checksum,
+            },
+          })
+        )
+
+        // the metadata file
+        await handler.writeFile(
+          `metadata.json`,
+          JSON.stringify({
+            mode: 'delta',
+            vhds: [
+              `${basePath}/grandchild.vhd` + (useAlias ? '.alias.vhd' : ''), // grand child should not be merged
+              `${basePath}/child.vhd` + (useAlias ? '.alias.vhd' : ''),
+              `${basePath}/clean.vhd` + (useAlias ? '.alias.vhd' : ''),
+            ],
+          })
+        )
+        await adapter.cleanVm('/', { remove: true, merge: true })
+
+        // broken vhd, non referenced, abandonned should be deleted ( alias and data)
+        // ancestor and child should be merged
+        // grand child and clean vhd should not have changed
+        const survivors = await handler.list(basePath)
+        // console.log(survivors)
+        if (useAlias) {
+          // the goal of the alias : do not move a full folder
+          expect(survivors).toContain('ancestor.vhd.data')
+          expect(survivors).toContain('grandchild.vhd.data')
+          expect(survivors).toContain('cleanAncestor.vhd.data')
+          expect(survivors).toContain('clean.vhd.alias.vhd')
+          expect(survivors).toContain('child.vhd.alias.vhd')
+          expect(survivors).toContain('grandchild.vhd.alias.vhd')
+          expect(survivors.length).toEqual(6)
+        } else {
+          expect(survivors).toContain('clean.vhd')
+          expect(survivors).toContain('child.vhd')
+          expect(survivors).toContain('grandchild.vhd')
+          expect(survivors.length).toEqual(3)
+        }
+      })
+    }
+  }
+})

@xen-orchestra/backups/_cleanVm.js

@@ -1,13 +1,14 @@
 const assert = require('assert')
 const sum = require('lodash/sum')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { VhdFile, mergeVhd } = require('vhd-lib')
+const { Constants, mergeVhd, openVhd, VhdAbstract, VhdFile } = require('vhd-lib')
 const { dirname, resolve } = require('path')
-const { DISK_TYPE_DIFFERENCING } = require('vhd-lib/dist/_constants.js')
+const { DISK_TYPES } = Constants
 const { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } = require('./_backupType.js')
 const { limitConcurrency } = require('limit-concurrency-decorator')
 
 const { Task } = require('./Task.js')
+const { Disposable } = require('promise-toolbox')
 
 // chain is an array of VHDs from child to parent
 //
@@ -65,12 +66,12 @@ async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
     clearInterval(handle)
 
     await Promise.all([
-      handler.rename(parent, child),
+      VhdAbstract.rename(handler, parent, child),
       asyncMap(children.slice(0, -1), child => {
         onLog(`the VHD ${child} is unused`)
         if (remove) {
           onLog(`deleting unused VHD ${child}`)
-          return handler.unlink(child)
+          return VhdAbstract.unlink(handler, child)
         }
       }),
     ])
@@ -137,53 +138,55 @@ exports.cleanVm = async function cleanVm(
   // remove broken VHDs
   await asyncMap(vhdsList.vhds, async path => {
     try {
-      const vhd = new VhdFile(handler, path)
-      await vhd.readHeaderAndFooter(!vhdsList.interruptedVhds.has(path))
-      vhds.add(path)
-      if (vhd.footer.diskType === DISK_TYPE_DIFFERENCING) {
-        const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
-        vhdParents[path] = parent
-        if (parent in vhdChildren) {
-          const error = new Error('this script does not support multiple VHD children')
-          error.parent = parent
-          error.child1 = vhdChildren[parent]
-          error.child2 = path
-          throw error // should we throw?
+      await Disposable.use(openVhd(handler, path, { checkSecondFooter: !vhdsList.interruptedVhds.has(path) }), vhd => {
+        vhds.add(path)
+        if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {
+          const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
+          vhdParents[path] = parent
+          if (parent in vhdChildren) {
+            const error = new Error('this script does not support multiple VHD children')
+            error.parent = parent
+            error.child1 = vhdChildren[parent]
+            error.child2 = path
+            throw error // should we throw?
+          }
+          vhdChildren[parent] = path
         }
-        vhdChildren[parent] = path
-      }
+      })
     } catch (error) {
       onLog(`error while checking the VHD with path ${path}`, { error })
       if (error?.code === 'ERR_ASSERTION' && remove) {
         onLog(`deleting broken ${path}`)
-        await handler.unlink(path)
+        return VhdAbstract.unlink(handler, path)
       }
     }
   })
 
+  // @todo : add check for data folder of alias not referenced in a valid alias
+
   // remove VHDs with missing ancestors
   {
     const deletions = []
 
     // return true if the VHD has been deleted or is missing
-    const deleteIfOrphan = vhd => {
-      const parent = vhdParents[vhd]
+    const deleteIfOrphan = vhdPath => {
+      const parent = vhdParents[vhdPath]
       if (parent === undefined) {
         return
       }
 
       // no longer needs to be checked
-      delete vhdParents[vhd]
+      delete vhdParents[vhdPath]
 
       deleteIfOrphan(parent)
 
       if (!vhds.has(parent)) {
-        vhds.delete(vhd)
+        vhds.delete(vhdPath)
 
-        onLog(`the parent ${parent} of the VHD ${vhd} is missing`)
+        onLog(`the parent ${parent} of the VHD ${vhdPath} is missing`)
         if (remove) {
-          onLog(`deleting orphan VHD ${vhd}`)
-          deletions.push(handler.unlink(vhd))
+          onLog(`deleting orphan VHD ${vhdPath}`)
+          deletions.push(VhdAbstract.unlink(handler, vhdPath))
         }
       }
     }
@@ -253,15 +256,26 @@ exports.cleanVm = async function cleanVm(
         const { vhds } = metadata
         return Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
       })()
-
       // FIXME: find better approach by keeping as much of the backup as
       // possible (existing disks) even if one disk is missing
       if (linkedVhds.every(_ => vhds.has(_))) {
         linkedVhds.forEach(_ => unusedVhds.delete(_))
 
-        size = await asyncMap(linkedVhds, vhd => handler.getSize(vhd)).then(sum, error => {
-          onLog(`failed to get size of ${json}`, { error })
-        })
+        // checking the size of a vhd directory is costly
+        // 1 Http Query per 1000 blocks
+        // we only check size of all the vhd are VhdFiles
+
+        const shouldComputeSize = linkedVhds.every(vhd => vhd instanceof VhdFile)
+        if (shouldComputeSize) {
+          try {
+            await Disposable.use(Disposable.all(linkedVhds.map(vhdPath => openVhd(handler, vhdPath))), async vhds => {
+              const sizes = await asyncMap(vhds, vhd => vhd.getSize())
+              size = sum(sizes)
+            })
+          } catch (error) {
+            onLog(`failed to get size of ${json}`, { error })
+          }
+        }
       } else {
         onLog(`Some VHDs linked to the metadata ${json} are missing`)
         if (remove) {
@@ -324,7 +338,7 @@ exports.cleanVm = async function cleanVm(
       onLog(`the VHD ${vhd} is unused`)
       if (remove) {
         onLog(`deleting unused VHD ${vhd}`)
-        unusedVhdsDeletion.push(handler.unlink(vhd))
+        unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
       }
     }
 

@xen-orchestra/backups/package.json

@@ -8,7 +8,7 @@
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
-  "version": "0.15.1",
+  "version": "0.16.0",
   "engines": {
     "node": ">=14.6"
   },
@@ -20,7 +20,7 @@
     "@vates/disposable": "^0.1.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/fs": "^0.19.1",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/template": "^0.1.0",
     "compare-versions": "^4.0.1",
@@ -35,11 +35,12 @@
     "promise-toolbox": "^0.20.0",
     "proper-lockfile": "^4.1.2",
     "pump": "^3.0.0",
-    "vhd-lib": "^1.3.0",
+    "uuid": "^8.3.2",
+    "vhd-lib": "^2.0.1",
     "yazl": "^2.5.1"
   },
   "peerDependencies": {
-    "@xen-orchestra/xapi": "^0.8.0"
+    "@xen-orchestra/xapi": "^0.8.4"
   },
   "license": "AGPL-3.0-or-later",
   "author": {

@xen-orchestra/backups/tests.fixtures.js

@@ -0,0 +1,92 @@
+// a valid footer of a 2
+exports.VHDFOOTER = {
+  cookie: 'conectix',
+  features: 2,
+  fileFormatVersion: 65536,
+  dataOffset: 512,
+  timestamp: 0,
+  creatorApplication: 'caml',
+  creatorVersion: 1,
+  creatorHostOs: 0,
+  originalSize: 53687091200,
+  currentSize: 53687091200,
+  diskGeometry: { cylinders: 25700, heads: 16, sectorsPerTrackCylinder: 255 },
+  diskType: 3,
+  checksum: 4294962945,
+  uuid: Buffer.from('d8dbcad85265421e8b298d99c2eec551', 'utf-8'),
+  saved: '',
+  hidden: '',
+  reserved: '',
+}
+exports.VHDHEADER = {
+  cookie: 'cxsparse',
+  dataOffset: undefined,
+  tableOffset: 2048,
+  headerVersion: 65536,
+  maxTableEntries: 25600,
+  blockSize: 2097152,
+  checksum: 4294964241,
+  parentUuid: null,
+  parentTimestamp: 0,
+  reserved1: 0,
+  parentUnicodeName: '',
+  parentLocatorEntry: [
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+    {
+      platformCode: 0,
+      platformDataSpace: 0,
+      platformDataLength: 0,
+      reserved: 0,
+      platformDataOffset: 0,
+    },
+  ],
+  reserved2: '',
+}

@xen-orchestra/backups/writers/DeltaBackupWriter.js

@@ -3,7 +3,7 @@ const map = require('lodash/map.js')
 const mapValues = require('lodash/mapValues.js')
 const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { chainVhd, checkVhdChain, VhdFile } = require('vhd-lib')
+const { chainVhd, checkVhdChain, openVhd, VhdAbstract } = require('vhd-lib')
 const { createLogger } = require('@xen-orchestra/log')
 const { dirname } = require('path')
 
@@ -16,6 +16,7 @@ const { MixinBackupWriter } = require('./_MixinBackupWriter.js')
 const { AbstractDeltaWriter } = require('./_AbstractDeltaWriter.js')
 const { checkVhd } = require('./_checkVhd.js')
 const { packUuid } = require('./_packUuid.js')
+const { Disposable } = require('promise-toolbox')
 
 const { warn } = createLogger('xo:backups:DeltaBackupWriter')
 
@@ -37,13 +38,13 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
         await asyncMap(vhds, async path => {
           try {
             await checkVhdChain(handler, path)
-
-            const vhd = new VhdFile(handler, path)
-            await vhd.readHeaderAndFooter()
-            found = found || vhd.footer.uuid.equals(packUuid(baseUuid))
+            await Disposable.use(
+              openVhd(handler, path),
+              vhd => (found = found || vhd.footer.uuid.equals(packUuid(baseUuid)))
+            )
           } catch (error) {
             warn('checkBaseVdis', { error })
-            await ignoreErrors.call(handler.unlink(path))
+            await ignoreErrors.call(VhdAbstract.unlink(handler, path))
           }
         })
       } catch (error) {
@@ -144,7 +145,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
               // don't do delta for it
               vdi.uuid
             : vdi.$snapshot_of$uuid
-        }/${basename}.vhd`
+        }/${adapter.getVhdFileName(basename)}`
     )
 
     const metadataFilename = `${backupDir}/${basename}.json`
@@ -188,7 +189,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
             await checkVhd(handler, parentPath)
           }
 
-          await adapter.outputStream(path, deltaExport.streams[`${id}.vhd`], {
+          await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
             // no checksum for VHDs, because they will be invalidated by
             // merges and chainings
             checksum: false,
@@ -200,11 +201,11 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           }
 
           // set the correct UUID in the VHD
-          const vhd = new VhdFile(handler, path)
-          await vhd.readHeaderAndFooter()
-          vhd.footer.uuid = packUuid(vdi.uuid)
-          await vhd.readBlockAllocationTable() // required by writeFooter()
-          await vhd.writeFooter()
+          await Disposable.use(openVhd(handler, path), async vhd => {
+            vhd.footer.uuid = packUuid(vdi.uuid)
+            await vhd.readBlockAllocationTable() // required by writeFooter()
+            await vhd.writeFooter()
+          })
         })
       )
       return {

@xen-orchestra/backups/writers/_checkVhd.js

@@ -1,5 +1,6 @@
-const Vhd = require('vhd-lib').VhdFile
+const openVhd = require('vhd-lib').openVhd
+const Disposable = require('promise-toolbox/Disposable')
 
 exports.checkVhd = async function checkVhd(handler, path) {
-  await new Vhd(handler, path).readHeaderAndFooter()
+  await Disposable.use(openVhd(handler, path), () => {})
 }

@xen-orchestra/cron/src/index.spec.js

@@ -2,6 +2,8 @@
 
 import { createSchedule } from './'
 
+jest.useFakeTimers()
+
 const wrap = value => () => value
 
 describe('issues', () => {

@xen-orchestra/fs/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "@xen-orchestra/fs",
-  "version": "0.18.0",
+  "version": "0.19.1",
   "license": "AGPL-3.0-or-later",
   "description": "The File System for Xen Orchestra backups.",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
@@ -33,7 +33,7 @@
     "proper-lockfile": "^4.1.2",
     "readable-stream": "^3.0.6",
     "through2": "^4.0.2",
-    "xo-remote-parser": "^0.7.0"
+    "xo-remote-parser": "^0.8.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

@xen-orchestra/fs/src/abstract.js

@@ -76,6 +76,7 @@ export default class RemoteHandlerAbstract {
 
     const sharedLimit = limitConcurrency(options.maxParallelOperations ?? DEFAULT_MAX_PARALLEL_OPERATIONS)
     this.closeFile = sharedLimit(this.closeFile)
+    this.copy = sharedLimit(this.copy)
     this.getInfo = sharedLimit(this.getInfo)
     this.getSize = sharedLimit(this.getSize)
     this.list = sharedLimit(this.list)
@@ -307,6 +308,17 @@ export default class RemoteHandlerAbstract {
     return p
   }
 
+  async copy(oldPath, newPath, { checksum = false } = {}) {
+    oldPath = normalizePath(oldPath)
+    newPath = normalizePath(newPath)
+
+    let p = timeout.call(this._copy(oldPath, newPath), this._timeout)
+    if (checksum) {
+      p = Promise.all([p, this._copy(checksumFile(oldPath), checksumFile(newPath))])
+    }
+    return p
+  }
+
   async rmdir(dir) {
     await timeout.call(this._rmdir(normalizePath(dir)).catch(ignoreEnoent), this._timeout)
   }
@@ -519,6 +531,9 @@ export default class RemoteHandlerAbstract {
   async _rename(oldPath, newPath) {
     throw new Error('Not implemented')
   }
+  async _copy(oldPath, newPath) {
+    throw new Error('Not implemented')
+  }
 
   async _rmdir(dir) {
     throw new Error('Not implemented')

@xen-orchestra/fs/src/local.js

@@ -33,6 +33,10 @@ export default class LocalHandler extends RemoteHandlerAbstract {
     return fs.close(fd)
   }
 
+  async _copy(oldPath, newPath) {
+    return fs.copy(this._getFilePath(oldPath), this._getFilePath(newPath))
+  }
+
   async _createReadStream(file, options) {
     if (typeof file === 'string') {
       const stream = fs.createReadStream(this._getFilePath(file), options)

@xen-orchestra/fs/src/s3.js

@@ -1,6 +1,7 @@
 import aws from '@sullux/aws-sdk'
 import assert from 'assert'
 import http from 'http'
+import https from 'https'
 import { parse } from 'xo-remote-parser'
 
 import RemoteHandlerAbstract from './abstract'
@@ -16,7 +17,7 @@ const IDEAL_FRAGMENT_SIZE = Math.ceil(MAX_OBJECT_SIZE / MAX_PARTS_COUNT) // the
 export default class S3Handler extends RemoteHandlerAbstract {
   constructor(remote, _opts) {
     super(remote)
-    const { host, path, username, password, protocol, region } = parse(remote.url)
+    const { allowUnauthorized, host, path, username, password, protocol, region } = parse(remote.url)
     const params = {
       accessKeyId: username,
       apiVersion: '2006-03-01',
@@ -29,8 +30,13 @@ export default class S3Handler extends RemoteHandlerAbstract {
       },
     }
     if (protocol === 'http') {
-      params.httpOptions.agent = new http.Agent()
+      params.httpOptions.agent = new http.Agent({ keepAlive: true })
       params.sslEnabled = false
+    } else if (protocol === 'https') {
+      params.httpOptions.agent = new https.Agent({
+        rejectUnauthorized: !allowUnauthorized,
+        keepAlive: true,
+      })
     }
     if (region !== undefined) {
       params.region = region
@@ -51,6 +57,27 @@ export default class S3Handler extends RemoteHandlerAbstract {
     return { Bucket: this._bucket, Key: this._dir + file }
   }
 
+  async _copy(oldPath, newPath) {
+    const size = await this._getSize(oldPath)
+    const multipartParams = await this._s3.createMultipartUpload({ ...this._createParams(newPath) })
+    const param2 = { ...multipartParams, CopySource: `/${this._bucket}/${this._dir}${oldPath}` }
+    try {
+      const parts = []
+      let start = 0
+      while (start < size) {
+        const range = `bytes=${start}-${Math.min(start + MAX_PART_SIZE, size) - 1}`
+        const partParams = { ...param2, PartNumber: parts.length + 1, CopySourceRange: range }
+        const upload = await this._s3.uploadPartCopy(partParams)
+        parts.push({ ETag: upload.CopyPartResult.ETag, PartNumber: partParams.PartNumber })
+        start += MAX_PART_SIZE
+      }
+      await this._s3.completeMultipartUpload({ ...multipartParams, MultipartUpload: { Parts: parts } })
+    } catch (e) {
+      await this._s3.abortMultipartUpload(multipartParams)
+      throw e
+    }
+  }
+
   async _isNotEmptyDir(path) {
     const result = await this._s3.listObjectsV2({
       Bucket: this._bucket,
@@ -147,25 +174,9 @@ export default class S3Handler extends RemoteHandlerAbstract {
     // nothing to do, directories do not exist, they are part of the files' path
   }
 
+  // s3 doesn't have a rename operation, so copy + delete source
   async _rename(oldPath, newPath) {
-    const size = await this._getSize(oldPath)
-    const multipartParams = await this._s3.createMultipartUpload({ ...this._createParams(newPath) })
-    const param2 = { ...multipartParams, CopySource: `/${this._bucket}/${this._dir}${oldPath}` }
-    try {
-      const parts = []
-      let start = 0
-      while (start < size) {
-        const range = `bytes=${start}-${Math.min(start + MAX_PART_SIZE, size) - 1}`
-        const partParams = { ...param2, PartNumber: parts.length + 1, CopySourceRange: range }
-        const upload = await this._s3.uploadPartCopy(partParams)
-        parts.push({ ETag: upload.CopyPartResult.ETag, PartNumber: partParams.PartNumber })
-        start += MAX_PART_SIZE
-      }
-      await this._s3.completeMultipartUpload({ ...multipartParams, MultipartUpload: { Parts: parts } })
-    } catch (e) {
-      await this._s3.abortMultipartUpload(multipartParams)
-      throw e
-    }
+    await this.copy(oldPath, newPath)
     await this._s3.deleteObject(this._createParams(oldPath))
   }
 
@@ -211,6 +222,23 @@ export default class S3Handler extends RemoteHandlerAbstract {
     // nothing to do, directories do not exist, they are part of the files' path
   }
 
+  // reimplement _rmTree to handle efficiantly path with more than 1000 entries in trees
+  // @todo : use parallel processing for unlink
+  async _rmTree(path) {
+    let NextContinuationToken
+    do {
+      const result = await this._s3.listObjectsV2({
+        Bucket: this._bucket,
+        Prefix: this._dir + path + '/',
+        ContinuationToken: NextContinuationToken,
+      })
+      NextContinuationToken = result.isTruncated ? null : result.NextContinuationToken
+      for (const path of result.Contents) {
+        await this._unlink(path)
+      }
+    } while (NextContinuationToken !== null)
+  }
+
   async _write(file, buffer, position) {
     if (typeof file !== 'string') {
       file = file.fd

@xen-orchestra/log/USAGE.md

@@ -48,6 +48,10 @@ configure([
     // if filter is a string, then it is pattern
     // (https://github.com/visionmedia/debug#wildcards) which is
     // matched against the namespace of the logs
+    //
+    // If it's an array, it will be handled as an array of filters
+    // and the transport will be used if any one of them match the
+    // current log
     filter: process.env.DEBUG,
 
     transport: transportConsole(),

@xen-orchestra/mixins/Config.js

@@ -13,7 +13,7 @@ module.exports = class Config {
     const watchers = (this._watchers = new Set())
 
     app.hooks.on('start', async () => {
-      app.hooks.on(
+      app.hooks.once(
         'stop',
         await watch({ appDir, appName, ignoreUnknownFormats: true }, (error, config) => {
           if (error != null) {

@xen-orchestra/proxy/package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "@xen-orchestra/proxy",
-  "version": "0.15.2",
+  "version": "0.15.3",
   "license": "AGPL-3.0-or-later",
   "description": "XO Proxy used to remotely execute backup jobs",
   "keywords": [
@@ -31,13 +31,13 @@
     "@vates/decorate-with": "^0.1.0",
     "@vates/disposable": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.15.1",
-    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/backups": "^0.16.0",
+    "@xen-orchestra/fs": "^0.19.1",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/mixin": "^0.1.0",
     "@xen-orchestra/mixins": "^0.1.1",
     "@xen-orchestra/self-signed": "^0.1.0",
-    "@xen-orchestra/xapi": "^0.8.0",
+    "@xen-orchestra/xapi": "^0.8.4",
     "ajv": "^8.0.3",
     "app-conf": "^0.9.0",
     "async-iterator-to-stream": "^1.1.0",
@@ -47,7 +47,7 @@
     "golike-defer": "^0.5.1",
     "http-server-plus": "^0.11.0",
     "json-rpc-protocol": "^0.13.1",
-    "jsonrpc-websocket-client": "^0.6.0",
+    "jsonrpc-websocket-client": "^0.7.2",
     "koa": "^2.5.1",
     "koa-compress": "^5.0.1",
     "koa-helmet": "^5.1.0",

@xen-orchestra/proxy/src/app/mixins/authentication.mjs

@@ -1,3 +1,4 @@
+import assert from 'assert'
 import fse from 'fs-extra'
 import xdg from 'xdg-basedir'
 import { createLogger } from '@xen-orchestra/log'
@@ -10,33 +11,48 @@ const { warn } = createLogger('xo:proxy:authentication')
 const isValidToken = t => typeof t === 'string' && t.length !== 0
 
 export default class Authentication {
-  constructor(_, { appName, config: { authenticationToken: token } }) {
-    if (!isValidToken(token)) {
-      token = JSON.parse(execFileSync('xenstore-read', ['vm-data/xo-proxy-authenticationToken']))
+  #token
 
-      if (!isValidToken(token)) {
-        throw new Error('missing authenticationToken in configuration')
-      }
+  constructor(app, { appName, config: { authenticationToken: token } }) {
+    const setToken = ({ token }) => {
+      assert(isValidToken(token), 'invalid authentication token: ' + token)
+
+      // save this token in the automatically handled conf file
+      fse.outputFileSync(
+        // this file must take precedence over normal user config
+        `${xdg.config}/${appName}/config.z-auto.json`,
+        JSON.stringify({ authenticationToken: token }),
+        { mode: 0o600 }
+      )
+
+      this.#token = token
+    }
+
+    if (isValidToken(token)) {
+      this.#token = token
+    } else {
+      setToken({ token: JSON.parse(execFileSync('xenstore-read', ['vm-data/xo-proxy-authenticationToken'])) })
 
       try {
-        // save this token in the automatically handled conf file
-        fse.outputFileSync(
-          // this file must take precedence over normal user config
-          `${xdg.config}/${appName}/config.z-auto.json`,
-          JSON.stringify({ authenticationToken: token }),
-          { mode: 0o600 }
-        )
         execFileSync('xenstore-rm', ['vm-data/xo-proxy-authenticationToken'])
       } catch (error) {
         warn('failed to remove token from XenStore', { error })
       }
     }
 
-    this._token = token
+    app.api.addMethod('authentication.setToken', setToken, {
+      description: 'change the authentication token used by this XO Proxy',
+      params: {
+        token: {
+          type: 'string',
+          minLength: 1,
+        },
+      },
+    })
   }
 
   async findProfile(credentials) {
-    if (credentials?.authenticationToken === this._token) {
+    if (credentials?.authenticationToken === this.#token) {
       return new Profile()
     }
   }

@xen-orchestra/upload-ova/package.json

@@ -44,7 +44,7 @@
     "pw": "^0.0.4",
     "strip-indent": "^3.0.0",
     "xdg-basedir": "^4.0.0",
-    "xo-lib": "^0.10.1",
+    "xo-lib": "^0.11.1",
     "xo-vmdk-to-vhd": "^2.0.0"
   },
   "devDependencies": {

@xen-orchestra/xapi/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/xapi",
-  "version": "0.8.0",
+  "version": "0.8.4",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/xapi",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {

@xen-orchestra/xapi/src/index.js

@@ -98,18 +98,16 @@ function removeWatcher(predicate, cb) {
 
 class Xapi extends Base {
   constructor({
-    callRetryWhenTooManyPendingTasks,
+    callRetryWhenTooManyPendingTasks = { delay: 5e3, tries: 10 },
     ignoreNobakVdis,
     maxUncoalescedVdis,
-    vdiDestroyRetryWhenInUse,
+    vdiDestroyRetryWhenInUse = { delay: 5e3, tries: 10 },
     ...opts
   }) {
     assert.notStrictEqual(ignoreNobakVdis, undefined)
 
     super(opts)
     this._callRetryWhenTooManyPendingTasks = {
-      delay: 5e3,
-      tries: 10,
       ...callRetryWhenTooManyPendingTasks,
       onRetry,
       when: { code: 'TOO_MANY_PENDING_TASKS' },
@@ -117,8 +115,6 @@ class Xapi extends Base {
     this._ignoreNobakVdis = ignoreNobakVdis
     this._maxUncoalescedVdis = maxUncoalescedVdis
     this._vdiDestroyRetryWhenInUse = {
-      delay: 5e3,
-      retries: 10,
       ...vdiDestroyRetryWhenInUse,
       onRetry,
       when: { code: 'VDI_IN_USE' },

@xen-orchestra/xapi/src/vm.js

@@ -99,6 +99,7 @@ module.exports = class Vm {
       // should coalesce
       const children = childrenMap[vdi.uuid]
       if (
+        children !== undefined && // unused unmanaged VDI, will be GC-ed
         children.length === 1 &&
         !children[0].managed && // some SRs do not coalesce the leaf
         tolerance-- <= 0

CHANGELOG.md

@@ -1,9 +1,42 @@
-## **5.64.0** (2021-10-29)
-
 # ChangeLog
 
+## **5.65.0** (2021-11-30)
+
 <img id="latest" src="https://badgen.net/badge/channel/latest/yellow" alt="Channel: latest" />
 
+### Highlights
+
+- [VM] Ability to export a snapshot's memory (PR [#6015](https://github.com/vatesfr/xen-orchestra/pull/6015))
+- [Cloud config] Ability to create a network cloud config template and reuse it in the VM creation [#5931](https://github.com/vatesfr/xen-orchestra/issues/5931) (PR [#5979](https://github.com/vatesfr/xen-orchestra/pull/5979))
+- [Backup/logs] identify XAPI errors (PR [#6001](https://github.com/vatesfr/xen-orchestra/pull/6001))
+- [lite] Highlight selected VM (PR [#5939](https://github.com/vatesfr/xen-orchestra/pull/5939))
+
+### Enhancements
+
+- [S3] Ability to authorize self signed certificates for S3 remote (PR [#5961](https://github.com/vatesfr/xen-orchestra/pull/5961))
+
+### Bug fixes
+
+- [Import/VM] Fix the import of OVA files (PR [#5976](https://github.com/vatesfr/xen-orchestra/pull/5976))
+
+### Packages to release
+
+- @vates/async-each 0.1.0
+- xo-remote-parser 0.8.4
+- @xen-orchestra/fs 0.19.0
+- @xen-orchestra/xapi patch
+- vhd-lib 2.0.1
+- @xen-orchestra/backups 0.16.0
+- xo-lib 0.11.1
+- @xen-orchestra/proxy 0.15.3
+- xo-server 5.84.1
+- vhd-cli 0.6.0
+- xo-web 5.90.0
+
+## **5.64.0** (2021-10-29)
+
+<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
+
 ## Highlights
 
 - [Netbox] Support older versions of Netbox and prevent "active is not a valid choice" error [#5898](https://github.com/vatesfr/xen-orchestra/issues/5898) (PR [#5946](https://github.com/vatesfr/xen-orchestra/pull/5946))
@@ -45,8 +78,6 @@
 
 ## **5.63.0** (2021-09-30)
 
-<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
-
 ### Highlights
 
 - [Backup] Go back to previous page instead of going to the overview after editing a job: keeps current filters and page (PR [#5913](https://github.com/vatesfr/xen-orchestra/pull/5913))

CHANGELOG.unreleased.md

@@ -11,7 +11,9 @@
 
 > Users must be able to say: “I had this issue, happy to know it's fixed”
 
-[Import/VM] Fix the import of OVA files (PR [#5976](https://github.com/vatesfr/xen-orchestra/pull/5976))
+- [Delta Backup Restoration] Fix assertion error [Forum #5257](https://xcp-ng.org/forum/topic/5257/problems-building-from-source/16)
+- [Delta Backup Restoration] `TypeError: this disposable has already been disposed` [Forum #5257](https://xcp-ng.org/forum/topic/5257/problems-building-from-source/20)
+- [Delta Backup Restoration] Fix error `ENOENT: no such file '/xo-vm-backups/../parentLocatorEntryN' with vhd having empty parent locator
 
 ### Packages to release
 
@@ -30,6 +32,8 @@
 >
 > In case of conflict, the highest (lowest in previous list) `$version` wins.
 
-- @xen-orchestra/fs minor
+- xo-vmdk-to-vhd patch
+- vhd-lib patch
+- @xen-orchestra/backups patch
+- @xen-orchestra/proxy patch
 - xo-server patch
-- vhd-cli minor

docs/advanced.md

@@ -327,6 +327,8 @@ Synchronize your pools, VMs, network interfaces and IP addresses with your [Netb
 
 ![](./assets/netbox.png)
 
+### Netbox side
+
 - Go to your Netbox interface
 - Configure prefixes:
   - Go to IPAM > Prefixes > Add
@@ -339,13 +341,19 @@ XO will try to find the right prefix for each IP address. If it can't find a pre
 - Generate a token:
   - Go to Admin > Tokens > Add token
   - Create a token with "Write enabled"
-- Add a UUID custom field:
+- Add a UUID custom field (for **Netbox 2.x**):
   - Got to Admin > Custom fields > Add custom field
   - Create a custom field called "uuid" (lower case!)
   - Assign it to object types `virtualization > cluster` and `virtualization > virtual machine`
 
 ![](./assets/customfield.png)
 
+:::tip
+In Netbox 3.x, custom fields can be found directly in the site (no need to go in the admin section). It's available in "Other/Customization/Custom Fields". After creation of the `uuid` field, assign it to the object types `virtualization > cluster` and `virtualization > virtual machine`.
+:::
+
+### In Xen Orchestra
+
 - Go to Xen Orchestra > Settings > Plugins > Netbox and fill out the configuration:
   - Endpoint: the URL of your Netbox instance (e.g.: `https://netbox.company.net`)
   - Unauthorized certificate: only for HTTPS, enable this option if your Netbox instance uses a self-signed SSL certificate

docs/backups.md

@@ -26,6 +26,12 @@ Each backups' job execution is identified by a `runId`. You can find this `runId
 
 ![](./assets/log-runId.png)
 
+## Exclude disks
+
+During a backup job, you can avoid saving all disks of the VM. To do that is trivial: just edit the VM disk name and add `[NOBAK]` before the current name, eg: `data-disk` will become `[NOBAK] data-disk` (with a space or not, doesn't matter).
+
+The disks marked with `[NOBAK]` will be now ignored in all following backups.
+
 ## Schedule
 
 :::tip

docs/delta_backups.md

@@ -43,12 +43,6 @@ Just go into your "Backup" view, and select Delta Backup. Then, it's the same as
 
 Unlike other types of backup jobs which delete the associated snapshot when the job is done and it has been exported, delta backups always keep a snapshot of every VM in the backup job, and uses it for the delta. Do not delete these snapshots!
 
-## Exclude disks
-
-During a delta backup job, you can avoid saving all disks of the VM. To do that is trivial: just edit the VM disk name and add `[NOBAK]` before the current name, eg: `data-disk` will become `[NOBAK] data-disk` (with a space or not, doesn't matter).
-
-The disks marked with `[NOBAK]` will be now ignored in all following backups.
-
 ## Delta backup initial seed
 
 If you don't want to do an initial full directly toward the destination, you can create a local delta backup first, then transfer the files to your destination.

docs/xoa.md

@@ -61,7 +61,7 @@ Please only use this if you have issues with [the default way to deploy XOA](ins
 Alternatively, you can deploy it by connecting to your XenServer host and executing the following:
 
 ```
-bash -c "$(curl -sS https://xoa.io/deploy)"
+bash -c "$(wget -qO- https://xoa.io/deploy)"
 ```
 
 :::tip
@@ -78,7 +78,7 @@ curl: (35) error:1407742E:SSL routines:SSL23_GET_SERVER_HELLO:tlsv1 alert protoc
 It means that the secure HTTPS protocol is not supported, you can bypass this using the unsecure command instead:
 
 ```
-bash -c "$(curl -sS http://xoa.io/deploy)"
+bash -c "$(wget -qO- http://xoa.io/deploy)"
 ```
 
 :::

docs/xosan.md

@@ -318,7 +318,7 @@ XOSAN is a 100% software defined solution for XenServer hyperconvergence. You ca
 
 You will need to be registered on our website in order to use Xen Orchestra. If you are not yet registered, [here is the way](https://xen-orchestra.com/#!/signup)
 
-SSH in your XenServer and use the command line `bash -c "$(curl -sS https://xoa.io/deploy)"` - it will deploy Xen Orchestra Appliance on your XenServer infrastructure which is required to use XOSAN.
+SSH in your XenServer and use the command line `bash -c "$(wget -qO- https://xoa.io/deploy)"` - it will deploy Xen Orchestra Appliance on your XenServer infrastructure which is required to use XOSAN.
 
 > Note: You can also download the XVA file and follow [these instructions](https://xen-orchestra.com/docs/xoa.html#the-alternative).
 

package.json

@@ -19,7 +19,7 @@
     "handlebars": "^4.7.6",
     "husky": "^4.2.5",
     "jest": "^27.3.1",
-    "lint-staged": "^11.1.2",
+    "lint-staged": "^12.0.3",
     "lodash": "^4.17.4",
     "prettier": "^2.0.5",
     "promise-toolbox": "^0.20.0",
@@ -61,8 +61,7 @@
       "/xo-server-test/",
       "/xo-web/"
     ],
-    "testRegex": "\\.spec\\.js$",
-    "timers": "fake"
+    "testRegex": "\\.spec\\.js$"
   },
   "lint-staged": {
     "*.{md,ts,ts}": "prettier --write"

packages/vhd-cli/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "vhd-cli",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "license": "ISC",
   "description": "Tools to read/create and merge VHD files",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-cli",
@@ -24,11 +24,12 @@
     "node": ">=8.10"
   },
   "dependencies": {
-    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/fs": "^0.19.1",
     "cli-progress": "^3.1.0",
     "exec-promise": "^0.7.0",
     "getopts": "^2.2.3",
-    "vhd-lib": "^1.3.0"
+    "human-format": "^0.11.0",
+    "vhd-lib": "^2.0.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/vhd-cli/src/commands/info.js

@@ -1,6 +1,34 @@
-import { VhdFile } from 'vhd-lib'
+import { Constants, VhdFile } from 'vhd-lib'
 import { getHandler } from '@xen-orchestra/fs'
 import { resolve } from 'path'
+import humanFormat from 'human-format'
+import invert from 'lodash/invert.js'
+
+const { PLATFORMS } = Constants
+
+const DISK_TYPES_MAP = invert(Constants.DISK_TYPES)
+const PLATFORMS_MAP = invert(PLATFORMS)
+
+const MAPPERS = {
+  bytes: humanFormat.bytes,
+  date: _ => (_ !== 0 ? new Date(_) : 0),
+  diskType: _ => DISK_TYPES_MAP[_],
+  platform: _ => PLATFORMS_MAP[_],
+}
+function mapProperties(object, mapping) {
+  const result = { ...object }
+  for (const prop of Object.keys(mapping)) {
+    const value = object[prop]
+    if (value !== undefined) {
+      let mapper = mapping[prop]
+      if (typeof mapper === 'string') {
+        mapper = MAPPERS[mapper]
+      }
+      result[prop] = mapper(value)
+    }
+  }
+  return result
+}
 
 export default async args => {
   const vhd = new VhdFile(getHandler({ url: 'file:///' }), resolve(args[0]))
@@ -12,6 +40,26 @@ export default async args => {
     await vhd.readHeaderAndFooter(false)
   }
 
-  console.log(vhd.header)
-  console.log(vhd.footer)
+  console.log(
+    mapProperties(vhd.footer, {
+      currentSize: 'bytes',
+      diskType: 'diskType',
+      originalSize: 'bytes',
+      timestamp: 'date',
+    })
+  )
+
+  console.log(
+    mapProperties(vhd.header, {
+      blockSize: 'bytes',
+      parentTimestamp: 'date',
+      parentLocatorEntry: _ =>
+        _.filter(_ => _.platformCode !== PLATFORMS.NONE) // hide empty
+          .map(_ =>
+            mapProperties(_, {
+              platformCode: 'platform',
+            })
+          ),
+    })
+  )
 }

packages/vhd-cli/src/commands/raw.js

@@ -1,13 +1,18 @@
-import { createContentStream } from 'vhd-lib'
-import { getHandler } from '@xen-orchestra/fs'
+import { openVhd } from 'vhd-lib'
+import { getSyncedHandler } from '@xen-orchestra/fs'
 import { resolve } from 'path'
 
 import { writeStream } from '../_utils'
+import { Disposable } from 'promise-toolbox'
 
 export default async args => {
   if (args.length < 2 || args.some(_ => _ === '-h' || _ === '--help')) {
     return `Usage: ${this.command} <input VHD> [<output raw>]`
   }
 
-  await writeStream(createContentStream(getHandler({ url: 'file:///' }), resolve(args[0])), args[1])
+  await Disposable.use(async function* () {
+    const handler = getSyncedHandler({ url: 'file:///' })
+    const vhd = openVhd(handler, resolve(args[0]))
+    await writeStream(vhd.rawContent())
+  })
 }

packages/vhd-lib/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "vhd-lib",
-  "version": "1.3.0",
+  "version": "2.0.1",
   "license": "AGPL-3.0-or-later",
   "description": "Primitives for VHD file handling",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-lib",
@@ -16,6 +16,7 @@
     "node": ">=10"
   },
   "dependencies": {
+    "@vates/async-each": "^0.1.0",
     "@vates/read-chunk": "^0.1.2",
     "@xen-orchestra/log": "^0.3.0",
     "async-iterator-to-stream": "^1.0.2",
@@ -30,7 +31,7 @@
     "@babel/cli": "^7.0.0",
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
-    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/fs": "^0.19.1",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^7.0.2",
     "execa": "^5.0.0",

packages/vhd-lib/src/Vhd/VhdAbstract.integ.spec.js

@@ -0,0 +1,269 @@
+/* eslint-env jest */
+
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import fs from 'fs-extra'
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { Disposable, pFromCallback } from 'promise-toolbox'
+
+import { openVhd } from '../index'
+import { checkFile, createRandomFile, convertFromRawToVhd, createRandomVhdDirectory } from '../tests/utils'
+import { VhdAbstract } from './VhdAbstract'
+import { BLOCK_UNUSED, FOOTER_SIZE, HEADER_SIZE, SECTOR_SIZE } from '../_constants'
+import { unpackHeader, unpackFooter } from './_utils'
+
+let tempDir
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+const streamToBuffer = stream => {
+  let buffer = Buffer.alloc(0)
+
+  return new Promise((resolve, reject) => {
+    stream.on('data', data => (buffer = Buffer.concat([buffer, data])))
+    stream.on('end', () => resolve(buffer))
+  })
+}
+
+test('It creates an alias', async () => {
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file://' + tempDir })
+    const aliasPath = `alias/alias.alias.vhd`
+    const aliasFsPath = `${tempDir}/${aliasPath}`
+    await fs.mkdirp(`${tempDir}/alias`)
+
+    const testOneCombination = async ({ targetPath, targetContent }) => {
+      await VhdAbstract.createAlias(handler, aliasPath, targetPath)
+      // alias file is created
+      expect(await fs.exists(aliasFsPath)).toEqual(true)
+      // content is the target path relative to the alias location
+      const content = await fs.readFile(aliasFsPath, 'utf-8')
+      expect(content).toEqual(targetContent)
+      // create alias fails if alias already exists, remove it before next loop step
+      await fs.unlink(aliasFsPath)
+    }
+
+    const combinations = [
+      { targetPath: `targets.vhd`, targetContent: `../targets.vhd` },
+      { targetPath: `alias/targets.vhd`, targetContent: `targets.vhd` },
+      { targetPath: `alias/sub/targets.vhd`, targetContent: `sub/targets.vhd` },
+      { targetPath: `sibling/targets.vhd`, targetContent: `../sibling/targets.vhd` },
+    ]
+
+    for (const { targetPath, targetContent } of combinations) {
+      await testOneCombination({ targetPath, targetContent })
+    }
+  })
+})
+
+test('alias must have *.alias.vhd extension', async () => {
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    const aliasPath = `${tempDir}/invalidalias.vhd`
+    const targetPath = `${tempDir}/targets.vhd`
+    expect(async () => await VhdAbstract.createAlias(handler, aliasPath, targetPath)).rejects.toThrow()
+
+    expect(await fs.exists(aliasPath)).toEqual(false)
+  })
+})
+
+test('alias must not be chained', async () => {
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    const aliasPath = `${tempDir}/valid.alias.vhd`
+    const targetPath = `${tempDir}/an.other.valid.alias.vhd`
+    expect(async () => await VhdAbstract.createAlias(handler, aliasPath, targetPath)).rejects.toThrow()
+    expect(await fs.exists(aliasPath)).toEqual(false)
+  })
+})
+
+test('It rename and unlink a VHDFile', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    const { size } = await fs.stat(vhdFileName)
+    const targetFileName = `${tempDir}/renamed.vhd`
+
+    await VhdAbstract.rename(handler, vhdFileName, targetFileName)
+    expect(await fs.exists(vhdFileName)).toEqual(false)
+    const { size: renamedSize } = await fs.stat(targetFileName)
+    expect(size).toEqual(renamedSize)
+    await VhdAbstract.unlink(handler, targetFileName)
+    expect(await fs.exists(targetFileName)).toEqual(false)
+  })
+})
+
+test('It rename and unlink a VhdDirectory', async () => {
+  const initalSize = 4
+  const vhdDirectory = `${tempDir}/randomfile.dir`
+  await createRandomVhdDirectory(vhdDirectory, initalSize)
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    const vhd = yield openVhd(handler, vhdDirectory)
+    expect(vhd.header.cookie).toEqual('cxsparse')
+    expect(vhd.footer.cookie).toEqual('conectix')
+
+    const targetFileName = `${tempDir}/renamed.vhd`
+    // it should clean an existing directory
+    await fs.mkdir(targetFileName)
+    await fs.writeFile(`${targetFileName}/dummy`, 'I exists')
+    await VhdAbstract.rename(handler, vhdDirectory, targetFileName)
+    expect(await fs.exists(vhdDirectory)).toEqual(false)
+    expect(await fs.exists(targetFileName)).toEqual(true)
+    expect(await fs.exists(`${targetFileName}/dummy`)).toEqual(false)
+    await VhdAbstract.unlink(handler, targetFileName)
+    expect(await fs.exists(targetFileName)).toEqual(false)
+  })
+})
+
+test('It create , rename and unlink alias', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  const aliasFileName = `${tempDir}/aliasFileName.alias.vhd`
+  const aliasFileNameRenamed = `${tempDir}/aliasFileNameRenamed.alias.vhd`
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    await VhdAbstract.createAlias(handler, aliasFileName, vhdFileName)
+    expect(await fs.exists(aliasFileName)).toEqual(true)
+    expect(await fs.exists(vhdFileName)).toEqual(true)
+
+    await VhdAbstract.rename(handler, aliasFileName, aliasFileNameRenamed)
+    expect(await fs.exists(aliasFileName)).toEqual(false)
+    expect(await fs.exists(vhdFileName)).toEqual(true)
+    expect(await fs.exists(aliasFileNameRenamed)).toEqual(true)
+
+    await VhdAbstract.unlink(handler, aliasFileNameRenamed)
+    expect(await fs.exists(aliasFileName)).toEqual(false)
+    expect(await fs.exists(vhdFileName)).toEqual(false)
+    expect(await fs.exists(aliasFileNameRenamed)).toEqual(false)
+  })
+})
+
+test('it can create a vhd stream', async () => {
+  const initialNbBlocks = 3
+  const initalSize = initialNbBlocks * 2
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/vhd.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  const bat = Buffer.alloc(512)
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file://' + tempDir })
+
+    const vhd = yield openVhd(handler, 'vhd.vhd')
+
+    // mark first block as unused
+    await handler.read('vhd.vhd', bat, vhd.header.tableOffset)
+    bat.writeUInt32BE(BLOCK_UNUSED, 0)
+    await handler.write('vhd.vhd', bat, vhd.header.tableOffset)
+
+    // read our modified bat
+    await vhd.readBlockAllocationTable()
+    const stream = vhd.stream()
+
+    // read all the stream into a buffer
+
+    const buffer = await streamToBuffer(stream)
+    const length = buffer.length
+    const bufFooter = buffer.slice(0, FOOTER_SIZE)
+
+    // footer is still valid
+    expect(() => unpackFooter(bufFooter)).not.toThrow()
+    const footer = unpackFooter(bufFooter)
+
+    // header is still valid
+    const bufHeader = buffer.slice(FOOTER_SIZE, HEADER_SIZE + FOOTER_SIZE)
+    expect(() => unpackHeader(bufHeader, footer)).not.toThrow()
+
+    // 1 deleted block should be in ouput
+    const start = FOOTER_SIZE + HEADER_SIZE + vhd.batSize
+    expect(length).toEqual(start + (initialNbBlocks - 1) * vhd.fullBlockSize + FOOTER_SIZE)
+    expect(stream.length).toEqual(buffer.length)
+    // blocks
+    const blockBuf = Buffer.alloc(vhd.sectorsPerBlock * SECTOR_SIZE, 0)
+    for (let i = 1; i < initialNbBlocks; i++) {
+      const blockDataStart = start + (i - 1) * vhd.fullBlockSize + 512 /* block bitmap */
+      const blockDataEnd = blockDataStart + vhd.sectorsPerBlock * SECTOR_SIZE
+      const content = buffer.slice(blockDataStart, blockDataEnd)
+      await handler.read('randomfile', blockBuf, i * vhd.sectorsPerBlock * SECTOR_SIZE)
+      expect(content).toEqual(blockBuf)
+    }
+    // footer
+    const endFooter = buffer.slice(length - FOOTER_SIZE)
+    expect(bufFooter).toEqual(endFooter)
+
+    await handler.writeFile('out.vhd', buffer)
+    // check that the vhd is still valid
+    await checkFile(`${tempDir}/out.vhd`)
+  })
+})
+
+it('can stream content', async () => {
+  const initalSizeMb = 5 // 2 block and an half
+  const initialNbBlocks = Math.ceil(initalSizeMb / 2)
+  const initialByteSize = initalSizeMb * 1024 * 1024
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSizeMb)
+  const vhdFileName = `${tempDir}/vhd.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  const bat = Buffer.alloc(512)
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file://' + tempDir })
+
+    const vhd = yield openVhd(handler, 'vhd.vhd')
+    // mark first block as unused
+    await handler.read('vhd.vhd', bat, vhd.header.tableOffset)
+    bat.writeUInt32BE(BLOCK_UNUSED, 0)
+    await handler.write('vhd.vhd', bat, vhd.header.tableOffset)
+
+    // read our modified block allocation table
+    await vhd.readBlockAllocationTable()
+    const stream = vhd.rawContent()
+    const buffer = await streamToBuffer(stream)
+
+    // qemu can modify size, to align it to geometry
+
+    // check that data didn't change
+    const blockDataLength = vhd.sectorsPerBlock * SECTOR_SIZE
+
+    // first block should be empty
+    const EMPTY = Buffer.alloc(blockDataLength, 0)
+    const firstBlock = buffer.slice(0, blockDataLength)
+    // using buffer1 toEquals buffer2 make jest crash trying to stringify it on failure
+    expect(firstBlock.equals(EMPTY)).toEqual(true)
+
+    let remainingLength = initialByteSize - blockDataLength // already checked the first block
+    for (let i = 1; i < initialNbBlocks; i++) {
+      // last block will be truncated
+      const blockSize = Math.min(blockDataLength, remainingLength - blockDataLength)
+      const blockDataStart = i * blockDataLength // first block have been deleted
+      const blockDataEnd = blockDataStart + blockSize
+      const content = buffer.slice(blockDataStart, blockDataEnd)
+
+      const blockBuf = Buffer.alloc(blockSize, 0)
+
+      await handler.read('randomfile', blockBuf, i * blockDataLength)
+      expect(content.equals(blockBuf)).toEqual(true)
+      remainingLength -= blockSize
+    }
+  })
+})

packages/vhd-lib/src/Vhd/VhdAbstract.js

@@ -1,26 +1,38 @@
-import { computeBatSize, sectorsRoundUpNoZero, sectorsToBytes } from './_utils'
-import { PLATFORM_NONE, SECTOR_SIZE, PLATFORM_W2KU, PARENT_LOCATOR_ENTRIES } from '../_constants'
+import { computeBatSize, computeSectorOfBitmap, computeSectorsPerBlock, sectorsToBytes } from './_utils'
+import { PLATFORMS, SECTOR_SIZE, PARENT_LOCATOR_ENTRIES, FOOTER_SIZE, HEADER_SIZE, BLOCK_UNUSED } from '../_constants'
 import assert from 'assert'
+import path from 'path'
+import asyncIteratorToStream from 'async-iterator-to-stream'
+import { checksumStruct, fuFooter, fuHeader } from '../_structs'
+import { isVhdAlias, resolveAlias } from '../_resolveAlias'
 
 export class VhdAbstract {
   #header
-  bitmapSize
   footer
-  fullBlockSize
-  sectorsOfBitmap
-  sectorsPerBlock
+
+  get bitmapSize() {
+    return sectorsToBytes(this.sectorsOfBitmap)
+  }
+
+  get fullBlockSize() {
+    return sectorsToBytes(this.sectorsOfBitmap + this.sectorsPerBlock)
+  }
 
   get header() {
     assert.notStrictEqual(this.#header, undefined, `header must be read before it's used`)
     return this.#header
   }
 
+  get sectorsOfBitmap() {
+    return computeSectorOfBitmap(this.header.blockSize)
+  }
+
+  get sectorsPerBlock() {
+    return computeSectorsPerBlock(this.header.blockSize)
+  }
+
   set header(header) {
     this.#header = header
-    this.sectorsPerBlock = header.blockSize / SECTOR_SIZE
-    this.sectorsOfBitmap = sectorsRoundUpNoZero(this.sectorsPerBlock >> 3)
-    this.fullBlockSize = sectorsToBytes(this.sectorsOfBitmap + this.sectorsPerBlock)
-    this.bitmapSize = sectorsToBytes(this.sectorsOfBitmap)
   }
 
   /**
@@ -80,8 +92,10 @@ export class VhdAbstract {
    *
    * @returns {number} the merged data size
    */
-  coalesceBlock(child, blockId) {
-    throw new Error(`coalescing the block ${blockId} from ${child} is not implemented`)
+  async coalesceBlock(child, blockId) {
+    const block = await child.readBlock(blockId)
+    await this.writeEntireBlock(block)
+    return block.data.length
   }
 
   /**
@@ -114,7 +128,7 @@ export class VhdAbstract {
     return computeBatSize(this.header.maxTableEntries)
   }
 
-  async writeParentLocator({ id, platformCode = PLATFORM_NONE, data = Buffer.alloc(0) }) {
+  async writeParentLocator({ id, platformCode = PLATFORMS.NONE, data = Buffer.alloc(0) }) {
     assert(id >= 0, 'parent Locator id must be a positive number')
     assert(id < PARENT_LOCATOR_ENTRIES, `parent Locator id  must be less than ${PARENT_LOCATOR_ENTRIES}`)
 
@@ -143,14 +157,14 @@ export class VhdAbstract {
   async setUniqueParentLocator(fileNameString) {
     await this.writeParentLocator({
       id: 0,
-      code: PLATFORM_W2KU,
+      platformCode: PLATFORMS.W2KU,
       data: Buffer.from(fileNameString, 'utf16le'),
     })
 
     for (let i = 1; i < PARENT_LOCATOR_ENTRIES; i++) {
       await this.writeParentLocator({
         id: i,
-        code: PLATFORM_NONE,
+        platformCode: PLATFORMS.NONE,
         data: Buffer.alloc(0),
       })
     }
@@ -164,4 +178,138 @@ export class VhdAbstract {
       }
     }
   }
+
+  static async rename(handler, sourcePath, targetPath) {
+    try {
+      // delete target if it already exists
+      await VhdAbstract.unlink(handler, targetPath)
+    } catch (e) {}
+    await handler.rename(sourcePath, targetPath)
+  }
+
+  static async unlink(handler, path) {
+    const resolved = await resolveAlias(handler, path)
+    try {
+      await handler.unlink(resolved)
+    } catch (err) {
+      if (err.code === 'EISDIR') {
+        await handler.rmtree(resolved)
+      } else {
+        throw err
+      }
+    }
+
+    // also delete the alias file
+    if (path !== resolved) {
+      await handler.unlink(path)
+    }
+  }
+
+  static async createAlias(handler, aliasPath, targetPath) {
+    if (!isVhdAlias(aliasPath)) {
+      throw new Error(`Alias must be named *.alias.vhd,  ${aliasPath} given`)
+    }
+    if (isVhdAlias(targetPath)) {
+      throw new Error(`Chaining alias is forbidden ${aliasPath} to ${targetPath}`)
+    }
+    // aliasPath and targetPath are absolute path from the root of the handler
+    // normalize them so they can't  escape this dir
+    const aliasDir = path.dirname(path.resolve('/', aliasPath))
+    // only store the relative path from alias to target
+    const relativePathToTarget = path.relative(aliasDir, path.resolve('/', targetPath))
+    await handler.writeFile(aliasPath, relativePathToTarget)
+  }
+
+  stream() {
+    const { footer, batSize } = this
+    const { ...header } = this.header // copy since we don't ant to modifiy the current header
+    const rawFooter = fuFooter.pack(footer)
+    checksumStruct(rawFooter, fuFooter)
+
+    // compute parent locator place and size
+    // update them in header
+    // update checksum in header
+
+    let offset = FOOTER_SIZE + HEADER_SIZE + batSize
+    for (let i = 0; i < PARENT_LOCATOR_ENTRIES; i++) {
+      const { ...entry } = header.parentLocatorEntry[i]
+      if (entry.platformDataSpace > 0) {
+        entry.platformDataOffset = offset
+        offset += entry.platformDataSpace
+      }
+      header.parentLocatorEntry[i] = entry
+    }
+
+    const rawHeader = fuHeader.pack(header)
+    checksumStruct(rawHeader, fuHeader)
+
+    assert.strictEqual(offset % SECTOR_SIZE, 0)
+
+    const bat = Buffer.allocUnsafe(batSize)
+    let offsetSector = offset / SECTOR_SIZE
+    const blockSizeInSectors = this.fullBlockSize / SECTOR_SIZE
+    let fileSize = offsetSector * SECTOR_SIZE + FOOTER_SIZE /* the footer at the end */
+    // compute BAT , blocks starts after parent locator entries
+    for (let i = 0; i < header.maxTableEntries; i++) {
+      if (this.containsBlock(i)) {
+        bat.writeUInt32BE(offsetSector, i * 4)
+        offsetSector += blockSizeInSectors
+        fileSize += this.fullBlockSize
+      } else {
+        bat.writeUInt32BE(BLOCK_UNUSED, i * 4)
+      }
+    }
+
+    const self = this
+    async function* iterator() {
+      yield rawFooter
+      yield rawHeader
+      yield bat
+
+      // yield parent locator entries
+      for (let i = 0; i < PARENT_LOCATOR_ENTRIES; i++) {
+        if (header.parentLocatorEntry[i].platformDataSpace > 0) {
+          const parentLocator = await self.readParentLocator(i)
+          // @ todo pad to platformDataSpace
+          yield parentLocator.data
+        }
+      }
+
+      // yield all blocks
+      // since contains() can be costly for synthetic vhd, use the computed bat
+      for (let i = 0; i < header.maxTableEntries; i++) {
+        if (bat.readUInt32BE(i * 4) !== BLOCK_UNUSED) {
+          const block = await self.readBlock(i)
+          yield block.buffer
+        }
+      }
+      // yield footer again
+      yield rawFooter
+    }
+
+    const stream = asyncIteratorToStream(iterator())
+    stream.length = fileSize
+    return stream
+  }
+
+  rawContent() {
+    const { header, footer } = this
+    const { blockSize } = header
+    const self = this
+    async function* iterator() {
+      const nBlocks = header.maxTableEntries
+      let remainingSize = footer.currentSize
+      const EMPTY = Buffer.alloc(blockSize, 0)
+      for (let blockId = 0; blockId < nBlocks; ++blockId) {
+        let buffer = self.containsBlock(blockId) ? (await self.readBlock(blockId)).data : EMPTY
+        // the last block can be truncated since raw size is not a multiple of blockSize
+        buffer = remainingSize < blockSize ? buffer.slice(0, remainingSize) : buffer
+        remainingSize -= blockSize
+        yield buffer
+      }
+    }
+    const stream = asyncIteratorToStream(iterator())
+    stream.length = footer.currentSize
+    return stream
+  }
 }

packages/vhd-lib/src/Vhd/VhdDirectory.integ.spec.js

@@ -0,0 +1,67 @@
+/* eslint-env jest */
+
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { getHandler } from '@xen-orchestra/fs'
+import { Disposable, pFromCallback } from 'promise-toolbox'
+
+import { openVhd } from '../openVhd'
+import { createRandomFile, convertFromRawToVhd, convertToVhdDirectory } from '../tests/utils'
+
+let tempDir = null
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+test('Can coalesce block', async () => {
+  const initalSize = 4
+  const parentrawFileName = `${tempDir}/randomfile`
+  const parentFileName = `${tempDir}/parent.vhd`
+  const parentDirectoryName = `${tempDir}/parent.dir.vhd`
+
+  await createRandomFile(parentrawFileName, initalSize)
+  await convertFromRawToVhd(parentrawFileName, parentFileName)
+  await convertToVhdDirectory(parentrawFileName, parentFileName, parentDirectoryName)
+
+  const childrawFileName = `${tempDir}/randomfile`
+  const childFileName = `${tempDir}/childFile.vhd`
+  await createRandomFile(childrawFileName, initalSize)
+  await convertFromRawToVhd(childrawFileName, childFileName)
+  const childRawDirectoryName = `${tempDir}/randomFile2.vhd`
+  const childDirectoryFileName = `${tempDir}/childDirFile.vhd`
+  const childDirectoryName = `${tempDir}/childDir.vhd`
+  await createRandomFile(childRawDirectoryName, initalSize)
+  await convertFromRawToVhd(childRawDirectoryName, childDirectoryFileName)
+  await convertToVhdDirectory(childRawDirectoryName, childDirectoryFileName, childDirectoryName)
+
+  await Disposable.use(async function* () {
+    const handler = getHandler({ url: 'file://' })
+    const parentVhd = yield openVhd(handler, parentDirectoryName, { flags: 'w' })
+    await parentVhd.readBlockAllocationTable()
+    const childFileVhd = yield openVhd(handler, childFileName)
+    await childFileVhd.readBlockAllocationTable()
+    const childDirectoryVhd = yield openVhd(handler, childDirectoryName)
+    await childDirectoryVhd.readBlockAllocationTable()
+
+    await parentVhd.coalesceBlock(childFileVhd, 0)
+    await parentVhd.writeFooter()
+    await parentVhd.writeBlockAllocationTable()
+    let parentBlockData = (await parentVhd.readBlock(0)).data
+    let childBlockData = (await childFileVhd.readBlock(0)).data
+    expect(parentBlockData.equals(childBlockData)).toEqual(true)
+
+    await parentVhd.coalesceBlock(childDirectoryVhd, 0)
+    await parentVhd.writeFooter()
+    await parentVhd.writeBlockAllocationTable()
+    parentBlockData = (await parentVhd.readBlock(0)).data
+    childBlockData = (await childDirectoryVhd.readBlock(0)).data
+    expect(parentBlockData).toEqual(childBlockData)
+  })
+})

packages/vhd-lib/src/Vhd/VhdDirectory.js

@@ -1,4 +1,4 @@
-import { buildHeader, buildFooter } from './_utils'
+import { unpackHeader, unpackFooter, sectorsToBytes } from './_utils'
 import { createLogger } from '@xen-orchestra/log'
 import { fuFooter, fuHeader, checksumStruct } from '../_structs'
 import { test, set as setBitmap } from '../_bitmap'
@@ -39,8 +39,8 @@ export class VhdDirectory extends VhdAbstract {
     this.#uncheckedBlockTable = blockTable
   }
 
-  static async open(handler, path) {
-    const vhd = new VhdDirectory(handler, path)
+  static async open(handler, path, { flags = 'r+' } = {}) {
+    const vhd = new VhdDirectory(handler, path, { flags })
 
     // openning a file for reading does not trigger EISDIR as long as we don't really read from it :
     // https://man7.org/linux/man-pages/man2/open.2.html
@@ -54,19 +54,20 @@ export class VhdDirectory extends VhdAbstract {
     }
   }
 
-  static async create(handler, path) {
+  static async create(handler, path, { flags = 'wx+' } = {}) {
     await handler.mkdir(path)
-    const vhd = new VhdDirectory(handler, path)
+    const vhd = new VhdDirectory(handler, path, { flags })
     return {
       dispose: () => {},
       value: vhd,
     }
   }
 
-  constructor(handler, path) {
+  constructor(handler, path, opts) {
     super()
     this._handler = handler
     this._path = path
+    this._opts = opts
   }
 
   async readBlockAllocationTable() {
@@ -78,13 +79,13 @@ export class VhdDirectory extends VhdAbstract {
     return test(this.#blockTable, blockId)
   }
 
-  getChunkPath(partName) {
+  _getChunkPath(partName) {
     return this._path + '/' + partName
   }
 
   async _readChunk(partName) {
     // here we can implement compression and / or crypto
-    const buffer = await this._handler.readFile(this.getChunkPath(partName))
+    const buffer = await this._handler.readFile(this._getChunkPath(partName))
 
     return {
       buffer: Buffer.from(buffer),
@@ -92,20 +93,14 @@ export class VhdDirectory extends VhdAbstract {
   }
 
   async _writeChunk(partName, buffer) {
-    assert(Buffer.isBuffer(buffer))
+    assert.notStrictEqual(
+      this._opts?.flags,
+      'r',
+      `Can't write a chunk ${partName} in ${this._path} with read permission`
+    )
     // here we can implement compression and / or crypto
 
-    // chunks can be in sub directories :  create direcotries if necessary
-    const pathParts = partName.split('/')
-    let currentPath = this._path
-
-    // the last one is the file name
-    for (let i = 0; i < pathParts.length - 1; i++) {
-      currentPath += '/' + pathParts[i]
-      await this._handler.mkdir(currentPath)
-    }
-
-    return this._handler.writeFile(this.getChunkPath(partName), buffer)
+    return this._handler.outputFile(this._getChunkPath(partName), buffer, this._opts)
   }
 
   // put block in subdirectories to limit impact when doing directory listing
@@ -118,8 +113,8 @@ export class VhdDirectory extends VhdAbstract {
   async readHeaderAndFooter() {
     const { buffer: bufHeader } = await this._readChunk('header')
     const { buffer: bufFooter } = await this._readChunk('footer')
-    const footer = buildFooter(bufFooter)
-    const header = buildHeader(bufHeader, footer)
+    const footer = unpackFooter(bufFooter)
+    const header = unpackHeader(bufHeader, footer)
 
     this.footer = footer
     this.header = header
@@ -167,11 +162,18 @@ export class VhdDirectory extends VhdAbstract {
     return this._writeChunk('bat', this.#blockTable)
   }
 
-  // only works if data are in the same bucket
+  // only works if data are in the same handler
   // and if the full block is modified in child ( which is the case whit xcp)
 
-  coalesceBlock(child, blockId) {
-    this._handler.copy(child.getChunkPath(blockId), this.getChunkPath(blockId))
+  async coalesceBlock(child, blockId) {
+    if (!(child instanceof VhdDirectory) || this._handler !== child._handler) {
+      return super.coalesceBlock(child, blockId)
+    }
+    await this._handler.copy(
+      child._getChunkPath(child._getBlockPath(blockId)),
+      this._getChunkPath(this._getBlockPath(blockId))
+    )
+    return sectorsToBytes(this.sectorsPerBlock)
   }
 
   async writeEntireBlock(block) {
@@ -180,7 +182,14 @@ export class VhdDirectory extends VhdAbstract {
   }
 
   async _readParentLocatorData(id) {
-    return (await this._readChunk('parentLocatorEntry' + id)).buffer
+    try {
+      return (await this._readChunk('parentLocatorEntry' + id)).buffer
+    } catch (e) {
+      if (e.code === 'ENOENT') {
+        return Buffer.alloc(0)
+      }
+      throw e
+    }
   }
 
   async _writeParentLocatorData(id, data) {

packages/vhd-lib/src/Vhd/VhdFile.integ.spec.js

@@ -0,0 +1,237 @@
+/* eslint-env jest */
+
+import execa from 'execa'
+import fs from 'fs-extra'
+import getStream from 'get-stream'
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { getHandler } from '@xen-orchestra/fs'
+import { Disposable, pFromCallback } from 'promise-toolbox'
+import { randomBytes } from 'crypto'
+
+import { VhdFile } from './VhdFile'
+import { openVhd } from '../openVhd'
+
+import { SECTOR_SIZE } from '../_constants'
+import {
+  checkFile,
+  createRandomFile,
+  convertFromRawToVhd,
+  convertToVhdDirectory,
+  recoverRawContent,
+} from '../tests/utils'
+
+let tempDir = null
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+test('respect the checkSecondFooter flag', async () => {
+  const initalSize = 0
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+
+  const handler = getHandler({ url: `file://${tempDir}` })
+
+  const size = await handler.getSize('randomfile.vhd')
+  const fd = await handler.openFile('randomfile.vhd', 'r+')
+  const buffer = Buffer.alloc(512, 0)
+  // add a fake footer at the end
+  handler.write(fd, buffer, size)
+  await handler.closeFile(fd)
+  // not using openVhd to be able to call readHeaderAndFooter separatly
+  const vhd = new VhdFile(handler, 'randomfile.vhd')
+
+  await expect(async () => await vhd.readHeaderAndFooter()).rejects.toThrow()
+  await expect(async () => await vhd.readHeaderAndFooter(true)).rejects.toThrow()
+  await expect(await vhd.readHeaderAndFooter(false)).toEqual(undefined)
+})
+
+test('blocks can be moved', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  const handler = getHandler({ url: 'file://' })
+  const originalSize = await handler.getSize(rawFileName)
+  const newVhd = new VhdFile(handler, vhdFileName)
+  await newVhd.readHeaderAndFooter()
+  await newVhd.readBlockAllocationTable()
+  await newVhd._freeFirstBlockSpace(8000000)
+  const recoveredFileName = `${tempDir}/recovered`
+  await recoverRawContent(vhdFileName, recoveredFileName, originalSize)
+  expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(rawFileName))
+})
+
+test('the BAT MSB is not used for sign', async () => {
+  const randomBuffer = await pFromCallback(cb => randomBytes(SECTOR_SIZE, cb))
+  const emptyFileName = `${tempDir}/empty.vhd`
+  await execa('qemu-img', ['create', '-fvpc', emptyFileName, '1.8T'])
+  const handler = getHandler({ url: 'file://' })
+  const vhd = new VhdFile(handler, emptyFileName)
+  await vhd.readHeaderAndFooter()
+  await vhd.readBlockAllocationTable()
+  // we want the bit 31 to be on, to prove it's not been used for sign
+  const hugeWritePositionSectors = Math.pow(2, 31) + 200
+  await vhd.writeData(hugeWritePositionSectors, randomBuffer)
+  await checkFile(emptyFileName)
+  // here we are moving the first sector very far in the VHD to prove the BAT doesn't use signed int32
+  const hugePositionBytes = hugeWritePositionSectors * SECTOR_SIZE
+  await vhd._freeFirstBlockSpace(hugePositionBytes)
+  await vhd.writeFooter()
+
+  // we recover the data manually for speed reasons.
+  // fs.write() with offset is way faster than qemu-img when there is a 1.5To
+  // hole before the block of data
+  const recoveredFileName = `${tempDir}/recovered`
+  const recoveredFile = await fs.open(recoveredFileName, 'w')
+  try {
+    const vhd2 = new VhdFile(handler, emptyFileName)
+    await vhd2.readHeaderAndFooter()
+    await vhd2.readBlockAllocationTable()
+    for (let i = 0; i < vhd.header.maxTableEntries; i++) {
+      if (vhd.containsBlock(i)) {
+        const block = (await vhd2.readBlock(i)).data
+        await fs.write(recoveredFile, block, 0, block.length, vhd2.header.blockSize * i)
+      }
+    }
+  } finally {
+    fs.close(recoveredFile)
+  }
+  const recovered = await getStream.buffer(
+    await fs.createReadStream(recoveredFileName, {
+      start: hugePositionBytes,
+      end: hugePositionBytes + randomBuffer.length - 1,
+    })
+  )
+  expect(recovered).toEqual(randomBuffer)
+})
+
+test('writeData on empty file', async () => {
+  const mbOfRandom = 3
+  const rawFileName = `${tempDir}/randomfile`
+  const emptyFileName = `${tempDir}/empty.vhd`
+  await createRandomFile(rawFileName, mbOfRandom)
+  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 'M'])
+  const randomData = await fs.readFile(rawFileName)
+  const handler = getHandler({ url: 'file://' })
+  const originalSize = await handler.getSize(rawFileName)
+  const newVhd = new VhdFile(handler, emptyFileName)
+  await newVhd.readHeaderAndFooter()
+  await newVhd.readBlockAllocationTable()
+  await newVhd.writeData(0, randomData)
+  const recoveredFileName = `${tempDir}/recovered`
+  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
+  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
+})
+
+test('writeData in 2 non-overlaping operations', async () => {
+  const mbOfRandom = 3
+  const rawFileName = `${tempDir}/randomfile`
+  const emptyFileName = `${tempDir}/empty.vhd`
+  const recoveredFileName = `${tempDir}/recovered`
+  await createRandomFile(rawFileName, mbOfRandom)
+  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 'M'])
+  const randomData = await fs.readFile(rawFileName)
+  const handler = getHandler({ url: 'file://' })
+  const originalSize = await handler.getSize(rawFileName)
+  const newVhd = new VhdFile(handler, emptyFileName)
+  await newVhd.readHeaderAndFooter()
+  await newVhd.readBlockAllocationTable()
+  const splitPointSectors = 2
+  await newVhd.writeData(0, randomData.slice(0, splitPointSectors * 512))
+  await newVhd.writeData(splitPointSectors, randomData.slice(splitPointSectors * 512))
+  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
+  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
+})
+
+test('writeData in 2 overlaping operations', async () => {
+  const mbOfRandom = 3
+  const rawFileName = `${tempDir}/randomfile`
+  const emptyFileName = `${tempDir}/empty.vhd`
+  const recoveredFileName = `${tempDir}/recovered`
+  await createRandomFile(rawFileName, mbOfRandom)
+  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 'M'])
+  const randomData = await fs.readFile(rawFileName)
+  const handler = getHandler({ url: 'file://' })
+  const originalSize = await handler.getSize(rawFileName)
+  const newVhd = new VhdFile(handler, emptyFileName)
+  await newVhd.readHeaderAndFooter()
+  await newVhd.readBlockAllocationTable()
+  const endFirstWrite = 3
+  const startSecondWrite = 2
+  await newVhd.writeData(0, randomData.slice(0, endFirstWrite * 512))
+  await newVhd.writeData(startSecondWrite, randomData.slice(startSecondWrite * 512))
+  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
+  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
+})
+
+test('BAT can be extended and blocks moved', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  const recoveredFileName = `${tempDir}/recovered`
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await createRandomFile(rawFileName, initalSize)
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  const handler = getHandler({ url: 'file://' })
+  const originalSize = await handler.getSize(rawFileName)
+  const newVhd = new VhdFile(handler, vhdFileName)
+  await newVhd.readHeaderAndFooter()
+  await newVhd.readBlockAllocationTable()
+  await newVhd.ensureBatSize(2000)
+  await newVhd.writeBlockAllocationTable()
+  await recoverRawContent(vhdFileName, recoveredFileName, originalSize)
+  expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(rawFileName))
+})
+
+test('Can coalesce block', async () => {
+  const initalSize = 4
+  const parentrawFileName = `${tempDir}/randomfile`
+  const parentFileName = `${tempDir}/parent.vhd`
+  await createRandomFile(parentrawFileName, initalSize)
+  await convertFromRawToVhd(parentrawFileName, parentFileName)
+  const childrawFileName = `${tempDir}/randomfile`
+  const childFileName = `${tempDir}/childFile.vhd`
+  await createRandomFile(childrawFileName, initalSize)
+  await convertFromRawToVhd(childrawFileName, childFileName)
+  const childRawDirectoryName = `${tempDir}/randomFile2.vhd`
+  const childDirectoryFileName = `${tempDir}/childDirFile.vhd`
+  const childDirectoryName = `${tempDir}/childDir.vhd`
+  await createRandomFile(childRawDirectoryName, initalSize)
+  await convertFromRawToVhd(childRawDirectoryName, childDirectoryFileName)
+  await convertToVhdDirectory(childRawDirectoryName, childDirectoryFileName, childDirectoryName)
+
+  await Disposable.use(async function* () {
+    const handler = getHandler({ url: 'file://' })
+    const parentVhd = yield openVhd(handler, parentFileName, { flags: 'r+' })
+    await parentVhd.readBlockAllocationTable()
+    const childFileVhd = yield openVhd(handler, childFileName)
+    await childFileVhd.readBlockAllocationTable()
+    const childDirectoryVhd = yield openVhd(handler, childDirectoryName)
+    await childDirectoryVhd.readBlockAllocationTable()
+
+    await parentVhd.coalesceBlock(childFileVhd, 0)
+    await parentVhd.writeFooter()
+    await parentVhd.writeBlockAllocationTable()
+    let parentBlockData = (await parentVhd.readBlock(0)).data
+    let childBlockData = (await childFileVhd.readBlock(0)).data
+    expect(parentBlockData).toEqual(childBlockData)
+
+    await parentVhd.coalesceBlock(childDirectoryVhd, 0)
+    await parentVhd.writeFooter()
+    await parentVhd.writeBlockAllocationTable()
+    parentBlockData = (await parentVhd.readBlock(0)).data
+    childBlockData = (await childDirectoryVhd.readBlock(0)).data
+    expect(parentBlockData).toEqual(childBlockData)
+  })
+})

packages/vhd-lib/src/Vhd/VhdFile.js

@@ -1,15 +1,8 @@
-import {
-  BLOCK_UNUSED,
-  FOOTER_SIZE,
-  HEADER_SIZE,
-  PLATFORM_NONE,
-  SECTOR_SIZE,
-  PARENT_LOCATOR_ENTRIES,
-} from '../_constants'
-import { computeBatSize, sectorsToBytes, buildHeader, buildFooter, BUF_BLOCK_UNUSED } from './_utils'
+import { BLOCK_UNUSED, FOOTER_SIZE, HEADER_SIZE, PLATFORMS, SECTOR_SIZE, PARENT_LOCATOR_ENTRIES } from '../_constants'
+import { computeBatSize, sectorsToBytes, unpackHeader, unpackFooter, BUF_BLOCK_UNUSED } from './_utils'
 import { createLogger } from '@xen-orchestra/log'
 import { fuFooter, fuHeader, checksumStruct } from '../_structs'
-import { set as mapSetBit, test as mapTestBit } from '../_bitmap'
+import { set as mapSetBit } from '../_bitmap'
 import { VhdAbstract } from './VhdAbstract'
 import assert from 'assert'
 import getFirstAndLastBlocks from '../_getFirstAndLastBlocks'
@@ -78,23 +71,23 @@ export class VhdFile extends VhdAbstract {
     return super.header
   }
 
-  static async open(handler, path) {
-    const fd = await handler.openFile(path, 'r+')
+  static async open(handler, path, { flags, checkSecondFooter = true } = {}) {
+    const fd = await handler.openFile(path, flags ?? 'r+')
     const vhd = new VhdFile(handler, fd)
     // openning a file for reading does not trigger EISDIR as long as we don't really read from it :
     // https://man7.org/linux/man-pages/man2/open.2.html
     // EISDIR pathname refers to a directory and the access requested
     // involved writing (that is, O_WRONLY or O_RDWR is set).
     // reading the header ensure we have a well formed file immediatly
-    await vhd.readHeaderAndFooter()
+    await vhd.readHeaderAndFooter(checkSecondFooter)
     return {
       dispose: () => handler.closeFile(fd),
       value: vhd,
     }
   }
 
-  static async create(handler, path) {
-    const fd = await handler.openFile(path, 'wx')
+  static async create(handler, path, { flags } = {}) {
+    const fd = await handler.openFile(path, flags ?? 'wx')
     const vhd = new VhdFile(handler, fd)
     return {
       dispose: () => handler.closeFile(fd),
@@ -129,7 +122,7 @@ export class VhdFile extends VhdAbstract {
     for (let i = 0; i < PARENT_LOCATOR_ENTRIES; i++) {
       const entry = header.parentLocatorEntry[i]
 
-      if (entry.platformCode !== PLATFORM_NONE) {
+      if (entry.platformCode !== PLATFORMS.NONE) {
         end = Math.max(end, entry.platformDataOffset + sectorsToBytes(entry.platformDataSpace))
       }
     }
@@ -177,8 +170,8 @@ export class VhdFile extends VhdAbstract {
     const bufFooter = buf.slice(0, FOOTER_SIZE)
     const bufHeader = buf.slice(FOOTER_SIZE)
 
-    const footer = buildFooter(bufFooter)
-    const header = buildHeader(bufHeader, footer)
+    const footer = unpackFooter(bufFooter)
+    const header = unpackHeader(bufHeader, footer)
 
     if (checkSecondFooter) {
       const size = await this._handler.getSize(this._path)
@@ -343,47 +336,6 @@ export class VhdFile extends VhdAbstract {
     )
   }
 
-  async coalesceBlock(child, blockId) {
-    const block = await child.readBlock(blockId)
-    const { bitmap, data } = block
-
-    debug(`coalesceBlock block=${blockId}`)
-
-    // For each sector of block data...
-    const { sectorsPerBlock } = child
-    let parentBitmap = null
-    for (let i = 0; i < sectorsPerBlock; i++) {
-      // If no changes on one sector, skip.
-      if (!mapTestBit(bitmap, i)) {
-        continue
-      }
-      let endSector = i + 1
-
-      // Count changed sectors.
-      while (endSector < sectorsPerBlock && mapTestBit(bitmap, endSector)) {
-        ++endSector
-      }
-
-      // Write n sectors into parent.
-      debug(`coalesceBlock: write sectors=${i}...${endSector}`)
-
-      const isFullBlock = i === 0 && endSector === sectorsPerBlock
-      if (isFullBlock) {
-        await this.writeEntireBlock(block)
-      } else {
-        if (parentBitmap === null) {
-          parentBitmap = (await this.readBlock(blockId, true)).bitmap
-        }
-        await this._writeBlockSectors(block, i, endSector, parentBitmap)
-      }
-
-      i = endSector
-    }
-
-    // Return the merged data size
-    return data.length
-  }
-
   // Write a context footer. (At the end and beginning of a vhd file.)
   async writeFooter(onlyEndFooter = false) {
     const { footer } = this
@@ -478,11 +430,27 @@ export class VhdFile extends VhdAbstract {
         // new parent locator length is smaller than available space : keep it in place
         position = header.parentLocatorEntry[parentLocatorId].platformDataOffset
       } else {
-        // new parent locator length is bigger than available space : move it to the end
-        position = this._getEndOfData()
+        const firstAndLastBlocks = getFirstAndLastBlocks(this.#blockTable)
+        if (firstAndLastBlocks === undefined) {
+          // no block in data : put the parent locatorn entry at the end
+          position = this._getEndOfData()
+        } else {
+          // need more size
+
+          // since there can be multiple parent locator entry, we can't extend the entry in place
+          // move the first(s) block(s) at the end of the data
+          // move the parent locator to the  precedent position of the first block
+          const { firstSector } = firstAndLastBlocks
+          await this._freeFirstBlockSpace(header.parentLocatorEntry[parentLocatorId].platformDataSpace)
+          position = sectorsToBytes(firstSector)
+        }
       }
       await this._write(data, position)
       header.parentLocatorEntry[parentLocatorId].platformDataOffset = position
     }
   }
+
+  async getSize() {
+    return await this._handler.getSize(this._path)
+  }
 }

packages/vhd-lib/src/Vhd/VhdSynthetic.integ.spec.js

@@ -0,0 +1,83 @@
+/* eslint-env jest */
+
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { Disposable, pFromCallback } from 'promise-toolbox'
+import { getSyncedHandler } from '@xen-orchestra/fs'
+
+import { SECTOR_SIZE, PLATFORMS } from '../_constants'
+import { createRandomFile, convertFromRawToVhd } from '../tests/utils'
+import { openVhd, chainVhd } from '..'
+import { VhdSynthetic } from './VhdSynthetic'
+
+let tempDir = null
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+test('It can read block and parent locator from a synthetic vhd', async () => {
+  const bigRawFileName = `/bigrandomfile`
+  await createRandomFile(`${tempDir}/${bigRawFileName}`, 8)
+  const bigVhdFileName = `/bigrandomfile.vhd`
+  await convertFromRawToVhd(`${tempDir}/${bigRawFileName}`, `${tempDir}/${bigVhdFileName}`)
+
+  const smallRawFileName = `/smallrandomfile`
+  await createRandomFile(`${tempDir}/${smallRawFileName}`, 4)
+  const smallVhdFileName = `/smallrandomfile.vhd`
+  await convertFromRawToVhd(`${tempDir}/${smallRawFileName}`, `${tempDir}/${smallVhdFileName}`)
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
+    // ensure the two VHD are linked, with the child of type DISK_TYPES.DIFFERENCING
+    await chainVhd(handler, bigVhdFileName, handler, smallVhdFileName, true)
+
+    const [smallVhd, bigVhd] = yield Disposable.all([
+      openVhd(handler, smallVhdFileName),
+      openVhd(handler, bigVhdFileName),
+    ])
+    // add parent locato
+    // this will also scramble the block inside the vhd files
+    await bigVhd.writeParentLocator({
+      id: 0,
+      platformCode: PLATFORMS.W2KU,
+      data: Buffer.from('I am in the big one'),
+    })
+    const syntheticVhd = new VhdSynthetic([smallVhd, bigVhd])
+    await syntheticVhd.readBlockAllocationTable()
+
+    expect(syntheticVhd.header.diskType).toEqual(bigVhd.header.diskType)
+    expect(syntheticVhd.header.parentTimestamp).toEqual(bigVhd.header.parentTimestamp)
+
+    // first two block should be from small
+    const buf = Buffer.alloc(syntheticVhd.sectorsPerBlock * SECTOR_SIZE, 0)
+    let content = (await syntheticVhd.readBlock(0)).data
+    await handler.read(smallRawFileName, buf, 0)
+    expect(content).toEqual(buf)
+
+    content = (await syntheticVhd.readBlock(1)).data
+    await handler.read(smallRawFileName, buf, buf.length)
+    expect(content).toEqual(buf)
+
+    // the next one from big
+
+    content = (await syntheticVhd.readBlock(2)).data
+    await handler.read(bigRawFileName, buf, buf.length * 2)
+    expect(content).toEqual(buf)
+
+    content = (await syntheticVhd.readBlock(3)).data
+    await handler.read(bigRawFileName, buf, buf.length * 3)
+    expect(content).toEqual(buf)
+
+    // the parent locator should the one of the root vhd
+    const parentLocator = await syntheticVhd.readParentLocator(0)
+    expect(parentLocator.platformCode).toEqual(PLATFORMS.W2KU)
+    expect(Buffer.from(parentLocator.data, 'utf-8').toString()).toEqual('I am in the big one')
+  })
+})

packages/vhd-lib/src/Vhd/VhdSynthetic.js

@@ -0,0 +1,93 @@
+import * as UUID from 'uuid'
+import { asyncMap } from '@xen-orchestra/async-map'
+import { VhdAbstract } from './VhdAbstract'
+import { DISK_TYPES, FOOTER_SIZE, HEADER_SIZE } from '../_constants'
+
+import assert from 'assert'
+
+export class VhdSynthetic extends VhdAbstract {
+  #vhds = []
+  set header(_) {
+    throw new Error('Header is read only for VhdSynthetic')
+  }
+
+  get header() {
+    // this the VHD we want to synthetize
+    const vhd = this.#vhds[0]
+
+    // this is the root VHD
+    const rootVhd = this.#vhds[this.#vhds.length - 1]
+
+    // data of our synthetic VHD
+    // TODO: set parentLocatorEntry-s in header
+    return {
+      ...vhd.header,
+      tableOffset: FOOTER_SIZE + HEADER_SIZE,
+      parentTimestamp: rootVhd.header.parentTimestamp,
+      parentUnicodeName: rootVhd.header.parentUnicodeName,
+      parentUuid: rootVhd.header.parentUuid,
+    }
+  }
+
+  set footer(_) {
+    throw new Error('Footer is read only for VhdSynthetic')
+  }
+
+  get footer() {
+    // this is the root VHD
+    const rootVhd = this.#vhds[this.#vhds.length - 1]
+    return {
+      ...this.#vhds[0].footer,
+      dataOffset: FOOTER_SIZE,
+      diskType: rootVhd.footer.diskType,
+    }
+  }
+
+  static async open(vhds) {
+    const vhd = new VhdSynthetic(vhds)
+    return {
+      dispose: () => {},
+      value: vhd,
+    }
+  }
+  /**
+   * @param {Array<VhdAbstract>} vhds the chain of Vhds used to compute this Vhd, from the deepest child (in position 0), to the root (in the last position)
+   * only the last one can have any type. Other must have type DISK_TYPES.DIFFERENCING (delta)
+   */
+  constructor(vhds) {
+    assert(vhds.length > 0)
+    super()
+    this.#vhds = vhds
+  }
+
+  async readBlockAllocationTable() {
+    await asyncMap(this.#vhds, vhd => vhd.readBlockAllocationTable())
+  }
+
+  containsBlock(blockId) {
+    return this.#vhds.some(vhd => vhd.containsBlock(blockId))
+  }
+
+  async readHeaderAndFooter() {
+    const vhds = this.#vhds
+
+    await asyncMap(vhds, vhd => vhd.readHeaderAndFooter())
+
+    for (let i = 0, n = vhds.length - 1; i < n; ++i) {
+      const child = vhds[i]
+      const parent = vhds[i + 1]
+      assert.strictEqual(child.footer.diskType, DISK_TYPES.DIFFERENCING)
+      assert.strictEqual(UUID.stringify(child.header.parentUuid), UUID.stringify(parent.footer.uuid))
+    }
+  }
+
+  async readBlock(blockId, onlyBitmap = false) {
+    const index = this.#vhds.findIndex(vhd => vhd.containsBlock(blockId))
+    // only read the content of the first vhd containing this block
+    return await this.#vhds[index].readBlock(blockId, onlyBitmap)
+  }
+
+  _readParentLocatorData(id) {
+    return this.#vhds[this.#vhds.length - 1]._readParentLocatorData(id)
+  }
+}

packages/vhd-lib/src/Vhd/_utils.js

@@ -6,6 +6,11 @@ import checkHeader from '../_checkHeader'
 
 export const computeBatSize = entries => sectorsToBytes(sectorsRoundUpNoZero(entries * 4))
 
+export const computeSectorsPerBlock = blockSize => blockSize / SECTOR_SIZE
+// one bit per sector
+export const computeBlockBitmapSize = blockSize => computeSectorsPerBlock(blockSize) >>> 3
+export const computeSectorOfBitmap = blockSize => sectorsRoundUpNoZero(computeBlockBitmapSize(blockSize))
+
 // Sectors conversions.
 export const sectorsRoundUpNoZero = bytes => Math.ceil(bytes / SECTOR_SIZE) || 1
 export const sectorsToBytes = sectors => sectors * SECTOR_SIZE
@@ -27,7 +32,7 @@ BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
  * @param {Object} footer
  * @returns {Object} the parsed header
  */
-export const buildHeader = (bufHeader, footer) => {
+export const unpackHeader = (bufHeader, footer) => {
   assertChecksum('header', bufHeader, fuHeader)
 
   const header = fuHeader.unpack(bufHeader)
@@ -43,7 +48,7 @@ export const buildHeader = (bufHeader, footer) => {
  * @returns {Object} the parsed footer
  */
 
-export const buildFooter = bufFooter => {
+export const unpackFooter = bufFooter => {
   assertChecksum('footer', bufFooter, fuFooter)
 
   const footer = fuFooter.unpack(bufFooter)

packages/vhd-lib/src/_constants.js

@@ -12,19 +12,27 @@ export const DEFAULT_BLOCK_SIZE = 0x00200000 // from the spec
 export const FOOTER_COOKIE = 'conectix'
 export const HEADER_COOKIE = 'cxsparse'
 
-export const DISK_TYPE_FIXED = 2
-export const DISK_TYPE_DYNAMIC = 3
-export const DISK_TYPE_DIFFERENCING = 4
+export const DISK_TYPES = {
+  __proto__: null,
+
+  FIXED: 2,
+  DYNAMIC: 3,
+  DIFFERENCING: 4,
+}
 
 export const PARENT_LOCATOR_ENTRIES = 8
 
-export const PLATFORM_NONE = 0
-export const PLATFORM_WI2R = 0x57693272
-export const PLATFORM_WI2K = 0x5769326b
-export const PLATFORM_W2RU = 0x57327275
-export const PLATFORM_W2KU = 0x57326b75
-export const PLATFORM_MAC = 0x4d616320
-export const PLATFORM_MACX = 0x4d616358
+export const PLATFORMS = {
+  __proto__: null,
+
+  NONE: 0,
+  WI2R: 0x57693272,
+  WI2K: 0x5769326b,
+  W2RU: 0x57327275,
+  W2KU: 0x57326b75,
+  MAC: 0x4d616320,
+  MACX: 0x4d616358,
+}
 
 export const FILE_FORMAT_VERSION = 1 << 16
 export const HEADER_VERSION = 1 << 16

packages/vhd-lib/src/_createFooterHeader.integ.spec.js

@@ -0,0 +1,10 @@
+/* eslint-env jest */
+import { createFooter } from './_createFooterHeader'
+
+test('createFooter() does not crash', () => {
+  createFooter(104448, Math.floor(Date.now() / 1000), {
+    cylinders: 3,
+    heads: 4,
+    sectorsPerTrack: 17,
+  })
+})

packages/vhd-lib/src/_createFooterHeader.js

@@ -4,17 +4,17 @@ import { checksumStruct, fuFooter, fuHeader } from './_structs'
 import {
   CREATOR_APPLICATION,
   DEFAULT_BLOCK_SIZE as VHD_BLOCK_SIZE_BYTES,
-  DISK_TYPE_FIXED,
+  DISK_TYPES,
   FILE_FORMAT_VERSION,
   FOOTER_COOKIE,
   FOOTER_SIZE,
   HEADER_COOKIE,
   HEADER_SIZE,
   HEADER_VERSION,
-  PLATFORM_WI2K,
+  PLATFORMS,
 } from './_constants'
 
-export function createFooter(size, timestamp, geometry, dataOffset, diskType = DISK_TYPE_FIXED) {
+export function createFooter(size, timestamp, geometry, dataOffset, diskType = DISK_TYPES.FIXED) {
   const footer = fuFooter.pack({
     cookie: FOOTER_COOKIE,
     features: 2,
@@ -22,7 +22,7 @@ export function createFooter(size, timestamp, geometry, dataOffset, diskType = D
     dataOffset,
     timestamp,
     creatorApplication: CREATOR_APPLICATION,
-    creatorHostOs: PLATFORM_WI2K, // it looks like everybody is using Wi2k
+    creatorHostOs: PLATFORMS.WI2K, // it looks like everybody is using Wi2k
     originalSize: size,
     currentSize: size,
     diskGeometry: geometry,

packages/vhd-lib/src/_resolveAlias.integ.spec.js

@@ -0,0 +1,56 @@
+/* eslint-env jest */
+
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { Disposable, pFromCallback } from 'promise-toolbox'
+
+import { isVhdAlias, resolveAlias } from './_resolveAlias'
+
+let tempDir
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+test('is vhd alias recognize only *.alias.vhd files', () => {
+  expect(isVhdAlias('filename.alias.vhd')).toEqual(true)
+  expect(isVhdAlias('alias.vhd')).toEqual(false)
+  expect(isVhdAlias('filename.vhd')).toEqual(false)
+  expect(isVhdAlias('filename.alias.vhd.other')).toEqual(false)
+})
+
+test('resolve return the path in argument for a non alias file ', async () => {
+  expect(await resolveAlias(null, 'filename.vhd')).toEqual('filename.vhd')
+})
+test('resolve get the path of the target file for an alias', async () => {
+  await Disposable.use(async function* () {
+    // same directory
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    const tempDirFomRemoteUrl = tempDir.slice(1) // remove the / which is included in the remote url
+    const alias = `${tempDirFomRemoteUrl}/alias.alias.vhd`
+    await handler.writeFile(alias, 'target.vhd')
+    expect(await resolveAlias(handler, alias)).toEqual(`${tempDirFomRemoteUrl}/target.vhd`)
+
+    // different directory
+    await handler.mkdir(`${tempDirFomRemoteUrl}/sub/`)
+    await handler.writeFile(alias, 'sub/target.vhd', { flags: 'w' })
+    expect(await resolveAlias(handler, alias)).toEqual(`${tempDirFomRemoteUrl}/sub/target.vhd`)
+  })
+})
+
+test('resolve throws an error an alias to an alias', async () => {
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    const alias = `${tempDir}/alias.alias.vhd`
+    const target = `${tempDir}/target.alias.vhd`
+    await handler.writeFile(alias, target)
+    expect(async () => await resolveAlias(handler, alias)).rejects.toThrow(Error)
+  })
+})

packages/vhd-lib/src/_resolveAlias.js

@@ -0,0 +1,18 @@
+import resolveRelativeFromFile from './_resolveRelativeFromFile'
+
+export function isVhdAlias(filename) {
+  return filename.endsWith('.alias.vhd')
+}
+
+export async function resolveAlias(handler, filename) {
+  if (!isVhdAlias(filename)) {
+    return filename
+  }
+  const aliasContent = (await handler.readFile(filename)).toString().trim()
+  // also handle circular references and unreasonnably long chains
+  if (isVhdAlias(aliasContent)) {
+    throw new Error(`Chaining alias is forbidden ${filename} to ${aliasContent}`)
+  }
+  // the target is relative to the alias location
+  return resolveRelativeFromFile(filename, aliasContent)
+}

packages/vhd-lib/src/_structs.js

@@ -33,7 +33,7 @@ export const fuFooter = fu.struct([
     fu.uint8('heads'), // 58
     fu.uint8('sectorsPerTrackCylinder'), // 59
   ]),
-  fu.uint32('diskType'), // 60 Disk type, must be equal to HARD_DISK_TYPE_DYNAMIC/HARD_DISK_TYPE_DIFFERENCING.
+  fu.uint32('diskType'), // 60 Disk type, must be equal to DYNAMIC/DIFFERENCING.
   fu.uint32('checksum'), // 64
   fu.byte('uuid', 16), // 68
   fu.char('saved'), // 84

packages/vhd-lib/src/chain.js

@@ -1,28 +1,30 @@
 import { dirname, relative } from 'path'
 
-import { VhdFile } from './'
-import { DISK_TYPE_DIFFERENCING } from './_constants'
+import { openVhd } from './'
+import { DISK_TYPES } from './_constants'
+import { Disposable } from 'promise-toolbox'
 
 export default async function chain(parentHandler, parentPath, childHandler, childPath, force = false) {
-  const parentVhd = new VhdFile(parentHandler, parentPath)
-  const childVhd = new VhdFile(childHandler, childPath)
+  await Disposable.use(
+    [openVhd(parentHandler, parentPath), openVhd(childHandler, childPath)],
+    async ([parentVhd, childVhd]) => {
+      await childVhd.readHeaderAndFooter()
+      const { header, footer } = childVhd
 
-  await childVhd.readHeaderAndFooter()
-  const { header, footer } = childVhd
+      if (footer.diskType !== DISK_TYPES.DIFFERENCING) {
+        if (!force) {
+          throw new Error('cannot chain disk of type ' + footer.diskType)
+        }
+        footer.diskType = DISK_TYPES.DIFFERENCING
+      }
+      await childVhd.readBlockAllocationTable()
 
-  if (footer.diskType !== DISK_TYPE_DIFFERENCING) {
-    if (!force) {
-      throw new Error('cannot chain disk of type ' + footer.diskType)
+      const parentName = relative(dirname(childPath), parentPath)
+      header.parentUuid = parentVhd.footer.uuid
+      header.parentUnicodeName = parentName
+      await childVhd.setUniqueParentLocator(parentName)
+      await childVhd.writeHeader()
+      await childVhd.writeFooter()
     }
-    footer.diskType = DISK_TYPE_DIFFERENCING
-  }
-
-  await Promise.all([childVhd.readBlockAllocationTable(), parentVhd.readHeaderAndFooter()])
-
-  const parentName = relative(dirname(childPath), parentPath)
-  header.parentUuid = parentVhd.footer.uuid
-  header.parentUnicodeName = parentName
-  await childVhd.setUniqueParentLocator(parentName)
-  await childVhd.writeHeader()
-  await childVhd.writeFooter()
+  )
 }

packages/vhd-lib/src/checkChain.js

@@ -1,16 +1,14 @@
-import { VhdFile } from '.'
+import { openVhd } from '.'
 import resolveRelativeFromFile from './_resolveRelativeFromFile'
-import { DISK_TYPE_DYNAMIC } from './_constants'
+import { DISK_TYPES } from './_constants'
+import { Disposable } from 'promise-toolbox'
 
 export default async function checkChain(handler, path) {
-  while (true) {
-    const vhd = new VhdFile(handler, path)
-    await vhd.readHeaderAndFooter()
-
-    if (vhd.footer.diskType === DISK_TYPE_DYNAMIC) {
-      break
-    }
-
-    path = resolveRelativeFromFile(path, vhd.header.parentUnicodeName)
-  }
+  await Disposable.use(function* () {
+    let vhd
+    do {
+      vhd = yield openVhd(handler, path)
+      path = resolveRelativeFromFile(path, vhd.header.parentUnicodeName)
+    } while (vhd.footer.diskType !== DISK_TYPES.DYNAMIC)
+  })
 }

packages/vhd-lib/src/checkFooter.js

@@ -1,17 +1,11 @@
 import assert from 'assert'
 
-import {
-  DISK_TYPE_DIFFERENCING,
-  DISK_TYPE_DYNAMIC,
-  FILE_FORMAT_VERSION,
-  FOOTER_COOKIE,
-  FOOTER_SIZE,
-} from './_constants'
+import { DISK_TYPES, FILE_FORMAT_VERSION, FOOTER_COOKIE, FOOTER_SIZE } from './_constants'
 
 export default footer => {
   assert.strictEqual(footer.cookie, FOOTER_COOKIE)
   assert.strictEqual(footer.dataOffset, FOOTER_SIZE)
   assert.strictEqual(footer.fileFormatVersion, FILE_FORMAT_VERSION)
   assert(footer.originalSize <= footer.currentSize)
-  assert(footer.diskType === DISK_TYPE_DIFFERENCING || footer.diskType === DISK_TYPE_DYNAMIC)
+  assert(footer.diskType === DISK_TYPES.DIFFERENCING || footer.diskType === DISK_TYPES.DYNAMIC)
 }

packages/vhd-lib/src/createContentStream.js

@@ -1,31 +0,0 @@
-import asyncIteratorToStream from 'async-iterator-to-stream'
-
-import { VhdFile } from '.'
-
-export default asyncIteratorToStream(async function* (handler, path) {
-  const fd = await handler.openFile(path, 'r')
-  try {
-    const vhd = new VhdFile(handler, fd)
-    await vhd.readHeaderAndFooter()
-    await vhd.readBlockAllocationTable()
-    const {
-      footer: { currentSize },
-      header: { blockSize },
-    } = vhd
-    const nFullBlocks = Math.floor(currentSize / blockSize)
-    const nLeftoverBytes = currentSize % blockSize
-
-    const emptyBlock = Buffer.alloc(blockSize)
-    for (let i = 0; i < nFullBlocks; ++i) {
-      yield vhd.containsBlock(i) ? (await vhd.readBlock(i)).data : emptyBlock
-    }
-    if (nLeftoverBytes !== 0) {
-      yield (vhd.containsBlock(nFullBlocks) ? (await vhd.readBlock(nFullBlocks)).data : emptyBlock).slice(
-        0,
-        nLeftoverBytes
-      )
-    }
-  } finally {
-    await handler.closeFile(fd)
-  }
-})

packages/vhd-lib/src/createReadableRawStream.integ.spec.js

@@ -8,7 +8,6 @@ import { pipeline } from 'readable-stream'
 
 import { createReadableRawStream, createReadableSparseStream } from './'
 
-import { createFooter } from './_createFooterHeader'
 import { checkFile, convertFromVhdToRaw } from './tests/utils'
 
 let tempDir = null
@@ -21,14 +20,6 @@ afterEach(async () => {
   await pFromCallback(cb => rimraf(tempDir, cb))
 })
 
-test('createFooter() does not crash', () => {
-  createFooter(104448, Math.floor(Date.now() / 1000), {
-    cylinders: 3,
-    heads: 4,
-    sectorsPerTrack: 17,
-  })
-})
-
 test('ReadableRawVHDStream does not crash', async () => {
   const data = [
     {
@@ -103,6 +94,7 @@ test('ReadableSparseVHDStream can handle a sparse file', async () => {
     },
   ]
   const fileSize = blockSize * 110
+
   const stream = await createReadableSparseStream(
     fileSize,
     blockSize,
@@ -111,6 +103,7 @@ test('ReadableSparseVHDStream can handle a sparse file', async () => {
   )
   expect(stream.length).toEqual(4197888)
   const pipe = stream.pipe(createWriteStream(`${tempDir}/output.vhd`))
+
   await fromEvent(pipe, 'finish')
   await checkFile(`${tempDir}/output.vhd`)
   await convertFromVhdToRaw(`${tempDir}/output.vhd`, `${tempDir}/out1.raw`)

packages/vhd-lib/src/createReadableSparseStream.js

@@ -7,7 +7,7 @@ import { createFooter, createHeader } from './_createFooterHeader'
 import {
   BLOCK_UNUSED,
   DEFAULT_BLOCK_SIZE as VHD_BLOCK_SIZE_BYTES,
-  DISK_TYPE_DYNAMIC,
+  DISK_TYPES,
   FOOTER_SIZE,
   HEADER_SIZE,
   SECTOR_SIZE,
@@ -73,7 +73,7 @@ export default async function createReadableStream(diskSize, fragmentSize, fragm
   const firstBlockPosition = batPosition + tablePhysicalSizeBytes
   const geometry = computeGeometryForSize(diskSize)
   const actualSize = geometry.actualSize
-  const footer = createFooter(actualSize, Math.floor(Date.now() / 1000), geometry, FOOTER_SIZE, DISK_TYPE_DYNAMIC)
+  const footer = createFooter(actualSize, Math.floor(Date.now() / 1000), geometry, FOOTER_SIZE, DISK_TYPES.DYNAMIC)
   const header = createHeader(maxTableEntries, batPosition, VHD_BLOCK_SIZE_BYTES)
   const bitmapSize = Math.ceil(VHD_BLOCK_SIZE_SECTORS / 8 / SECTOR_SIZE) * SECTOR_SIZE
   const bat = Buffer.alloc(tablePhysicalSizeBytes, 0xff)

packages/vhd-lib/src/createSyntheticStream.js

@@ -1,165 +0,0 @@
-import asyncIteratorToStream from 'async-iterator-to-stream'
-import { createLogger } from '@xen-orchestra/log'
-
-import resolveRelativeFromFile from './_resolveRelativeFromFile'
-
-import { VhdFile } from '.'
-import { BLOCK_UNUSED, DISK_TYPE_DYNAMIC, FOOTER_SIZE, HEADER_SIZE, SECTOR_SIZE } from './_constants'
-import { fuFooter, fuHeader, checksumStruct } from './_structs'
-import { test as mapTestBit } from './_bitmap'
-
-const { warn } = createLogger('vhd-lib:createSyntheticStream')
-
-export default async function createSyntheticStream(handler, paths) {
-  const fds = []
-  const cleanup = () => {
-    for (let i = 0, n = fds.length; i < n; ++i) {
-      handler.closeFile(fds[i]).catch(error => {
-        warn('error while closing file', {
-          error,
-          fd: fds[i],
-        })
-      })
-    }
-  }
-  try {
-    const vhds = []
-    const open = async path => {
-      const fd = await handler.openFile(path, 'r')
-      fds.push(fd)
-      const vhd = new VhdFile(handler, fd)
-      vhds.push(vhd)
-      await vhd.readHeaderAndFooter()
-      await vhd.readBlockAllocationTable()
-
-      return vhd
-    }
-    if (typeof paths === 'string') {
-      let path = paths
-      let vhd
-      while ((vhd = await open(path)).footer.diskType !== DISK_TYPE_DYNAMIC) {
-        path = resolveRelativeFromFile(path, vhd.header.parentUnicodeName)
-      }
-    } else {
-      for (const path of paths) {
-        await open(path)
-      }
-    }
-    const nVhds = vhds.length
-
-    // this the VHD we want to synthetize
-    const vhd = vhds[0]
-
-    // this is the root VHD
-    const rootVhd = vhds[nVhds - 1]
-
-    // data of our synthetic VHD
-    // TODO: set parentLocatorEntry-s in header
-    let header = {
-      ...vhd.header,
-      tableOffset: FOOTER_SIZE + HEADER_SIZE,
-      parentTimestamp: rootVhd.header.parentTimestamp,
-      parentUnicodeName: rootVhd.header.parentUnicodeName,
-      parentUuid: rootVhd.header.parentUuid,
-    }
-
-    const bat = Buffer.allocUnsafe(vhd.batSize)
-    let footer = {
-      ...vhd.footer,
-      dataOffset: FOOTER_SIZE,
-      diskType: rootVhd.footer.diskType,
-    }
-    const sectorsPerBlockData = vhd.sectorsPerBlock
-    const sectorsPerBlock = sectorsPerBlockData + vhd.bitmapSize / SECTOR_SIZE
-
-    const nBlocks = Math.ceil(footer.currentSize / header.blockSize)
-
-    const blocksOwner = new Array(nBlocks)
-    let blockOffset = Math.ceil((header.tableOffset + bat.length) / SECTOR_SIZE)
-    for (let iBlock = 0; iBlock < nBlocks; ++iBlock) {
-      let blockSector = BLOCK_UNUSED
-      for (let i = 0; i < nVhds; ++i) {
-        if (vhds[i].containsBlock(iBlock)) {
-          blocksOwner[iBlock] = i
-          blockSector = blockOffset
-          blockOffset += sectorsPerBlock
-          break
-        }
-      }
-      bat.writeUInt32BE(blockSector, iBlock * 4)
-    }
-    const fileSize = blockOffset * SECTOR_SIZE + FOOTER_SIZE
-
-    const iterator = function* () {
-      try {
-        footer = fuFooter.pack(footer)
-        checksumStruct(footer, fuFooter)
-        yield footer
-
-        header = fuHeader.pack(header)
-        checksumStruct(header, fuHeader)
-        yield header
-
-        yield bat
-
-        // TODO: for generic usage the bitmap needs to be properly computed for each block
-        const bitmap = Buffer.alloc(vhd.bitmapSize, 0xff)
-        for (let iBlock = 0; iBlock < nBlocks; ++iBlock) {
-          const owner = blocksOwner[iBlock]
-          if (owner === undefined) {
-            continue
-          }
-
-          yield bitmap
-
-          const blocksByVhd = new Map()
-          const emitBlockSectors = function* (iVhd, i, n) {
-            const vhd = vhds[iVhd]
-            const isRootVhd = vhd === rootVhd
-            if (!vhd.containsBlock(iBlock)) {
-              if (isRootVhd) {
-                yield Buffer.alloc((n - i) * SECTOR_SIZE)
-              } else {
-                yield* emitBlockSectors(iVhd + 1, i, n)
-              }
-              return
-            }
-            let block = blocksByVhd.get(vhd)
-            if (block === undefined) {
-              block = yield vhd.readBlock(iBlock)
-              blocksByVhd.set(vhd, block)
-            }
-            const { bitmap, data } = block
-            if (isRootVhd) {
-              yield data.slice(i * SECTOR_SIZE, n * SECTOR_SIZE)
-              return
-            }
-            while (i < n) {
-              const hasData = mapTestBit(bitmap, i)
-              const start = i
-              do {
-                ++i
-              } while (i < n && mapTestBit(bitmap, i) === hasData)
-              if (hasData) {
-                yield data.slice(start * SECTOR_SIZE, i * SECTOR_SIZE)
-              } else {
-                yield* emitBlockSectors(iVhd + 1, start, i)
-              }
-            }
-          }
-          yield* emitBlockSectors(owner, 0, sectorsPerBlockData)
-        }
-        yield footer
-      } finally {
-        cleanup()
-      }
-    }
-
-    const stream = asyncIteratorToStream(iterator())
-    stream.length = fileSize
-    return stream
-  } catch (e) {
-    cleanup()
-    throw e
-  }
-}

packages/vhd-lib/src/createVhdDirectoryFromStream.js

@@ -0,0 +1,168 @@
+import { VhdDirectory } from './'
+import { BLOCK_UNUSED, FOOTER_SIZE, HEADER_SIZE, SECTOR_SIZE } from './_constants'
+import { readChunk } from '@vates/read-chunk'
+import assert from 'assert'
+import { Disposable } from 'promise-toolbox'
+import { unpackFooter, unpackHeader, computeBlockBitmapSize } from './Vhd/_utils'
+import { asyncEach } from '@vates/async-each'
+
+const cappedBufferConcat = (buffers, maxSize) => {
+  let buffer = Buffer.concat(buffers)
+  if (buffer.length > maxSize) {
+    buffer = buffer.slice(buffer.length - maxSize)
+  }
+  return buffer
+}
+
+async function* parse(stream) {
+  let bytesRead = 0
+
+  // handle empty space between elements
+  // ensure we read stream in order
+  async function read(offset, size) {
+    assert(bytesRead <= offset, `offset is ${offset} but we already read ${bytesRead} bytes`)
+    if (bytesRead < offset) {
+      // empty spaces
+      await read(bytesRead, offset - bytesRead)
+    }
+    const buf = await readChunk(stream, size)
+    assert.strictEqual(buf.length, size, `read ${buf.length} instead of ${size}`)
+    bytesRead += size
+    return buf
+  }
+
+  const bufFooter = await read(0, FOOTER_SIZE)
+
+  const footer = unpackFooter(bufFooter)
+  yield { type: 'footer', footer, offset: 0 }
+
+  const bufHeader = await read(FOOTER_SIZE, HEADER_SIZE)
+  const header = unpackHeader(bufHeader, footer)
+
+  yield { type: 'header', header, offset: SECTOR_SIZE }
+  const blockSize = header.blockSize
+  assert.strictEqual(blockSize % SECTOR_SIZE, 0)
+
+  const blockBitmapSize = computeBlockBitmapSize(blockSize)
+  const blockAndBitmapSize = blockBitmapSize + blockSize
+
+  const index = []
+
+  for (const parentLocatorId in header.parentLocatorEntry) {
+    const parentLocatorEntry = header.parentLocatorEntry[parentLocatorId]
+    // empty parent locator entry, does not exist in the content
+    if (parentLocatorEntry.platformDataSpace === 0) {
+      continue
+    }
+    index.push({
+      ...parentLocatorEntry,
+      type: 'parentLocator',
+      offset: parentLocatorEntry.platformDataOffset,
+      size: parentLocatorEntry.platformDataLength,
+      id: parentLocatorId,
+    })
+  }
+
+  const batOffset = header.tableOffset
+  const batSize = Math.max(1, Math.ceil((header.maxTableEntries * 4) / SECTOR_SIZE)) * SECTOR_SIZE
+
+  index.push({
+    type: 'bat',
+    offset: batOffset,
+    size: batSize,
+  })
+
+  // sometimes some parent locator are before the BAT
+  index.sort((a, b) => a.offset - b.offset)
+  while (index.length > 0) {
+    const item = index.shift()
+    const buffer = await read(item.offset, item.size)
+    if (item.type === 'bat') {
+      // found the BAT : read it and ad block to index
+      for (let blockCounter = 0; blockCounter < header.maxTableEntries; blockCounter++) {
+        const batEntrySector = buffer.readUInt32BE(blockCounter * 4)
+        // unallocated block, no need to export it
+        if (batEntrySector !== BLOCK_UNUSED) {
+          const batEntryBytes = batEntrySector * SECTOR_SIZE
+          // ensure the block is not before the bat
+          assert.ok(batEntryBytes >= batOffset + batSize)
+          index.push({
+            type: 'block',
+            id: blockCounter,
+            offset: batEntryBytes,
+            size: blockAndBitmapSize,
+          })
+        }
+      }
+      // sort again index to ensure block and parent locator are in the right order
+      index.sort((a, b) => a.offset - b.offset)
+    } else {
+      yield { ...item, buffer }
+    }
+  }
+
+  /**
+   * the second footer is at filesize - 512 , there can be empty spaces between last block
+   * and the start of the footer
+   *
+   * we read till the end of the stream, and use the last 512 bytes as the footer
+   */
+  const bufFooterEnd = await readLastSector(stream)
+  assert(bufFooter.equals(bufFooterEnd), 'footer1 !== footer2')
+}
+
+function readLastSector(stream) {
+  return new Promise((resolve, reject) => {
+    let bufFooterEnd = Buffer.alloc(0)
+    stream.on('data', chunk => {
+      if (chunk.length > 0) {
+        bufFooterEnd = cappedBufferConcat([bufFooterEnd, chunk], SECTOR_SIZE)
+      }
+    })
+
+    stream.on('end', () => resolve(bufFooterEnd))
+    stream.on('error', reject)
+  })
+}
+
+const buildVhd = Disposable.wrap(async function* (handler, path, inputStream, { concurrency }) {
+  const vhd = yield VhdDirectory.create(handler, path)
+  await asyncEach(
+    parse(inputStream),
+    async function (item) {
+      switch (item.type) {
+        case 'footer':
+          vhd.footer = item.footer
+          break
+        case 'header':
+          vhd.header = item.header
+          break
+        case 'parentLocator':
+          await vhd.writeParentLocator({ ...item, data: item.buffer })
+          break
+        case 'block':
+          await vhd.writeEntireBlock(item)
+          break
+        default:
+          throw new Error(`unhandled type of block generated by parser : ${item.type} while generating ${path}`)
+      }
+    },
+    {
+      concurrency,
+    }
+  )
+  await Promise.all([vhd.writeFooter(), vhd.writeHeader(), vhd.writeBlockAllocationTable()])
+})
+
+export async function createVhdDirectoryFromStream(handler, path, inputStream, { validator, concurrency = 16 } = {}) {
+  try {
+    await buildVhd(handler, path, inputStream, { concurrency })
+    if (validator !== undefined) {
+      await validator.call(this, path)
+    }
+  } catch (error) {
+    // cleanup on error
+    await handler.rmTree(path)
+    throw error
+  }
+}

packages/vhd-lib/src/index.js

@@ -1,14 +1,15 @@
 export { default as chainVhd } from './chain'
 export { default as checkFooter } from './checkFooter'
 export { default as checkVhdChain } from './checkChain'
-export { default as createContentStream } from './createContentStream'
 export { default as createReadableRawStream } from './createReadableRawStream'
 export { default as createReadableSparseStream } from './createReadableSparseStream'
-export { default as createSyntheticStream } from './createSyntheticStream'
 export { default as createVhdStreamWithLength } from './createVhdStreamWithLength'
+export { createVhdDirectoryFromStream } from './createVhdDirectoryFromStream'
 export { default as mergeVhd } from './merge'
 export { default as peekFooterFromVhdStream } from './peekFooterFromVhdStream'
 export { openVhd } from './openVhd'
+export { VhdAbstract } from './Vhd/VhdAbstract'
 export { VhdDirectory } from './Vhd/VhdDirectory'
 export { VhdFile } from './Vhd/VhdFile'
+export { VhdSynthetic } from './Vhd/VhdSynthetic'
 export * as Constants from './_constants'

packages/vhd-lib/src/merge.integ.spec.js

@@ -1,19 +1,14 @@
 /* eslint-env jest */
 
-import execa from 'execa'
 import fs from 'fs-extra'
-import getStream from 'get-stream'
 import rimraf from 'rimraf'
 import tmp from 'tmp'
 import { getHandler } from '@xen-orchestra/fs'
 import { pFromCallback } from 'promise-toolbox'
-import { pipeline } from 'readable-stream'
-import { randomBytes } from 'crypto'
 
-import { VhdFile, chainVhd, createSyntheticStream, mergeVhd as vhdMerge } from './index'
+import { VhdFile, chainVhd, mergeVhd as vhdMerge } from './index'
 
-import { SECTOR_SIZE } from './_constants'
-import { checkFile, createRandomFile, convertFromRawToVhd, recoverRawContent } from './tests/utils'
+import { checkFile, createRandomFile, convertFromRawToVhd } from './tests/utils'
 
 let tempDir = null
 
@@ -27,229 +22,136 @@ afterEach(async () => {
   await pFromCallback(cb => rimraf(tempDir, cb))
 })
 
-test('blocks can be moved', async () => {
-  const initalSize = 4
-  const rawFileName = `${tempDir}/randomfile`
-  await createRandomFile(rawFileName, initalSize)
-  const vhdFileName = `${tempDir}/randomfile.vhd`
-  await convertFromRawToVhd(rawFileName, vhdFileName)
-  const handler = getHandler({ url: 'file://' })
-  const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new VhdFile(handler, vhdFileName)
-  await newVhd.readHeaderAndFooter()
-  await newVhd.readBlockAllocationTable()
-  await newVhd._freeFirstBlockSpace(8000000)
-  const recoveredFileName = `${tempDir}/recovered`
-  await recoverRawContent(vhdFileName, recoveredFileName, originalSize)
-  expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(rawFileName))
-})
-
-test('the BAT MSB is not used for sign', async () => {
-  const randomBuffer = await pFromCallback(cb => randomBytes(SECTOR_SIZE, cb))
-  const emptyFileName = `${tempDir}/empty.vhd`
-  await execa('qemu-img', ['create', '-fvpc', emptyFileName, '1.8T'])
-  const handler = getHandler({ url: 'file://' })
-  const vhd = new VhdFile(handler, emptyFileName)
-  await vhd.readHeaderAndFooter()
-  await vhd.readBlockAllocationTable()
-  // we want the bit 31 to be on, to prove it's not been used for sign
-  const hugeWritePositionSectors = Math.pow(2, 31) + 200
-  await vhd.writeData(hugeWritePositionSectors, randomBuffer)
-  await checkFile(emptyFileName)
-  // here we are moving the first sector very far in the VHD to prove the BAT doesn't use signed int32
-  const hugePositionBytes = hugeWritePositionSectors * SECTOR_SIZE
-  await vhd._freeFirstBlockSpace(hugePositionBytes)
-
-  // we recover the data manually for speed reasons.
-  // fs.write() with offset is way faster than qemu-img when there is a 1.5To
-  // hole before the block of data
-  const recoveredFileName = `${tempDir}/recovered`
-  const recoveredFile = await fs.open(recoveredFileName, 'w')
-  try {
-    const vhd2 = new VhdFile(handler, emptyFileName)
-    await vhd2.readHeaderAndFooter()
-    await vhd2.readBlockAllocationTable()
-    for (let i = 0; i < vhd.header.maxTableEntries; i++) {
-      if (vhd.containsBlock(i)) {
-        const block = (await vhd2.readBlock(i)).data
-        await fs.write(recoveredFile, block, 0, block.length, vhd2.header.blockSize * i)
-      }
-    }
-  } finally {
-    fs.close(recoveredFile)
-  }
-  const recovered = await getStream.buffer(
-    await fs.createReadStream(recoveredFileName, {
-      start: hugePositionBytes,
-      end: hugePositionBytes + randomBuffer.length - 1,
-    })
-  )
-  expect(recovered).toEqual(randomBuffer)
-})
-
-test('writeData on empty file', async () => {
-  const mbOfRandom = 3
-  const rawFileName = `${tempDir}/randomfile`
-  const emptyFileName = `${tempDir}/empty.vhd`
-  await createRandomFile(rawFileName, mbOfRandom)
-  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 'M'])
-  const randomData = await fs.readFile(rawFileName)
-  const handler = getHandler({ url: 'file://' })
-  const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new VhdFile(handler, emptyFileName)
-  await newVhd.readHeaderAndFooter()
-  await newVhd.readBlockAllocationTable()
-  await newVhd.writeData(0, randomData)
-  const recoveredFileName = `${tempDir}/recovered`
-  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
-  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
-})
-
-test('writeData in 2 non-overlaping operations', async () => {
-  const mbOfRandom = 3
-  const rawFileName = `${tempDir}/randomfile`
-  const emptyFileName = `${tempDir}/empty.vhd`
-  const recoveredFileName = `${tempDir}/recovered`
-  await createRandomFile(rawFileName, mbOfRandom)
-  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 'M'])
-  const randomData = await fs.readFile(rawFileName)
-  const handler = getHandler({ url: 'file://' })
-  const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new VhdFile(handler, emptyFileName)
-  await newVhd.readHeaderAndFooter()
-  await newVhd.readBlockAllocationTable()
-  const splitPointSectors = 2
-  await newVhd.writeData(0, randomData.slice(0, splitPointSectors * 512))
-  await newVhd.writeData(splitPointSectors, randomData.slice(splitPointSectors * 512))
-  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
-  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
-})
-
-test('writeData in 2 overlaping operations', async () => {
-  const mbOfRandom = 3
-  const rawFileName = `${tempDir}/randomfile`
-  const emptyFileName = `${tempDir}/empty.vhd`
-  const recoveredFileName = `${tempDir}/recovered`
-  await createRandomFile(rawFileName, mbOfRandom)
-  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 'M'])
-  const randomData = await fs.readFile(rawFileName)
-  const handler = getHandler({ url: 'file://' })
-  const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new VhdFile(handler, emptyFileName)
-  await newVhd.readHeaderAndFooter()
-  await newVhd.readBlockAllocationTable()
-  const endFirstWrite = 3
-  const startSecondWrite = 2
-  await newVhd.writeData(0, randomData.slice(0, endFirstWrite * 512))
-  await newVhd.writeData(startSecondWrite, randomData.slice(startSecondWrite * 512))
-  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
-  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
-})
-
-test('BAT can be extended and blocks moved', async () => {
-  const initalSize = 4
-  const rawFileName = `${tempDir}/randomfile`
-  const recoveredFileName = `${tempDir}/recovered`
-  const vhdFileName = `${tempDir}/randomfile.vhd`
-  await createRandomFile(rawFileName, initalSize)
-  await convertFromRawToVhd(rawFileName, vhdFileName)
-  const handler = getHandler({ url: 'file://' })
-  const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new VhdFile(handler, vhdFileName)
-  await newVhd.readHeaderAndFooter()
-  await newVhd.readBlockAllocationTable()
-  await newVhd.ensureBatSize(2000)
-  await recoverRawContent(vhdFileName, recoveredFileName, originalSize)
-  expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(rawFileName))
-})
-
-test('coalesce works with empty parent files', async () => {
-  const mbOfRandom = 2
-  const rawFileName = `${tempDir}/randomfile`
-  const emptyFileName = `${tempDir}/empty.vhd`
-  const vhdFileName = `${tempDir}/randomfile.vhd`
-  const recoveredFileName = `${tempDir}/recovered`
-  await createRandomFile(rawFileName, mbOfRandom)
-  await convertFromRawToVhd(rawFileName, vhdFileName)
-  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 1 + 'M'])
-  await checkFile(vhdFileName)
-  await checkFile(emptyFileName)
-  const handler = getHandler({ url: 'file://' })
-  const originalSize = await handler._getSize(rawFileName)
-  await chainVhd(handler, emptyFileName, handler, vhdFileName, true)
-  await checkFile(vhdFileName)
-  await checkFile(emptyFileName)
-  await vhdMerge(handler, emptyFileName, handler, vhdFileName)
-  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
-  expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(rawFileName))
-})
-
-test('coalesce works in normal cases', async () => {
-  const mbOfRandom = 5
-  const randomFileName = `${tempDir}/randomfile`
-  const random2FileName = `${tempDir}/randomfile2`
-  const smallRandomFileName = `${tempDir}/small_randomfile`
+test('merge works in normal cases', async () => {
+  const mbOfFather = 8
+  const mbOfChildren = 4
+  const parentRandomFileName = `${tempDir}/randomfile`
+  const childRandomFileName = `${tempDir}/small_randomfile`
   const parentFileName = `${tempDir}/parent.vhd`
   const child1FileName = `${tempDir}/child1.vhd`
-  const child2FileName = `${tempDir}/child2.vhd`
-  const recoveredFileName = `${tempDir}/recovered`
-  await createRandomFile(randomFileName, mbOfRandom)
-  await createRandomFile(smallRandomFileName, Math.ceil(mbOfRandom / 2))
-  await execa('qemu-img', ['create', '-fvpc', parentFileName, mbOfRandom + 1 + 'M'])
-  await convertFromRawToVhd(randomFileName, child1FileName)
   const handler = getHandler({ url: 'file://' })
-  await execa('vhd-util', ['snapshot', '-n', child2FileName, '-p', child1FileName])
-  const vhd = new VhdFile(handler, child2FileName)
-  await vhd.readHeaderAndFooter()
-  await vhd.readBlockAllocationTable()
-  vhd.footer.creatorApplication = 'xoa'
-  await vhd.writeFooter()
 
-  const originalSize = await handler._getSize(randomFileName)
+  await createRandomFile(parentRandomFileName, mbOfFather)
+  await convertFromRawToVhd(parentRandomFileName, parentFileName)
+
+  await createRandomFile(childRandomFileName, mbOfChildren)
+  await convertFromRawToVhd(childRandomFileName, child1FileName)
   await chainVhd(handler, parentFileName, handler, child1FileName, true)
-  await execa('vhd-util', ['check', '-t', '-n', child1FileName])
-  await chainVhd(handler, child1FileName, handler, child2FileName, true)
-  await execa('vhd-util', ['check', '-t', '-n', child2FileName])
-  const smallRandom = await fs.readFile(smallRandomFileName)
-  const newVhd = new VhdFile(handler, child2FileName)
-  await newVhd.readHeaderAndFooter()
-  await newVhd.readBlockAllocationTable()
-  await newVhd.writeData(5, smallRandom)
-  await checkFile(child2FileName)
-  await checkFile(child1FileName)
-  await checkFile(parentFileName)
+
+  // merge
   await vhdMerge(handler, parentFileName, handler, child1FileName)
+
+  // check that vhd is still valid
   await checkFile(parentFileName)
-  await chainVhd(handler, parentFileName, handler, child2FileName, true)
-  await checkFile(child2FileName)
-  await vhdMerge(handler, parentFileName, handler, child2FileName)
-  await checkFile(parentFileName)
-  await recoverRawContent(parentFileName, recoveredFileName, originalSize)
-  await execa('cp', [randomFileName, random2FileName])
-  const fd = await fs.open(random2FileName, 'r+')
-  try {
-    await fs.write(fd, smallRandom, 0, smallRandom.length, 5 * SECTOR_SIZE)
-  } finally {
-    await fs.close(fd)
+
+  const parentVhd = new VhdFile(handler, parentFileName)
+  await parentVhd.readHeaderAndFooter()
+  await parentVhd.readBlockAllocationTable()
+
+  let offset = 0
+  // check that the data are the same as source
+  for await (const block of parentVhd.blocks()) {
+    const blockContent = block.data
+    const file = offset < mbOfChildren * 1024 * 1024 ? childRandomFileName : parentRandomFileName
+    const buffer = Buffer.alloc(blockContent.length)
+    const fd = await fs.open(file, 'r')
+    await fs.read(fd, buffer, 0, buffer.length, offset)
+
+    expect(buffer.equals(blockContent)).toEqual(true)
+    offset += parentVhd.header.blockSize
   }
-  expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(random2FileName))
 })
 
-test.only('createSyntheticStream passes vhd-util check', async () => {
-  const initalSize = 4
-  const rawFileName = `${tempDir}/randomfile`
-  const vhdFileName = `${tempDir}/randomfile.vhd`
-  const recoveredVhdFileName = `${tempDir}/recovered.vhd`
-  await createRandomFile(rawFileName, initalSize)
-  await convertFromRawToVhd(rawFileName, vhdFileName)
-  await checkFile(vhdFileName)
-  const handler = getHandler({ url: 'file://' })
-  const stream = await createSyntheticStream(handler, vhdFileName)
-  const expectedVhdSize = (await fs.stat(vhdFileName)).size
-  expect(stream.length).toEqual((await fs.stat(vhdFileName)).size)
-  await pFromCallback(cb => pipeline(stream, fs.createWriteStream(recoveredVhdFileName), cb))
-  await checkFile(recoveredVhdFileName)
-  const stats = await fs.stat(recoveredVhdFileName)
-  expect(stats.size).toEqual(expectedVhdSize)
-  await execa('qemu-img', ['compare', recoveredVhdFileName, rawFileName])
+test('it can resume a merge ', async () => {
+  const mbOfFather = 8
+  const mbOfChildren = 4
+  const parentRandomFileName = `${tempDir}/randomfile`
+  const childRandomFileName = `${tempDir}/small_randomfile`
+  const handler = getHandler({ url: `file://${tempDir}` })
+
+  await createRandomFile(`${tempDir}/randomfile`, mbOfFather)
+  await convertFromRawToVhd(`${tempDir}/randomfile`, `${tempDir}/parent.vhd`)
+  const parentVhd = new VhdFile(handler, 'parent.vhd')
+  await parentVhd.readHeaderAndFooter()
+
+  await createRandomFile(`${tempDir}/small_randomfile`, mbOfChildren)
+  await convertFromRawToVhd(`${tempDir}/small_randomfile`, `${tempDir}/child1.vhd`)
+  await chainVhd(handler, 'parent.vhd', handler, 'child1.vhd', true)
+  const childVhd = new VhdFile(handler, 'child1.vhd')
+  await childVhd.readHeaderAndFooter()
+
+  await handler.writeFile(
+    '.parent.vhd.merge.json',
+    JSON.stringify({
+      parent: {
+        header: parentVhd.header.checksum,
+      },
+      child: {
+        header: 'NOT CHILD HEADER ',
+      },
+    })
+  )
+  // expect merge to fail since child header is not ok
+  await expect(async () => await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd')).rejects.toThrow()
+
+  await handler.unlink('.parent.vhd.merge.json')
+  await handler.writeFile(
+    '.parent.vhd.merge.json',
+    JSON.stringify({
+      parent: {
+        header: 'NOT PARENT HEADER',
+      },
+      child: {
+        header: childVhd.header.checksum,
+      },
+    })
+  )
+  // expect merge to fail since parent header is not ok
+  await expect(async () => await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd')).rejects.toThrow()
+
+  // break the end footer of parent
+  const size = await handler.getSize('parent.vhd')
+  const fd = await handler.openFile('parent.vhd', 'r+')
+  const buffer = Buffer.alloc(512, 0)
+  // add a fake footer at the end
+  handler.write(fd, buffer, size)
+  await handler.closeFile(fd)
+  // check vhd should fail
+  await expect(async () => await parentVhd.readHeaderAndFooter()).rejects.toThrow()
+
+  await handler.unlink('.parent.vhd.merge.json')
+  await handler.writeFile(
+    '.parent.vhd.merge.json',
+    JSON.stringify({
+      parent: {
+        header: parentVhd.header.checksum,
+      },
+      child: {
+        header: childVhd.header.checksum,
+      },
+      currentBlock: 1,
+    })
+  )
+
+  // really merge
+  await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd')
+
+  // reload header footer and block allocation table , they should succed
+  await parentVhd.readHeaderAndFooter()
+  await parentVhd.readBlockAllocationTable()
+  let offset = 0
+  // check that the data are the same as source
+  for await (const block of parentVhd.blocks()) {
+    const blockContent = block.data
+    // first block is marked as already merged, should not be modified
+    // second block should come from children
+    // then two block only in parent
+    const file = block.id === 1 ? childRandomFileName : parentRandomFileName
+    const buffer = Buffer.alloc(blockContent.length)
+    const fd = await fs.open(file, 'r')
+    await fs.read(fd, buffer, 0, buffer.length, offset)
+
+    expect(buffer.equals(blockContent)).toEqual(true)
+    offset += parentVhd.header.blockSize
+  }
 })

packages/vhd-lib/src/merge.js

@@ -5,9 +5,10 @@ import noop from './_noop'
 import { createLogger } from '@xen-orchestra/log'
 import { limitConcurrency } from 'limit-concurrency-decorator'
 
-import { VhdFile } from '.'
+import { openVhd } from '.'
 import { basename, dirname } from 'path'
-import { DISK_TYPE_DIFFERENCING, DISK_TYPE_DYNAMIC } from './_constants'
+import { DISK_TYPES } from './_constants'
+import { Disposable } from 'promise-toolbox'
 
 const { warn } = createLogger('vhd-lib:merge')
 
@@ -23,109 +24,99 @@ export default limitConcurrency(2)(async function merge(
 ) {
   const mergeStatePath = dirname(parentPath) + '/' + '.' + basename(parentPath) + '.merge.json'
 
-  const parentFd = await parentHandler.openFile(parentPath, 'r+')
-  try {
-    const parentVhd = new VhdFile(parentHandler, parentFd)
-    const childFd = await childHandler.openFile(childPath, 'r')
-    try {
-      const childVhd = new VhdFile(childHandler, childFd)
-
-      let mergeState = await parentHandler.readFile(mergeStatePath).catch(error => {
-        if (error.code !== 'ENOENT') {
-          throw error
-        }
-        // no merge state in case of missing file
-      })
-
-      // Reading footer and header.
-      await Promise.all([
-        parentVhd.readHeaderAndFooter(
-          // dont check VHD is complete if recovering a merge
-          mergeState === undefined
-        ),
-
-        childVhd.readHeaderAndFooter(),
-      ])
-
-      if (mergeState !== undefined) {
-        mergeState = JSON.parse(mergeState)
-
-        // ensure the correct merge will be continued
-        assert.strictEqual(parentVhd.header.checksum, mergeState.parent.header)
-        assert.strictEqual(childVhd.header.checksum, mergeState.child.header)
-      } else {
-        assert.strictEqual(childVhd.header.blockSize, parentVhd.header.blockSize)
-
-        const parentDiskType = parentVhd.footer.diskType
-        assert(parentDiskType === DISK_TYPE_DIFFERENCING || parentDiskType === DISK_TYPE_DYNAMIC)
-        assert.strictEqual(childVhd.footer.diskType, DISK_TYPE_DIFFERENCING)
+  return await Disposable.use(async function* () {
+    let mergeState = await parentHandler.readFile(mergeStatePath).catch(error => {
+      if (error.code !== 'ENOENT') {
+        throw error
       }
+      // no merge state in case of missing file
+    })
+    // during merging, the end footer of the parent can be overwritten by new blocks
+    // we should use it as a way to check vhd health
+    const parentVhd = yield openVhd(parentHandler, parentPath, {
+      flags: 'r+',
+      checkSecondFooter: mergeState === undefined,
+    })
+    const childVhd = yield openVhd(childHandler, childPath)
+    if (mergeState !== undefined) {
+      mergeState = JSON.parse(mergeState)
 
-      // Read allocation table of child/parent.
-      await Promise.all([parentVhd.readBlockAllocationTable(), childVhd.readBlockAllocationTable()])
+      // ensure the correct merge will be continued
+      assert.strictEqual(parentVhd.header.checksum, mergeState.parent.header)
+      assert.strictEqual(childVhd.header.checksum, mergeState.child.header)
+    } else {
+      assert.strictEqual(childVhd.header.blockSize, parentVhd.header.blockSize)
 
-      const { maxTableEntries } = childVhd.header
-
-      if (mergeState === undefined) {
-        await parentVhd.ensureBatSize(childVhd.header.maxTableEntries)
-
-        mergeState = {
-          child: { header: childVhd.header.checksum },
-          parent: { header: parentVhd.header.checksum },
-          currentBlock: 0,
-          mergedDataSize: 0,
-        }
-
-        // finds first allocated block for the 2 following loops
-        while (mergeState.currentBlock < maxTableEntries && !childVhd.containsBlock(mergeState.currentBlock)) {
-          ++mergeState.currentBlock
-        }
-      }
-
-      // counts number of allocated blocks
-      let nBlocks = 0
-      for (let block = mergeState.currentBlock; block < maxTableEntries; block++) {
-        if (childVhd.containsBlock(block)) {
-          nBlocks += 1
-        }
-      }
-
-      onProgress({ total: nBlocks, done: 0 })
-
-      // merges blocks
-      for (let i = 0; i < nBlocks; ++i, ++mergeState.currentBlock) {
-        while (!childVhd.containsBlock(mergeState.currentBlock)) {
-          ++mergeState.currentBlock
-        }
-
-        await parentHandler.writeFile(mergeStatePath, JSON.stringify(mergeState), { flags: 'w' }).catch(warn)
-
-        mergeState.mergedDataSize += await parentVhd.coalesceBlock(childVhd, mergeState.currentBlock)
-        onProgress({
-          total: nBlocks,
-          done: i + 1,
-        })
-      }
-
-      const cFooter = childVhd.footer
-      const pFooter = parentVhd.footer
-
-      pFooter.currentSize = cFooter.currentSize
-      pFooter.diskGeometry = { ...cFooter.diskGeometry }
-      pFooter.originalSize = cFooter.originalSize
-      pFooter.timestamp = cFooter.timestamp
-      pFooter.uuid = cFooter.uuid
-
-      // necessary to update values and to recreate the footer after block
-      // creation
-      await parentVhd.writeFooter()
-
-      return mergeState.mergedDataSize
-    } finally {
-      await childHandler.closeFile(childFd)
+      const parentDiskType = parentVhd.footer.diskType
+      assert(parentDiskType === DISK_TYPES.DIFFERENCING || parentDiskType === DISK_TYPES.DYNAMIC)
+      assert.strictEqual(childVhd.footer.diskType, DISK_TYPES.DIFFERENCING)
     }
-  } finally {
+
+    // Read allocation table of child/parent.
+    await Promise.all([parentVhd.readBlockAllocationTable(), childVhd.readBlockAllocationTable()])
+
+    const { maxTableEntries } = childVhd.header
+
+    if (mergeState === undefined) {
+      await parentVhd.ensureBatSize(childVhd.header.maxTableEntries)
+
+      mergeState = {
+        child: { header: childVhd.header.checksum },
+        parent: { header: parentVhd.header.checksum },
+        currentBlock: 0,
+        mergedDataSize: 0,
+      }
+
+      // finds first allocated block for the 2 following loops
+      while (mergeState.currentBlock < maxTableEntries && !childVhd.containsBlock(mergeState.currentBlock)) {
+        ++mergeState.currentBlock
+      }
+    }
+
+    // counts number of allocated blocks
+    let nBlocks = 0
+    for (let block = mergeState.currentBlock; block < maxTableEntries; block++) {
+      if (childVhd.containsBlock(block)) {
+        nBlocks += 1
+      }
+    }
+
+    onProgress({ total: nBlocks, done: 0 })
+
+    // merges blocks
+    for (let i = 0; i < nBlocks; ++i, ++mergeState.currentBlock) {
+      while (!childVhd.containsBlock(mergeState.currentBlock)) {
+        ++mergeState.currentBlock
+      }
+
+      await parentHandler.writeFile(mergeStatePath, JSON.stringify(mergeState), { flags: 'w' }).catch(warn)
+
+      mergeState.mergedDataSize += await parentVhd.coalesceBlock(childVhd, mergeState.currentBlock)
+      onProgress({
+        total: nBlocks,
+        done: i + 1,
+      })
+    }
+
+    // some blocks could have been created or moved in parent : write bat
+    await parentVhd.writeBlockAllocationTable()
+
+    const cFooter = childVhd.footer
+    const pFooter = parentVhd.footer
+
+    pFooter.currentSize = cFooter.currentSize
+    pFooter.diskGeometry = { ...cFooter.diskGeometry }
+    pFooter.originalSize = cFooter.originalSize
+    pFooter.timestamp = cFooter.timestamp
+    pFooter.uuid = cFooter.uuid
+
+    // necessary to update values and to recreate the footer after block
+    // creation
+    await parentVhd.writeFooter()
+
+    // should be a disposable
     parentHandler.unlink(mergeStatePath).catch(warn)
-    await parentHandler.closeFile(parentFd)
-  }
+
+    return mergeState.mergedDataSize
+  })
 })

packages/vhd-lib/src/openVhd.integ.spec.js

@@ -0,0 +1,62 @@
+/* eslint-env jest */
+
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { Disposable, pFromCallback } from 'promise-toolbox'
+
+import { openVhd } from './index'
+import { createRandomFile, convertFromRawToVhd, createRandomVhdDirectory } from './tests/utils'
+
+import { VhdAbstract } from './Vhd/VhdAbstract'
+
+let tempDir
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+test('It opens a vhd file ( alias or not)', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file://' })
+    const vhd = yield openVhd(handler, vhdFileName)
+    expect(vhd.header.cookie).toEqual('cxsparse')
+    expect(vhd.footer.cookie).toEqual('conectix')
+
+    const aliasFileName = `${tempDir}/out.alias.vhd`
+    await VhdAbstract.createAlias(handler, aliasFileName, vhdFileName)
+    const alias = yield openVhd(handler, aliasFileName)
+    expect(alias.header.cookie).toEqual('cxsparse')
+    expect(alias.footer.cookie).toEqual('conectix')
+  })
+})
+
+test('It opens a vhd directory', async () => {
+  const initalSize = 4
+  const vhdDirectory = `${tempDir}/randomfile.dir`
+  await createRandomVhdDirectory(vhdDirectory, initalSize)
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file://' })
+    const vhd = yield openVhd(handler, vhdDirectory)
+    expect(vhd.header.cookie).toEqual('cxsparse')
+    expect(vhd.footer.cookie).toEqual('conectix')
+
+    const aliasFileName = `${tempDir}/out.alias.vhd`
+    await VhdAbstract.createAlias(handler, aliasFileName, vhdDirectory)
+    const alias = yield openVhd(handler, aliasFileName)
+    expect(alias.header.cookie).toEqual('cxsparse')
+    expect(alias.footer.cookie).toEqual('conectix')
+  })
+})

packages/vhd-lib/src/openVhd.js

@@ -1,12 +1,14 @@
+import { resolveAlias } from './_resolveAlias'
 import { VhdFile, VhdDirectory } from './'
 
-export async function openVhd(handler, path) {
+export async function openVhd(handler, path, opts) {
+  const resolved = await resolveAlias(handler, path)
   try {
-    return await VhdFile.open(handler, path)
+    return await VhdFile.open(handler, resolved, opts)
   } catch (e) {
     if (e.code !== 'EISDIR') {
       throw e
     }
-    return await VhdDirectory.open(handler, path)
+    return await VhdDirectory.open(handler, resolved, opts)
   }
 }

packages/vhd-lib/src/tests/utils.js

@@ -42,9 +42,57 @@ export async function convertFromVmdkToRaw(vmdkName, rawName) {
 }
 
 export async function recoverRawContent(vhdName, rawName, originalSize) {
+  // todo should use createContentStream
   await checkFile(vhdName)
   await convertFromVhdToRaw(vhdName, rawName)
   if (originalSize !== undefined) {
     await execa('truncate', ['-s', originalSize, rawName])
   }
 }
+
+// @ todo how can I call vhd-cli copy from here
+export async function convertToVhdDirectory(rawFileName, vhdFileName, path) {
+  fs.mkdirp(path)
+
+  const srcVhd = await fs.open(vhdFileName, 'r')
+
+  const footer = Buffer.alloc(512)
+  await fs.read(srcVhd, footer, 0, footer.length, 0)
+  await fs.writeFile(path + '/footer', footer)
+
+  const header = Buffer.alloc(1024)
+  await fs.read(srcVhd, header, 0, header.length, 512)
+  await fs.writeFile(path + '/header', header)
+
+  await fs.close(srcVhd)
+
+  // a BAT , with at most 512 blocks of 2MB
+  const bat = Buffer.alloc(512, 1)
+  await fs.writeFile(path + '/bat', bat)
+
+  // copy blocks
+  const srcRaw = await fs.open(rawFileName, 'r')
+  const blockDataSize = 512 * 4096
+  // make a block bitmap full of 1, marking all sectors of the block as used
+  const bitmap = Buffer.alloc(512, 255)
+  await fs.mkdir(path + '/blocks/')
+  await fs.mkdir(path + '/blocks/0/')
+  const stats = await fs.stat(rawFileName)
+
+  const sizeMB = stats.size / 1024 / 1024
+  for (let i = 0, offset = 0; i < sizeMB; i++, offset += blockDataSize) {
+    const blockData = Buffer.alloc(blockDataSize)
+    await fs.read(srcRaw, blockData, offset)
+    await fs.writeFile(path + '/blocks/0/' + i, Buffer.concat([bitmap, blockData]))
+  }
+  await fs.close(srcRaw)
+}
+
+export async function createRandomVhdDirectory(path, sizeMB) {
+  fs.mkdirp(path)
+  const rawFileName = `${path}/temp.raw`
+  await createRandomFile(rawFileName, sizeMB)
+  const vhdFileName = `${path}/temp.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  await convertToVhdDirectory(rawFileName, vhdFileName, path)
+}

packages/xen-api/USAGE.md

@@ -34,6 +34,7 @@ Options:
 - `auth`: credentials used to sign in (can also be specified in the URL)
 - `readOnly = false`: if true, no methods with side-effects can be called
 - `callTimeout`: number of milliseconds after which a call is considered failed (can also be a map of timeouts by methods)
+- `httpProxy`: URL of the HTTP/HTTPS proxy used to reach the host, can include credentials
 
 ```js
 // Force connection.

packages/xen-api/examples/package.json

@@ -8,6 +8,6 @@
     "promise-toolbox": "^0.19.2",
     "readable-stream": "^3.1.1",
     "throttle": "^1.0.3",
-    "vhd-lib": "^1.3.0"
+    "vhd-lib": "^2.0.1"
   }
 }

packages/xo-cli/README.md

@@ -20,9 +20,12 @@ Installation of the [npm package](https://npmjs.org/package/xo-cli):
 > xo-cli --help
 Usage:
 
-  xo-cli --register [--expiresIn duration] <XO-Server URL> <username> [<password>]
+  xo-cli --register [--allowUnauthorized] [--expiresIn duration] <XO-Server URL> <username> [<password>]
     Registers the XO instance to use.
 
+    --allowUnauthorized, --au
+      Accept invalid certificate (e.g. self-signed).
+
     --expiresIn duration
       Can be used to change the validity duration of the
       authorization token (default: one month).

packages/xo-cli/USAGE.md

@@ -2,9 +2,12 @@
 > xo-cli --help
 Usage:
 
-  xo-cli --register [--expiresIn duration] <XO-Server URL> <username> [<password>]
+  xo-cli --register [--allowUnauthorized] [--expiresIn duration] <XO-Server URL> <username> [<password>]
     Registers the XO instance to use.
 
+    --allowUnauthorized, --au
+      Accept invalid certificate (e.g. self-signed).
+
     --expiresIn duration
       Can be used to change the validity duration of the
       authorization token (default: one month).

packages/xo-cli/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "xo-cli",
-  "version": "0.11.1",
+  "version": "0.12.0",
   "license": "AGPL-3.0-or-later",
   "description": "Basic CLI for Xen-Orchestra",
   "keywords": [
@@ -34,6 +34,7 @@
     "chalk": "^4.1.0",
     "exec-promise": "^0.7.0",
     "fs-extra": "^10.0.0",
+    "getopts": "^2.3.0",
     "http-request-plus": "^0.13.0",
     "human-format": "^0.11.0",
     "l33teral": "^3.0.3",
@@ -47,7 +48,7 @@
     "pw": "^0.0.4",
     "strip-indent": "^3.0.0",
     "xdg-basedir": "^4.0.0",
-    "xo-lib": "^0.10.1"
+    "xo-lib": "^0.11.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/xo-cli/src/index.js

@@ -14,6 +14,7 @@ const chalk = require('chalk')
 const forEach = require('lodash/forEach')
 const fromCallback = require('promise-toolbox/fromCallback')
 const getKeys = require('lodash/keys')
+const getopts = require('getopts')
 const hrp = require('http-request-plus')
 const humanFormat = require('human-format')
 const identity = require('lodash/identity')
@@ -35,7 +36,7 @@ const config = require('./config')
 // ===================================================================
 
 async function connect() {
-  const { server, token } = await config.load()
+  const { allowUnauthorized, server, token } = await config.load()
   if (server === undefined) {
     throw new Error('no server to connect to!')
   }
@@ -44,7 +45,7 @@ async function connect() {
     throw new Error('no token available')
   }
 
-  const xo = new Xo({ url: server })
+  const xo = new Xo({ rejectUnauthorized: !allowUnauthorized, url: server })
   await xo.open()
   await xo.signIn({ token })
   return xo
@@ -150,9 +151,12 @@ const help = wrap(
       `
     Usage:
 
-      $name --register [--expiresIn duration] <XO-Server URL> <username> [<password>]
+      $name --register [--allowUnauthorized] [--expiresIn duration] <XO-Server URL> <username> [<password>]
         Registers the XO instance to use.
 
+        --allowUnauthorized, --au
+          Accept invalid certificate (e.g. self-signed).
+
         --expiresIn duration
           Can be used to change the validity duration of the
           authorization token (default: one month).
@@ -231,29 +235,35 @@ exports = module.exports = main
 exports.help = help
 
 async function register(args) {
-  let expiresIn
-  if (args[0] === '--expiresIn') {
-    expiresIn = args[1]
-    args = args.slice(2)
-  }
+  const {
+    allowUnauthorized,
+    expiresIn,
+    _: [
+      url,
+      email,
+      password = await new Promise(function (resolve) {
+        process.stdout.write('Password: ')
+        pw(resolve)
+      }),
+    ],
+  } = getopts(args, {
+    alias: {
+      allowUnauthorized: 'au',
+    },
+    boolean: ['allowUnauthorized'],
+    stopEarly: true,
+    string: ['expiresIn'],
+  })
 
-  const [
-    url,
-    email,
-    password = await new Promise(function (resolve) {
-      process.stdout.write('Password: ')
-      pw(resolve)
-    }),
-  ] = args
-
-  const xo = new Xo({ url })
+  const xo = new Xo({ rejectUnauthorized: !allowUnauthorized, url })
   await xo.open()
   await xo.signIn({ email, password })
   console.log('Successfully logged with', xo.user.email)
 
   await config.set({
+    allowUnauthorized,
     server: url,
-    token: await xo.call('token.create', { expiresIn }),
+    token: await xo.call('token.create', { expiresIn: expiresIn === '' ? undefined : expiresIn }),
   })
 }
 exports.register = register

packages/xo-import-servers-csv/package.json

@@ -37,7 +37,7 @@
     "csv-parser": "^3.0.0",
     "exec-promise": "^0.7.0",
     "through2": "^4.0.2",
-    "xo-lib": "^0.10.1"
+    "xo-lib": "^0.11.1"
   },
   "devDependencies": {
     "@types/node": "^16.11.6",

packages/xo-lib/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "xo-lib",
-  "version": "0.10.1",
+  "version": "0.11.1",
   "license": "AGPL-3.0-or-later",
   "description": "Library to connect to XO-Server",
   "keywords": [
@@ -26,10 +26,10 @@
     "> 2%"
   ],
   "engines": {
-    "node": ">=6"
+    "node": ">=10"
   },
   "dependencies": {
-    "jsonrpc-websocket-client": "^0.6.0",
+    "jsonrpc-websocket-client": "^0.7.2",
     "lodash": "^4.17.2",
     "make-error": "^1.0.4"
   },

packages/xo-lib/src/index.js

@@ -13,8 +13,9 @@ export class XoError extends BaseError {}
 // -------------------------------------------------------------------
 
 export default class Xo extends JsonRpcWebSocketClient {
-  constructor({ credentials, url = '.' } = {}) {
-    super(`${trimEnd(url, '/')}/api/`)
+  constructor({ credentials, url = '.', ...opts } = {}) {
+    opts.url = `${trimEnd(url, '/')}/api/`
+    super(opts)
 
     this._credentials = credentials
     this._user = null

packages/xo-remote-parser/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "xo-remote-parser",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "license": "AGPL-3.0-or-later",
   "description": "Parse and format XO remote URLs",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-remote-parser",

packages/xo-remote-parser/src/index.js

@@ -2,7 +2,7 @@ import filter from 'lodash/filter'
 import map from 'lodash/map'
 import trim from 'lodash/trim'
 import trimStart from 'lodash/trimStart'
-import Url from 'url-parse'
+import urlParser from 'url-parse'
 
 const NFS_RE = /^([^:]+):(?:(\d+):)?([^:]+)$/
 const SMB_RE = /^([^:]+):(.+)@([^@]+)\\\\([^\0]+)(?:\0(.*))?$/
@@ -38,7 +38,7 @@ export const parse = string => {
     object.username = username
     object.password = password
   } else if (type === 's3' || type === 's3+http') {
-    const parsed = new Url(string)
+    const parsed = urlParser(string, true)
     object.protocol = parsed.protocol === 's3:' ? 'https' : 'http'
     object.type = 's3'
     object.region = parsed.hash.length === 0 ? undefined : parsed.hash.slice(1) // remove '#'
@@ -46,11 +46,24 @@ export const parse = string => {
     object.path = parsed.pathname
     object.username = parsed.username
     object.password = decodeURIComponent(parsed.password)
+    const qs = parsed.query
+    object.allowUnauthorized = qs.allowUnauthorized === 'true'
   }
   return object
 }
 
-export const format = ({ type, host, path, port, username, password, domain, protocol = type, region }) => {
+export const format = ({
+  type,
+  host,
+  path,
+  port,
+  username,
+  password,
+  domain,
+  protocol = type,
+  region,
+  allowUnauthorized = false,
+}) => {
   type === 'local' && (type = 'file')
   let string = `${type}://`
   if (type === 'nfs') {
@@ -71,6 +84,10 @@ export const format = ({ type, host, path, port, username, password, domain, pro
     path = `/${path}`
   }
   string += path
+
+  if (type === 's3' && allowUnauthorized === true) {
+    string += `?allowUnauthorized=true`
+  }
   if (type === 's3' && region !== undefined) {
     string += `#${region}`
   }

packages/xo-remote-parser/src/index.spec.js

@@ -54,6 +54,20 @@ const data = deepFreeze({
       username: 'AKIAS',
       password: 'XSuBupZ0mJlu+',
       region: undefined,
+      allowUnauthorized: false,
+    },
+  },
+  's3 accepting self signed ': {
+    string: 's3://AKIAS:XSuBupZ0mJlu%2B@s3-us-west-2.amazonaws.com/test-bucket/dir?allowUnauthorized=true',
+    object: {
+      type: 's3',
+      protocol: 'https',
+      host: 's3-us-west-2.amazonaws.com',
+      path: '/test-bucket/dir',
+      username: 'AKIAS',
+      password: 'XSuBupZ0mJlu+',
+      region: undefined,
+      allowUnauthorized: true,
     },
   },
 })
@@ -97,6 +111,33 @@ const parseData = deepFreeze({
       region: 'reg1',
       username: 'Administrator',
       password: 'password',
+      allowUnauthorized: false,
+    },
+  },
+  'S3 accepting self signed certificate': {
+    string: 's3+http://Administrator:password@192.168.100.225/bucket/dir?allowUnauthorized=true#reg1',
+    object: {
+      type: 's3',
+      host: '192.168.100.225',
+      protocol: 'http',
+      path: '/bucket/dir',
+      region: 'reg1',
+      username: 'Administrator',
+      password: 'password',
+      allowUnauthorized: true,
+    },
+    'S3 with broken allowUnauthorized': {
+      string: 's3+http://Administrator:password@192.168.100.225/bucket/dir?allowUnauthorized=notTrue#reg1',
+      object: {
+        type: 's3',
+        host: '192.168.100.225',
+        protocol: 'http',
+        path: '/bucket/dir',
+        region: 'reg1',
+        username: 'Administrator',
+        password: 'password',
+        allowUnauthorized: false,
+      },
     },
   },
 })
@@ -111,6 +152,19 @@ const formatData = deepFreeze({
       path: '/var/lib/xoa/backup',
     },
   },
+  'S3 with broken allowUnauthorized': {
+    string: 's3+http://Administrator:password@192.168.100.225/bucket/dir#reg1',
+    object: {
+      type: 's3',
+      host: '192.168.100.225',
+      protocol: 'http',
+      path: '/bucket/dir',
+      region: 'reg1',
+      username: 'Administrator',
+      password: 'password',
+      allowUnauthorized: 'notTrue',
+    },
+  },
 })
 
 // -------------------------------------------------------------------

packages/xo-server-test/package.json

@@ -36,7 +36,7 @@
     "promise-toolbox": "^0.20.0",
     "xo-collection": "^0.5.0",
     "xo-common": "^0.7.0",
-    "xo-lib": "^0.10.1"
+    "xo-lib": "^0.11.1"
   },
   "scripts": {
     "dev-test": "jest --bail --watch",

packages/xo-server-usage-report/package.json

@@ -32,7 +32,7 @@
     "@xen-orchestra/async-map": "^0.1.2",
     "@xen-orchestra/cron": "^1.0.6",
     "@xen-orchestra/log": "^0.3.0",
-    "csv-stringify": "^5.5.0",
+    "csv-stringify": "^6.0.0",
     "handlebars": "^4.0.6",
     "html-minifier": "^4.0.0",
     "human-format": "^0.11.0",

packages/xo-server/package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "xo-server",
-  "version": "5.83.0",
+  "version": "5.84.1",
   "license": "AGPL-3.0-or-later",
   "description": "Server part of Xen-Orchestra",
   "keywords": [
@@ -35,17 +35,17 @@
     "@vates/parse-duration": "^0.1.1",
     "@vates/read-chunk": "^0.1.2",
     "@xen-orchestra/async-map": "^0.1.2",
-    "@xen-orchestra/backups": "^0.15.1",
+    "@xen-orchestra/backups": "^0.16.0",
     "@xen-orchestra/cron": "^1.0.6",
     "@xen-orchestra/defined": "^0.0.1",
     "@xen-orchestra/emit-async": "^0.1.0",
-    "@xen-orchestra/fs": "^0.18.0",
+    "@xen-orchestra/fs": "^0.19.1",
     "@xen-orchestra/log": "^0.3.0",
     "@xen-orchestra/mixin": "^0.1.0",
     "@xen-orchestra/mixins": "^0.1.1",
     "@xen-orchestra/self-signed": "^0.1.0",
     "@xen-orchestra/template": "^0.1.0",
-    "@xen-orchestra/xapi": "^0.8.0",
+    "@xen-orchestra/xapi": "^0.8.4",
     "ajv": "^8.0.3",
     "app-conf": "^0.9.0",
     "async-iterator-to-stream": "^1.0.1",
@@ -54,6 +54,7 @@
     "blocked-at": "^1.2.0",
     "bluebird": "^3.5.1",
     "body-parser": "^1.18.2",
+    "complex-matcher": "^0.7.0",
     "compression": "^1.7.3",
     "connect-flash": "^0.1.1",
     "content-type": "^1.0.4",
@@ -122,14 +123,14 @@
     "unzipper": "^0.10.5",
     "uuid": "^8.3.1",
     "value-matcher": "^0.2.0",
-    "vhd-lib": "^1.3.0",
+    "vhd-lib": "^2.0.1",
     "ws": "^8.2.3",
     "xdg-basedir": "^4.0.0",
     "xen-api": "^0.35.1",
     "xo-acl-resolver": "^0.4.1",
     "xo-collection": "^0.5.0",
     "xo-common": "^0.7.0",
-    "xo-remote-parser": "^0.7.0",
+    "xo-remote-parser": "^0.8.0",
     "xo-vmdk-to-vhd": "^2.0.0"
   },
   "devDependencies": {

packages/xo-server/src/api/cloud-config.mjs

@@ -4,7 +4,13 @@ export function getAll() {
 
 getAll.description = 'Gets all existing cloud configs templates'
 
-export function create(props) {
+export function getAllNetworkConfigs() {
+  return this.getAllNetworkConfigs()
+}
+
+getAllNetworkConfigs.description = 'Gets all existing network config templates'
+
+export async function create(props) {
   return this.createCloudConfig(props)
 }
 
@@ -15,6 +21,17 @@ create.params = {
   template: { type: 'string' },
 }
 
+export function createNetworkConfig(props) {
+  return this.createCloudConfig({ ...props, type: 'network' })
+}
+
+createNetworkConfig.permission = 'admin'
+createNetworkConfig.description = 'Creates a new network config template'
+createNetworkConfig.params = {
+  name: { type: 'string' },
+  template: { type: 'string' },
+}
+
 export function update(props) {
   return this.updateCloudConfig(props)
 }

packages/xo-server/src/api/pool.mjs

@@ -1,4 +1,5 @@
 import { format } from 'json-rpc-peer'
+import { Ref } from 'xen-api'
 
 // ===================================================================
 
@@ -9,6 +10,7 @@ export async function set({
   name_label: nameLabel,
   backupNetwork,
   migrationNetwork,
+  suspendSr,
 }) {
   pool = this.getXapiObject(pool)
 
@@ -17,6 +19,7 @@ export async function set({
     nameLabel !== undefined && pool.set_name_label(nameLabel),
     migrationNetwork !== undefined && pool.update_other_config('xo:migrationNetwork', migrationNetwork),
     backupNetwork !== undefined && pool.update_other_config('xo:backupNetwork', backupNetwork),
+    suspendSr !== undefined && pool.$call('set_suspend_image_SR', suspendSr === null ? Ref.EMPTY : suspendSr._xapiRef),
   ])
 }
 
@@ -40,10 +43,15 @@ set.params = {
     type: ['string', 'null'],
     optional: true,
   },
+  suspendSr: {
+    type: ['string', 'null'],
+    optional: true,
+  },
 }
 
 set.resolve = {
   pool: ['id', 'pool', 'administrate'],
+  suspendSr: ['suspendSr', 'SR', 'administrate'],
 }
 
 // -------------------------------------------------------------------

packages/xo-server/src/api/vm.mjs

@@ -9,6 +9,7 @@ import { FAIL_ON_QUEUE } from 'limit-concurrency-decorator'
 import { format } from 'json-rpc-peer'
 import { ignoreErrors } from 'promise-toolbox'
 import { invalidParameters, noSuchObject, operationFailed, unauthorized } from 'xo-common/api-errors.js'
+import { Ref } from 'xen-api'
 
 import { forEach, map, mapFilter, parseSize, safeDateFormat } from '../utils.mjs'
 
@@ -534,6 +535,11 @@ export const set = defer(async function ($defer, params) {
     await this.shareVmResourceSet(vmId)
   }
 
+  const suspendSr = extract(params, 'suspendSr')
+  if (suspendSr !== undefined) {
+    await xapi.call('VM.set_suspend_SR', VM._xapiRef, suspendSr === null ? Ref.EMPTY : suspendSr._xapiRef)
+  }
+
   return xapi.editVm(vmId, params, async (limits, vm) => {
     const resourceSet = xapi.xo.getData(vm, 'resourceSet')
 
@@ -632,10 +638,13 @@ set.params = {
   virtualizationMode: { type: 'string', optional: true },
 
   blockedOperations: { type: 'object', optional: true },
+
+  suspendSr: { type: ['string', 'null'], optional: true },
 }
 
 set.resolve = {
   VM: ['id', ['VM', 'VM-snapshot', 'VM-template'], 'administrate'],
+  suspendSr: ['suspendSr', 'SR', 'administrate'],
 }
 
 // -------------------------------------------------------------------

packages/xo-server/src/api/xo.mjs

@@ -1,3 +1,4 @@
+import * as CM from 'complex-matcher'
 import getStream from 'get-stream'
 import { fromCallback } from 'promise-toolbox'
 import { pipeline } from 'readable-stream'
@@ -52,6 +53,10 @@ function handleGetAllObjects(req, res, { filter, limit }) {
 }
 
 export function getAllObjects({ filter, limit, ndjson = false }) {
+  if (typeof filter === 'string') {
+    filter = CM.parse(filter).createPredicate()
+  }
+
   return ndjson
     ? this.registerHttpRequest(handleGetAllObjects, {
         filter,
@@ -63,7 +68,7 @@ export function getAllObjects({ filter, limit, ndjson = false }) {
 getAllObjects.description = 'Returns all XO objects'
 
 getAllObjects.params = {
-  filter: { type: 'object', optional: true },
+  filter: { type: ['object', 'string'], optional: true },
   limit: { type: 'number', optional: true },
   ndjson: { type: 'boolean', optional: true },
 }

packages/xo-server/src/xapi-object-to-xo.mjs

@@ -471,6 +471,7 @@ const TRANSFORMS = {
 
       vm.snapshot_time = toTimestamp(obj.snapshot_time)
       vm.$snapshot_of = link(obj, 'snapshot_of')
+      vm.suspendVdi = link(obj, 'suspend_VDI')
     } else if (obj.is_a_template) {
       const defaultTemplate = isDefaultTemplate(obj)
       vm.type += '-template'

packages/xo-server/src/xapi/index.mjs

@@ -161,7 +161,15 @@ export default class Xapi extends XapiBase {
   // =================================================================
 
   async joinPool(masterAddress, masterUsername, masterPassword, force = false) {
-    await this.call(force ? 'pool.join_force' : 'pool.join', masterAddress, masterUsername, masterPassword)
+    try {
+      await this.call(force ? 'pool.join_force' : 'pool.join', masterAddress, masterUsername, masterPassword)
+    } catch (error) {
+      const params = error?.call?.params
+      if (Array.isArray(params)) {
+        params[2] = '* obfuscated *'
+      }
+      throw error
+    }
   }
 
   // =================================================================

packages/xo-server/src/xo-mixins/api.mjs

@@ -115,7 +115,10 @@ async function resolveParams(method, params) {
   const permissions = []
   forEach(resolve, ([param, types, permission = 'administrate'], key) => {
     const id = params[param]
-    if (id === undefined) {
+    if (
+      id === undefined || // optional param not used
+      id === null // explicit value to unset
+    ) {
       return
     }
 

packages/xo-server/src/xo-mixins/cloud-configs.mjs

@@ -41,8 +41,12 @@ export default class {
     return this._db.remove(id)
   }
 
-  getAllCloudConfigs() {
-    return this._db.get()
+  async getAllCloudConfigs() {
+    return (await this._db.get()).filter(({ type }) => type === undefined)
+  }
+
+  async getAllNetworkConfigs() {
+    return (await this._db.get()).filter(({ type }) => type === 'network')
   }
 
   async getCloudConfig(id) {

packages/xo-server/src/xo-mixins/proxies.mjs

@@ -23,7 +23,7 @@ import { timeout } from 'promise-toolbox'
 
 import Collection from '../collection/redis.mjs'
 import patch from '../patch.mjs'
-import { debounceWithKey } from '../_pDebounceWithKey.mjs'
+import { debounceWithKey, REMOVE_CACHE_ENTRY } from '../_pDebounceWithKey.mjs'
 import { extractIpFromVmNetworks } from '../_extractIpFromVmNetworks.mjs'
 import { generateToken } from '../utils.mjs'
 
@@ -191,6 +191,8 @@ export default class Proxy {
       await xapi.startVm(vmUuid)
     }
 
+    this.getProxyApplianceUpdaterState(REMOVE_CACHE_ENTRY, id)
+
     await xapi._waitObjectState(vmUuid, vm => extractIpFromVmNetworks(vm.$guest_metrics?.networks) !== undefined)
   }
 

packages/xo-vmdk-to-vhd/package.json

@@ -25,7 +25,7 @@
     "lodash": "^4.17.15",
     "pako": "^2.0.4",
     "promise-toolbox": "^0.20.0",
-    "vhd-lib": "^1.3.0",
+    "vhd-lib": "^2.0.1",
     "xml2js": "^0.4.23"
   },
   "devDependencies": {
@@ -37,7 +37,6 @@
     "execa": "^5.0.0",
     "fs-extra": "^10.0.0",
     "get-stream": "^6.0.0",
-    "promise-toolbox": "^0.19.2",
     "rimraf": "^3.0.0",
     "tmp": "^0.2.1"
   },

packages/xo-vmdk-to-vhd/src/definitions.js

@@ -35,7 +35,7 @@ export function parseU64b(buffer, offset, valueName) {
   return value
 }
 
-export function parseHeader(buffer) {
+export function unpackHeader(buffer) {
   const magicString = buffer.slice(0, 4).toString('ascii')
   if (magicString !== 'KDMV') {
     throw new Error('not a VMDK file')

packages/xo-vmdk-to-vhd/src/vmdk-read.js

@@ -1,7 +1,7 @@
 import assert from 'assert'
 import zlib from 'zlib'
 
-import { compressionDeflate, parseHeader, parseU64b } from './definitions'
+import { compressionDeflate, unpackHeader, parseU64b } from './definitions'
 import { VirtualBuffer } from './virtual-buffer'
 
 const SECTOR_SIZE = 512
@@ -132,7 +132,7 @@ export default class VMDKDirectParser {
     if (version !== 1 && version !== 3) {
       throw new Error('unsupported VMDK version ' + version + ', only version 1 and 3 are supported')
     }
-    this.header = parseHeader(headerBuffer)
+    this.header = unpackHeader(headerBuffer)
     // I think the multiplications are OK, because the descriptor is always at the beginning of the file
     const descriptorLength = this.header.descriptorSizeSectors * SECTOR_SIZE
     const descriptorBuffer = await this.virtualBuffer.readChunk(descriptorLength, 'descriptor')

packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js

@@ -24,6 +24,15 @@ afterEach(async () => {
   await pFromCallback(cb => rimraf(tmpDir, cb))
 })
 
+function bufferToArray(buffer) {
+  const view = new DataView(buffer)
+  const res = []
+  for (let i = 0; i < buffer.byteLength; i += 4) {
+    res.push(view.getUint32(i, true))
+  }
+  return res
+}
+
 function createFileAccessor(file) {
   return async (start, end) => {
     if (start < 0 || end < 0) {
@@ -52,7 +61,7 @@ test('VMDK to VHD can convert a random data file with VMDKDirectParser', async (
     })
     const result = await readVmdkGrainTable(createFileAccessor(vmdkFileName))
     const pipe = (
-      await vmdkToVhd(createReadStream(vmdkFileName), result.grainLogicalAddressList, result.grainFileOffsetList)
+      await vmdkToVhd(createReadStream(vmdkFileName), bufferToArray(result.grainLogicalAddressList), bufferToArray(result.grainFileOffsetList))
     ).pipe(createWriteStream(vhdFileName))
     await fromEvent(pipe, 'finish')
     await execa('vhd-util', ['check', '-p', '-b', '-t', '-n', vhdFileName])

packages/xo-web-6/.babelrc.js

@@ -1,4 +0,0 @@
-const config = (module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json')))
-
-// https://webpack.js.org/guides/tree-shaking/
-config.presets.find(_ => _[0] === '@babel/preset-env')[1].modules = false

packages/xo-web-6/package.json

@@ -1,56 +0,0 @@
-{
-  "name": "xo-web-6",
-  "version": "0.0.0",
-  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-web-6",
-  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
-  "repository": {
-    "directory": "packages/xo-web-6",
-    "type": "git",
-    "url": "https://github.com/vatesfr/xen-orchestra.git"
-  },
-  "devDependencies": {
-    "@babel/core": "^7.3.4",
-    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.2.0",
-    "@babel/plugin-proposal-optional-chaining": "^7.2.0",
-    "@babel/plugin-syntax-dynamic-import": "^7.2.0",
-    "@babel/preset-env": "^7.3.4",
-    "@babel/preset-react": "^7.0.0",
-    "@babel/preset-typescript": "^7.3.3",
-    "@emotion/core": "^10.0.10",
-    "@emotion/styled": "^10.0.10",
-    "@types/react": "^16.8.13",
-    "@types/react-dom": "^16.8.4",
-    "@types/react-helmet": "^5.0.8",
-    "@types/react-router-dom": "^4.3.2",
-    "@types/webpack-env": "^1.13.9",
-    "babel-loader": "^8.0.5",
-    "clean-webpack-plugin": "^2.0.0",
-    "cross-env": "^5.2.0",
-    "html-webpack-plugin": "^3.2.0",
-    "json-rpc-protocol": "^0.13.1",
-    "reaclette": "^0.9.0-0",
-    "react": "^16.8.4",
-    "react-dom": "^16.8.4",
-    "react-helmet-async": "^1.0.2",
-    "react-router-dom": "^5.0.0",
-    "typescript": "^3.4.4",
-    "webpack": "^4.29.6",
-    "webpack-cli": "^3.2.3",
-    "webpack-dev-server": "^3.2.1"
-  },
-  "browserslist": "> 0.5%, last 2 versions, Firefox ESR, not dead",
-  "scripts": {
-    "build": "cross-env NODE_ENV=production webpack",
-    "start": "cross-env NODE_ENV=development webpack-dev-server --hot",
-    "start:open": "npm run start -- --open"
-  },
-  "private": true,
-  "author": {
-    "name": "Vates SAS",
-    "url": "https://vates.fr"
-  },
-  "license": "AGPL-3.0-or-later",
-  "engines": {
-    "node": ">=8.10"
-  }
-}

packages/xo-web-6/public/index.html

@@ -1,16 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <meta
-      name="viewport"
-      content="width=device-width, initial-scale=1, shrink-to-fit=no"
-    />
-    <meta name="theme-color" content="#000000" />
-    <title>Xen Orchestra</title>
-  </head>
-  <body>
-    <noscript>You need to enable JavaScript to run this app.</noscript>
-    <div id="root">The application is loading…</div>
-  </body>
-</html>

packages/xo-web-6/src/App/Bar/index.tsx

@@ -1,11 +0,0 @@
-import React from 'react'
-import { Helmet } from 'react-helmet-async'
-
-export default () => (
-  <>
-    <Helmet>
-      <title>Bar</title>
-    </Helmet>
-    Welcome in Bar
-  </>
-)