feat(vhd-{cli,lib}): implement chunking and copy command (#5919)

@xen-orchestra/backups/RemoteAdapter.js

@@ -6,7 +6,7 @@ const pDefer = require('promise-toolbox/defer.js')
 const pump = require('pump')
 const { basename, dirname, join, normalize, resolve } = require('path')
 const { createLogger } = require('@xen-orchestra/log')
-const { createSyntheticStream, mergeVhd, default: Vhd } = require('vhd-lib')
+const { createSyntheticStream, mergeVhd, VhdFile } = require('vhd-lib')
 const { deduped } = require('@vates/disposable/deduped.js')
 const { execFile } = require('child_process')
 const { readdir, stat } = require('fs-extra')
@@ -86,7 +86,7 @@ class RemoteAdapter {
       }),
       async path => {
         try {
-          const vhd = new Vhd(handler, path)
+          const vhd = new VhdFile(handler, path)
           await vhd.readHeaderAndFooter()
           return {
             footer: vhd.footer,

@xen-orchestra/backups/_cleanVm.js

@@ -1,7 +1,7 @@
 const assert = require('assert')
 const sum = require('lodash/sum')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { default: Vhd, mergeVhd } = require('vhd-lib')
+const { VhdFile, mergeVhd } = require('vhd-lib')
 const { dirname, resolve } = require('path')
 const { DISK_TYPE_DIFFERENCING } = require('vhd-lib/dist/_constants.js')
 const { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } = require('./_backupType.js')
@@ -135,7 +135,7 @@ exports.cleanVm = async function cleanVm(
   // remove broken VHDs
   await asyncMap(vhdsList.vhds, async path => {
     try {
-      const vhd = new Vhd(handler, path)
+      const vhd = new VhdFile(handler, path)
       await vhd.readHeaderAndFooter(!vhdsList.interruptedVhds.has(path))
       vhds.add(path)
       if (vhd.footer.diskType === DISK_TYPE_DIFFERENCING) {

@xen-orchestra/backups/writers/DeltaBackupWriter.js

@@ -3,7 +3,7 @@ const map = require('lodash/map.js')
 const mapValues = require('lodash/mapValues.js')
 const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
 const { asyncMap } = require('@xen-orchestra/async-map')
-const { chainVhd, checkVhdChain, default: Vhd } = require('vhd-lib')
+const { chainVhd, checkVhdChain, VhdFile } = require('vhd-lib')
 const { createLogger } = require('@xen-orchestra/log')
 const { dirname } = require('path')
 
@@ -38,7 +38,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           try {
             await checkVhdChain(handler, path)
 
-            const vhd = new Vhd(handler, path)
+            const vhd = new VhdFile(handler, path)
             await vhd.readHeaderAndFooter()
             found = found || vhd.footer.uuid.equals(packUuid(baseUuid))
           } catch (error) {
@@ -200,7 +200,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
           }
 
           // set the correct UUID in the VHD
-          const vhd = new Vhd(handler, path)
+          const vhd = new VhdFile(handler, path)
           await vhd.readHeaderAndFooter()
           vhd.footer.uuid = packUuid(vdi.uuid)
           await vhd.readBlockAllocationTable() // required by writeFooter()

@xen-orchestra/backups/writers/_checkVhd.js

@@ -1,4 +1,4 @@
-const Vhd = require('vhd-lib').default
+const Vhd = require('vhd-lib').VhdFile
 
 exports.checkVhd = async function checkVhd(handler, path) {
   await new Vhd(handler, path).readHeaderAndFooter()

CHANGELOG.unreleased.md

@@ -34,7 +34,9 @@
 >
 > In case of conflict, the highest (lowest in previous list) `$version` wins.
 
+- vhd-lib minor
 - @xen-orchestra/backup minor
 - @xen-orchestra/proxy minor
+- vhd-cli minor
 - xo-server patch
 - xo-web minor

packages/vhd-cli/src/commands/check.js

@@ -1,9 +1,9 @@
-import Vhd, { checkVhdChain } from 'vhd-lib'
+import { VhdFile, checkVhdChain } from 'vhd-lib'
 import getopts from 'getopts'
 import { getHandler } from '@xen-orchestra/fs'
 import { resolve } from 'path'
 
-const checkVhd = (handler, path) => new Vhd(handler, path).readHeaderAndFooter()
+const checkVhd = (handler, path) => new VhdFile(handler, path).readHeaderAndFooter()
 
 export default async rawArgs => {
   const { chain, _: args } = getopts(rawArgs, {

packages/vhd-cli/src/commands/copy.js

@@ -0,0 +1,63 @@
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { resolve } from 'path'
+import { VhdDirectory, VhdFile } from 'vhd-lib'
+import Disposable from 'promise-toolbox/Disposable'
+import getopts from 'getopts'
+
+export default async rawArgs => {
+  const {
+    directory,
+    help,
+    _: args,
+  } = getopts(rawArgs, {
+    alias: {
+      directory: 'd',
+      help: 'h',
+    },
+    boolean: ['directory', 'force'],
+    default: {
+      directory: false,
+      help: false,
+    },
+  })
+  if (args.length < 2 || help) {
+    return `Usage: index.js copy <source VHD> <destination> --directory --force`
+  }
+  const [sourcePath, destPath] = args
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file://' })
+    const resolvedSourcePath = resolve(sourcePath)
+    let src
+    try {
+      src = yield VhdFile.open(handler, resolvedSourcePath)
+    } catch (e) {
+      if (e.code === 'EISDIR') {
+        src = yield VhdDirectory.open(handler, resolvedSourcePath)
+      } else {
+        throw e
+      }
+    }
+    await src.readBlockAllocationTable()
+    const resolvedDestPath = resolve(destPath)
+    const dest = yield directory
+      ? VhdDirectory.create(handler, resolvedDestPath)
+      : VhdFile.create(handler, resolvedDestPath)
+    // copy data
+    dest.header = src.header
+    dest.footer = src.footer
+
+    for await (const block of src.blocks()) {
+      await dest.writeEntireBlock(block)
+    }
+
+    // copy parent locators
+    for (let parentLocatorId = 0; parentLocatorId < 8; parentLocatorId++) {
+      const parentLocator = await src.readParentLocator(parentLocatorId)
+      await dest.writeParentLocator(parentLocator)
+    }
+    await dest.writeFooter()
+    await dest.writeHeader()
+    await dest.writeBlockAllocationTable()
+  })
+}

packages/vhd-cli/src/commands/info.js

@@ -1,9 +1,9 @@
-import Vhd from 'vhd-lib'
+import { VhdFile } from 'vhd-lib'
 import { getHandler } from '@xen-orchestra/fs'
 import { resolve } from 'path'
 
 export default async args => {
-  const vhd = new Vhd(getHandler({ url: 'file:///' }), resolve(args[0]))
+  const vhd = new VhdFile(getHandler({ url: 'file:///' }), resolve(args[0]))
 
   try {
     await vhd.readHeaderAndFooter()

packages/vhd-cli/src/commands/repl.js

@@ -2,7 +2,7 @@ import { asCallback, fromCallback, fromEvent } from 'promise-toolbox'
 import { getHandler } from '@xen-orchestra/fs'
 import { relative } from 'path'
 import { start as createRepl } from 'repl'
-import Vhd, * as vhdLib from 'vhd-lib'
+import * as vhdLib from 'vhd-lib'
 
 export default async args => {
   const cwd = process.cwd()
@@ -14,7 +14,7 @@ export default async args => {
     })
     Object.assign(repl.context, vhdLib)
     repl.context.handler = handler
-    repl.context.open = path => new Vhd(handler, relative(cwd, path))
+    repl.context.open = path => new vhdLib.VhdFile(handler, relative(cwd, path))
 
     // Make the REPL waits for promise completion.
     repl.eval = (evaluate => (cmd, context, filename, cb) => {

packages/vhd-lib/src/Vhd/VhdAbstract.js

@@ -0,0 +1,167 @@
+import { computeBatSize, sectorsRoundUpNoZero, sectorsToBytes } from './_utils'
+import { PLATFORM_NONE, SECTOR_SIZE, PLATFORM_W2KU, PARENT_LOCATOR_ENTRIES } from '../_constants'
+import assert from 'assert'
+
+export class VhdAbstract {
+  #header
+  bitmapSize
+  footer
+  fullBlockSize
+  sectorsOfBitmap
+  sectorsPerBlock
+
+  get header() {
+    assert.notStrictEqual(this.#header, undefined, `header must be read before it's used`)
+    return this.#header
+  }
+
+  set header(header) {
+    this.#header = header
+    this.sectorsPerBlock = header.blockSize / SECTOR_SIZE
+    this.sectorsOfBitmap = sectorsRoundUpNoZero(this.sectorsPerBlock >> 3)
+    this.fullBlockSize = sectorsToBytes(this.sectorsOfBitmap + this.sectorsPerBlock)
+    this.bitmapSize = sectorsToBytes(this.sectorsOfBitmap)
+  }
+
+  /**
+   * instantiate a Vhd
+   *
+   * @returns {AbstractVhd}
+   */
+  static async open() {
+    throw new Error('open not implemented')
+  }
+
+  /**
+   * Check if this vhd contains a block with id blockId
+   * Must be called after readBlockAllocationTable
+   *
+   * @param {number} blockId
+   * @returns {boolean}
+   *
+   */
+  containsBlock(blockId) {
+    throw new Error(`checking if this vhd contains the block ${blockId} is not implemented`)
+  }
+
+  /**
+   * Read the header and the footer
+   * check their integrity
+   * if checkSecondFooter also checks that the footer at the end is equal to the one at the beginning
+   *
+   * @param {boolean} checkSecondFooter
+   */
+  readHeaderAndFooter(checkSecondFooter = true) {
+    throw new Error(
+      `reading and checking footer, ${checkSecondFooter ? 'second footer,' : ''} and header is not implemented`
+    )
+  }
+
+  readBlockAllocationTable() {
+    throw new Error(`reading block allocation table is not implemented`)
+  }
+
+  /**
+   *
+   * @param {number} blockId
+   * @param {boolean} onlyBitmap
+   * @returns {Buffer}
+   */
+  readBlock(blockId, onlyBitmap = false) {
+    throw new Error(`reading  ${onlyBitmap ? 'bitmap of block' : 'block'} ${blockId} is not implemented`)
+  }
+
+  /**
+   * coalesce the block with id blockId from the child vhd into
+   * this vhd
+   *
+   * @param {AbstractVhd} child
+   * @param {number} blockId
+   *
+   * @returns {number} the merged data size
+   */
+  coalesceBlock(child, blockId) {
+    throw new Error(`coalescing the block ${blockId} from ${child} is not implemented`)
+  }
+
+  /**
+   * ensure the bat size can store at least entries block
+   * move blocks if needed
+   * @param {number} entries
+   */
+  ensureBatSize(entries) {
+    throw new Error(`ensuring batSize can store at least  ${entries} is not implemented`)
+  }
+
+  // Write a context footer. (At the end and beginning of a vhd file.)
+  writeFooter(onlyEndFooter = false) {
+    throw new Error(`writing footer   ${onlyEndFooter ? 'only at end' : 'on both side'} is not implemented`)
+  }
+
+  writeHeader() {
+    throw new Error(`writing header is not implemented`)
+  }
+
+  _writeParentLocatorData(parentLocatorId, platformDataOffset, data) {
+    throw new Error(`write Parent locator ${parentLocatorId} is not implemented`)
+  }
+
+  _readParentLocatorData(parentLocatorId, platformDataOffset, platformDataSpace) {
+    throw new Error(`read Parent locator ${parentLocatorId} is not implemented`)
+  }
+  // common
+  get batSize() {
+    return computeBatSize(this.header.maxTableEntries)
+  }
+
+  async writeParentLocator({ id, platformCode = PLATFORM_NONE, data = Buffer.alloc(0) }) {
+    assert(id >= 0, 'parent Locator id must be a positive number')
+    assert(id < PARENT_LOCATOR_ENTRIES, `parent Locator id  must be less than ${PARENT_LOCATOR_ENTRIES}`)
+
+    await this._writeParentLocatorData(id, data)
+
+    const entry = this.header.parentLocatorEntry[id]
+    const dataSpaceSectors = Math.ceil(data.length / SECTOR_SIZE)
+    entry.platformCode = platformCode
+    entry.platformDataSpace = dataSpaceSectors * SECTOR_SIZE
+    entry.platformDataLength = data.length
+  }
+
+  async readParentLocator(id) {
+    assert(id >= 0, 'parent Locator id must be a positive number')
+    assert(id < PARENT_LOCATOR_ENTRIES, `parent Locator id  must be less than ${PARENT_LOCATOR_ENTRIES}`)
+    const data = await this._readParentLocatorData(id)
+    // offset is storage specific, don't expose it
+    const { platformCode } = this.header.parentLocatorEntry[id]
+    return {
+      platformCode,
+      id,
+      data
+    }
+  }
+
+  async setUniqueParentLocator(fileNameString) {
+    await this.writeParentLocator({
+      id: 0,
+      code: PLATFORM_W2KU,
+      data: Buffer.from(fileNameString, 'utf16le')
+    })
+
+    for (let i = 1; i < PARENT_LOCATOR_ENTRIES; i++) {
+      await this.writeParentLocator({
+        id: i,
+        code: PLATFORM_NONE,
+        data: Buffer.alloc(0)
+      })
+    }
+  }
+
+  async *blocks() {
+    const nBlocks = this.header.maxTableEntries
+    for (let blockId = 0; blockId < nBlocks; ++blockId) {
+      if (await this.containsBlock(blockId)) {
+        yield await this.readBlock(blockId)
+      }
+    }
+  }
+}

packages/vhd-lib/src/Vhd/VhdDirectory.js

@@ -0,0 +1,190 @@
+import { buildHeader, buildFooter } from './_utils'
+import { createLogger } from '@xen-orchestra/log'
+import { fuFooter, fuHeader, checksumStruct } from '../_structs'
+import { test, set as setBitmap } from '../_bitmap'
+import { VhdAbstract } from './VhdAbstract'
+import assert from 'assert'
+
+const { debug } = createLogger('vhd-lib:VhdDirectory')
+
+// ===================================================================
+// Directory format
+// <path>
+// ├─ header // raw content of the header
+// ├─ footer // raw content of the footer
+// ├─ bat // bit array. A zero bit indicates at a position that this block is not present
+// ├─ parentLocatorEntry{0-7} // data of a parent locator
+// ├─ blocks // blockId is the position in the BAT
+//    └─ <the first to  {blockId.length -3} numbers of blockId >
+//         └─ <the three last numbers  of blockID >  // block content.
+
+export class VhdDirectory extends VhdAbstract {
+  #uncheckedBlockTable
+
+  set header(header) {
+    super.header = header
+    this.#blocktable = Buffer.alloc(header.maxTableEntries)
+  }
+
+  get header() {
+    return super.header
+  }
+
+  get #blocktable() {
+    assert.notStrictEqual(this.#blockTable, undefined, 'Block table must be initialized before access')
+    return this.#uncheckedBlockTable
+  }
+
+  set #blocktable(blocktable) {
+    this.#uncheckedBlockTable = blocktable
+  }
+
+  static async open(handler, path) {
+    const vhd = new VhdDirectory(handler, path)
+
+    // openning a file for reading does not trigger EISDIR as long as we don't really read from it :
+    // https://man7.org/linux/man-pages/man2/open.2.html
+    // EISDIR pathname refers to a directory and the access requested
+    // involved writing (that is, O_WRONLY or O_RDWR is set).
+    // reading the header ensure we have a well formed directory immediatly
+    await vhd.readHeaderAndFooter()
+    return {
+      dispose: () => {},
+      value: vhd
+    }
+  }
+
+  static async create(handler, path) {
+    await handler.mkdir(path)
+    const vhd = new VhdDirectory(handler, path)
+    return {
+      dispose: () => {},
+      value: vhd
+    }
+  }
+
+  constructor(handler, path) {
+    super()
+    this._handler = handler
+    this._path = path
+  }
+
+  async readBlockAllocationTable() {
+    const { buffer } = await this._readChunk('bat')
+    this.#blockTable = buffer
+  }
+
+  containsBlock(blockId) {
+    return test(this.#blockTable, blockId)
+  }
+
+  getChunkPath(partName) {
+    return this._path + '/' + partName
+  }
+
+  async _readChunk(partName) {
+    // here we can implement compression and / or crypto
+    const buffer = await this._handler.readFile(this.getChunkPath(partName))
+
+    return {
+      buffer: Buffer.from(buffer)
+    }
+  }
+
+  async _writeChunk(partName, buffer) {
+    assert(Buffer.isBuffer(buffer))
+    // here we can implement compression and / or crypto
+
+    // chunks can be in sub directories :  create direcotries if necessary
+    const pathParts = partName.split('/')
+    let currentPath = this._path
+
+    // the last one is the file name
+    for (let i = 0; i < pathParts.length - 1; i++) {
+      currentPath += '/' + pathParts[i]
+      await this._handler.mkdir(currentPath)
+    }
+
+    return this._handler.writeFile(this.getChunkPath(partName), buffer)
+  }
+
+  // put block in subdirectories to limit impact when doing directory listing
+  _getBlockPath(blockId) {
+    const blockPrefix = Math.floor(blockId / 1e3)
+    const blockSuffix = blockId - blockPrefix * 1e3
+    return `blocks/${blockPrefix}/${blockSuffix}`
+  }
+
+  async readHeaderAndFooter() {
+    const { buffer: bufHeader } = await this._readChunk('header')
+    const { buffer: bufFooter } = await this._readChunk('footer')
+    const footer = buildFooter(bufFooter)
+    const header = buildHeader(bufHeader, footer)
+
+    this.footer = footer
+    this.header = header
+  }
+
+  async readBlock(blockId, onlyBitmap = false) {
+    if (onlyBitmap) {
+      throw new Error(`reading 'bitmap of block' ${blockId} in a VhdDirectory is not implemented`)
+    }
+    const { buffer } = await this._readChunk(this._getBlockPath(blockId))
+    return {
+      id: blockId,
+      bitmap: buffer.slice(0, this.bitmapSize),
+      data: buffer.slice(this.bitmapSize),
+      buffer
+    }
+  }
+  ensureBatSize() {
+    // nothing to do in directory mode
+  }
+
+  async writeFooter() {
+    const { footer } = this
+
+    const rawFooter = fuFooter.pack(footer)
+
+    footer.checksum = checksumStruct(rawFooter, fuFooter)
+    debug(`Write footer  (checksum=${footer.checksum}). (data=${rawFooter.toString('hex')})`)
+
+    await this._writeChunk('footer', rawFooter)
+  }
+
+  writeHeader() {
+    const { header } = this
+    const rawHeader = fuHeader.pack(header)
+    header.checksum = checksumStruct(rawHeader, fuHeader)
+    debug(`Write header  (checksum=${header.checksum}). (data=${rawHeader.toString('hex')})`)
+    return this._writeChunk('header', rawHeader)
+  }
+
+  writeBlockAllocationTable() {
+    assert.notStrictEqual(this.#blockTable, undefined, 'Block allocation table has not been read')
+    assert.notStrictEqual(this.#blockTable.length, 0, 'Block allocation table is empty')
+
+    return this._writeChunk('bat', this.#blockTable)
+  }
+
+  // only works if data are in the same bucket
+  // and if the full block is modified in child ( which is the case whit xcp)
+
+  coalesceBlock(child, blockId) {
+    this._handler.copy(child.getChunkPath(blockId), this.getChunkPath(blockId))
+  }
+
+  async writeEntireBlock(block) {
+    await this._writeChunk(this._getBlockPath(block.id), block.buffer)
+    setBitmap(this.#blockTable, block.id)
+  }
+
+  async _readParentLocatorData(id) {
+    return (await this._readChunk('parentLocatorEntry' + id)).buffer
+  }
+
+  async _writeParentLocatorData(id, data) {
+    await this._writeChunk('parentLocatorEntry' + id, data)
+    this.header.parentLocatorEntry[id].platformDataOffset = 0
+  }
+}

packages/vhd-lib/src/Vhd/VhdFile.js

@@ -1,22 +1,20 @@
-import assert from 'assert'
-import { createLogger } from '@xen-orchestra/log'
-
-import checkFooter from './checkFooter'
-import checkHeader from './_checkHeader'
-import getFirstAndLastBlocks from './_getFirstAndLastBlocks'
-import { fuFooter, fuHeader, checksumStruct, unpackField } from './_structs'
-import { set as mapSetBit, test as mapTestBit } from './_bitmap'
 import {
   BLOCK_UNUSED,
   FOOTER_SIZE,
   HEADER_SIZE,
-  PARENT_LOCATOR_ENTRIES,
   PLATFORM_NONE,
-  PLATFORM_W2KU,
   SECTOR_SIZE,
-} from './_constants'
+  PARENT_LOCATOR_ENTRIES
+} from '../_constants'
+import { computeBatSize, sectorsToBytes, buildHeader, buildFooter, BUF_BLOCK_UNUSED } from './_utils'
+import { createLogger } from '@xen-orchestra/log'
+import { fuFooter, fuHeader, checksumStruct } from '../_structs'
+import { set as mapSetBit, test as mapTestBit } from '../_bitmap'
+import { VhdAbstract } from './VhdAbstract'
+import assert from 'assert'
+import getFirstAndLastBlocks from '../_getFirstAndLastBlocks'
 
-const { debug } = createLogger('vhd-lib:Vhd')
+const { debug } = createLogger('vhd-lib:VhdFile')
 
 // ===================================================================
 //
@@ -28,22 +26,6 @@ const { debug } = createLogger('vhd-lib:Vhd')
 //
 // ===================================================================
 
-const computeBatSize = entries => sectorsToBytes(sectorsRoundUpNoZero(entries * 4))
-
-// Sectors conversions.
-const sectorsRoundUpNoZero = bytes => Math.ceil(bytes / SECTOR_SIZE) || 1
-const sectorsToBytes = sectors => sectors * SECTOR_SIZE
-
-const assertChecksum = (name, buf, struct) => {
-  const actual = unpackField(struct.fields.checksum, buf)
-  const expected = checksumStruct(buf, struct)
-  assert.strictEqual(actual, expected, `invalid ${name} checksum ${actual}, expected ${expected}`)
-}
-
-// unused block as buffer containing a uint32BE
-const BUF_BLOCK_UNUSED = Buffer.allocUnsafe(4)
-BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
-
 // ===================================================================
 
 // Format:
@@ -68,12 +50,60 @@ BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
 // - parentLocatorSize(i) = header.parentLocatorEntry[i].platformDataSpace * sectorSize
 // - sectorSize = 512
 
-export default class Vhd {
+export class VhdFile extends VhdAbstract {
+  #uncheckedBlockTable
+
+  get #blocktable() {
+    assert.notStrictEqual(this.#blockTable, undefined, 'Block table must be initialized before access')
+    return this.#uncheckedBlockTable
+  }
+
+  set #blocktable(blocktable) {
+    this.#uncheckedBlockTable = blocktable
+  }
+
   get batSize() {
     return computeBatSize(this.header.maxTableEntries)
   }
 
+  set header(header) {
+    super.header = header
+    const size = this.batSize
+    this.#blockTable = Buffer.alloc(size)
+    for (let i = 0; i < this.header.maxTableEntries; i++) {
+      this.#blockTable.writeUInt32BE(BLOCK_UNUSED, i * 4)
+    }
+  }
+  get header() {
+    return super.header
+  }
+
+  static async open(handler, path) {
+    const fd = await handler.openFile(path, 'r+')
+    const vhd = new VhdFile(handler, fd)
+    // openning a file for reading does not trigger EISDIR as long as we don't really read from it :
+    // https://man7.org/linux/man-pages/man2/open.2.html
+    // EISDIR pathname refers to a directory and the access requested
+    // involved writing (that is, O_WRONLY or O_RDWR is set).
+    // reading the header ensure we have a well formed file immediatly
+    await vhd.readHeaderAndFooter()
+    return {
+      dispose: () => handler.closeFile(fd),
+      value: vhd
+    }
+  }
+
+  static async create(handler, path) {
+    const fd = await handler.openFile(path, 'wx')
+    const vhd = new VhdFile(handler, fd)
+    return {
+      dispose: () => handler.closeFile(fd),
+      value: vhd
+    }
+  }
+
   constructor(handler, path) {
+    super()
     this._handler = handler
     this._path = path
   }
@@ -87,11 +117,6 @@ export default class Vhd {
     assert.strictEqual(bytesRead, n)
     return buffer
   }
-
-  containsBlock(id) {
-    return this._getBatEntry(id) !== BLOCK_UNUSED
-  }
-
   // Returns the first address after metadata. (In bytes)
   _getEndOfHeaders() {
     const { header } = this
@@ -114,17 +139,24 @@ export default class Vhd {
     return end
   }
 
+  // return the first sector (bitmap) of a block
+  _getBatEntry(blockId) {
+    const i = blockId * 4
+    const blockTable = this.#blockTable
+    return i < blockTable.length ? blockTable.readUInt32BE(i) : BLOCK_UNUSED
+  }
+
   // Returns the first sector after data.
   _getEndOfData() {
     let end = Math.ceil(this._getEndOfHeaders() / SECTOR_SIZE)
 
-    const fullBlockSize = this.sectorsOfBitmap + this.sectorsPerBlock
+    const sectorsOfFullBlock = this.sectorsOfBitmap + this.sectorsPerBlock
     const { maxTableEntries } = this.header
     for (let i = 0; i < maxTableEntries; i++) {
       const blockAddr = this._getBatEntry(i)
 
       if (blockAddr !== BLOCK_UNUSED) {
-        end = Math.max(end, blockAddr + fullBlockSize)
+        end = Math.max(end, blockAddr + sectorsOfFullBlock)
       }
     }
 
@@ -133,7 +165,11 @@ export default class Vhd {
     return sectorsToBytes(end)
   }
 
-  // TODO: extract the checks into reusable functions:
+  containsBlock(id) {
+    return this._getBatEntry(id) !== BLOCK_UNUSED
+  }
+
+  // TODO:
   // - better human reporting
   // - auto repair if possible
   async readHeaderAndFooter(checkSecondFooter = true) {
@@ -141,50 +177,25 @@ export default class Vhd {
     const bufFooter = buf.slice(0, FOOTER_SIZE)
     const bufHeader = buf.slice(FOOTER_SIZE)
 
-    assertChecksum('footer', bufFooter, fuFooter)
-    assertChecksum('header', bufHeader, fuHeader)
+    const footer = buildFooter(bufFooter)
+    const header = buildHeader(bufHeader, footer)
 
     if (checkSecondFooter) {
       const size = await this._handler.getSize(this._path)
       assert(bufFooter.equals(await this._read(size - FOOTER_SIZE, FOOTER_SIZE)), 'footer1 !== footer2')
     }
 
-    const footer = (this.footer = fuFooter.unpack(bufFooter))
-    checkFooter(footer)
-
-    const header = (this.header = fuHeader.unpack(bufHeader))
-    checkHeader(header, footer)
-
-    // Compute the number of sectors in one block.
-    // Default: One block contains 4096 sectors of 512 bytes.
-    const sectorsPerBlock = (this.sectorsPerBlock = header.blockSize / SECTOR_SIZE)
-
-    // Compute bitmap size in sectors.
-    // Default: 1.
-    const sectorsOfBitmap = (this.sectorsOfBitmap = sectorsRoundUpNoZero(sectorsPerBlock >> 3))
-
-    // Full block size => data block size + bitmap size.
-    this.fullBlockSize = sectorsToBytes(sectorsPerBlock + sectorsOfBitmap)
-
-    // In bytes.
-    // Default: 512.
-    this.bitmapSize = sectorsToBytes(sectorsOfBitmap)
+    this.footer = footer
+    this.header = header
   }
 
   // Returns a buffer that contains the block allocation table of a vhd file.
   async readBlockAllocationTable() {
     const { header } = this
-    this.blockTable = await this._read(header.tableOffset, header.maxTableEntries * 4)
+    this.#blockTable = await this._read(header.tableOffset, header.maxTableEntries * 4)
   }
 
-  // return the first sector (bitmap) of a block
-  _getBatEntry(blockId) {
-    const i = blockId * 4
-    const { blockTable } = this
-    return i < blockTable.length ? blockTable.readUInt32BE(i) : BLOCK_UNUSED
-  }
-
-  _readBlock(blockId, onlyBitmap = false) {
+  readBlock(blockId, onlyBitmap = false) {
     const blockAddr = this._getBatEntry(blockId)
     if (blockAddr === BLOCK_UNUSED) {
       throw new Error(`no such block ${blockId}`)
@@ -197,7 +208,7 @@ export default class Vhd {
             id: blockId,
             bitmap: buf.slice(0, this.bitmapSize),
             data: buf.slice(this.bitmapSize),
-            buffer: buf,
+            buffer: buf
           }
     )
   }
@@ -214,7 +225,7 @@ export default class Vhd {
   }
 
   async _freeFirstBlockSpace(spaceNeededBytes) {
-    const firstAndLastBlocks = getFirstAndLastBlocks(this.blockTable)
+    const firstAndLastBlocks = getFirstAndLastBlocks(this.#blockTable)
     if (firstAndLastBlocks === undefined) {
       return
     }
@@ -249,8 +260,8 @@ export default class Vhd {
     const newBatSize = computeBatSize(entries)
     await this._freeFirstBlockSpace(newBatSize - this.batSize)
     const maxTableEntries = (header.maxTableEntries = entries)
-    const prevBat = this.blockTable
-    const bat = (this.blockTable = Buffer.allocUnsafe(newBatSize))
+    const prevBat = this.#blockTable
+    const bat = (this.#blockTable = Buffer.allocUnsafe(newBatSize))
     prevBat.copy(bat)
     bat.fill(BUF_BLOCK_UNUSED, prevMaxTableEntries * 4)
     debug(`ensureBatSize: extend BAT ${prevMaxTableEntries} -> ${maxTableEntries}`)
@@ -264,7 +275,7 @@ export default class Vhd {
   // set the first sector (bitmap) of a block
   _setBatEntry(block, blockSector) {
     const i = block * 4
-    const { blockTable } = this
+    const blockTable = this.#blockTable
 
     blockTable.writeUInt32BE(blockSector, i)
 
@@ -298,7 +309,7 @@ export default class Vhd {
     await this._write(bitmap, sectorsToBytes(blockAddr))
   }
 
-  async _writeEntireBlock(block) {
+  async writeEntireBlock(block) {
     let blockAddr = this._getBatEntry(block.id)
 
     if (blockAddr === BLOCK_UNUSED) {
@@ -314,7 +325,7 @@ export default class Vhd {
       blockAddr = await this._createBlock(block.id)
       parentBitmap = Buffer.alloc(this.bitmapSize, 0)
     } else if (parentBitmap === undefined) {
-      parentBitmap = (await this._readBlock(block.id, true)).bitmap
+      parentBitmap = (await this.readBlock(block.id, true)).bitmap
     }
 
     const offset = blockAddr + this.sectorsOfBitmap + beginSectorId
@@ -333,7 +344,7 @@ export default class Vhd {
   }
 
   async coalesceBlock(child, blockId) {
-    const block = await child._readBlock(blockId)
+    const block = await child.readBlock(blockId)
     const { bitmap, data } = block
 
     debug(`coalesceBlock block=${blockId}`)
@@ -358,10 +369,10 @@ export default class Vhd {
 
       const isFullBlock = i === 0 && endSector === sectorsPerBlock
       if (isFullBlock) {
-        await this._writeEntireBlock(block)
+        await this.writeEntireBlock(block)
       } else {
         if (parentBitmap === null) {
-          parentBitmap = (await this._readBlock(blockId, true)).bitmap
+          parentBitmap = (await this.readBlock(blockId, true)).bitmap
         }
         await this._writeBlockSectors(block, i, endSector, parentBitmap)
       }
@@ -399,6 +410,13 @@ export default class Vhd {
     return this._write(rawHeader, offset)
   }
 
+  writeBlockAllocationTable() {
+    const header = this.header
+    const blockTable = this.#blockTable
+    debug(`Write BlockAllocationTable at: ${header.tableOffset} ). (data=${blockTable.toString('hex')})`)
+    return this._write(blockTable, header.tableOffset)
+  }
+
   async writeData(offsetSectors, buffer) {
     const bufferSizeSectors = Math.ceil(buffer.length / SECTOR_SIZE)
     const startBlock = Math.floor(offsetSectors / this.sectorsPerBlock)
@@ -436,26 +454,35 @@ export default class Vhd {
       const deltaSectors = neededSectors - currentSpace
       await this._freeFirstBlockSpace(sectorsToBytes(deltaSectors))
       this.header.tableOffset += sectorsToBytes(deltaSectors)
-      await this._write(this.blockTable, this.header.tableOffset)
+      await this._write(this.#blockTable, this.header.tableOffset)
     }
     return firstLocatorOffset
   }
 
-  async setUniqueParentLocator(fileNameString) {
+  async _readParentLocatorData(parentLocatorId) {
+    const { platformDataOffset, platformDataLength } = this.header.parentLocatorEntry[parentLocatorId]
+    if (platformDataLength > 0) {
+      return (await this._read(platformDataOffset, platformDataLength)).buffer
+    }
+    return Buffer.alloc(0)
+  }
+
+  async _writeParentLocatorData(parentLocatorId, data) {
+    let position
     const { header } = this
-    header.parentLocatorEntry[0].platformCode = PLATFORM_W2KU
-    const encodedFilename = Buffer.from(fileNameString, 'utf16le')
-    const dataSpaceSectors = Math.ceil(encodedFilename.length / SECTOR_SIZE)
-    const position = await this._ensureSpaceForParentLocators(dataSpaceSectors)
-    await this._write(encodedFilename, position)
-    header.parentLocatorEntry[0].platformDataSpace = dataSpaceSectors * SECTOR_SIZE
-    header.parentLocatorEntry[0].platformDataLength = encodedFilename.length
-    header.parentLocatorEntry[0].platformDataOffset = position
-    for (let i = 1; i < 8; i++) {
-      header.parentLocatorEntry[i].platformCode = PLATFORM_NONE
-      header.parentLocatorEntry[i].platformDataSpace = 0
-      header.parentLocatorEntry[i].platformDataLength = 0
-      header.parentLocatorEntry[i].platformDataOffset = 0
+    if (data.length === 0) {
+      // reset offset if data is empty
+      header.parentLocatorEntry[parentLocatorId].platformDataOffset = 0
+    } else {
+      if (data.length <= header.parentLocatorEntry[parentLocatorId].platformDataSpace) {
+        // new parent locator length is smaller than available space : keep it in place
+        position = header.parentLocatorEntry[parentLocatorId].platformDataOffset
+      } else {
+        // new parent locator length is bigger than available space : move it to the end
+        position = this._getEndOfData()
+      }
+      await this._write(data, position)
+      header.parentLocatorEntry[parentLocatorId].platformDataOffset = position
     }
   }
 }

packages/vhd-lib/src/Vhd/_utils.js

@@ -0,0 +1,52 @@
+import assert from 'assert'
+import { BLOCK_UNUSED, SECTOR_SIZE } from '../_constants'
+import { fuFooter, fuHeader, checksumStruct, unpackField } from '../_structs'
+import checkFooter from '../checkFooter'
+import checkHeader from '../_checkHeader'
+
+export const computeBatSize = entries => sectorsToBytes(sectorsRoundUpNoZero(entries * 4))
+
+// Sectors conversions.
+export const sectorsRoundUpNoZero = bytes => Math.ceil(bytes / SECTOR_SIZE) || 1
+export const sectorsToBytes = sectors => sectors * SECTOR_SIZE
+
+export const assertChecksum = (name, buf, struct) => {
+  const actual = unpackField(struct.fields.checksum, buf)
+  const expected = checksumStruct(buf, struct)
+  assert.strictEqual(actual, expected, `invalid ${name} checksum ${actual}, expected ${expected}`)
+}
+
+// unused block as buffer containing a uint32BE
+export const BUF_BLOCK_UNUSED = Buffer.allocUnsafe(4)
+BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
+
+/**
+ * Check and parse the header buffer to build an header object
+ *
+ * @param {Buffer} bufHeader
+ * @param {Object} footer
+ * @returns {Object} the parsed header
+ */
+export const buildHeader = (bufHeader, footer) => {
+  assertChecksum('header', bufHeader, fuHeader)
+
+  const header = fuHeader.unpack(bufHeader)
+  checkHeader(header, footer)
+  return header
+}
+
+/**
+ * Check and parse the footer buffer to build a footer object
+ *
+ * @param {Buffer} bufHeader
+ * @param {Object} footer
+ * @returns {Object} the parsed footer
+ */
+
+export const buildFooter = bufFooter => {
+  assertChecksum('footer', bufFooter, fuFooter)
+
+  const footer = fuFooter.unpack(bufFooter)
+  checkFooter(footer)
+  return footer
+}

packages/vhd-lib/src/chain.js

@@ -1,11 +1,11 @@
 import { dirname, relative } from 'path'
 
-import Vhd from './vhd'
+import { VhdFile } from './'
 import { DISK_TYPE_DIFFERENCING } from './_constants'
 
 export default async function chain(parentHandler, parentPath, childHandler, childPath, force = false) {
-  const parentVhd = new Vhd(parentHandler, parentPath)
-  const childVhd = new Vhd(childHandler, childPath)
+  const parentVhd = new VhdFile(parentHandler, parentPath)
+  const childVhd = new VhdFile(childHandler, childPath)
 
   await childVhd.readHeaderAndFooter()
   const { header, footer } = childVhd

packages/vhd-lib/src/checkChain.js

@@ -1,10 +1,10 @@
-import Vhd from './vhd'
+import { VhdFile } from '.'
 import resolveRelativeFromFile from './_resolveRelativeFromFile'
 import { DISK_TYPE_DYNAMIC } from './_constants'
 
 export default async function checkChain(handler, path) {
   while (true) {
-    const vhd = new Vhd(handler, path)
+    const vhd = new VhdFile(handler, path)
     await vhd.readHeaderAndFooter()
 
     if (vhd.footer.diskType === DISK_TYPE_DYNAMIC) {

packages/vhd-lib/src/createContentStream.js

@@ -1,11 +1,11 @@
 import asyncIteratorToStream from 'async-iterator-to-stream'
 
-import Vhd from './vhd'
+import { VhdFile } from '.'
 
 export default asyncIteratorToStream(async function* (handler, path) {
   const fd = await handler.openFile(path, 'r')
   try {
-    const vhd = new Vhd(handler, fd)
+    const vhd = new VhdFile(handler, fd)
     await vhd.readHeaderAndFooter()
     await vhd.readBlockAllocationTable()
     const {
@@ -17,10 +17,10 @@ export default asyncIteratorToStream(async function* (handler, path) {
 
     const emptyBlock = Buffer.alloc(blockSize)
     for (let i = 0; i < nFullBlocks; ++i) {
-      yield vhd.containsBlock(i) ? (await vhd._readBlock(i)).data : emptyBlock
+      yield vhd.containsBlock(i) ? (await vhd.readBlock(i)).data : emptyBlock
     }
     if (nLeftoverBytes !== 0) {
-      yield (vhd.containsBlock(nFullBlocks) ? (await vhd._readBlock(nFullBlocks)).data : emptyBlock).slice(
+      yield (vhd.containsBlock(nFullBlocks) ? (await vhd.readBlock(nFullBlocks)).data : emptyBlock).slice(
         0,
         nLeftoverBytes
       )

packages/vhd-lib/src/createSyntheticStream.js

@@ -3,7 +3,7 @@ import { createLogger } from '@xen-orchestra/log'
 
 import resolveRelativeFromFile from './_resolveRelativeFromFile'
 
-import Vhd from './vhd'
+import { VhdFile } from '.'
 import { BLOCK_UNUSED, DISK_TYPE_DYNAMIC, FOOTER_SIZE, HEADER_SIZE, SECTOR_SIZE } from './_constants'
 import { fuFooter, fuHeader, checksumStruct } from './_structs'
 import { test as mapTestBit } from './_bitmap'
@@ -27,7 +27,7 @@ export default async function createSyntheticStream(handler, paths) {
     const open = async path => {
       const fd = await handler.openFile(path, 'r')
       fds.push(fd)
-      const vhd = new Vhd(handler, fd)
+      const vhd = new VhdFile(handler, fd)
       vhds.push(vhd)
       await vhd.readHeaderAndFooter()
       await vhd.readBlockAllocationTable()
@@ -126,7 +126,7 @@ export default async function createSyntheticStream(handler, paths) {
             }
             let block = blocksByVhd.get(vhd)
             if (block === undefined) {
-              block = yield vhd._readBlock(iBlock)
+              block = yield vhd.readBlock(iBlock)
               blocksByVhd.set(vhd, block)
             }
             const { bitmap, data } = block

packages/vhd-lib/src/index.js

@@ -1,11 +1,12 @@
-export { default } from './vhd'
 export { default as chainVhd } from './chain'
+export { default as checkFooter } from './checkFooter'
 export { default as checkVhdChain } from './checkChain'
 export { default as createContentStream } from './createContentStream'
 export { default as createReadableRawStream } from './createReadableRawStream'
 export { default as createReadableSparseStream } from './createReadableSparseStream'
 export { default as createSyntheticStream } from './createSyntheticStream'
-export { default as mergeVhd } from './merge'
 export { default as createVhdStreamWithLength } from './createVhdStreamWithLength'
+export { default as mergeVhd } from './merge'
 export { default as peekFooterFromVhdStream } from './peekFooterFromVhdStream'
-export { default as checkFooter } from './checkFooter'
+export { VhdDirectory } from './Vhd/VhdDirectory'
+export { VhdFile } from './Vhd/VhdFile'

packages/vhd-lib/src/merge.integ.spec.js

@@ -11,7 +11,7 @@ import { pFromCallback } from 'promise-toolbox'
 import { pipeline } from 'readable-stream'
 import { randomBytes } from 'crypto'
 
-import Vhd, { chainVhd, createSyntheticStream, mergeVhd as vhdMerge } from './index'
+import { VhdFile, chainVhd, createSyntheticStream, mergeVhd as vhdMerge } from './index'
 
 import { SECTOR_SIZE } from './_constants'
 
@@ -61,7 +61,7 @@ test('blocks can be moved', async () => {
   await convertFromRawToVhd(rawFileName, vhdFileName)
   const handler = getHandler({ url: 'file://' })
   const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new Vhd(handler, vhdFileName)
+  const newVhd = new VhdFile(handler, vhdFileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   await newVhd._freeFirstBlockSpace(8000000)
@@ -75,7 +75,7 @@ test('the BAT MSB is not used for sign', async () => {
   const emptyFileName = `${tempDir}/empty.vhd`
   await execa('qemu-img', ['create', '-fvpc', emptyFileName, '1.8T'])
   const handler = getHandler({ url: 'file://' })
-  const vhd = new Vhd(handler, emptyFileName)
+  const vhd = new VhdFile(handler, emptyFileName)
   await vhd.readHeaderAndFooter()
   await vhd.readBlockAllocationTable()
   // we want the bit 31 to be on, to prove it's not been used for sign
@@ -92,13 +92,12 @@ test('the BAT MSB is not used for sign', async () => {
   const recoveredFileName = `${tempDir}/recovered`
   const recoveredFile = await fs.open(recoveredFileName, 'w')
   try {
-    const vhd2 = new Vhd(handler, emptyFileName)
+    const vhd2 = new VhdFile(handler, emptyFileName)
     await vhd2.readHeaderAndFooter()
     await vhd2.readBlockAllocationTable()
     for (let i = 0; i < vhd.header.maxTableEntries; i++) {
-      const entry = vhd._getBatEntry(i)
-      if (entry !== 0xffffffff) {
-        const block = (await vhd2._readBlock(i)).data
+      if (vhd.containsBlock(i)) {
+        const block = (await vhd2.readBlock(i)).data
         await fs.write(recoveredFile, block, 0, block.length, vhd2.header.blockSize * i)
       }
     }
@@ -123,7 +122,7 @@ test('writeData on empty file', async () => {
   const randomData = await fs.readFile(rawFileName)
   const handler = getHandler({ url: 'file://' })
   const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new Vhd(handler, emptyFileName)
+  const newVhd = new VhdFile(handler, emptyFileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   await newVhd.writeData(0, randomData)
@@ -142,7 +141,7 @@ test('writeData in 2 non-overlaping operations', async () => {
   const randomData = await fs.readFile(rawFileName)
   const handler = getHandler({ url: 'file://' })
   const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new Vhd(handler, emptyFileName)
+  const newVhd = new VhdFile(handler, emptyFileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   const splitPointSectors = 2
@@ -162,7 +161,7 @@ test('writeData in 2 overlaping operations', async () => {
   const randomData = await fs.readFile(rawFileName)
   const handler = getHandler({ url: 'file://' })
   const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new Vhd(handler, emptyFileName)
+  const newVhd = new VhdFile(handler, emptyFileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   const endFirstWrite = 3
@@ -182,7 +181,7 @@ test('BAT can be extended and blocks moved', async () => {
   await convertFromRawToVhd(rawFileName, vhdFileName)
   const handler = getHandler({ url: 'file://' })
   const originalSize = await handler.getSize(rawFileName)
-  const newVhd = new Vhd(handler, vhdFileName)
+  const newVhd = new VhdFile(handler, vhdFileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   await newVhd.ensureBatSize(2000)
@@ -226,7 +225,7 @@ test('coalesce works in normal cases', async () => {
   await convertFromRawToVhd(randomFileName, child1FileName)
   const handler = getHandler({ url: 'file://' })
   await execa('vhd-util', ['snapshot', '-n', child2FileName, '-p', child1FileName])
-  const vhd = new Vhd(handler, child2FileName)
+  const vhd = new VhdFile(handler, child2FileName)
   await vhd.readHeaderAndFooter()
   await vhd.readBlockAllocationTable()
   vhd.footer.creatorApplication = 'xoa'
@@ -238,7 +237,7 @@ test('coalesce works in normal cases', async () => {
   await chainVhd(handler, child1FileName, handler, child2FileName, true)
   await execa('vhd-util', ['check', '-t', '-n', child2FileName])
   const smallRandom = await fs.readFile(smallRandomFileName)
-  const newVhd = new Vhd(handler, child2FileName)
+  const newVhd = new VhdFile(handler, child2FileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   await newVhd.writeData(5, smallRandom)

packages/vhd-lib/src/merge.js

@@ -5,7 +5,7 @@ import noop from './_noop'
 import { createLogger } from '@xen-orchestra/log'
 import { limitConcurrency } from 'limit-concurrency-decorator'
 
-import Vhd from './vhd'
+import { VhdFile } from '.'
 import { basename, dirname } from 'path'
 import { DISK_TYPE_DIFFERENCING, DISK_TYPE_DYNAMIC } from './_constants'
 
@@ -25,10 +25,10 @@ export default limitConcurrency(2)(async function merge(
 
   const parentFd = await parentHandler.openFile(parentPath, 'r+')
   try {
-    const parentVhd = new Vhd(parentHandler, parentFd)
+    const parentVhd = new VhdFile(parentHandler, parentFd)
     const childFd = await childHandler.openFile(childPath, 'r')
     try {
-      const childVhd = new Vhd(childHandler, childFd)
+      const childVhd = new VhdFile(childHandler, childFd)
 
       let mergeState = await parentHandler.readFile(mergeStatePath).catch(error => {
         if (error.code !== 'ENOENT') {