fix(vhd-lib/createVhdStreamWithLength): handle empty VHD (#4107)

Fixes #4105

CHANGELOG.unreleased.md

@@ -4,7 +4,10 @@
 
 ### Bug fixes
 
+- [Continuous Replication] Fix VHD size guess for empty files [#4105](https://github.com/vatesfr/xen-orchestra/issues/4105)  (PR [#4107](https://github.com/vatesfr/xen-orchestra/pull/4107))
+
 ### Released packages
 
+- vhd-lib v0.6.1
 - xo-server v5.39.0
 - xo-web v5.39.0

packages/vhd-lib/src/_getFirstAndLastBlocks.js

@@ -19,9 +19,7 @@ export default bat => {
     j += 4
 
     if (j === n) {
-      const error = new Error('no allocated block found')
+      return
-      error.noBlock = true
-      throw error
     }
   }
   lastSector = firstSector

packages/vhd-lib/src/createVhdStreamWithLength.integ.spec.js

@@ -23,71 +23,110 @@ afterEach(async () => {
   await pFromCallback(cb => rimraf(tempDir, cb))
 })
 
-async function convertFromRawToVhd(rawName, vhdName) {
+const RAW = 'raw'
-  await execa('qemu-img', ['convert', '-f', 'raw', '-Ovpc', rawName, vhdName])
+const VHD = 'vpc'
-}
+const convert = (inputFormat, inputFile, outputFormat, outputFile) =>
+  execa('qemu-img', [
+    'convert',
+    '-f',
+    inputFormat,
+    '-O',
+    outputFormat,
+    inputFile,
+    outputFile,
+  ])
+
+const createRandomStream = asyncIteratorToStream(function*(size) {
+  let requested = Math.min(size, yield)
+  while (size > 0) {
+    const buf = Buffer.allocUnsafe(requested)
+    for (let i = 0; i < requested; ++i) {
+      buf[i] = Math.floor(Math.random() * 256)
+    }
+    requested = Math.min((size -= requested), yield buf)
+  }
+})
 
 async function createRandomFile(name, size) {
-  const createRandomStream = asyncIteratorToStream(function*(size) {
-    while (size-- > 0) {
-      yield Buffer.from([Math.floor(Math.random() * 256)])
-    }
-  })
   const input = await createRandomStream(size)
   await pFromCallback(cb => pipeline(input, fs.createWriteStream(name), cb))
 }
 
-test('createVhdStreamWithLength can extract length', async () => {
+const forOwn = (object, cb) =>
-  const initialSize = 4 * 1024
+  Object.keys(object).forEach(key => cb(object[key], key, object))
-  const rawFileName = `${tempDir}/randomfile`
-  const vhdName = `${tempDir}/randomfile.vhd`
-  const outputVhdName = `${tempDir}/output.vhd`
-  await createRandomFile(rawFileName, initialSize)
-  await convertFromRawToVhd(rawFileName, vhdName)
-  const vhdSize = fs.statSync(vhdName).size
-  const result = await createVhdStreamWithLength(
-    await createReadStream(vhdName)
-  )
-  expect(result.length).toEqual(vhdSize)
-  const outputFileStream = await createWriteStream(outputVhdName)
-  await pFromCallback(cb => pipeline(result, outputFileStream, cb))
-  const outputSize = fs.statSync(outputVhdName).size
-  expect(outputSize).toEqual(vhdSize)
-})
 
-test('createVhdStreamWithLength can skip blank after last block and before footer', async () => {
+describe('createVhdStreamWithLength', () => {
-  const initialSize = 4 * 1024
+  forOwn(
-  const rawFileName = `${tempDir}/randomfile`
+    {
-  const vhdName = `${tempDir}/randomfile.vhd`
+      // qemu-img requires this length or it fill with null bytes which breaks
-  const outputVhdName = `${tempDir}/output.vhd`
+      // the test
-  await createRandomFile(rawFileName, initialSize)
+      'can extract length': 34816,
-  await convertFromRawToVhd(rawFileName, vhdName)
+
-  const vhdSize = fs.statSync(vhdName).size
+      'can handle empty file': 0,
-  // read file footer
+    },
-  const footer = await getStream.buffer(
+    (size, title) =>
-    createReadStream(vhdName, { start: vhdSize - FOOTER_SIZE })
+      it(title, async () => {
+        const inputRaw = `${tempDir}/input.raw`
+        await createRandomFile(inputRaw, size)
+
+        const inputVhd = `${tempDir}/input.vhd`
+        await convert(RAW, inputRaw, VHD, inputVhd)
+
+        const result = await createVhdStreamWithLength(
+          await createReadStream(inputVhd)
+        )
+        const { length } = result
+
+        const outputVhd = `${tempDir}/output.vhd`
+        await pFromCallback(
+          pipeline.bind(undefined, result, await createWriteStream(outputVhd))
+        )
+
+        // ensure the guessed length correspond to the stream length
+        const { size: outputSize } = await fs.stat(outputVhd)
+        expect(length).toEqual(outputSize)
+
+        // ensure the generated VHD is correct and contains the same data
+        const outputRaw = `${tempDir}/output.raw`
+        await convert(VHD, outputVhd, RAW, outputRaw)
+        await execa('cmp', [inputRaw, outputRaw])
+      })
   )
 
-  // we'll override the footer
+  it('can skip blank after the last block and before the footer', async () => {
-  const endOfFile = await createWriteStream(vhdName, {
+    const initialSize = 4 * 1024
-    flags: 'r+',
+    const rawFileName = `${tempDir}/randomfile`
-    start: vhdSize - FOOTER_SIZE,
+    const vhdName = `${tempDir}/randomfile.vhd`
+    const outputVhdName = `${tempDir}/output.vhd`
+    await createRandomFile(rawFileName, initialSize)
+    await convert(RAW, rawFileName, VHD, vhdName)
+    const { size: vhdSize } = await fs.stat(vhdName)
+    // read file footer
+    const footer = await getStream.buffer(
+      createReadStream(vhdName, { start: vhdSize - FOOTER_SIZE })
+    )
+
+    // we'll override the footer
+    const endOfFile = await createWriteStream(vhdName, {
+      flags: 'r+',
+      start: vhdSize - FOOTER_SIZE,
+    })
+    // write a blank over the previous footer
+    await pFromCallback(cb => endOfFile.write(Buffer.alloc(FOOTER_SIZE), cb))
+    // write the footer after the new blank
+    await pFromCallback(cb => endOfFile.end(footer, cb))
+    const { size: longerSize } = await fs.stat(vhdName)
+    // check input file has been lengthened
+    expect(longerSize).toEqual(vhdSize + FOOTER_SIZE)
+    const result = await createVhdStreamWithLength(
+      await createReadStream(vhdName)
+    )
+    expect(result.length).toEqual(vhdSize)
+    const outputFileStream = await createWriteStream(outputVhdName)
+    await pFromCallback(cb => pipeline(result, outputFileStream, cb))
+    const { size: outputSize } = await fs.stat(outputVhdName)
+    // check out file has been shortened again
+    expect(outputSize).toEqual(vhdSize)
+    await execa('qemu-img', ['compare', outputVhdName, vhdName])
   })
-  // write a blank over the previous footer
-  await pFromCallback(cb => endOfFile.write(Buffer.alloc(FOOTER_SIZE), cb))
-  // write the footer after the new blank
-  await pFromCallback(cb => endOfFile.end(footer, cb))
-  const longerSize = fs.statSync(vhdName).size
-  // check input file has been lengthened
-  expect(longerSize).toEqual(vhdSize + FOOTER_SIZE)
-  const result = await createVhdStreamWithLength(
-    await createReadStream(vhdName)
-  )
-  expect(result.length).toEqual(vhdSize)
-  const outputFileStream = await createWriteStream(outputVhdName)
-  await pFromCallback(cb => pipeline(result, outputFileStream, cb))
-  const outputSize = fs.statSync(outputVhdName).size
-  // check out file has been shortened again
-  expect(outputSize).toEqual(vhdSize)
-  await execa('qemu-img', ['compare', outputVhdName, vhdName])
 })

packages/vhd-lib/src/createVhdStreamWithLength.js

@@ -63,10 +63,14 @@ export default async function createVhdStreamWithLength(stream) {
     stream.unshift(buf)
   }
 
+  const firstAndLastBlocks = getFirstAndLastBlocks(table)
   const footerOffset =
-    getFirstAndLastBlocks(table).lastSector * SECTOR_SIZE +
+    firstAndLastBlocks !== undefined
-    Math.ceil(header.blockSize / SECTOR_SIZE / 8 / SECTOR_SIZE) * SECTOR_SIZE +
+      ? firstAndLastBlocks.lastSector * SECTOR_SIZE +
-    header.blockSize
+        Math.ceil(header.blockSize / SECTOR_SIZE / 8 / SECTOR_SIZE) *
+          SECTOR_SIZE +
+        header.blockSize
+      : Math.ceil(streamPosition / SECTOR_SIZE) * SECTOR_SIZE
 
   // ignore any data after footerOffset and push footerBuffer
   //

packages/vhd-lib/src/vhd.js

@@ -253,43 +253,37 @@ export default class Vhd {
   }
 
   async _freeFirstBlockSpace(spaceNeededBytes) {
-    try {
+    const firstAndLastBlocks = getFirstAndLastBlocks(this.blockTable)
-      const { first, firstSector, lastSector } = getFirstAndLastBlocks(
+    if (firstAndLastBlocks === undefined) {
-        this.blockTable
+      return
+    }
+
+    const { first, firstSector, lastSector } = firstAndLastBlocks
+    const tableOffset = this.header.tableOffset
+    const { batSize } = this
+    const newMinSector = Math.ceil(
+      (tableOffset + batSize + spaceNeededBytes) / SECTOR_SIZE
+    )
+    if (
+      tableOffset + batSize + spaceNeededBytes >=
+      sectorsToBytes(firstSector)
+    ) {
+      const { fullBlockSize } = this
+      const newFirstSector = Math.max(
+        lastSector + fullBlockSize / SECTOR_SIZE,
+        newMinSector
       )
-      const tableOffset = this.header.tableOffset
+      debug(
-      const { batSize } = this
+        `freeFirstBlockSpace: move first block ${firstSector} -> ${newFirstSector}`
-      const newMinSector = Math.ceil(
-        (tableOffset + batSize + spaceNeededBytes) / SECTOR_SIZE
       )
-      if (
+      // copy the first block at the end
-        tableOffset + batSize + spaceNeededBytes >=
+      const block = await this._read(sectorsToBytes(firstSector), fullBlockSize)
-        sectorsToBytes(firstSector)
+      await this._write(block, sectorsToBytes(newFirstSector))
-      ) {
+      await this._setBatEntry(first, newFirstSector)
-        const { fullBlockSize } = this
+      await this.writeFooter(true)
-        const newFirstSector = Math.max(
+      spaceNeededBytes -= this.fullBlockSize
-          lastSector + fullBlockSize / SECTOR_SIZE,
+      if (spaceNeededBytes > 0) {
-          newMinSector
+        return this._freeFirstBlockSpace(spaceNeededBytes)
-        )
-        debug(
-          `freeFirstBlockSpace: move first block ${firstSector} -> ${newFirstSector}`
-        )
-        // copy the first block at the end
-        const block = await this._read(
-          sectorsToBytes(firstSector),
-          fullBlockSize
-        )
-        await this._write(block, sectorsToBytes(newFirstSector))
-        await this._setBatEntry(first, newFirstSector)
-        await this.writeFooter(true)
-        spaceNeededBytes -= this.fullBlockSize
-        if (spaceNeededBytes > 0) {
-          return this._freeFirstBlockSpace(spaceNeededBytes)
-        }
-      }
-    } catch (e) {
-      if (!e.noBlock) {
-        throw e
       }
     }
   }