fix(vhd-lib/createVhdStreamWithLength): handle empty VHD (#4107)

Fixes #4105

CHANGELOG.unreleased.md

@@ -4,7 +4,10 @@
 
 ### Bug fixes
 
+- [Continuous Replication] Fix VHD size guess for empty files [#4105](https://github.com/vatesfr/xen-orchestra/issues/4105)  (PR [#4107](https://github.com/vatesfr/xen-orchestra/pull/4107))
+
 ### Released packages
 
+- vhd-lib v0.6.1
 - xo-server v5.39.0
 - xo-web v5.39.0

packages/vhd-lib/src/_getFirstAndLastBlocks.js

@@ -19,9 +19,7 @@ export default bat => {
     j += 4
 
     if (j === n) {
-      const error = new Error('no allocated block found')
+      return
-      error.noBlock = true
-      throw error
     }
   }
   lastSector = firstSector

packages/vhd-lib/src/createVhdStreamWithLength.integ.spec.js

@@ -23,46 +23,84 @@ afterEach(async () => {
   await pFromCallback(cb => rimraf(tempDir, cb))
 })
 
-async function convertFromRawToVhd(rawName, vhdName) {
+const RAW = 'raw'
-  await execa('qemu-img', ['convert', '-f', 'raw', '-Ovpc', rawName, vhdName])
+const VHD = 'vpc'
-}
+const convert = (inputFormat, inputFile, outputFormat, outputFile) =>
+  execa('qemu-img', [
+    'convert',
+    '-f',
+    inputFormat,
+    '-O',
+    outputFormat,
+    inputFile,
+    outputFile,
+  ])
+
+const createRandomStream = asyncIteratorToStream(function*(size) {
+  let requested = Math.min(size, yield)
+  while (size > 0) {
+    const buf = Buffer.allocUnsafe(requested)
+    for (let i = 0; i < requested; ++i) {
+      buf[i] = Math.floor(Math.random() * 256)
+    }
+    requested = Math.min((size -= requested), yield buf)
+  }
+})
 
 async function createRandomFile(name, size) {
-  const createRandomStream = asyncIteratorToStream(function*(size) {
-    while (size-- > 0) {
-      yield Buffer.from([Math.floor(Math.random() * 256)])
-    }
-  })
   const input = await createRandomStream(size)
   await pFromCallback(cb => pipeline(input, fs.createWriteStream(name), cb))
 }
 
-test('createVhdStreamWithLength can extract length', async () => {
+const forOwn = (object, cb) =>
-  const initialSize = 4 * 1024
+  Object.keys(object).forEach(key => cb(object[key], key, object))
-  const rawFileName = `${tempDir}/randomfile`
-  const vhdName = `${tempDir}/randomfile.vhd`
-  const outputVhdName = `${tempDir}/output.vhd`
-  await createRandomFile(rawFileName, initialSize)
-  await convertFromRawToVhd(rawFileName, vhdName)
-  const vhdSize = fs.statSync(vhdName).size
-  const result = await createVhdStreamWithLength(
-    await createReadStream(vhdName)
-  )
-  expect(result.length).toEqual(vhdSize)
-  const outputFileStream = await createWriteStream(outputVhdName)
-  await pFromCallback(cb => pipeline(result, outputFileStream, cb))
-  const outputSize = fs.statSync(outputVhdName).size
-  expect(outputSize).toEqual(vhdSize)
-})
 
-test('createVhdStreamWithLength can skip blank after last block and before footer', async () => {
+describe('createVhdStreamWithLength', () => {
+  forOwn(
+    {
+      // qemu-img requires this length or it fill with null bytes which breaks
+      // the test
+      'can extract length': 34816,
+
+      'can handle empty file': 0,
+    },
+    (size, title) =>
+      it(title, async () => {
+        const inputRaw = `${tempDir}/input.raw`
+        await createRandomFile(inputRaw, size)
+
+        const inputVhd = `${tempDir}/input.vhd`
+        await convert(RAW, inputRaw, VHD, inputVhd)
+
+        const result = await createVhdStreamWithLength(
+          await createReadStream(inputVhd)
+        )
+        const { length } = result
+
+        const outputVhd = `${tempDir}/output.vhd`
+        await pFromCallback(
+          pipeline.bind(undefined, result, await createWriteStream(outputVhd))
+        )
+
+        // ensure the guessed length correspond to the stream length
+        const { size: outputSize } = await fs.stat(outputVhd)
+        expect(length).toEqual(outputSize)
+
+        // ensure the generated VHD is correct and contains the same data
+        const outputRaw = `${tempDir}/output.raw`
+        await convert(VHD, outputVhd, RAW, outputRaw)
+        await execa('cmp', [inputRaw, outputRaw])
+      })
+  )
+
+  it('can skip blank after the last block and before the footer', async () => {
     const initialSize = 4 * 1024
     const rawFileName = `${tempDir}/randomfile`
     const vhdName = `${tempDir}/randomfile.vhd`
     const outputVhdName = `${tempDir}/output.vhd`
     await createRandomFile(rawFileName, initialSize)
-  await convertFromRawToVhd(rawFileName, vhdName)
+    await convert(RAW, rawFileName, VHD, vhdName)
-  const vhdSize = fs.statSync(vhdName).size
+    const { size: vhdSize } = await fs.stat(vhdName)
     // read file footer
     const footer = await getStream.buffer(
       createReadStream(vhdName, { start: vhdSize - FOOTER_SIZE })
@@ -77,7 +115,7 @@ test('createVhdStreamWithLength can skip blank after last block and before foote
     await pFromCallback(cb => endOfFile.write(Buffer.alloc(FOOTER_SIZE), cb))
     // write the footer after the new blank
     await pFromCallback(cb => endOfFile.end(footer, cb))
-  const longerSize = fs.statSync(vhdName).size
+    const { size: longerSize } = await fs.stat(vhdName)
     // check input file has been lengthened
     expect(longerSize).toEqual(vhdSize + FOOTER_SIZE)
     const result = await createVhdStreamWithLength(
@@ -86,8 +124,9 @@ test('createVhdStreamWithLength can skip blank after last block and before foote
     expect(result.length).toEqual(vhdSize)
     const outputFileStream = await createWriteStream(outputVhdName)
     await pFromCallback(cb => pipeline(result, outputFileStream, cb))
-  const outputSize = fs.statSync(outputVhdName).size
+    const { size: outputSize } = await fs.stat(outputVhdName)
     // check out file has been shortened again
     expect(outputSize).toEqual(vhdSize)
     await execa('qemu-img', ['compare', outputVhdName, vhdName])
+  })
 })

packages/vhd-lib/src/createVhdStreamWithLength.js

@@ -63,10 +63,14 @@ export default async function createVhdStreamWithLength(stream) {
     stream.unshift(buf)
   }
 
+  const firstAndLastBlocks = getFirstAndLastBlocks(table)
   const footerOffset =
-    getFirstAndLastBlocks(table).lastSector * SECTOR_SIZE +
+    firstAndLastBlocks !== undefined
-    Math.ceil(header.blockSize / SECTOR_SIZE / 8 / SECTOR_SIZE) * SECTOR_SIZE +
+      ? firstAndLastBlocks.lastSector * SECTOR_SIZE +
+        Math.ceil(header.blockSize / SECTOR_SIZE / 8 / SECTOR_SIZE) *
+          SECTOR_SIZE +
         header.blockSize
+      : Math.ceil(streamPosition / SECTOR_SIZE) * SECTOR_SIZE
 
   // ignore any data after footerOffset and push footerBuffer
   //

packages/vhd-lib/src/vhd.js

@@ -253,10 +253,12 @@ export default class Vhd {
   }
 
   async _freeFirstBlockSpace(spaceNeededBytes) {
-    try {
+    const firstAndLastBlocks = getFirstAndLastBlocks(this.blockTable)
-      const { first, firstSector, lastSector } = getFirstAndLastBlocks(
+    if (firstAndLastBlocks === undefined) {
-        this.blockTable
+      return
-      )
+    }
+
+    const { first, firstSector, lastSector } = firstAndLastBlocks
     const tableOffset = this.header.tableOffset
     const { batSize } = this
     const newMinSector = Math.ceil(
@@ -275,10 +277,7 @@ export default class Vhd {
         `freeFirstBlockSpace: move first block ${firstSector} -> ${newFirstSector}`
       )
       // copy the first block at the end
-        const block = await this._read(
+      const block = await this._read(sectorsToBytes(firstSector), fullBlockSize)
-          sectorsToBytes(firstSector),
-          fullBlockSize
-        )
       await this._write(block, sectorsToBytes(newFirstSector))
       await this._setBatEntry(first, newFirstSector)
       await this.writeFooter(true)
@@ -287,11 +286,6 @@ export default class Vhd {
         return this._freeFirstBlockSpace(spaceNeededBytes)
       }
     }
-    } catch (e) {
-      if (!e.noBlock) {
-        throw e
-      }
-    }
   }
 
   async ensureBatSize(entries) {