From 447f2f95066a59489792e1731180155abcbe0a7e Mon Sep 17 00:00:00 2001 From: Nicolas Raynaud Date: Mon, 1 Apr 2019 07:53:03 -0700 Subject: [PATCH] fix(vhd-lib/createVhdStreamWithLength): handle empty VHD (#4107) Fixes #4105 --- CHANGELOG.unreleased.md | 3 + .../vhd-lib/src/_getFirstAndLastBlocks.js | 4 +- .../createVhdStreamWithLength.integ.spec.js | 153 +++++++++++------- .../vhd-lib/src/createVhdStreamWithLength.js | 10 +- packages/vhd-lib/src/vhd.js | 64 ++++---- 5 files changed, 136 insertions(+), 98 deletions(-) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index f490560d9..f75739a39 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -4,7 +4,10 @@ ### Bug fixes +- [Continuous Replication] Fix VHD size guess for empty files [#4105](https://github.com/vatesfr/xen-orchestra/issues/4105) (PR [#4107](https://github.com/vatesfr/xen-orchestra/pull/4107)) + ### Released packages +- vhd-lib v0.6.1 - xo-server v5.39.0 - xo-web v5.39.0 diff --git a/packages/vhd-lib/src/_getFirstAndLastBlocks.js b/packages/vhd-lib/src/_getFirstAndLastBlocks.js index dadd29ee0..81896f7c9 100644 --- a/packages/vhd-lib/src/_getFirstAndLastBlocks.js +++ b/packages/vhd-lib/src/_getFirstAndLastBlocks.js @@ -19,9 +19,7 @@ export default bat => { j += 4 if (j === n) { - const error = new Error('no allocated block found') - error.noBlock = true - throw error + return } } lastSector = firstSector diff --git a/packages/vhd-lib/src/createVhdStreamWithLength.integ.spec.js b/packages/vhd-lib/src/createVhdStreamWithLength.integ.spec.js index 1737fa599..7b6682d39 100644 --- a/packages/vhd-lib/src/createVhdStreamWithLength.integ.spec.js +++ b/packages/vhd-lib/src/createVhdStreamWithLength.integ.spec.js @@ -23,71 +23,110 @@ afterEach(async () => { await pFromCallback(cb => rimraf(tempDir, cb)) }) -async function convertFromRawToVhd(rawName, vhdName) { - await execa('qemu-img', ['convert', '-f', 'raw', '-Ovpc', rawName, vhdName]) -} +const RAW = 'raw' +const VHD = 'vpc' +const convert = (inputFormat, inputFile, outputFormat, outputFile) => + execa('qemu-img', [ + 'convert', + '-f', + inputFormat, + '-O', + outputFormat, + inputFile, + outputFile, + ]) + +const createRandomStream = asyncIteratorToStream(function*(size) { + let requested = Math.min(size, yield) + while (size > 0) { + const buf = Buffer.allocUnsafe(requested) + for (let i = 0; i < requested; ++i) { + buf[i] = Math.floor(Math.random() * 256) + } + requested = Math.min((size -= requested), yield buf) + } +}) async function createRandomFile(name, size) { - const createRandomStream = asyncIteratorToStream(function*(size) { - while (size-- > 0) { - yield Buffer.from([Math.floor(Math.random() * 256)]) - } - }) const input = await createRandomStream(size) await pFromCallback(cb => pipeline(input, fs.createWriteStream(name), cb)) } -test('createVhdStreamWithLength can extract length', async () => { - const initialSize = 4 * 1024 - const rawFileName = `${tempDir}/randomfile` - const vhdName = `${tempDir}/randomfile.vhd` - const outputVhdName = `${tempDir}/output.vhd` - await createRandomFile(rawFileName, initialSize) - await convertFromRawToVhd(rawFileName, vhdName) - const vhdSize = fs.statSync(vhdName).size - const result = await createVhdStreamWithLength( - await createReadStream(vhdName) - ) - expect(result.length).toEqual(vhdSize) - const outputFileStream = await createWriteStream(outputVhdName) - await pFromCallback(cb => pipeline(result, outputFileStream, cb)) - const outputSize = fs.statSync(outputVhdName).size - expect(outputSize).toEqual(vhdSize) -}) +const forOwn = (object, cb) => + Object.keys(object).forEach(key => cb(object[key], key, object)) -test('createVhdStreamWithLength can skip blank after last block and before footer', async () => { - const initialSize = 4 * 1024 - const rawFileName = `${tempDir}/randomfile` - const vhdName = `${tempDir}/randomfile.vhd` - const outputVhdName = `${tempDir}/output.vhd` - await createRandomFile(rawFileName, initialSize) - await convertFromRawToVhd(rawFileName, vhdName) - const vhdSize = fs.statSync(vhdName).size - // read file footer - const footer = await getStream.buffer( - createReadStream(vhdName, { start: vhdSize - FOOTER_SIZE }) +describe('createVhdStreamWithLength', () => { + forOwn( + { + // qemu-img requires this length or it fill with null bytes which breaks + // the test + 'can extract length': 34816, + + 'can handle empty file': 0, + }, + (size, title) => + it(title, async () => { + const inputRaw = `${tempDir}/input.raw` + await createRandomFile(inputRaw, size) + + const inputVhd = `${tempDir}/input.vhd` + await convert(RAW, inputRaw, VHD, inputVhd) + + const result = await createVhdStreamWithLength( + await createReadStream(inputVhd) + ) + const { length } = result + + const outputVhd = `${tempDir}/output.vhd` + await pFromCallback( + pipeline.bind(undefined, result, await createWriteStream(outputVhd)) + ) + + // ensure the guessed length correspond to the stream length + const { size: outputSize } = await fs.stat(outputVhd) + expect(length).toEqual(outputSize) + + // ensure the generated VHD is correct and contains the same data + const outputRaw = `${tempDir}/output.raw` + await convert(VHD, outputVhd, RAW, outputRaw) + await execa('cmp', [inputRaw, outputRaw]) + }) ) - // we'll override the footer - const endOfFile = await createWriteStream(vhdName, { - flags: 'r+', - start: vhdSize - FOOTER_SIZE, + it('can skip blank after the last block and before the footer', async () => { + const initialSize = 4 * 1024 + const rawFileName = `${tempDir}/randomfile` + const vhdName = `${tempDir}/randomfile.vhd` + const outputVhdName = `${tempDir}/output.vhd` + await createRandomFile(rawFileName, initialSize) + await convert(RAW, rawFileName, VHD, vhdName) + const { size: vhdSize } = await fs.stat(vhdName) + // read file footer + const footer = await getStream.buffer( + createReadStream(vhdName, { start: vhdSize - FOOTER_SIZE }) + ) + + // we'll override the footer + const endOfFile = await createWriteStream(vhdName, { + flags: 'r+', + start: vhdSize - FOOTER_SIZE, + }) + // write a blank over the previous footer + await pFromCallback(cb => endOfFile.write(Buffer.alloc(FOOTER_SIZE), cb)) + // write the footer after the new blank + await pFromCallback(cb => endOfFile.end(footer, cb)) + const { size: longerSize } = await fs.stat(vhdName) + // check input file has been lengthened + expect(longerSize).toEqual(vhdSize + FOOTER_SIZE) + const result = await createVhdStreamWithLength( + await createReadStream(vhdName) + ) + expect(result.length).toEqual(vhdSize) + const outputFileStream = await createWriteStream(outputVhdName) + await pFromCallback(cb => pipeline(result, outputFileStream, cb)) + const { size: outputSize } = await fs.stat(outputVhdName) + // check out file has been shortened again + expect(outputSize).toEqual(vhdSize) + await execa('qemu-img', ['compare', outputVhdName, vhdName]) }) - // write a blank over the previous footer - await pFromCallback(cb => endOfFile.write(Buffer.alloc(FOOTER_SIZE), cb)) - // write the footer after the new blank - await pFromCallback(cb => endOfFile.end(footer, cb)) - const longerSize = fs.statSync(vhdName).size - // check input file has been lengthened - expect(longerSize).toEqual(vhdSize + FOOTER_SIZE) - const result = await createVhdStreamWithLength( - await createReadStream(vhdName) - ) - expect(result.length).toEqual(vhdSize) - const outputFileStream = await createWriteStream(outputVhdName) - await pFromCallback(cb => pipeline(result, outputFileStream, cb)) - const outputSize = fs.statSync(outputVhdName).size - // check out file has been shortened again - expect(outputSize).toEqual(vhdSize) - await execa('qemu-img', ['compare', outputVhdName, vhdName]) }) diff --git a/packages/vhd-lib/src/createVhdStreamWithLength.js b/packages/vhd-lib/src/createVhdStreamWithLength.js index ac19de8e0..0712ce878 100644 --- a/packages/vhd-lib/src/createVhdStreamWithLength.js +++ b/packages/vhd-lib/src/createVhdStreamWithLength.js @@ -63,10 +63,14 @@ export default async function createVhdStreamWithLength(stream) { stream.unshift(buf) } + const firstAndLastBlocks = getFirstAndLastBlocks(table) const footerOffset = - getFirstAndLastBlocks(table).lastSector * SECTOR_SIZE + - Math.ceil(header.blockSize / SECTOR_SIZE / 8 / SECTOR_SIZE) * SECTOR_SIZE + - header.blockSize + firstAndLastBlocks !== undefined + ? firstAndLastBlocks.lastSector * SECTOR_SIZE + + Math.ceil(header.blockSize / SECTOR_SIZE / 8 / SECTOR_SIZE) * + SECTOR_SIZE + + header.blockSize + : Math.ceil(streamPosition / SECTOR_SIZE) * SECTOR_SIZE // ignore any data after footerOffset and push footerBuffer // diff --git a/packages/vhd-lib/src/vhd.js b/packages/vhd-lib/src/vhd.js index 6f67b6cf0..c80331aae 100644 --- a/packages/vhd-lib/src/vhd.js +++ b/packages/vhd-lib/src/vhd.js @@ -253,43 +253,37 @@ export default class Vhd { } async _freeFirstBlockSpace(spaceNeededBytes) { - try { - const { first, firstSector, lastSector } = getFirstAndLastBlocks( - this.blockTable + const firstAndLastBlocks = getFirstAndLastBlocks(this.blockTable) + if (firstAndLastBlocks === undefined) { + return + } + + const { first, firstSector, lastSector } = firstAndLastBlocks + const tableOffset = this.header.tableOffset + const { batSize } = this + const newMinSector = Math.ceil( + (tableOffset + batSize + spaceNeededBytes) / SECTOR_SIZE + ) + if ( + tableOffset + batSize + spaceNeededBytes >= + sectorsToBytes(firstSector) + ) { + const { fullBlockSize } = this + const newFirstSector = Math.max( + lastSector + fullBlockSize / SECTOR_SIZE, + newMinSector ) - const tableOffset = this.header.tableOffset - const { batSize } = this - const newMinSector = Math.ceil( - (tableOffset + batSize + spaceNeededBytes) / SECTOR_SIZE + debug( + `freeFirstBlockSpace: move first block ${firstSector} -> ${newFirstSector}` ) - if ( - tableOffset + batSize + spaceNeededBytes >= - sectorsToBytes(firstSector) - ) { - const { fullBlockSize } = this - const newFirstSector = Math.max( - lastSector + fullBlockSize / SECTOR_SIZE, - newMinSector - ) - debug( - `freeFirstBlockSpace: move first block ${firstSector} -> ${newFirstSector}` - ) - // copy the first block at the end - const block = await this._read( - sectorsToBytes(firstSector), - fullBlockSize - ) - await this._write(block, sectorsToBytes(newFirstSector)) - await this._setBatEntry(first, newFirstSector) - await this.writeFooter(true) - spaceNeededBytes -= this.fullBlockSize - if (spaceNeededBytes > 0) { - return this._freeFirstBlockSpace(spaceNeededBytes) - } - } - } catch (e) { - if (!e.noBlock) { - throw e + // copy the first block at the end + const block = await this._read(sectorsToBytes(firstSector), fullBlockSize) + await this._write(block, sectorsToBytes(newFirstSector)) + await this._setBatEntry(first, newFirstSector) + await this.writeFooter(true) + spaceNeededBytes -= this.fullBlockSize + if (spaceNeededBytes > 0) { + return this._freeFirstBlockSpace(spaceNeededBytes) } } }