From 005a9fdc01066accda331f442f5101770eb7aa1d Mon Sep 17 00:00:00 2001 From: Nicolas Raynaud Date: Fri, 18 May 2018 05:02:19 -0700 Subject: [PATCH] fix(xo-vmdk-to-vhd): various bugs (#2961) --- .../vhd-lib/src/createReadableSparseStream.js | 113 ++++++++---------- packages/vhd-lib/src/vhd.integ.spec.js | 6 +- packages/xo-vmdk-to-vhd/package.json | 1 - .../src/virtual-buffer.integ.spec.js | 2 +- .../src/vmdk-read.integ.spec.js | 2 +- packages/xo-vmdk-to-vhd/src/vmdk-read.js | 12 +- .../src/vmdk-to-vhd.integ.spec.js | 5 +- packages/xo-web/src/xo-app/vm-import/index.js | 7 +- 8 files changed, 65 insertions(+), 83 deletions(-) diff --git a/packages/vhd-lib/src/createReadableSparseStream.js b/packages/vhd-lib/src/createReadableSparseStream.js index 943674e8f..cc81f8b55 100644 --- a/packages/vhd-lib/src/createReadableSparseStream.js +++ b/packages/vhd-lib/src/createReadableSparseStream.js @@ -1,3 +1,4 @@ +import assert from 'assert' import asyncIteratorToStream from 'async-iterator-to-stream' import computeGeometryForSize from './_computeGeometryForSize' @@ -25,62 +26,16 @@ function createBAT ( bat, bitmapSize ) { - const vhdOccupationTable = [] let currentVhdPositionSector = firstBlockPosition / SECTOR_SIZE blockAddressList.forEach(blockPosition => { - const scaled = blockPosition / VHD_BLOCK_SIZE_BYTES - const vhdTableIndex = Math.floor(scaled) + assert.strictEqual(blockPosition % 512, 0) + const vhdTableIndex = Math.floor(blockPosition / VHD_BLOCK_SIZE_BYTES) if (bat.readUInt32BE(vhdTableIndex * 4) === BLOCK_UNUSED) { bat.writeUInt32BE(currentVhdPositionSector, vhdTableIndex * 4) currentVhdPositionSector += (bitmapSize + VHD_BLOCK_SIZE_BYTES) / SECTOR_SIZE } - // not using bit operators to avoid the int32 coercion, that way we can go to 53 bits - vhdOccupationTable[vhdTableIndex] = - (vhdOccupationTable[vhdTableIndex] || 0) + - Math.pow(2, (scaled % 1) * ratio) }) - return vhdOccupationTable -} - -function createBitmap (bitmapSize, ratio, vhdOccupationBucket) { - const bitmap = Buffer.alloc(bitmapSize) - for (let i = 0; i < VHD_BLOCK_SIZE_SECTORS / ratio; i++) { - // do not shift to avoid int32 coercion - if ((vhdOccupationBucket * Math.pow(2, -i)) & 1) { - for (let j = 0; j < ratio; j++) { - setBitmap(bitmap, i * ratio + j) - } - } - } - return bitmap -} - -function * yieldIfNotEmpty (buffer) { - if (buffer.length > 0) { - yield buffer - } -} - -async function * generateFileContent ( - blockIterator, - bitmapSize, - ratio, - vhdOccupationTable -) { - let currentVhdBlockIndex = -1 - let currentBlockBuffer = Buffer.alloc(0) - for await (const next of blockIterator) { - const batEntry = Math.floor(next.offsetBytes / VHD_BLOCK_SIZE_BYTES) - if (batEntry !== currentVhdBlockIndex) { - yield * yieldIfNotEmpty(currentBlockBuffer) - currentBlockBuffer = Buffer.alloc(VHD_BLOCK_SIZE_BYTES) - currentVhdBlockIndex = batEntry - yield createBitmap(bitmapSize, ratio, vhdOccupationTable[batEntry]) - } - next.data.copy(currentBlockBuffer, next.offsetBytes % VHD_BLOCK_SIZE_BYTES) - } - yield * yieldIfNotEmpty(currentBlockBuffer) } export default asyncIteratorToStream(async function * ( @@ -123,21 +78,49 @@ export default asyncIteratorToStream(async function * ( const bitmapSize = Math.ceil(VHD_BLOCK_SIZE_SECTORS / 8 / SECTOR_SIZE) * SECTOR_SIZE const bat = Buffer.alloc(tablePhysicalSizeBytes, 0xff) - const vhdOccupationTable = createBAT( - firstBlockPosition, - blockAddressList, - ratio, - bat, - bitmapSize - ) - yield footer - yield header - yield bat - yield * generateFileContent( - blockIterator, - bitmapSize, - ratio, - vhdOccupationTable - ) - yield footer + createBAT(firstBlockPosition, blockAddressList, ratio, bat, bitmapSize) + let position = 0 + function * yieldAndTrack (buffer, expectedPosition) { + if (expectedPosition !== undefined) { + assert.strictEqual(position, expectedPosition) + } + if (buffer.length > 0) { + yield buffer + position += buffer.length + } + } + async function * generateFileContent (blockIterator, bitmapSize, ratio) { + let currentBlock = -1 + let currentVhdBlockIndex = -1 + let currentBlockWithBitmap = Buffer.alloc(0) + for await (const next of blockIterator) { + currentBlock++ + assert.strictEqual(blockAddressList[currentBlock], next.offsetBytes) + const batIndex = Math.floor(next.offsetBytes / VHD_BLOCK_SIZE_BYTES) + if (batIndex !== currentVhdBlockIndex) { + if (currentVhdBlockIndex >= 0) { + yield * yieldAndTrack( + currentBlockWithBitmap, + bat.readUInt32BE(currentVhdBlockIndex * 4) * 512 + ) + } + currentBlockWithBitmap = Buffer.alloc(bitmapSize + VHD_BLOCK_SIZE_BYTES) + currentVhdBlockIndex = batIndex + } + const blockOffset = (next.offsetBytes / 512) % VHD_BLOCK_SIZE_SECTORS + for (let bitPos = 0; bitPos < VHD_BLOCK_SIZE_SECTORS / ratio; bitPos++) { + setBitmap(currentBlockWithBitmap, blockOffset + bitPos) + } + next.data.copy( + currentBlockWithBitmap, + bitmapSize + next.offsetBytes % VHD_BLOCK_SIZE_BYTES + ) + } + yield * yieldAndTrack(currentBlockWithBitmap) + } + yield * yieldAndTrack(footer, 0) + yield * yieldAndTrack(header, FOOTER_SIZE) + yield * yieldAndTrack(bat, FOOTER_SIZE + HEADER_SIZE) + yield * generateFileContent(blockIterator, bitmapSize, ratio) + yield * yieldAndTrack(footer) }) diff --git a/packages/vhd-lib/src/vhd.integ.spec.js b/packages/vhd-lib/src/vhd.integ.spec.js index 8412fcd37..f828f51ab 100644 --- a/packages/vhd-lib/src/vhd.integ.spec.js +++ b/packages/vhd-lib/src/vhd.integ.spec.js @@ -102,15 +102,15 @@ test('ReadableSparseVHDStream can handle a sparse file', async () => { data: Buffer.alloc(blockSize, 'azerzaerazeraze', 'ascii'), }, { - offsetBytes: blockSize * 5, + offsetBytes: blockSize * 100, data: Buffer.alloc(blockSize, 'gdfslkdfguer', 'ascii'), }, ] - const fileSize = blockSize * 10 + const fileSize = blockSize * 110 const stream = createReadableSparseVHDStream( fileSize, blockSize, - [100, 700], + blocks.map(b => b.offsetBytes), blocks ) const pipe = stream.pipe(createWriteStream('output.vhd')) diff --git a/packages/xo-vmdk-to-vhd/package.json b/packages/xo-vmdk-to-vhd/package.json index 5a25f9179..15910babd 100644 --- a/packages/xo-vmdk-to-vhd/package.json +++ b/packages/xo-vmdk-to-vhd/package.json @@ -25,7 +25,6 @@ "dependencies": { "@babel/runtime": "^7.0.0-beta.44", "child-process-promise": "^2.0.3", - "fs-promise": "^2.0.0", "pipette": "^0.9.3", "promise-toolbox": "^0.9.5", "tmp": "^0.0.33", diff --git a/packages/xo-vmdk-to-vhd/src/virtual-buffer.integ.spec.js b/packages/xo-vmdk-to-vhd/src/virtual-buffer.integ.spec.js index 40d66b190..cfb2ef287 100644 --- a/packages/xo-vmdk-to-vhd/src/virtual-buffer.integ.spec.js +++ b/packages/xo-vmdk-to-vhd/src/virtual-buffer.integ.spec.js @@ -1,6 +1,6 @@ /* eslint-env jest */ -import { createReadStream, readFile } from 'fs-promise' +import { createReadStream, readFile } from 'fs-extra' import { exec } from 'child-process-promise' import { fromCallback as pFromCallback } from 'promise-toolbox' import rimraf from 'rimraf' diff --git a/packages/xo-vmdk-to-vhd/src/vmdk-read.integ.spec.js b/packages/xo-vmdk-to-vhd/src/vmdk-read.integ.spec.js index 13568dd93..c4270efc6 100644 --- a/packages/xo-vmdk-to-vhd/src/vmdk-read.integ.spec.js +++ b/packages/xo-vmdk-to-vhd/src/vmdk-read.integ.spec.js @@ -1,6 +1,6 @@ /* eslint-env jest */ -import { createReadStream } from 'fs-promise' +import { createReadStream } from 'fs-extra' import { exec } from 'child-process-promise' import { fromCallback as pFromCallback } from 'promise-toolbox' import rimraf from 'rimraf' diff --git a/packages/xo-vmdk-to-vhd/src/vmdk-read.js b/packages/xo-vmdk-to-vhd/src/vmdk-read.js index bf059a4d2..afb855872 100644 --- a/packages/xo-vmdk-to-vhd/src/vmdk-read.js +++ b/packages/xo-vmdk-to-vhd/src/vmdk-read.js @@ -308,17 +308,15 @@ export class VMDKDirectParser { export async function readVmdkGrainTable (fileAccessor) { let headerBuffer = await fileAccessor(0, 512) - let grainDirAddr = headerBuffer.slice(56, 56 + 8) + let grainAddrBuffer = headerBuffer.slice(56, 56 + 8) if ( - new Int8Array(grainDirAddr).reduce((acc, val) => acc && val === -1, true) + new Int8Array(grainAddrBuffer).reduce((acc, val) => acc && val === -1, true) ) { headerBuffer = await fileAccessor(-1024, -1024 + 512) - grainDirAddr = new DataView(headerBuffer.slice(56, 56 + 8)).getUint32( - 0, - true - ) + grainAddrBuffer = headerBuffer.slice(56, 56 + 8) } - const grainDirPosBytes = grainDirAddr * 512 + const grainDirPosBytes = + new DataView(grainAddrBuffer).getUint32(0, true) * 512 const capacity = new DataView(headerBuffer.slice(12, 12 + 8)).getUint32(0, true) * 512 const grainSize = diff --git a/packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js b/packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js index da91e81bc..042750346 100644 --- a/packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js +++ b/packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js @@ -6,7 +6,7 @@ import getStream from 'get-stream' import rimraf from 'rimraf' import tmp from 'tmp' -import { createReadStream, createWriteStream, stat } from 'fs-promise' +import { createReadStream, createWriteStream, stat } from 'fs-extra' import { fromCallback as pFromCallback } from 'promise-toolbox' import convertFromVMDK, { readVmdkGrainTable } from '.' @@ -49,7 +49,7 @@ test('VMDK to VHD can convert a random data file with VMDKDirectParser', async ( const vhdFileName = 'from-vmdk-VMDKDirectParser.vhd' const reconvertedFromVhd = 'from-vhd.raw' const reconvertedFromVmdk = 'from-vhd-by-vbox.raw' - const dataSize = 8355840 // this number is an integer head/cylinder/count equation solution + const dataSize = 100 * 1024 * 1024 // this number is an integer head/cylinder/count equation solution try { await execa.shell( 'base64 /dev/urandom | head -c ' + dataSize + ' > ' + inputRawFileName @@ -82,6 +82,7 @@ test('VMDK to VHD can convert a random data file with VMDKDirectParser', async ( reconvertedFromVhd, ]) await execa('qemu-img', ['compare', inputRawFileName, vhdFileName]) + await execa('qemu-img', ['compare', vmdkFileName, vhdFileName]) } catch (error) { console.error(error.stdout) console.error(error.stderr) diff --git a/packages/xo-web/src/xo-app/vm-import/index.js b/packages/xo-web/src/xo-app/vm-import/index.js index 897854938..452a9c525 100644 --- a/packages/xo-web/src/xo-app/vm-import/index.js +++ b/packages/xo-web/src/xo-app/vm-import/index.js @@ -235,9 +235,10 @@ const parseFile = async (file, type, func) => { } } -const getRedirectionUrl = vms => vms.length === 1 - ? `/vms/${vms[0]}` - : `/home?s=${encodeURIComponent(`id:|(${vms.join(' ')})`)}&t=VM` +const getRedirectionUrl = vms => + vms.length === 1 + ? `/vms/${vms[0]}` + : `/home?s=${encodeURIComponent(`id:|(${vms.join(' ')})`)}&t=VM` export default class Import extends Component { constructor (props) {