fix(xo-vmdk-to-vhd): various bugs (#2961)

packages/vhd-lib/src/createReadableSparseStream.js

@@ -1,3 +1,4 @@
+import assert from 'assert'
 import asyncIteratorToStream from 'async-iterator-to-stream'
 
 import computeGeometryForSize from './_computeGeometryForSize'
@@ -25,62 +26,16 @@ function createBAT (
   bat,
   bitmapSize
 ) {
-  const vhdOccupationTable = []
   let currentVhdPositionSector = firstBlockPosition / SECTOR_SIZE
   blockAddressList.forEach(blockPosition => {
-    const scaled = blockPosition / VHD_BLOCK_SIZE_BYTES
-    const vhdTableIndex = Math.floor(scaled)
+    assert.strictEqual(blockPosition % 512, 0)
+    const vhdTableIndex = Math.floor(blockPosition / VHD_BLOCK_SIZE_BYTES)
     if (bat.readUInt32BE(vhdTableIndex * 4) === BLOCK_UNUSED) {
       bat.writeUInt32BE(currentVhdPositionSector, vhdTableIndex * 4)
       currentVhdPositionSector +=
         (bitmapSize + VHD_BLOCK_SIZE_BYTES) / SECTOR_SIZE
     }
-    // not using bit operators to avoid the int32 coercion, that way we can go to 53 bits
-    vhdOccupationTable[vhdTableIndex] =
-      (vhdOccupationTable[vhdTableIndex] || 0) +
-      Math.pow(2, (scaled % 1) * ratio)
   })
-  return vhdOccupationTable
-}
-
-function createBitmap (bitmapSize, ratio, vhdOccupationBucket) {
-  const bitmap = Buffer.alloc(bitmapSize)
-  for (let i = 0; i < VHD_BLOCK_SIZE_SECTORS / ratio; i++) {
-    // do not shift to avoid int32 coercion
-    if ((vhdOccupationBucket * Math.pow(2, -i)) & 1) {
-      for (let j = 0; j < ratio; j++) {
-        setBitmap(bitmap, i * ratio + j)
-      }
-    }
-  }
-  return bitmap
-}
-
-function * yieldIfNotEmpty (buffer) {
-  if (buffer.length > 0) {
-    yield buffer
-  }
-}
-
-async function * generateFileContent (
-  blockIterator,
-  bitmapSize,
-  ratio,
-  vhdOccupationTable
-) {
-  let currentVhdBlockIndex = -1
-  let currentBlockBuffer = Buffer.alloc(0)
-  for await (const next of blockIterator) {
-    const batEntry = Math.floor(next.offsetBytes / VHD_BLOCK_SIZE_BYTES)
-    if (batEntry !== currentVhdBlockIndex) {
-      yield * yieldIfNotEmpty(currentBlockBuffer)
-      currentBlockBuffer = Buffer.alloc(VHD_BLOCK_SIZE_BYTES)
-      currentVhdBlockIndex = batEntry
-      yield createBitmap(bitmapSize, ratio, vhdOccupationTable[batEntry])
-    }
-    next.data.copy(currentBlockBuffer, next.offsetBytes % VHD_BLOCK_SIZE_BYTES)
-  }
-  yield * yieldIfNotEmpty(currentBlockBuffer)
 }
 
 export default asyncIteratorToStream(async function * (
@@ -123,21 +78,49 @@ export default asyncIteratorToStream(async function * (
   const bitmapSize =
     Math.ceil(VHD_BLOCK_SIZE_SECTORS / 8 / SECTOR_SIZE) * SECTOR_SIZE
   const bat = Buffer.alloc(tablePhysicalSizeBytes, 0xff)
-  const vhdOccupationTable = createBAT(
-    firstBlockPosition,
-    blockAddressList,
-    ratio,
-    bat,
-    bitmapSize
-  )
-  yield footer
-  yield header
-  yield bat
-  yield * generateFileContent(
-    blockIterator,
-    bitmapSize,
-    ratio,
-    vhdOccupationTable
-  )
-  yield footer
+  createBAT(firstBlockPosition, blockAddressList, ratio, bat, bitmapSize)
+  let position = 0
+  function * yieldAndTrack (buffer, expectedPosition) {
+    if (expectedPosition !== undefined) {
+      assert.strictEqual(position, expectedPosition)
+    }
+    if (buffer.length > 0) {
+      yield buffer
+      position += buffer.length
+    }
+  }
+  async function * generateFileContent (blockIterator, bitmapSize, ratio) {
+    let currentBlock = -1
+    let currentVhdBlockIndex = -1
+    let currentBlockWithBitmap = Buffer.alloc(0)
+    for await (const next of blockIterator) {
+      currentBlock++
+      assert.strictEqual(blockAddressList[currentBlock], next.offsetBytes)
+      const batIndex = Math.floor(next.offsetBytes / VHD_BLOCK_SIZE_BYTES)
+      if (batIndex !== currentVhdBlockIndex) {
+        if (currentVhdBlockIndex >= 0) {
+          yield * yieldAndTrack(
+            currentBlockWithBitmap,
+            bat.readUInt32BE(currentVhdBlockIndex * 4) * 512
+          )
+        }
+        currentBlockWithBitmap = Buffer.alloc(bitmapSize + VHD_BLOCK_SIZE_BYTES)
+        currentVhdBlockIndex = batIndex
+      }
+      const blockOffset = (next.offsetBytes / 512) % VHD_BLOCK_SIZE_SECTORS
+      for (let bitPos = 0; bitPos < VHD_BLOCK_SIZE_SECTORS / ratio; bitPos++) {
+        setBitmap(currentBlockWithBitmap, blockOffset + bitPos)
+      }
+      next.data.copy(
+        currentBlockWithBitmap,
+        bitmapSize + next.offsetBytes % VHD_BLOCK_SIZE_BYTES
+      )
+    }
+    yield * yieldAndTrack(currentBlockWithBitmap)
+  }
+  yield * yieldAndTrack(footer, 0)
+  yield * yieldAndTrack(header, FOOTER_SIZE)
+  yield * yieldAndTrack(bat, FOOTER_SIZE + HEADER_SIZE)
+  yield * generateFileContent(blockIterator, bitmapSize, ratio)
+  yield * yieldAndTrack(footer)
 })

packages/vhd-lib/src/vhd.integ.spec.js

@@ -102,15 +102,15 @@ test('ReadableSparseVHDStream can handle a sparse file', async () => {
       data: Buffer.alloc(blockSize, 'azerzaerazeraze', 'ascii'),
     },
     {
-      offsetBytes: blockSize * 5,
+      offsetBytes: blockSize * 100,
       data: Buffer.alloc(blockSize, 'gdfslkdfguer', 'ascii'),
     },
   ]
-  const fileSize = blockSize * 10
+  const fileSize = blockSize * 110
   const stream = createReadableSparseVHDStream(
     fileSize,
     blockSize,
-    [100, 700],
+    blocks.map(b => b.offsetBytes),
     blocks
   )
   const pipe = stream.pipe(createWriteStream('output.vhd'))

packages/xo-vmdk-to-vhd/package.json

@@ -25,7 +25,6 @@
   "dependencies": {
     "@babel/runtime": "^7.0.0-beta.44",
     "child-process-promise": "^2.0.3",
-    "fs-promise": "^2.0.0",
     "pipette": "^0.9.3",
     "promise-toolbox": "^0.9.5",
     "tmp": "^0.0.33",

packages/xo-vmdk-to-vhd/src/virtual-buffer.integ.spec.js

@@ -1,6 +1,6 @@
 /* eslint-env jest */
 
-import { createReadStream, readFile } from 'fs-promise'
+import { createReadStream, readFile } from 'fs-extra'
 import { exec } from 'child-process-promise'
 import { fromCallback as pFromCallback } from 'promise-toolbox'
 import rimraf from 'rimraf'

packages/xo-vmdk-to-vhd/src/vmdk-read.integ.spec.js

@@ -1,6 +1,6 @@
 /* eslint-env jest */
 
-import { createReadStream } from 'fs-promise'
+import { createReadStream } from 'fs-extra'
 import { exec } from 'child-process-promise'
 import { fromCallback as pFromCallback } from 'promise-toolbox'
 import rimraf from 'rimraf'

packages/xo-vmdk-to-vhd/src/vmdk-read.js

@@ -308,17 +308,15 @@ export class VMDKDirectParser {
 
 export async function readVmdkGrainTable (fileAccessor) {
   let headerBuffer = await fileAccessor(0, 512)
-  let grainDirAddr = headerBuffer.slice(56, 56 + 8)
+  let grainAddrBuffer = headerBuffer.slice(56, 56 + 8)
   if (
-    new Int8Array(grainDirAddr).reduce((acc, val) => acc && val === -1, true)
+    new Int8Array(grainAddrBuffer).reduce((acc, val) => acc && val === -1, true)
   ) {
     headerBuffer = await fileAccessor(-1024, -1024 + 512)
-    grainDirAddr = new DataView(headerBuffer.slice(56, 56 + 8)).getUint32(
-      0,
-      true
-    )
+    grainAddrBuffer = headerBuffer.slice(56, 56 + 8)
   }
-  const grainDirPosBytes = grainDirAddr * 512
+  const grainDirPosBytes =
+    new DataView(grainAddrBuffer).getUint32(0, true) * 512
   const capacity =
     new DataView(headerBuffer.slice(12, 12 + 8)).getUint32(0, true) * 512
   const grainSize =

packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js

@@ -6,7 +6,7 @@ import getStream from 'get-stream'
 import rimraf from 'rimraf'
 import tmp from 'tmp'
 
-import { createReadStream, createWriteStream, stat } from 'fs-promise'
+import { createReadStream, createWriteStream, stat } from 'fs-extra'
 import { fromCallback as pFromCallback } from 'promise-toolbox'
 import convertFromVMDK, { readVmdkGrainTable } from '.'
 
@@ -49,7 +49,7 @@ test('VMDK to VHD can convert a random data file with VMDKDirectParser', async (
   const vhdFileName = 'from-vmdk-VMDKDirectParser.vhd'
   const reconvertedFromVhd = 'from-vhd.raw'
   const reconvertedFromVmdk = 'from-vhd-by-vbox.raw'
-  const dataSize = 8355840 // this number is an integer head/cylinder/count equation solution
+  const dataSize = 100 * 1024 * 1024 // this number is an integer head/cylinder/count equation solution
   try {
     await execa.shell(
       'base64 /dev/urandom | head -c ' + dataSize + ' > ' + inputRawFileName
@@ -82,6 +82,7 @@ test('VMDK to VHD can convert a random data file with VMDKDirectParser', async (
       reconvertedFromVhd,
     ])
     await execa('qemu-img', ['compare', inputRawFileName, vhdFileName])
+    await execa('qemu-img', ['compare', vmdkFileName, vhdFileName])
   } catch (error) {
     console.error(error.stdout)
     console.error(error.stderr)

packages/xo-web/src/xo-app/vm-import/index.js

@@ -235,9 +235,10 @@ const parseFile = async (file, type, func) => {
   }
 }
 
-const getRedirectionUrl = vms => vms.length === 1
-  ? `/vms/${vms[0]}`
-  : `/home?s=${encodeURIComponent(`id:|(${vms.join(' ')})`)}&t=VM`
+const getRedirectionUrl = vms =>
+  vms.length === 1
+    ? `/vms/${vms[0]}`
+    : `/home?s=${encodeURIComponent(`id:|(${vms.join(' ')})`)}&t=VM`
 
 export default class Import extends Component {
   constructor (props) {