feat(import/ova): allow import of gzipped vmdk disks (#5085)

CHANGELOG.unreleased.md

@@ -13,6 +13,7 @@
 - [New schedule] Enable 'Enable immediately after creation' by default (PR [#5111](https://github.com/vatesfr/xen-orchestra/pull/5111))
 - [Self Service] Ability to globally ignore snapshots in resource set quotas (PR [#5164](https://github.com/vatesfr/xen-orchestra/pull/5164))
 - [Home/VM, host] Ability to filter by power state (PR [#5118](https://github.com/vatesfr/xen-orchestra/pull/5118))
+- [Import/OVA] Allow for VMDK disks inside .ova files to be gzipped (PR [#5085](https://github.com/vatesfr/xen-orchestra/pull/5085))
 
 ### Bug fixes
 
@@ -39,6 +40,6 @@
 >
 > In case of conflict, the highest (lowest in previous list) `$version` wins.
 
-- xo-vmdk-to-vhd patch
+- xo-vmdk-to-vhd minor
 - xo-server minor
 - xo-web minor

packages/xo-server/src/xapi/index.js

@@ -1413,6 +1413,7 @@ export default class Xapi extends XapiBase {
 
     // 2. Create VDIs & Vifs.
     const vdis = {}
+    const compression = {}
     const vifDevices = await this.call('VM.get_allowed_VIF_devices', vm.$ref)
     await Promise.all(
       map(disks, async disk => {
@@ -1423,7 +1424,7 @@ export default class Xapi extends XapiBase {
           sr: sr.$ref,
         }))
         $defer.onFailure(() => this._deleteVdi(vdi.$ref))
-
+        compression[disk.path] = disk.compression
         return this.createVbd({
           userdevice: String(disk.position),
           vdi,
@@ -1454,12 +1455,12 @@ export default class Xapi extends XapiBase {
           stream.resume()
           return
         }
-
         const table = tables[entry.name]
         const vhdStream = await vmdkToVhd(
           stream,
           table.grainLogicalAddressList,
-          table.grainFileOffsetList
+          table.grainFileOffsetList,
+          compression[entry.name] === 'gzip'
         )
         try {
           await this._importVdiContent(vdi, vhdStream, VDI_FORMAT_VHD)

packages/xo-vmdk-to-vhd/package.json

@@ -28,6 +28,7 @@
     "child-process-promise": "^2.0.3",
     "core-js": "^3.0.0",
     "lodash": "^4.17.15",
+    "pako": "^1.0.11",
     "promise-toolbox": "^0.15.0",
     "vhd-lib": "^0.7.2",
     "xml2js": "^0.4.23"

packages/xo-vmdk-to-vhd/src/index.js

@@ -11,12 +11,14 @@ export {
 async function vmdkToVhd(
   vmdkReadStream,
   grainLogicalAddressList,
-  grainFileOffsetList
+  grainFileOffsetList,
+  gzipped = false
 ) {
   const parser = new VMDKDirectParser(
     vmdkReadStream,
     grainLogicalAddressList,
-    grainFileOffsetList
+    grainFileOffsetList,
+    gzipped
   )
   const header = await parser.readHeader()
   return createReadableSparseStream(

packages/xo-vmdk-to-vhd/src/ova.js

@@ -1,5 +1,7 @@
 import find from 'lodash/find'
 import forEach from 'lodash/forEach'
+import pako from 'pako'
+import sum from 'lodash/sum'
 import xml2js, { processors } from 'xml2js'
 
 import { readVmdkGrainTable } from '.'
@@ -108,11 +110,6 @@ function parseTarHeader(header, stringDeserializer) {
 }
 
 export class ParsableFile {
-  // noinspection JSMethodCanBeStatic
-  get size() {
-    return 0
-  }
-
   /** returns a ParsableFile */
   slice(start, end) {}
 
@@ -195,9 +192,9 @@ async function parseOVF(fileFragment, stringDeserializer) {
             capacity:
               disk.capacity * ((unit && allocationUnitsToFactor(unit)) || 1),
             path: file && file.href,
+            compression: file && file.compression,
           }
         })
-
         // Get hardware info: CPU, RAM, disks, networks...
         const handleItem = item => {
           const handler = RESOURCE_TYPE_TO_HANDLER[item.ResourceType]
@@ -218,6 +215,77 @@ async function parseOVF(fileFragment, stringDeserializer) {
   )
 }
 
+const GZIP_CHUNK_SIZE = 4 * 1024 * 1024
+
+async function parseGzipFromStart(start, end, fileSlice) {
+  let currentDeflatedPos = 0
+  let currentInflatedPos = 0
+  const inflate = new pako.Inflate()
+  const chunks = []
+  while (currentInflatedPos < end) {
+    const slice = fileSlice.slice(
+      currentDeflatedPos,
+      currentDeflatedPos + GZIP_CHUNK_SIZE
+    )
+    const compressed = await slice.read()
+    inflate.push(compressed, pako.Z_SYNC_FLUSH)
+    let chunk = inflate.result
+    const inflatedChunkEnd = currentInflatedPos + chunk.length
+    if (inflatedChunkEnd > start) {
+      if (currentInflatedPos < start) {
+        chunk = chunk.slice(start - currentInflatedPos)
+      }
+      if (inflatedChunkEnd > end) {
+        chunk = chunk.slice(0, -(inflatedChunkEnd - end))
+      }
+      chunks.push(chunk)
+    }
+    currentInflatedPos = inflatedChunkEnd
+    currentDeflatedPos += GZIP_CHUNK_SIZE
+  }
+  const resultBuffer = new Uint8Array(sum(chunks.map(c => c.length)))
+  let index = 0
+  chunks.forEach(c => {
+    resultBuffer.set(c, index)
+    index += c.length
+  })
+  return resultBuffer.buffer
+}
+
+// start and end are negative numbers
+// used with streamOptimized format where only the footer has the directory address filled
+async function parseGzipFromEnd(start, end, fileSlice, header) {
+  const l = end - start
+  const chunks = []
+  let savedSize = 0
+  let currentDeflatedPos = 0
+  const inflate = new pako.Inflate()
+  while (currentDeflatedPos < header.fileSize) {
+    const slice = fileSlice.slice(
+      currentDeflatedPos,
+      currentDeflatedPos + GZIP_CHUNK_SIZE
+    )
+    const compressed = await slice.read()
+    inflate.push(compressed, pako.Z_SYNC_FLUSH)
+    const chunk = inflate.result.slice()
+    chunks.push({ pos: currentDeflatedPos, buffer: chunk })
+    savedSize += chunk.length
+    if (savedSize - chunks[0].buffer.length >= l) {
+      savedSize -= chunks[0].buffer.length
+      chunks.shift()
+    }
+    currentDeflatedPos += GZIP_CHUNK_SIZE
+  }
+  let resultBuffer = new Uint8Array(sum(chunks.map(c => c.buffer.length)))
+  let index = 0
+  chunks.forEach(c => {
+    resultBuffer.set(c.buffer, index)
+    index += c.buffer.length
+  })
+  resultBuffer = resultBuffer.slice(start, end)
+  return resultBuffer.buffer
+}
+
 /**
  *
  * @param parsableFile: ParsableFile
@@ -262,6 +330,20 @@ export async function parseOVAFile(
         data.tables[header.fileName] = await readVmdkGrainTable(readFile)
       }
     }
+    if (!skipVmdk && header.fileName.toLowerCase().endsWith('.vmdk.gz')) {
+      const fileSlice = parsableFile.slice(offset, offset + header.fileSize)
+      const readFile = async (start, end) => {
+        if (start === end) {
+          return new Uint8Array(0)
+        }
+        if (start >= 0 && end >= 0) {
+          return parseGzipFromStart(start, end, fileSlice)
+        } else if (start < 0 && end < 0) {
+          return parseGzipFromEnd(start, end, fileSlice, header)
+        }
+      }
+      data.tables[header.fileName] = await readVmdkGrainTable(readFile)
+    }
     offset += Math.ceil(header.fileSize / 512) * 512
   }
   return data

packages/xo-vmdk-to-vhd/src/vmdk-read-table.js

@@ -40,6 +40,44 @@ export default async function readVmdkGrainTable(fileAccessor) {
   return (await readCapacityAndGrainTable(fileAccessor)).tablePromise
 }
 
+/**
+ * reading a big chunk of the file to memory before parsing is useful when the vmdk is gzipped and random access is costly
+ */
+async function grabTables(
+  grainDirectoryEntries,
+  grainDir,
+  grainTablePhysicalSize,
+  fileAccessor
+) {
+  const cachedGrainTables = []
+  let grainTableAddresMin = Infinity
+  let grainTableAddressMax = -Infinity
+  for (let i = 0; i < grainDirectoryEntries; i++) {
+    const grainTableAddr = grainDir[i] * SECTOR_SIZE
+    if (grainTableAddr !== 0) {
+      grainTableAddresMin = Math.min(grainTableAddresMin, grainTableAddr)
+      grainTableAddressMax = Math.max(
+        grainTableAddressMax,
+        grainTableAddr + grainTablePhysicalSize
+      )
+    }
+  }
+  const grainTableBuffer = await fileAccessor(
+    grainTableAddresMin,
+    grainTableAddressMax
+  )
+  for (let i = 0; i < grainDirectoryEntries; i++) {
+    const grainTableAddr = grainDir[i] * SECTOR_SIZE
+    if (grainTableAddr !== 0) {
+      const addr = grainTableAddr - grainTableAddresMin
+      cachedGrainTables[i] = new Uint32Array(
+        grainTableBuffer.slice(addr, addr + grainTablePhysicalSize)
+      )
+    }
+  }
+  return cachedGrainTables
+}
+
 /***
  *
  * @param fileAccessor: (start, end) => ArrayBuffer
@@ -84,18 +122,12 @@ export async function readCapacityAndGrainTable(fileAccessor) {
         grainDirPosBytes + grainDirectoryPhysicalSize
       )
     )
-    const cachedGrainTables = []
-    for (let i = 0; i < grainDirectoryEntries; i++) {
-      const grainTableAddr = grainDir[i] * SECTOR_SIZE
-      if (grainTableAddr !== 0) {
-        cachedGrainTables[i] = new Uint32Array(
-          await fileAccessor(
-            grainTableAddr,
-            grainTableAddr + grainTablePhysicalSize
-          )
-        )
-      }
-    }
+    const cachedGrainTables = await grabTables(
+      grainDirectoryEntries,
+      grainDir,
+      grainTablePhysicalSize,
+      fileAccessor
+    )
     const extractedGrainTable = []
     for (let i = 0; i < grainCount; i++) {
       const directoryEntry = Math.floor(i / numGTEsPerGT)

packages/xo-vmdk-to-vhd/src/vmdk-read.js

@@ -67,7 +67,17 @@ function alignSectors(number) {
 }
 
 export default class VMDKDirectParser {
-  constructor(readStream, grainLogicalAddressList, grainFileOffsetList) {
+  constructor(
+    readStream,
+    grainLogicalAddressList,
+    grainFileOffsetList,
+    gzipped = false
+  ) {
+    if (gzipped) {
+      const unzipStream = zlib.createGunzip()
+      readStream.pipe(unzipStream)
+      readStream = unzipStream
+    }
     this.grainLogicalAddressList = grainLogicalAddressList
     this.grainFileOffsetList = grainFileOffsetList
     this.virtualBuffer = new VirtualBuffer(readStream)

packages/xo-web/src/xo-app/vm-import/index.js

@@ -60,6 +60,7 @@ class VmData extends Component {
         descriptionLabel: PropTypes.string.isRequired,
         nameLabel: PropTypes.string.isRequired,
         path: PropTypes.string.isRequired,
+        compression: PropTypes.string,
       })
     ),
     memory: PropTypes.number,
@@ -73,13 +74,17 @@ class VmData extends Component {
     const { props, refs } = this
     return {
       descriptionLabel: refs.descriptionLabel.value,
-      disks: map(props.disks, ({ capacity, path, position }, diskId) => ({
-        capacity,
-        descriptionLabel: refs[`disk-description-${diskId}`].value,
-        nameLabel: refs[`disk-name-${diskId}`].value,
-        path,
-        position,
-      })),
+      disks: map(
+        props.disks,
+        ({ capacity, path, compression, position }, diskId) => ({
+          capacity,
+          descriptionLabel: refs[`disk-description-${diskId}`].value,
+          nameLabel: refs[`disk-name-${diskId}`].value,
+          path,
+          position,
+          compression,
+        })
+      ),
       memory: +refs.memory.value,
       nameLabel: refs.nameLabel.value,
       networks: map(props.networks, (_, networkId) => {

yarn.lock

@@ -13452,7 +13452,7 @@ pad@^3.2.0:
   dependencies:
     wcwidth "^1.0.1"
 
-pako@^1.0.3, pako@~1.0.5:
+pako@^1.0.11, pako@^1.0.3, pako@~1.0.5:
   version "1.0.11"
   resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf"
   integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==