feat(upload-ova): new CLI to import OVA VMs (#3630)

@xen-orchestra/upload-ova/.babelrc.js

@@ -0,0 +1,3 @@
+module.exports = require('../../@xen-orchestra/babel-config')(
+  require('./package.json')
+)

@xen-orchestra/upload-ova/.npmignore

@@ -0,0 +1,24 @@
+/benchmark/
+/benchmarks/
+*.bench.js
+*.bench.js.map
+
+/examples/
+example.js
+example.js.map
+*.example.js
+*.example.js.map
+
+/fixture/
+/fixtures/
+*.fixture.js
+*.fixture.js.map
+*.fixtures.js
+*.fixtures.js.map
+
+/test/
+/tests/
+*.spec.js
+*.spec.js.map
+
+__snapshots__/

@xen-orchestra/upload-ova/README.md

@@ -0,0 +1,62 @@
+# XO-UPLOAD-OVA
+
+> Basic CLI to upload ova files to Xen-Orchestra
+
+## Usage
+
+```
+Usage:
+
+  xo-upload-ova --register [--expiresIn duration] <XO-Server URL> <username> [<password>]
+    Registers the XO instance to use.
+
+    --expiresIn duration
+      Can be used to change the validity duration of the
+      authorization token (default: one month).
+
+  xo-upload-ova --unregister
+    Remove stored credentials.
+
+  xo-upload-ova --inspect <file>
+    Displays the data that would be imported from the ova.
+
+  xo-upload-ova --upload <file> <sr> [--override <key>=<value> [<key>=<value>]+]
+    Actually imports the VM contained in <file> to the Storage Repository <sr>.
+    Some parameters can be overridden from the file, consult --inspect to get the list.
+    Note: --override has to come last. By default arguments are string, prefix them with <json:> to type
+    them, ex. " --override nameLabel='new VM'  memory=json:67108864 disks.vmdisk1.capacity=json:134217728"
+
+xo-upload-ova v0.1.0
+
+```
+
+#### Register your XO instance
+
+```
+> xo-upload-ova --register http://xo.my-company.net admin@admin.net admin
+Successfully logged with admin@admin.net
+```
+
+Note: only a token will be saved in the configuration file.
+
+#### Import your .ova file
+
+```
+> xo-upload-ova --upload dsl.ova a7c630bf-b38c-489e-d3c3-e62507948980 --override 'nameLabel=dsl ' descriptionLabel='short desc' memory=json:671088640 disks.vmdisk1.descriptionLabel='disk description' disks.vmdisk1.capacity=json:1342177280
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+XO-UPLOAD-OVA is released under the [AGPL
+v3](http://www.gnu.org/licenses/agpl-3.0-standalone.html).

@xen-orchestra/upload-ova/package.json

@@ -0,0 +1,72 @@
+{
+  "name": "@xen-orchestra/upload-ova",
+  "version": "0.0.0",
+  "license": "AGPL-3.0",
+  "description": "Basic CLI to upload ova files to Xen-Orchestra",
+  "keywords": [
+    "import",
+    "orchestra",
+    "ova",
+    "xcp-ng",
+    "xcp",
+    "xen-orchestra",
+    "xen-server",
+    "xen",
+    "xo"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/upload-ova",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@xen-orchestra/upload-ova",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "preferGlobal": true,
+  "main": "dist/",
+  "bin": {
+    "xo-upload-ova": "dist/index.js"
+  },
+  "files": [
+    "dist/"
+  ],
+  "engines": {
+    "node": ">=8.10"
+  },
+  "dependencies": {
+    "@babel/polyfill": "^7.0.0",
+    "chalk": "^2.2.0",
+    "exec-promise": "^0.7.0",
+    "fs-extra": "^7.0.0",
+    "fs-promise": "^2.0.3",
+    "get-stream": "^4.1.0",
+    "http-request-plus": "^0.8.0",
+    "human-format": "^0.10.0",
+    "l33teral": "^3.0.3",
+    "lodash": "^4.17.4",
+    "nice-pipe": "0.0.0",
+    "pretty-ms": "^4.0.0",
+    "progress-stream": "^2.0.0",
+    "pw": "^0.0.4",
+    "strip-indent": "^2.0.0",
+    "xdg-basedir": "^3.0.0",
+    "xo-lib": "^0.9.0",
+    "xo-vmdk-to-vhd": "^0.1.6"
+  },
+  "devDependencies": {
+    "@babel/cli": "^7.0.0",
+    "@babel/core": "^7.0.0",
+    "@babel/preset-env": "^7.0.0",
+    "@babel/preset-flow": "^7.0.0",
+    "babel-plugin-lodash": "^3.3.2",
+    "cross-env": "^5.1.3",
+    "rimraf": "^2.6.2"
+  },
+  "scripts": {
+    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
+    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
+    "prebuild": "rimraf dist/",
+    "predev": "yarn run prebuild",
+    "prepublishOnly": "yarn run build"
+  },
+  "private": true
+}

@xen-orchestra/upload-ova/src/config.js

@@ -0,0 +1,44 @@
+'use strict'
+
+// ===================================================================
+
+import assign from 'lodash/assign'
+import l33t from 'l33teral'
+import xdgBasedir from 'xdg-basedir'
+
+import { mkdirp, readFile, writeFile } from 'fs-extra'
+
+const configPath = xdgBasedir.config + '/xo-upload-ova'
+const configFile = configPath + '/config.json'
+
+export async function load() {
+  try {
+    return JSON.parse(await readFile(configFile))
+  } catch (e) {
+    return {}
+  }
+}
+
+export async function get(path) {
+  const config = await load()
+  return l33t(config).tap(path)
+}
+
+export async function save(config) {
+  await mkdirp(configPath)
+  await writeFile(configFile, JSON.stringify(config))
+}
+
+export async function set(data) {
+  const config = await load()
+  await save(assign(config, data))
+}
+
+export async function unset(paths) {
+  const config = await load()
+  const l33tConfig = l33t(config)
+  ;[].concat(paths).forEach(function(path) {
+    l33tConfig.purge(path, true)
+  })
+  return save(config)
+}

@xen-orchestra/upload-ova/src/index.js

@@ -0,0 +1,306 @@
+/* eslint no-console: "off" */
+
+import chalk from 'chalk'
+import execPromise from 'exec-promise'
+import { createReadStream } from 'fs'
+import { stat } from 'fs-promise'
+import getStream from 'get-stream'
+import hrp from 'http-request-plus'
+import humanFormat from 'human-format'
+import l33t from 'l33teral'
+import isObject from 'lodash/isObject'
+import getKeys from 'lodash/keys'
+import startsWith from 'lodash/startsWith'
+import nicePipe from 'nice-pipe'
+import prettyMs from 'pretty-ms'
+import progressStream from 'progress-stream'
+import pw from 'pw'
+import stripIndent from 'strip-indent'
+import { URL } from 'url'
+import Xo from 'xo-lib'
+import { parseOVAFile } from 'xo-vmdk-to-vhd'
+
+import pkg from '../package'
+import {
+  load as loadConfig,
+  set as setConfig,
+  unset as unsetConfig,
+} from './config'
+
+function help() {
+  return stripIndent(
+    `
+    Usage:
+
+      $name --register [--expiresIn duration] <XO-Server URL> <username> [<password>]
+        Registers the XO instance to use.
+
+        --expiresIn duration
+          Can be used to change the validity duration of the
+          authorization token (default: one month).
+
+      $name --unregister
+        Remove stored credentials.
+
+      $name --inspect <file>
+        Displays the data that would be imported from the ova.
+
+      $name --upload <file> <sr> [--override <key>=<value> [<key>=<value>]+]
+        Actually imports the VM contained in <file> to the Storage Repository <sr>.
+        Some parameters can be overridden from the file, consult --inspect to get the list.
+        Note: --override has to come last. By default arguments are string, prefix them with <json:> to type
+        them, ex. " --override nameLabel='new VM'  memory=json:67108864 disks.vmdisk1.capacity=json:134217728"
+
+    $name v$version
+  `
+  ).replace(/<([^>]+)>|\$(\w+)/g, function(_, arg, key) {
+    if (arg) {
+      return '<' + chalk.yellow(arg) + '>'
+    }
+
+    if (key === 'name') {
+      return chalk.bold(pkg[key])
+    }
+
+    return pkg[key]
+  })
+}
+
+async function connect() {
+  const { server, token } = await loadConfig()
+  if (server === undefined) {
+    throw new Error('no server to connect to!')
+  }
+
+  if (token === undefined) {
+    throw new Error('no token available')
+  }
+
+  const xo = new Xo({ url: server })
+  await xo.open()
+  await xo.signIn({ token })
+  return xo
+}
+
+export function unregister() {
+  return unsetConfig(['server', 'token'])
+}
+
+export async function register(args) {
+  let expiresIn
+  if (args[0] === '--expiresIn') {
+    expiresIn = args[1]
+    args = args.slice(2)
+  }
+
+  const [
+    url,
+    email,
+    password = await new Promise(resolve => {
+      process.stdout.write('Password: ')
+      pw(resolve)
+    }),
+  ] = args
+
+  const xo = new Xo({ url })
+  await xo.open()
+  await xo.signIn({ email, password })
+  console.log('Successfully logged with', xo.user.email)
+
+  await setConfig({
+    server: url,
+    token: await xo.call('token.create', { expiresIn }),
+  })
+}
+
+function nodeStringDecoder(buffer, encoder) {
+  return Buffer.from(buffer).toString(encoder)
+}
+
+export async function inspect(args) {
+  const file = args[0]
+  const data = await parseOVAFile(
+    new NodeParsableFile(file, (await stat(file)).size),
+    nodeStringDecoder,
+    true
+  )
+  console.log('file metadata:', data)
+}
+
+function parseOverride(args) {
+  const flag = args.shift()
+  if (flag !== '--override') {
+    throw new Error('Third argument has to be --override')
+  }
+
+  if (args.length === 0) {
+    throw new Error('Missing actual override')
+  }
+  const overrides = {}
+  for (const definition of args) {
+    const index = definition.indexOf('=')
+    const key = definition.slice(0, index)
+    let value = definition.slice(index + 1)
+    if (startsWith(value, 'json:')) {
+      value = JSON.parse(value.slice(5))
+    }
+    overrides[key] = value
+  }
+  return overrides
+}
+
+export async function upload(args) {
+  const file = args.shift()
+  const srId = args.shift()
+  let overrides = {}
+  if (args.length > 1) {
+    overrides = parseOverride(args)
+  }
+
+  const data = await parseOVAFile(
+    new NodeParsableFile(file, (await stat(file)).size),
+    nodeStringDecoder
+  )
+  const params = { sr: srId }
+  const xo = await connect()
+  const getXoObject = async filter =>
+    Object.values(await xo.call('xo.getAllObjects', { filter }))[0]
+  const sr = await getXoObject({ id: srId })
+  const pool = await getXoObject({ id: sr.$poolId })
+  const master = await getXoObject({ id: pool.master })
+  const pif = await getXoObject({
+    type: 'PIF',
+    management: true,
+    $host: master.id,
+  })
+  data.networks = data.networks.map(() => pif.$network)
+  console.log('data', data)
+  const l33tData = l33t(data)
+  const overridesKeys = Object.keys(overrides)
+  const missingKeys = overridesKeys.filter(k => !l33tData.probe(k))
+  if (missingKeys.length) {
+    // eslint-disable-next-line no-throw-literal
+    throw `those override keys don't exist in the metadata: ${missingKeys}`
+  }
+  for (const key of overridesKeys) {
+    l33tData.plant(key, overrides[key])
+  }
+  data.disks = Object.values(data.disks)
+  params.data = l33tData.obj
+  params.type = 'ova'
+  const method = 'vm.import'
+
+  // FIXME: do not use private properties.
+  const baseUrl = xo._url.replace(/^ws/, 'http')
+
+  const result = await xo.call(method, params)
+  let keys, key, url
+  if (isObject(result) && (keys = getKeys(result)).length === 1) {
+    key = keys[0]
+
+    if (key === '$sendTo') {
+      if (typeof file !== 'string') {
+        // eslint-disable-next-line no-throw-literal
+        throw 'file parameter should be a path'
+      }
+      url = new URL(result[key], baseUrl)
+
+      const { size: length } = await stat(file)
+      const input = nicePipe([
+        createReadStream(file),
+        progressStream(
+          {
+            length,
+            time: 1e3,
+          },
+          printProgress
+        ),
+      ])
+
+      try {
+        return await hrp
+          .post(url.toString(), {
+            body: input,
+            headers: {
+              'content-length': length,
+            },
+          })
+          .readAll('utf-8')
+      } catch (e) {
+        console.log('ERROR', e)
+        console.log('ERROR content', await e.response.readAll('utf-8'))
+        throw e
+      }
+    }
+  }
+}
+
+export class NodeParsableFile {
+  constructor(fileName, fileLength = Infinity) {
+    this._fileName = fileName
+    this._start = 0
+    this._end = fileLength
+  }
+
+  slice(start, end) {
+    const newFile = new NodeParsableFile(this._fileName)
+    newFile._start = start < 0 ? this._end + start : this._start + start
+    newFile._end = end < 0 ? this._end + end : this._start + end
+    return newFile
+  }
+
+  async read() {
+    const result = await getStream.buffer(
+      createReadStream(this._fileName, {
+        start: this._start,
+        end: this._end - 1,
+      })
+    )
+    // crazy stuff to get a browser-compatible ArrayBuffer from a node buffer
+    // https://stackoverflow.com/a/31394257/72637
+    return result.buffer.slice(
+      result.byteOffset,
+      result.byteOffset + result.byteLength
+    )
+  }
+}
+
+const humanFormatOpts = {
+  unit: 'B',
+  scale: 'binary',
+}
+
+function printProgress(progress) {
+  if (progress.length) {
+    console.warn(
+      '%s% of %s @ %s/s - ETA %s',
+      Math.round(progress.percentage),
+      humanFormat(progress.length, humanFormatOpts),
+      humanFormat(progress.speed, humanFormatOpts),
+      prettyMs(progress.eta * 1e3)
+    )
+  } else {
+    console.warn(
+      '%s @ %s/s',
+      humanFormat(progress.transferred, humanFormatOpts),
+      humanFormat(progress.speed, humanFormatOpts)
+    )
+  }
+}
+
+export default async function main(args) {
+  if (!args || !args.length || args[0] === '-h' || args[0] === '--help') {
+    return help()
+  }
+  const fnName = args[0].replace(/^--|-\w/g, match =>
+    match === '--' ? '' : match[1].toUpperCase()
+  )
+  if (fnName in exports) {
+    return exports[fnName](args.slice(1))
+  }
+  return help()
+}
+
+if (!module.parent) {
+  execPromise(main)
+}

CHANGELOG.unreleased.md

@@ -8,6 +8,7 @@
 > Users must be able to say: “Nice enhancement, I'm eager to test it”
 
 - [Backup] **BETA** Ability to backup running VMs with their memory [#645](https://github.com/vatesfr/xen-orchestra/issues/645) (PR [#4252](https://github.com/vatesfr/xen-orchestra/pull/4252))
+- [Import] add CLI tool to import OVA files (PR [#3630](https://github.com/vatesfr/xen-orchestra/pull/3630))
 
 ### Bug fixes
 
@@ -20,5 +21,7 @@
 >
 > Rule of thumb: add packages on top.
 
+- xo-vmdk-to-vhd major
+- @xen-orchestra/upload-ova major
 - xo-server minor
 - xo-web minor

packages/xo-cli/README.md

@@ -117,12 +117,18 @@ encoding by prefixing with `json:`:
 > xo-cli vm.export vm=a01667e0-8e29-49fc-a550-17be4226783c @=vm.xva
 ```
 
-##### VM import
+##### XVA VM import
 
 ```
 > xo-cli vm.import sr=60a6939e-8b0a-4352-9954-5bde44bcdf7d @=vm.xva
 ```
 
+> Note: `xo-cli` only supports the import of XVA files. It will not import OVA files.
+
+##### OVA VM import
+
+A separate utility, [`xo-upload-ova`](https://github.com/vatesfr/xen-orchestra/blob/master/packages/@xen-orchestra/upload-ova/README.md), can be used to import `.ova` files.
+
 ## Development
 
 ```

packages/xo-server/src/xapi/index.js

@@ -9,7 +9,7 @@ import mixin from '@xen-orchestra/mixin'
 import ms from 'ms'
 import synchronized from 'decorator-synchronized'
 import tarStream from 'tar-stream'
-import vmdkToVhd from 'xo-vmdk-to-vhd'
+import { vmdkToVhd } from 'xo-vmdk-to-vhd'
 import {
   cancelable,
   defer,
@@ -1461,11 +1461,15 @@ export default class Xapi extends XapiBase {
           table.grainLogicalAddressList,
           table.grainFileOffsetList
         )
-        await this._importVdiContent(vdi, vhdStream, VDI_FORMAT_VHD)
+        try {
-
+          await this._importVdiContent(vdi, vhdStream, VDI_FORMAT_VHD)
-        // See: https://github.com/mafintosh/tar-stream#extracting
+          // See: https://github.com/mafintosh/tar-stream#extracting
-        // No import parallelization.
+          // No import parallelization.
-        cb()
+        } catch (e) {
+          reject(e)
+        } finally {
+          cb()
+        }
       })
       stream.pipe(extract)
     })

packages/xo-vmdk-to-vhd/README.md

@@ -15,11 +15,10 @@ Installation of the [npm package](https://npmjs.org/package/xo-vmdk-to-vhd):
 To convert a VMDK stream to a Fixed VHD stream without buffering the entire input or output:
 
 ```js
-import convertFromVMDK from 'xo-vmdk-to-vhd'
+import { vmdkToVhd } from 'xo-vmdk-to-vhd'
 import { createReadStream, createWriteStream } from 'fs'
-
 ;(async () => {
-  const stream = await convertFromVMDK(fs.createReadStream(vmdkFileName))
+  const stream = await vmdkToVhd(fs.createReadStream(vmdkFileName))
 
   stream.pipe(fs.createWriteStream(vhdFileName))
 })()
@@ -28,11 +27,11 @@ import { createReadStream, createWriteStream } from 'fs'
 or:
 
 ```js
-var convertFromVMDK = require('xo-vmdk-to-vhd').default
+var vmdkToVhd = require('xo-vmdk-to-vhd').vmdkToVhd
 var createReadStream = require('fs').createReadStream
 var createWriteStream = require('fs').createWriteStream
 
-convertFromVMDK(fs.createReadStream(vmdkFileName)).then(function(stream) {
+vmdkToVhd(fs.createReadStream(vmdkFileName)).then(function(stream) {
   stream.pipe(fs.createWriteStream(vhdFileName))
 })
 ```

packages/xo-vmdk-to-vhd/src/index.js

@@ -1,12 +1,14 @@
 import { createReadableSparseStream } from 'vhd-lib'
+import { parseOVAFile, ParsableFile } from './ova'
 
 import VMDKDirectParser from './vmdk-read'
+
 export {
   default as readVmdkGrainTable,
   readCapacityAndGrainTable,
 } from './vmdk-read-table'
 
-async function convertFromVMDK(
+async function vmdkToVhd(
   vmdkReadStream,
   grainLogicalAddressList,
   grainFileOffsetList
@@ -24,4 +26,5 @@ async function convertFromVMDK(
     parser.blockIterator()
   )
 }
-export { convertFromVMDK as default }
+
+export { ParsableFile, parseOVAFile, vmdkToVhd }

packages/xo-vmdk-to-vhd/src/ova.integ.spec.js

@@ -0,0 +1,240 @@
+/* eslint-env jest */
+
+import { exec } from 'child-process-promise'
+import { createReadStream } from 'fs'
+import { stat, writeFile } from 'fs-extra'
+import getStream from 'get-stream'
+import { pFromCallback } from 'promise-toolbox'
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+
+import { ParsableFile, parseOVAFile } from './ova'
+import readVmdkGrainTable from './vmdk-read-table'
+
+const initialDir = process.cwd()
+beforeEach(async () => {
+  const dir = await pFromCallback(cb => tmp.dir(cb))
+  process.chdir(dir)
+})
+
+afterEach(async () => {
+  const tmpDir = process.cwd()
+  process.chdir(initialDir)
+  await pFromCallback(cb => rimraf(tmpDir, cb))
+})
+
+export class NodeParsableFile extends ParsableFile {
+  constructor(fileName, fileLength = Infinity) {
+    super()
+    this._fileName = fileName
+    this._start = 0
+    this._end = fileLength
+  }
+
+  slice(start, end) {
+    const newFile = new NodeParsableFile(this._fileName)
+    newFile._start = start < 0 ? this._end + start : this._start + start
+    newFile._end = end < 0 ? this._end + end : this._start + end
+    return newFile
+  }
+
+  async read() {
+    const result = await getStream.buffer(
+      createReadStream(this._fileName, {
+        start: this._start,
+        end: this._end - 1,
+      })
+    )
+    // crazy stuff to get a browser-compatible ArrayBuffer from a node buffer
+    // https://stackoverflow.com/a/31394257/72637
+    return result.buffer.slice(
+      result.byteOffset,
+      result.byteOffset + result.byteLength
+    )
+  }
+}
+
+const vmdkFileName = 'random-data.vmdk'
+test('An ova file is parsed correctly', async () => {
+  const ovfName = 'test.ovf'
+  await writeFile(ovfName, xmlContent)
+  const rawFileName = 'random-data'
+  await exec(`base64 /dev/urandom | head -c 104448 > ${rawFileName}`)
+  await exec(
+    `rm -f ${vmdkFileName} && python /usr/share/pyshared/VMDKstream.py ${rawFileName} ${vmdkFileName}`
+  )
+  const ovaName = `test.ova`
+  await exec(`tar cf ${ovaName} ${ovfName} ${vmdkFileName}`)
+  const vmdkParsableFile = new NodeParsableFile(
+    vmdkFileName,
+    (await stat(vmdkFileName)).size
+  )
+  const directGrainTableFetch = await readVmdkGrainTable(async (start, end) =>
+    vmdkParsableFile.slice(start, end).read()
+  )
+  expect(directGrainTableFetch).toEqual(expectedResult.tables[vmdkFileName])
+  const data = await parseOVAFile(
+    new NodeParsableFile(ovaName),
+    (buffer, encoder) => {
+      return Buffer.from(buffer).toString(encoder)
+    }
+  )
+  expect(data).toEqual(expectedResult)
+})
+
+const expectedResult = {
+  tables: {
+    [vmdkFileName]: {
+      grainFileOffsetList: [65536, 115712],
+      grainLogicalAddressList: [0, 65536],
+    },
+  },
+  disks: {
+    vmdisk1: {
+      capacity: 134217728,
+      path: 'random-data.vmdk',
+      descriptionLabel: 'No description',
+      nameLabel: 'Hard Disk 1',
+      position: 0,
+    },
+  },
+  networks: ['LAN'],
+  nameLabel: 'dsl',
+  descriptionLabel: "NetworkJutsu's Damn Small Linux OVA",
+  nCpus: 1,
+  memory: 67108864,
+}
+const xmlContent = `<?xml version="1.0" encoding="UTF-8"?>
+<!--Generated by VMware ESX Server, User: root, UTC time: 2015-09-20T04:41:19.332937Z-->
+<Envelope vmw:buildId="build-2615704" xmlns="http://schemas.dmtf.org/ovf/envelope/1" xmlns:cim="http://schemas.dmtf.org/wbem/wscim/1/common" xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1" xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData" xmlns:vmw="http://www.vmware.com/schema/ovf" xmlns:vssd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <References>
+    <File ovf:href="${vmdkFileName}" ovf:id="file1" ovf:size="76934656" />
+  </References>
+  <DiskSection>
+    <Info>Virtual disk information</Info>
+    <Disk ovf:capacity="128" ovf:capacityAllocationUnits="byte * 2^20" ovf:diskId="vmdisk1" ovf:fileRef="file1" ovf:format="http://www.vmware.com/interfaces/specifications/vmdk.html#streamOptimized" ovf:populatedSize="82313216" />
+  </DiskSection>
+  <NetworkSection>
+    <Info>The list of logical networks</Info>
+    <Network ovf:name="LAN">
+      <Description>The LAN network</Description>
+    </Network>
+  </NetworkSection>
+  <VirtualSystem ovf:id="dsl">
+    <Info>A virtual machine</Info>
+    <Name>dsl</Name>
+    <OperatingSystemSection ovf:id="1" vmw:osType="otherGuest">
+      <Info>The kind of installed guest operating system</Info>
+    </OperatingSystemSection>
+    <VirtualHardwareSection>
+      <Info>Virtual hardware requirements</Info>
+      <System>
+        <vssd:ElementName>Virtual Hardware Family</vssd:ElementName>
+        <vssd:InstanceID>0</vssd:InstanceID>
+        <vssd:VirtualSystemIdentifier>dsl</vssd:VirtualSystemIdentifier>
+        <vssd:VirtualSystemType>vmx-11</vssd:VirtualSystemType>
+      </System>
+      <Item>
+        <rasd:AllocationUnits>hertz * 10^6</rasd:AllocationUnits>
+        <rasd:Description>Number of Virtual CPUs</rasd:Description>
+        <rasd:ElementName>1 virtual CPU(s)</rasd:ElementName>
+        <rasd:InstanceID>1</rasd:InstanceID>
+        <rasd:ResourceType>3</rasd:ResourceType>
+        <rasd:VirtualQuantity>1</rasd:VirtualQuantity>
+      </Item>
+      <Item>
+        <rasd:AllocationUnits>byte * 2^20</rasd:AllocationUnits>
+        <rasd:Description>Memory Size</rasd:Description>
+        <rasd:ElementName>64MB of memory</rasd:ElementName>
+        <rasd:InstanceID>2</rasd:InstanceID>
+        <rasd:ResourceType>4</rasd:ResourceType>
+        <rasd:VirtualQuantity>64</rasd:VirtualQuantity>
+      </Item>
+      <Item>
+        <rasd:Address>1</rasd:Address>
+        <rasd:Description>IDE Controller</rasd:Description>
+        <rasd:ElementName>VirtualIDEController 1</rasd:ElementName>
+        <rasd:InstanceID>3</rasd:InstanceID>
+        <rasd:ResourceType>5</rasd:ResourceType>
+      </Item>
+      <Item>
+        <rasd:Address>0</rasd:Address>
+        <rasd:Description>IDE Controller</rasd:Description>
+        <rasd:ElementName>VirtualIDEController 0</rasd:ElementName>
+        <rasd:InstanceID>4</rasd:InstanceID>
+        <rasd:ResourceType>5</rasd:ResourceType>
+      </Item>
+      <Item ovf:required="false">
+        <rasd:AutomaticAllocation>false</rasd:AutomaticAllocation>
+        <rasd:ElementName>VirtualVideoCard</rasd:ElementName>
+        <rasd:InstanceID>5</rasd:InstanceID>
+        <rasd:ResourceType>24</rasd:ResourceType>
+        <vmw:Config ovf:required="false" vmw:key="enable3DSupport" vmw:value="false" />
+        <vmw:Config ovf:required="false" vmw:key="enableMPTSupport" vmw:value="false" />
+        <vmw:Config ovf:required="false" vmw:key="use3dRenderer" vmw:value="automatic" />
+        <vmw:Config ovf:required="false" vmw:key="useAutoDetect" vmw:value="false" />
+        <vmw:Config ovf:required="false" vmw:key="videoRamSizeInKB" vmw:value="4096" />
+      </Item>
+      <Item ovf:required="false">
+        <rasd:AutomaticAllocation>false</rasd:AutomaticAllocation>
+        <rasd:ElementName>VirtualVMCIDevice</rasd:ElementName>
+        <rasd:InstanceID>6</rasd:InstanceID>
+        <rasd:ResourceSubType>vmware.vmci</rasd:ResourceSubType>
+        <rasd:ResourceType>1</rasd:ResourceType>
+        <vmw:Config ovf:required="false" vmw:key="allowUnrestrictedCommunication" vmw:value="false" />
+        <vmw:Config ovf:required="false" vmw:key="slotInfo.pciSlotNumber" vmw:value="33" />
+      </Item>
+      <Item ovf:required="false">
+        <rasd:AddressOnParent>0</rasd:AddressOnParent>
+        <rasd:AutomaticAllocation>false</rasd:AutomaticAllocation>
+        <rasd:ElementName>CD-ROM 1</rasd:ElementName>
+        <rasd:InstanceID>7</rasd:InstanceID>
+        <rasd:Parent>3</rasd:Parent>
+        <rasd:ResourceSubType>vmware.cdrom.remoteatapi</rasd:ResourceSubType>
+        <rasd:ResourceType>15</rasd:ResourceType>
+      </Item>
+      <Item>
+        <rasd:AddressOnParent>0</rasd:AddressOnParent>
+        <rasd:ElementName>Hard Disk 1</rasd:ElementName>
+        <rasd:HostResource>ovf:/disk/vmdisk1</rasd:HostResource>
+        <rasd:InstanceID>8</rasd:InstanceID>
+        <rasd:Parent>4</rasd:Parent>
+        <rasd:ResourceType>17</rasd:ResourceType>
+        <vmw:Config ovf:required="false" vmw:key="backing.writeThrough" vmw:value="false" />
+      </Item>
+      <Item>
+        <rasd:AddressOnParent>7</rasd:AddressOnParent>
+        <rasd:AutomaticAllocation>true</rasd:AutomaticAllocation>
+        <rasd:Connection>LAN</rasd:Connection>
+        <rasd:Description>PCNet32 ethernet adapter on "LAN"</rasd:Description>
+        <rasd:ElementName>Ethernet 1</rasd:ElementName>
+        <rasd:InstanceID>9</rasd:InstanceID>
+        <rasd:ResourceSubType>PCNet32</rasd:ResourceSubType>
+        <rasd:ResourceType>10</rasd:ResourceType>
+        <vmw:Config ovf:required="false" vmw:key="slotInfo.pciSlotNumber" vmw:value="32" />
+        <vmw:Config ovf:required="false" vmw:key="wakeOnLanEnabled" vmw:value="true" />
+      </Item>
+      <vmw:Config ovf:required="false" vmw:key="cpuHotAddEnabled" vmw:value="false" />
+      <vmw:Config ovf:required="false" vmw:key="cpuHotRemoveEnabled" vmw:value="false" />
+      <vmw:Config ovf:required="false" vmw:key="firmware" vmw:value="bios" />
+      <vmw:Config ovf:required="false" vmw:key="virtualICH7MPresent" vmw:value="false" />
+      <vmw:Config ovf:required="false" vmw:key="virtualSMCPresent" vmw:value="false" />
+      <vmw:Config ovf:required="false" vmw:key="memoryHotAddEnabled" vmw:value="false" />
+      <vmw:Config ovf:required="false" vmw:key="nestedHVEnabled" vmw:value="false" />
+      <vmw:Config ovf:required="false" vmw:key="powerOpInfo.powerOffType" vmw:value="soft" />
+      <vmw:Config ovf:required="false" vmw:key="powerOpInfo.resetType" vmw:value="soft" />
+      <vmw:Config ovf:required="false" vmw:key="powerOpInfo.standbyAction" vmw:value="powerOnSuspend" />
+      <vmw:Config ovf:required="false" vmw:key="powerOpInfo.suspendType" vmw:value="hard" />
+      <vmw:Config ovf:required="false" vmw:key="tools.afterPowerOn" vmw:value="true" />
+      <vmw:Config ovf:required="false" vmw:key="tools.afterResume" vmw:value="true" />
+      <vmw:Config ovf:required="false" vmw:key="tools.beforeGuestShutdown" vmw:value="true" />
+      <vmw:Config ovf:required="false" vmw:key="tools.beforeGuestStandby" vmw:value="true" />
+      <vmw:Config ovf:required="false" vmw:key="tools.syncTimeWithHost" vmw:value="false" />
+      <vmw:Config ovf:required="false" vmw:key="tools.toolsUpgradePolicy" vmw:value="manual" />
+    </VirtualHardwareSection>
+    <AnnotationSection ovf:required="false">
+      <Info>A human-readable annotation</Info>
+      <Annotation>NetworkJutsu's Damn Small Linux OVA</Annotation>
+    </AnnotationSection>
+  </VirtualSystem>
+</Envelope>`

packages/xo-vmdk-to-vhd/src/ova.js

@@ -0,0 +1,226 @@
+import find from 'lodash/find'
+import forEach from 'lodash/forEach'
+import xml2js from 'xml2js'
+
+import { readVmdkGrainTable } from '.'
+
+/********
+ *
+ * THIS FILE HAS TO WORK IN BOTH THE BROWSER AND NODE
+ *
+ ********/
+
+// See: http://opennodecloud.com/howto/2013/12/25/howto-ON-ovf-reference.html
+// See: http://www.dmtf.org/sites/default/files/standards/documents/DSP0243_1.0.0.pdf
+// See: http://www.dmtf.org/sites/default/files/standards/documents/DSP0243_2.1.0.pdf
+
+const MEMORY_UNIT_TO_FACTOR = {
+  k: 1024,
+  m: 1048576,
+  g: 1073741824,
+  t: 1099511627776,
+}
+const RESOURCE_TYPE_TO_HANDLER = {
+  // CPU.
+  '3': (data, { 'rasd:VirtualQuantity': nCpus }) => {
+    data.nCpus = +nCpus
+  },
+  // RAM.
+  '4': (
+    data,
+    { 'rasd:AllocationUnits': unit, 'rasd:VirtualQuantity': quantity }
+  ) => {
+    data.memory = quantity * allocationUnitsToFactor(unit)
+  },
+  // Network.
+  '10': (
+    { networks },
+    { 'rasd:AutomaticAllocation': enabled, 'rasd:Connection': name }
+  ) => {
+    if (enabled) {
+      networks.push(name)
+    }
+  },
+  // Disk.
+  '17': (
+    { disks },
+    {
+      'rasd:AddressOnParent': position,
+      'rasd:Description': description = 'No description',
+      'rasd:ElementName': name,
+      'rasd:HostResource': resource,
+    }
+  ) => {
+    const diskId = resource.match(/^(?:ovf:)?\/disk\/(.+)$/)
+    const disk = diskId && disks[diskId[1]]
+
+    if (disk) {
+      disk.descriptionLabel = description
+      disk.nameLabel = name
+      disk.position = +position
+    } else {
+      // TODO: Log error in U.I.
+      console.error(`No disk found: '${diskId}'.`)
+    }
+  },
+}
+
+function parseTarHeader(header, stringDeserializer) {
+  const fileName = stringDeserializer(header.slice(0, 100), 'ascii').split(
+    '\0'
+  )[0]
+  if (fileName.length === 0) {
+    return null
+  }
+  const fileSize = parseInt(
+    stringDeserializer(header.slice(124, 124 + 11), 'ascii'),
+    8
+  )
+  return { fileName, fileSize }
+}
+
+export class ParsableFile {
+  // noinspection JSMethodCanBeStatic
+  get size() {
+    return 0
+  }
+
+  /** returns a ParsableFile */
+  slice(start, end) {}
+
+  /** reads the fragment, returns an ArrayBuffer */
+  async read() {}
+}
+
+export const ensureArray = value => {
+  if (value === undefined) {
+    return []
+  }
+
+  return Array.isArray(value) ? value : [value]
+}
+
+const allocationUnitsToFactor = unit => {
+  const intValue = unit.match(/\^([0-9]+)$/)
+  return intValue != null
+    ? Math.pow(2, intValue[1])
+    : MEMORY_UNIT_TO_FACTOR[unit.charAt(0).toLowerCase()]
+}
+
+const filterDisks = disks => {
+  for (const diskId in disks) {
+    if (disks[diskId].position == null) {
+      // TODO: Log error in U.I.
+      console.error(`No position specified for '${diskId}'.`)
+      delete disks[diskId]
+    }
+  }
+}
+
+async function parseOVF(fileFragment, stringDeserializer) {
+  const xmlString = stringDeserializer(await fileFragment.read(), 'utf-8')
+  return new Promise((resolve, reject) =>
+    xml2js.parseString(
+      xmlString,
+      { mergeAttrs: true, explicitArray: false },
+      (err, res) => {
+        if (err) {
+          reject(err)
+          return
+        }
+
+        const {
+          Envelope: {
+            DiskSection: { Disk: disks },
+            References: { File: files },
+            VirtualSystem: system,
+          },
+        } = res
+
+        const data = {
+          disks: {},
+          networks: [],
+        }
+        const hardware = system.VirtualHardwareSection
+
+        // Get VM name/description.
+        data.nameLabel = hardware.System['vssd:VirtualSystemIdentifier']
+        data.descriptionLabel =
+          (system.AnnotationSection && system.AnnotationSection.Annotation) ||
+          (system.OperatingSystemSection &&
+            system.OperatingSystemSection.Description)
+
+        // Get disks.
+        forEach(ensureArray(disks), disk => {
+          const file = find(
+            ensureArray(files),
+            file => file['ovf:id'] === disk['ovf:fileRef']
+          )
+          const unit = disk['ovf:capacityAllocationUnits']
+
+          data.disks[disk['ovf:diskId']] = {
+            capacity:
+              disk['ovf:capacity'] *
+              ((unit && allocationUnitsToFactor(unit)) || 1),
+            path: file && file['ovf:href'],
+          }
+        })
+
+        // Get hardware info: CPU, RAM, disks, networks...
+        forEach(ensureArray(hardware.Item), item => {
+          const handler = RESOURCE_TYPE_TO_HANDLER[item['rasd:ResourceType']]
+          if (!handler) {
+            return
+          }
+          handler(data, item)
+        })
+
+        // Remove disks which not have a position.
+        // (i.e. no info in hardware.Item section.)
+        filterDisks(data.disks)
+        resolve(data)
+      }
+    )
+  )
+}
+
+/**
+ *
+ * @param parsableFile: ParsableFile
+ * @param stringDeserializer function (ArrayBuffer, encoding) => String
+ * @param skipVmdk if true avoid parsing the VMDK file tables
+ * @returns {Promise<{tables: {}}>}
+ */
+export async function parseOVAFile(
+  parsableFile,
+  stringDeserializer,
+  skipVmdk = false
+) {
+  let offset = 0
+  const HEADER_SIZE = 512
+  let data = { tables: {} }
+  while (true) {
+    const header = parseTarHeader(
+      await parsableFile.slice(offset, offset + HEADER_SIZE).read(),
+      stringDeserializer
+    )
+    offset += HEADER_SIZE
+    if (header === null) {
+      break
+    }
+    if (header.fileName.toLowerCase().endsWith('.ovf')) {
+      const res = await parseOVF(
+        parsableFile.slice(offset, offset + header.fileSize),
+        stringDeserializer
+      )
+      data = { ...data, ...res }
+    }
+    if (!skipVmdk && header.fileName.toLowerCase().endsWith('.vmdk')) {
+      const fileSlice = parsableFile.slice(offset, offset + header.fileSize)
+      const readFile = async (start, end) => fileSlice.slice(start, end).read()
+      data.tables[header.fileName] = await readVmdkGrainTable(readFile)
+    }
+    offset += Math.ceil(header.fileSize / 512) * 512
+  }
+  return data
+}

packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js

@@ -8,7 +8,7 @@ import tmp from 'tmp'
 
 import { createReadStream, createWriteStream, stat } from 'fs-extra'
 import { pFromCallback } from 'promise-toolbox'
-import convertFromVMDK, { readVmdkGrainTable } from '.'
+import { vmdkToVhd, readVmdkGrainTable } from '.'
 
 const initialDir = process.cwd()
 jest.setTimeout(100000)
@@ -66,7 +66,7 @@ test('VMDK to VHD can convert a random data file with VMDKDirectParser', async (
     )
     const result = await readVmdkGrainTable(createFileAccessor(vmdkFileName))
     const pipe = (
-      await convertFromVMDK(
+      await vmdkToVhd(
         createReadStream(vmdkFileName),
         result.grainLogicalAddressList,
         result.grainFileOffsetList

yarn.lock

@@ -7271,7 +7271,7 @@ get-stdin@^7.0.0:
   resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-7.0.0.tgz#8d5de98f15171a125c5e516643c7a6d0ea8a96f6"
   integrity sha512-zRKcywvrXlXsA0v0i9Io4KDRaAw7+a1ZpjRwl9Wox8PFlVCCHra7E9c4kqXCoCM9nR5tBkaTTZRBoCm60bFqTQ==
 
-get-stream@^4.0.0:
+get-stream@^4.0.0, get-stream@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5"
   integrity sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==