feat(xo-server): multi processes

If a single VM has been imported, redirect to its page.

If multiple VMs has been imported, redirect to the homepage with all other VMs filtered out.

.eslintrc.js

@@ -1,15 +1,22 @@
 module.exports = {
-  'extends': [
-    'standard',
-  ],
-  'parser': 'babel-eslint',
-  'rules': {
+  extends: ['standard', 'standard-jsx'],
+  globals: {
+    __DEV__: true,
+    $Dict: true,
+    $Diff: true,
+    $Exact: true,
+    $Keys: true,
+    $PropertyType: true,
+    $Shape: true,
+  },
+  parser: 'babel-eslint',
+  rules: {
     'comma-dangle': ['error', 'always-multiline'],
+    indent: 'off',
     'no-var': 'error',
     'node/no-extraneous-import': 'error',
     'node/no-extraneous-require': 'error',
-    'node/no-missing-import': 'error',
-    'node/no-missing-require': 'error',
     'prefer-const': 'error',
+    'react/jsx-indent': 'off',
   },
 }

.flowconfig

@@ -8,6 +8,7 @@
 [lints]
 
 [options]
+esproposal.decorators=ignore
 include_warnings=true
 module.use_strict=true
 

.gitignore

@@ -8,11 +8,20 @@
 /packages/*/dist/
 /packages/*/node_modules/
 
+/packages/vhd-cli/src/commands/index.js
+
+/packages/xen-api/plot.dat
+
 /packages/xo-server/.xo-server.*
 /packages/xo-server/src/api/index.js
 /packages/xo-server/src/xapi/mixins/index.js
 /packages/xo-server/src/xo-mixins/index.js
 
+/packages/xo-server-auth-ldap/ldap.cache.conf
+
+/packages/xo-web/src/common/intl/locales/index.js
+/packages/xo-web/src/common/themes/index.js
+
 npm-debug.log
 npm-debug.log.*
 pnpm-debug.log

.prettierrc.js

@@ -0,0 +1,5 @@
+module.exports = {
+  semi: false,
+  singleQuote: true,
+  trailingComma: 'es5',
+}

.travis.yml

@@ -1,12 +1,18 @@
 language: node_js
 node_js:
-  - stable
+  #- stable # disable for now due to an issue of indirect dep upath with Node 9
   - 8
   - 6
 
 # Use containers.
 # http://docs.travis-ci.com/user/workers/container-based-infrastructure/
 sudo: false
+addons:
+  apt:
+    packages:
+    - qemu-utils
+    - blktap-utils
+    - vmdk-stream-converter
 
 before_install:
   - curl -o- -L https://yarnpkg.com/install.sh | bash
@@ -14,3 +20,7 @@ before_install:
 
 cache:
   yarn: true
+
+script:
+  - yarn run test
+  - yarn run test-integration

@xen-orchestra/babel-config/index.js

@@ -0,0 +1,47 @@
+'use strict'
+
+const PLUGINS_RE = /^(?:@babel\/plugin-.+|babel-plugin-lodash)$/
+const PRESETS_RE = /^@babel\/preset-.+$/
+
+const NODE_ENV = process.env.NODE_ENV || 'development'
+const __PROD__ = NODE_ENV === 'production'
+const __TEST__ = NODE_ENV === 'test'
+
+module.exports = function (pkg, plugins, presets) {
+  plugins === undefined && (plugins = {})
+
+  presets === undefined && (presets = {})
+  presets['@babel/preset-env'] = {
+    debug: !__TEST__,
+    loose: true,
+    shippedProposals: true,
+    targets: __PROD__
+      ? (() => {
+        let node = (pkg.engines || {}).node
+        if (node !== undefined) {
+          const trimChars = '^=>~'
+          while (trimChars.includes(node[0])) {
+            node = node.slice(1)
+          }
+          return { node: node }
+        }
+      })()
+      : { browsers: '', node: 'current' },
+    useBuiltIns: '@babel/polyfill' in (pkg.dependencies || {}) && 'usage',
+  }
+
+  Object.keys(pkg.devDependencies || {}).forEach(name => {
+    if (!(name in presets) && PLUGINS_RE.test(name)) {
+      plugins[name] = {}
+    } else if (!(name in presets) && PRESETS_RE.test(name)) {
+      presets[name] = {}
+    }
+  })
+
+  return {
+    comments: !__PROD__,
+    ignore: __TEST__ ? undefined : [/\.spec\.js$/],
+    plugins: Object.keys(plugins).map(plugin => [plugin, plugins[plugin]]),
+    presets: Object.keys(presets).map(preset => [preset, presets[preset]]),
+  }
+}

@xen-orchestra/babel-config/package.json

@@ -0,0 +1,11 @@
+{
+  "private": true,
+  "name": "@xen-orchestra/babel-config",
+  "version": "0.0.0",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/babel-config",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  }
+}

@xen-orchestra/cron/.babelrc.js

@@ -1,41 +1,3 @@
-'use strict'
-
-const NODE_ENV = process.env.NODE_ENV || 'development'
-const __PROD__ = NODE_ENV === 'production'
-const __TEST__ = NODE_ENV === 'test'
-
-const pkg = require('./package')
-
-let nodeCompat = (pkg.engines || {}).node
-if (nodeCompat === undefined) {
-  nodeCompat = '6'
-} else {
-  const trimChars = '^=>~'
-  while (trimChars.includes(nodeCompat[0])) {
-    nodeCompat = nodeCompat.slice(1)
-  }
-}
-
-module.exports = {
-  comments: !__PROD__,
-  ignore: __TEST__ ? undefined : [/\.spec\.js$/],
-  plugins: ['lodash'],
-  presets: [
-    [
-      '@babel/env',
-      {
-        debug: !__TEST__,
-        loose: true,
-        shippedProposals: true,
-        targets: __PROD__
-          ? {
-            browsers: '>2%',
-            node: nodeCompat,
-          }
-          : { node: 'current' },
-        useBuiltIns: '@babel/polyfill' in (pkg.dependencies || {}) && 'usage',
-      },
-    ],
-    '@babel/flow',
-  ],
-}
+module.exports = require('../../@xen-orchestra/babel-config')(
+  require('./package.json')
+)

@xen-orchestra/cron/README.md

@@ -10,25 +10,6 @@ Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/cron)
 > npm install --save @xen-orchestra/cron
 ```
 
-## Usage
-
-```js
-import { createSchedule } from '@xen-orchestra/cron'
-
-const schedule = createSchedule('0 0 * * sun', 'America/New_York')
-
-schedule.next(2)
-// [ 2018-02-11T05:00:00.000Z, 2018-02-18T05:00:00.000Z ]
-
-const job = schedule.createJob(() => {
-  console.log(new Date())
-})
-
-job.start()
-
-job.stop()
-```
-
 ### Pattern syntax
 
 ```
@@ -60,6 +41,75 @@ Step values can be used in conjunctions with ranges. For instance,
 > [cron](https://github.com/kelektiv/node-cron), it does not appear to
 > be very standard though.
 
+### API
+
+`createSchedule(pattern: string, zone: string = 'utc'): Schedule`
+
+> Create a new schedule.
+
+- `pattern`: the pattern to use, see [the syntax](#pattern-syntax)
+- `zone`: the timezone to use, use `'local'` for the local timezone
+
+```js
+import { createSchedule } from '@xen-orchestra/cron'
+
+const schedule = createSchedule('0 0 * * sun', 'America/New_York')
+```
+
+`Schedule#createJob(fn: Function): Job`
+
+> Create a new job from this schedule.
+
+- `fn`: function to execute, if it returns a promise, it will be
+  awaited before scheduling the next run.
+
+```js
+const job = schedule.createJob(() => {
+  console.log(new Date())
+})
+```
+
+`Schedule#next(n: number): Array<Date>`
+
+> Returns the next dates matching this schedule.
+
+- `n`: number of dates to return
+
+```js
+schedule.next(2)
+// [ 2018-02-11T05:00:00.000Z, 2018-02-18T05:00:00.000Z ]
+```
+
+`Schedule#startJob(fn: Function): () => void`
+
+> Start a new job from this schedule and return a function to stop it.
+
+- `fn`: function to execute, if it returns a promise, it will be
+  awaited before scheduling the next run.
+
+```js
+const stopJob = schedule.startJob(() => {
+  console.log(new Date())
+})
+stopJob()
+```
+
+`Job#start(): void`
+
+> Start this job.
+
+```js
+job.start()
+```
+
+`Job#stop(): void`
+
+> Stop this job.
+
+```js
+job.stop()
+```
+
 ## Development
 
 ```
@@ -86,7 +136,7 @@ the code.
 
 You may:
 
-- report any [issue](https://github.com/vatesfr/xo-web/issues)
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
   you've encountered;
 - fork and create a pull request.
 

@xen-orchestra/cron/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/cron",
-  "version": "0.1.0",
+  "version": "1.0.3",
   "license": "ISC",
   "description": "Focused, well maintained, cron parser/scheduler",
   "keywords": [
@@ -14,8 +14,8 @@
     "scheduling",
     "task"
   ],
-  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/@xen-orchestra/cron",
-  "bugs": "https://github.com/vatesfr/xo-web/issues",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/cron",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
@@ -30,18 +30,21 @@
   "files": [
     "dist/"
   ],
+  "browserslist": [
+    ">2%"
+  ],
   "engines": {
     "node": ">=6"
   },
   "dependencies": {
     "lodash": "^4.17.4",
-    "luxon": "^0.4.0"
+    "moment-timezone": "^0.5.14"
   },
   "devDependencies": {
-    "@babel/cli": "7.0.0-beta.39",
-    "@babel/core": "7.0.0-beta.39",
-    "@babel/preset-env": "7.0.0-beta.39",
-    "@babel/preset-flow": "7.0.0-beta.39",
+    "@babel/cli": "7.0.0-beta.44",
+    "@babel/core": "7.0.0-beta.44",
+    "@babel/preset-env": "7.0.0-beta.44",
+    "@babel/preset-flow": "7.0.0-beta.44",
     "cross-env": "^5.1.3",
     "rimraf": "^2.6.2"
   },

@xen-orchestra/cron/src/index.js

@@ -1,23 +1,36 @@
-import { DateTime } from 'luxon'
+import moment from 'moment-timezone'
 
 import next from './next'
 import parse from './parse'
 
+const MAX_DELAY = 2 ** 31 - 1
+
 class Job {
   constructor (schedule, fn) {
-    const wrapper = scheduledRun => {
-      if (scheduledRun) {
-        fn()
+    const wrapper = () => {
+      const result = fn()
+      let then
+      if (result != null && typeof (then = result.then) === 'function') {
+        then.call(result, scheduleNext, scheduleNext)
+      } else {
+        scheduleNext()
       }
-      this._timeout = setTimeout(wrapper, schedule.next() - Date.now(), true)
     }
-    this._fn = wrapper
+    const scheduleNext = () => {
+      const delay = schedule._nextDelay()
+      this._timeout =
+        delay < MAX_DELAY
+          ? setTimeout(wrapper, delay)
+          : setTimeout(scheduleNext, MAX_DELAY)
+    }
+
+    this._scheduleNext = scheduleNext
     this._timeout = undefined
   }
 
   start () {
     this.stop()
-    this._fn()
+    this._scheduleNext()
   }
 
   stop () {
@@ -28,7 +41,10 @@ class Job {
 class Schedule {
   constructor (pattern, zone = 'utc') {
     this._schedule = parse(pattern)
-    this._dateTimeOpts = { zone }
+    this._createDate =
+      zone.toLowerCase() === 'utc'
+        ? moment.utc
+        : zone === 'local' ? moment : () => moment.tz(zone)
   }
 
   createJob (fn) {
@@ -38,12 +54,23 @@ class Schedule {
   next (n) {
     const dates = new Array(n)
     const schedule = this._schedule
-    let date = DateTime.fromObject(this._dateTimeOpts)
+    let date = this._createDate()
     for (let i = 0; i < n; ++i) {
-      dates[i] = (date = next(schedule, date)).toJSDate()
+      dates[i] = (date = next(schedule, date)).toDate()
     }
     return dates
   }
+
+  _nextDelay () {
+    const now = this._createDate()
+    return next(this._schedule, now) - now
+  }
+
+  startJob (fn) {
+    const job = this.createJob(fn)
+    job.start()
+    return job.stop.bind(job)
+  }
 }
 
 export const createSchedule = (...args) => new Schedule(...args)

@xen-orchestra/cron/src/next.js

@@ -1,50 +1,49 @@
+import moment from 'moment-timezone'
 import sortedIndex from 'lodash/sortedIndex'
-import { DateTime } from 'luxon'
 
 const NEXT_MAPPING = {
   month: { year: 1 },
-  day: { month: 1 },
-  weekday: { week: 1 },
+  date: { month: 1 },
+  day: { week: 1 },
   hour: { day: 1 },
   minute: { hour: 1 },
 }
 
-const getFirst = values => values !== undefined ? values[0] : 0
+const getFirst = values => (values !== undefined ? values[0] : 0)
 
 const setFirstAvailable = (date, unit, values) => {
   if (values === undefined) {
-    return date
+    return
   }
 
   const curr = date.get(unit)
   const next = values[sortedIndex(values, curr) % values.length]
   if (curr === next) {
-    return date
+    return
   }
 
-  const newDate = date.set({ [unit]: next })
-  return newDate > date ? newDate : newDate.plus(NEXT_MAPPING[unit])
+  const timestamp = +date
+  date.set(unit, next)
+  if (timestamp > +date) {
+    date.add(NEXT_MAPPING[unit])
+  }
+  return true
 }
 
 // returns the next run, after the passed date
 export default (schedule, fromDate) => {
-  let date = fromDate
+  let date = moment(fromDate)
     .set({
       second: 0,
       millisecond: 0,
     })
-    .plus({ minute: 1 })
+    .add({ minute: 1 })
 
   const { minute, hour, dayOfMonth, month, dayOfWeek } = schedule
-  date = setFirstAvailable(date, 'minute', minute)
+  setFirstAvailable(date, 'minute', minute)
 
-  let tmp
-
-  tmp = setFirstAvailable(date, 'hour', hour)
-  if (tmp !== date) {
-    date = tmp.set({
-      minute: getFirst(minute),
-    })
+  if (setFirstAvailable(date, 'hour', hour)) {
+    date.set('minute', getFirst(minute))
   }
 
   let loop
@@ -52,30 +51,30 @@ export default (schedule, fromDate) => {
   do {
     loop = false
 
-    tmp = setFirstAvailable(date, 'month', month)
-    if (tmp !== date) {
-      date = tmp.set({
-        day: 1,
+    if (setFirstAvailable(date, 'month', month)) {
+      date.set({
+        date: 1,
         hour: getFirst(hour),
         minute: getFirst(minute),
       })
     }
 
+    let newDate = date.clone()
     if (dayOfMonth === undefined) {
       if (dayOfWeek !== undefined) {
-        tmp = setFirstAvailable(date, 'weekday', dayOfWeek)
+        setFirstAvailable(newDate, 'day', dayOfWeek)
       }
     } else if (dayOfWeek === undefined) {
-      tmp = setFirstAvailable(date, 'day', dayOfMonth)
+      setFirstAvailable(newDate, 'date', dayOfMonth)
     } else {
-      tmp = DateTime.min(
-        setFirstAvailable(date, 'day', dayOfMonth),
-        setFirstAvailable(date, 'weekday', dayOfWeek)
-      )
+      const dateDay = newDate.clone()
+      setFirstAvailable(dateDay, 'date', dayOfMonth)
+      setFirstAvailable(newDate, 'day', dayOfWeek)
+      newDate = moment.min(dateDay, newDate)
     }
-    if (tmp !== date) {
-      loop = tmp.month !== date.month
-      date = tmp.set({
+    if (+date !== +newDate) {
+      loop = date.month() !== newDate.month()
+      date = newDate.set({
         hour: getFirst(hour),
         minute: getFirst(minute),
       })

@xen-orchestra/cron/src/next.spec.js

@@ -1,17 +1,15 @@
 /* eslint-env jest */
 
 import mapValues from 'lodash/mapValues'
-import { DateTime } from 'luxon'
+import moment from 'moment-timezone'
 
 import next from './next'
 import parse from './parse'
 
-const N = (pattern, fromDate = '2018-04-09T06:25') =>
-  next(parse(pattern), DateTime.fromISO(fromDate, { zone: 'utc' })).toISO({
-    includeOffset: false,
-    suppressMilliseconds: true,
-    suppressSeconds: true,
-  })
+const N = (pattern, fromDate = '2018-04-09T06:25') => {
+  const iso = next(parse(pattern), moment.utc(fromDate)).toISOString()
+  return iso.slice(0, iso.lastIndexOf(':'))
+}
 
 describe('next()', () => {
   mapValues(
@@ -43,4 +41,8 @@ describe('next()', () => {
       'no solutions found for this schedule'
     )
   })
+
+  it('select the first sunday of the month', () => {
+    expect(N('* * * * 0', '2018-03-31T00:00')).toBe('2018-04-01T00:00')
+  })
 })

@xen-orchestra/cron/src/parse.js

@@ -90,7 +90,7 @@ const createParser = ({ fields: [...fields], presets: { ...presets } }) => {
       if (!match('/')) {
         return
       }
-      [start, end] = field.range
+      ;[start, end] = field.range
       step = parseInteger()
     } else {
       start = parseValue()
@@ -173,18 +173,13 @@ export default createParser({
     {
       aliases: 'jan feb mar apr may jun jul aug sep oct nov dec'.split(' '),
       name: 'month',
-      range: [1, 12],
-
-      // this function is applied to numeric entries (not steps)
-      //
-      // currently parse month 0-11
-      post: value => value + 1,
+      range: [0, 11],
     },
     {
-      aliases: 'mon tue wen thu fri sat sun'.split(' '),
+      aliases: 'sun mon tue wen thu fri sat'.split(' '),
       name: 'dayOfWeek',
-      post: value => (value === 0 ? 7 : value),
-      range: [1, 7],
+      post: value => (value === 7 ? 0 : value),
+      range: [0, 6],
     },
   ],
   presets: {

@xen-orchestra/cron/src/parse.spec.js

@@ -8,29 +8,27 @@ describe('parse()', () => {
       minute: [0],
       hour: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
       dayOfMonth: [1, 11, 21, 31],
-      month: [1, 3, 5, 8, 11],
+      month: [0, 2, 4, 7, 10],
     })
   })
 
   it('correctly parse months', () => {
     expect(parse('* * * 0,11 *')).toEqual({
-      month: [1, 12],
+      month: [0, 11],
     })
     expect(parse('* * * jan,dec *')).toEqual({
-      month: [1, 12],
+      month: [0, 11],
     })
   })
 
   it('correctly parse days', () => {
     expect(parse('* * * * mon,sun')).toEqual({
-      dayOfWeek: [1, 7],
+      dayOfWeek: [0, 1],
     })
   })
 
   it('reports missing integer', () => {
-    expect(() => parse('*/a')).toThrow(
-      'minute: missing integer at character 2'
-    )
+    expect(() => parse('*/a')).toThrow('minute: missing integer at character 2')
     expect(() => parse('*')).toThrow('hour: missing integer at character 1')
   })
 
@@ -42,10 +40,10 @@ describe('parse()', () => {
 
   it('dayOfWeek: 0 and 7 bind to sunday', () => {
     expect(parse('* * * * 0')).toEqual({
-      dayOfWeek: [7],
+      dayOfWeek: [0],
     })
     expect(parse('* * * * 7')).toEqual({
-      dayOfWeek: [7],
+      dayOfWeek: [0],
     })
   })
 })

@xen-orchestra/fs/.babelrc.js

@@ -0,0 +1,3 @@
+module.exports = require('../../@xen-orchestra/babel-config')(
+  require('./package.json')
+)

@xen-orchestra/fs/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@xen-orchestra/fs",
+  "version": "0.0.0",
+  "license": "AGPL-3.0",
+  "description": "The File System for Xen Orchestra backups.",
+  "keywords": [],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "preferGlobal": true,
+  "main": "dist/",
+  "bin": {},
+  "files": [
+    "dist/"
+  ],
+  "engines": {
+    "node": ">=6"
+  },
+  "dependencies": {
+    "@babel/runtime": "^7.0.0-beta.44",
+    "@marsaud/smb2-promise": "^0.2.1",
+    "execa": "^0.10.0",
+    "fs-extra": "^5.0.0",
+    "get-stream": "^3.0.0",
+    "lodash": "^4.17.4",
+    "promise-toolbox": "^0.9.5",
+    "through2": "^2.0.3",
+    "tmp": "^0.0.33",
+    "xo-remote-parser": "^0.3"
+  },
+  "devDependencies": {
+    "@babel/cli": "7.0.0-beta.44",
+    "@babel/core": "7.0.0-beta.44",
+    "@babel/plugin-proposal-function-bind": "7.0.0-beta.44",
+    "@babel/plugin-transform-runtime": "^7.0.0-beta.44",
+    "@babel/preset-env": "7.0.0-beta.44",
+    "@babel/preset-flow": "7.0.0-beta.44",
+    "babel-plugin-lodash": "^3.3.2",
+    "cross-env": "^5.1.3",
+    "index-modules": "^0.3.0",
+    "rimraf": "^2.6.2"
+  },
+  "scripts": {
+    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
+    "clean": "rimraf dist/",
+    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
+    "prebuild": "yarn run clean",
+    "predev": "yarn run clean",
+    "prepare": "yarn run build"
+  }
+}

@xen-orchestra/fs/src/abstract.js

@@ -0,0 +1,288 @@
+// @flow
+
+import getStream from 'get-stream'
+import { randomBytes } from 'crypto'
+import { fromCallback, fromEvent, ignoreErrors } from 'promise-toolbox'
+import { type Readable, type Writable } from 'stream'
+import { parse } from 'xo-remote-parser'
+
+import { createChecksumStream, validChecksumOfReadStream } from './checksum'
+
+type Data = Buffer | Readable | string
+type FileDescriptor = {| fd: mixed, path: string |}
+type LaxReadable = Readable & Object
+type LaxWritable = Writable & Object
+
+type File = FileDescriptor | string
+
+const checksumFile = file => file + '.checksum'
+
+export default class RemoteHandlerAbstract {
+  _remote: Object
+  constructor (remote: any) {
+    this._remote = { ...remote, ...parse(remote.url) }
+    if (this._remote.type !== this.type) {
+      throw new Error('Incorrect remote type')
+    }
+  }
+
+  get type (): string {
+    throw new Error('Not implemented')
+  }
+
+  /**
+   * Asks the handler to sync the state of the effective remote with its' metadata
+   */
+  async sync (): Promise<mixed> {
+    return this._sync()
+  }
+
+  async _sync (): Promise<mixed> {
+    throw new Error('Not implemented')
+  }
+
+  /**
+   * Free the resources possibly dedicated to put the remote at work, when it is no more needed
+   */
+  async forget (): Promise<void> {
+    await this._forget()
+  }
+
+  async _forget (): Promise<void> {
+    throw new Error('Not implemented')
+  }
+
+  async test (): Promise<Object> {
+    const testFileName = `${Date.now()}.test`
+    const data = await fromCallback(cb => randomBytes(1024 * 1024, cb))
+    let step = 'write'
+    try {
+      await this.outputFile(testFileName, data)
+      step = 'read'
+      const read = await this.readFile(testFileName)
+      if (data.compare(read) !== 0) {
+        throw new Error('output and input did not match')
+      }
+      return {
+        success: true,
+      }
+    } catch (error) {
+      return {
+        success: false,
+        step,
+        file: testFileName,
+        error: error.message || String(error),
+      }
+    } finally {
+      ignoreErrors.call(this.unlink(testFileName))
+    }
+  }
+
+  async outputFile (file: string, data: Data, options?: Object): Promise<void> {
+    return this._outputFile(file, data, {
+      flags: 'wx',
+      ...options,
+    })
+  }
+
+  async _outputFile (file: string, data: Data, options?: Object): Promise<void> {
+    const stream = await this.createOutputStream(file, options)
+    const promise = fromEvent(stream, 'finish')
+    stream.end(data)
+    await promise
+  }
+
+  async readFile (file: string, options?: Object): Promise<Buffer> {
+    return this._readFile(file, options)
+  }
+
+  _readFile (file: string, options?: Object): Promise<Buffer> {
+    return this.createReadStream(file, options).then(getStream.buffer)
+  }
+
+  async rename (
+    oldPath: string,
+    newPath: string,
+    { checksum = false }: Object = {}
+  ) {
+    let p = this._rename(oldPath, newPath)
+    if (checksum) {
+      p = Promise.all([
+        p,
+        this._rename(checksumFile(oldPath), checksumFile(newPath)),
+      ])
+    }
+    return p
+  }
+
+  async _rename (oldPath: string, newPath: string) {
+    throw new Error('Not implemented')
+  }
+
+  async list (
+    dir: string = '.',
+    {
+      filter,
+      prependDir = false,
+    }: { filter?: (name: string) => boolean, prependDir?: boolean } = {}
+  ): Promise<string[]> {
+    const entries = await this._list(dir)
+
+    if (prependDir) {
+      entries.forEach((entry, i) => {
+        entries[i] = dir + '/' + entry
+      })
+    }
+
+    return filter === undefined ? entries : entries.filter(filter)
+  }
+
+  async _list (dir: string): Promise<string[]> {
+    throw new Error('Not implemented')
+  }
+
+  createReadStream (
+    file: string,
+    { checksum = false, ignoreMissingChecksum = false, ...options }: Object = {}
+  ): Promise<LaxReadable> {
+    const path = typeof file === 'string' ? file : file.path
+    const streamP = this._createReadStream(file, options).then(stream => {
+      // detect early errors
+      let promise = fromEvent(stream, 'readable')
+
+      // try to add the length prop if missing and not a range stream
+      if (
+        stream.length === undefined &&
+        options.end === undefined &&
+        options.start === undefined
+      ) {
+        promise = Promise.all([
+          promise,
+          ignoreErrors.call(
+            this.getSize(file).then(size => {
+              stream.length = size
+            })
+          ),
+        ])
+      }
+
+      return promise.then(() => stream)
+    })
+
+    if (!checksum) {
+      return streamP
+    }
+
+    // avoid a unhandled rejection warning
+    ignoreErrors.call(streamP)
+
+    return this.readFile(checksumFile(path)).then(
+      checksum =>
+        streamP.then(stream => {
+          const { length } = stream
+          stream = (validChecksumOfReadStream(
+            stream,
+            String(checksum).trim()
+          ): LaxReadable)
+          stream.length = length
+
+          return stream
+        }),
+      error => {
+        if (ignoreMissingChecksum && error && error.code === 'ENOENT') {
+          return streamP
+        }
+        throw error
+      }
+    )
+  }
+
+  async _createReadStream (
+    file: string,
+    options?: Object
+  ): Promise<LaxReadable> {
+    throw new Error('Not implemented')
+  }
+
+  async openFile (path: string, flags?: string): Promise<FileDescriptor> {
+    return { fd: await this._openFile(path, flags), path }
+  }
+
+  async _openFile (path: string, flags?: string): Promise<mixed> {
+    throw new Error('Not implemented')
+  }
+
+  async closeFile (fd: FileDescriptor): Promise<void> {
+    await this._closeFile(fd.fd)
+  }
+
+  async _closeFile (fd: mixed): Promise<void> {
+    throw new Error('Not implemented')
+  }
+
+  async refreshChecksum (path: string): Promise<void> {
+    const stream = (await this.createReadStream(path)).pipe(
+      createChecksumStream()
+    )
+    stream.resume() // start reading the whole file
+    await this.outputFile(checksumFile(path), await stream.checksum)
+  }
+
+  async createOutputStream (
+    file: File,
+    { checksum = false, ...options }: Object = {}
+  ): Promise<LaxWritable> {
+    const path = typeof file === 'string' ? file : file.path
+    const streamP = this._createOutputStream(file, {
+      flags: 'wx',
+      ...options,
+    })
+
+    if (!checksum) {
+      return streamP
+    }
+
+    const checksumStream = createChecksumStream()
+    const forwardError = error => {
+      checksumStream.emit('error', error)
+    }
+
+    const stream = await streamP
+    stream.on('error', forwardError)
+    checksumStream.pipe(stream)
+
+    // $FlowFixMe
+    checksumStream.checksumWritten = checksumStream.checksum
+      .then(value => this.outputFile(checksumFile(path), value))
+      .catch(forwardError)
+
+    return checksumStream
+  }
+
+  async _createOutputStream (
+    file: mixed,
+    options?: Object
+  ): Promise<LaxWritable> {
+    throw new Error('Not implemented')
+  }
+
+  async unlink (file: string, { checksum = true }: Object = {}): Promise<void> {
+    if (checksum) {
+      ignoreErrors.call(this._unlink(checksumFile(file)))
+    }
+
+    await this._unlink(file)
+  }
+
+  async _unlink (file: mixed): Promise<void> {
+    throw new Error('Not implemented')
+  }
+
+  async getSize (file: mixed): Promise<number> {
+    return this._getSize(file)
+  }
+
+  async _getSize (file: mixed): Promise<number> {
+    throw new Error('Not implemented')
+  }
+}

@xen-orchestra/fs/src/checksum.js

@@ -0,0 +1,100 @@
+// @flow
+
+// $FlowFixMe
+import through2 from 'through2'
+import { createHash } from 'crypto'
+import { defer, fromEvent } from 'promise-toolbox'
+import { invert } from 'lodash'
+import { type Readable, type Transform } from 'stream'
+
+// Format: $<algorithm>$<salt>$<encrypted>
+//
+// http://man7.org/linux/man-pages/man3/crypt.3.html#NOTES
+const ALGORITHM_TO_ID = {
+  md5: '1',
+  sha256: '5',
+  sha512: '6',
+}
+
+const ID_TO_ALGORITHM = invert(ALGORITHM_TO_ID)
+
+// Create a through stream which computes the checksum of all data going
+// through.
+//
+// The `checksum` attribute is a promise which resolves at the end of the stream
+// with a string representation of the checksum.
+//
+//    const source = ...
+//    const checksumStream = source.pipe(createChecksumStream())
+//    checksumStream.resume() // make the data flow without an output
+//    console.log(await checksumStream.checksum)
+export const createChecksumStream = (
+  algorithm: string = 'md5'
+): Transform & { checksum: Promise<string> } => {
+  const algorithmId = ALGORITHM_TO_ID[algorithm]
+
+  if (!algorithmId) {
+    throw new Error(`unknown algorithm: ${algorithm}`)
+  }
+
+  const hash = createHash(algorithm)
+  const { promise, resolve, reject } = defer()
+
+  const stream = through2(
+    (chunk, enc, callback) => {
+      hash.update(chunk)
+      callback(null, chunk)
+    },
+    callback => {
+      resolve(`$${algorithmId}$$${hash.digest('hex')}`)
+      callback()
+    }
+  ).once('error', reject)
+  stream.checksum = promise
+  return stream
+}
+
+// Check if the checksum of a readable stream is equals to an expected checksum.
+// The given stream is wrapped in a stream which emits an error event
+// if the computed checksum is not equals to the expected checksum.
+export const validChecksumOfReadStream = (
+  stream: Readable,
+  expectedChecksum: string
+): Readable & { checksumVerified: Promise<void> } => {
+  const algorithmId = expectedChecksum.slice(
+    1,
+    expectedChecksum.indexOf('$', 1)
+  )
+
+  if (!algorithmId) {
+    throw new Error(`unknown algorithm: ${algorithmId}`)
+  }
+
+  const hash = createHash(ID_TO_ALGORITHM[algorithmId])
+
+  const wrapper: any = stream.pipe(
+    through2(
+      { highWaterMark: 0 },
+      (chunk, enc, callback) => {
+        hash.update(chunk)
+        callback(null, chunk)
+      },
+      callback => {
+        const checksum = `$${algorithmId}$$${hash.digest('hex')}`
+
+        callback(
+          checksum !== expectedChecksum
+            ? new Error(
+              `Bad checksum (${checksum}), expected: ${expectedChecksum}`
+            )
+            : null
+        )
+      }
+    )
+  )
+
+  stream.on('error', error => wrapper.emit('error', error))
+  wrapper.checksumVerified = fromEvent(wrapper, 'end')
+
+  return wrapper
+}

@xen-orchestra/fs/src/fs.integ.spec.js

@@ -0,0 +1,26 @@
+/* eslint-env jest */
+
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+
+import { fromCallback as pFromCallback } from 'promise-toolbox'
+import { getHandler } from '.'
+
+const initialDir = process.cwd()
+
+beforeEach(async () => {
+  const dir = await pFromCallback(cb => tmp.dir(cb))
+  process.chdir(dir)
+})
+
+afterEach(async () => {
+  const tmpDir = process.cwd()
+  process.chdir(initialDir)
+  await pFromCallback(cb => rimraf(tmpDir, cb))
+})
+
+test("fs test doesn't crash", async () => {
+  const handler = getHandler({ url: 'file://' + process.cwd() })
+  const result = await handler.test()
+  expect(result.success).toBeTruthy()
+})

@xen-orchestra/fs/src/index.js

@@ -0,0 +1,26 @@
+// @flow
+
+import type RemoteHandler from './abstract'
+import RemoteHandlerLocal from './local'
+import RemoteHandlerNfs from './nfs'
+import RemoteHandlerSmb from './smb'
+
+export type { default as RemoteHandler } from './abstract'
+export type Remote = { url: string }
+
+const HANDLERS = {
+  file: RemoteHandlerLocal,
+  smb: RemoteHandlerSmb,
+  nfs: RemoteHandlerNfs,
+}
+
+export const getHandler = (remote: Remote): RemoteHandler => {
+  // FIXME: should be done in xo-remote-parser.
+  const type = remote.url.split('://')[0]
+
+  const Handler = HANDLERS[type]
+  if (!Handler) {
+    throw new Error('Unhandled remote type')
+  }
+  return new Handler(remote)
+}

@xen-orchestra/fs/src/local.js

@@ -63,13 +63,29 @@ export default class LocalHandler extends RemoteHandlerAbstract {
   }
 
   async _createReadStream (file, options) {
-    return fs.createReadStream(this._getFilePath(file), options)
+    if (typeof file === 'string') {
+      return fs.createReadStream(this._getFilePath(file), options)
+    } else {
+      return fs.createReadStream('', {
+        autoClose: false,
+        ...options,
+        fd: file.fd,
+      })
+    }
   }
 
   async _createOutputStream (file, options) {
-    const path = this._getFilePath(file)
-    await fs.ensureDir(dirname(path))
-    return fs.createWriteStream(path, options)
+    if (typeof file === 'string') {
+      const path = this._getFilePath(file)
+      await fs.ensureDir(dirname(path))
+      return fs.createWriteStream(path, options)
+    } else {
+      return fs.createWriteStream('', {
+        autoClose: false,
+        ...options,
+        fd: file.fd,
+      })
+    }
   }
 
   async _unlink (file) {
@@ -82,7 +98,17 @@ export default class LocalHandler extends RemoteHandlerAbstract {
   }
 
   async _getSize (file) {
-    const stats = await fs.stat(this._getFilePath(file))
+    const stats = await fs.stat(
+      this._getFilePath(typeof file === 'string' ? file : file.path)
+    )
     return stats.size
   }
+
+  async _openFile (path, flags) {
+    return fs.open(this._getFilePath(path), flags)
+  }
+
+  async _closeFile (fd) {
+    return fs.close(fd)
+  }
 }

@xen-orchestra/fs/src/nfs.js

@@ -17,7 +17,14 @@ export default class NfsHandler extends LocalHandler {
     let stdout
     const mounted = {}
     try {
-      stdout = await execa.stdout('findmnt', ['-P', '-t', 'nfs,nfs4', '--output', 'SOURCE,TARGET', '--noheadings'])
+      stdout = await execa.stdout('findmnt', [
+        '-P',
+        '-t',
+        'nfs,nfs4',
+        '--output',
+        'SOURCE,TARGET',
+        '--noheadings',
+      ])
       const regex = /^SOURCE="([^:]*):(.*)" TARGET="(.*)"$/
       forEach(stdout.split('\n'), m => {
         if (m) {
@@ -45,7 +52,14 @@ export default class NfsHandler extends LocalHandler {
 
   async _mount () {
     await fs.ensureDir(this._getRealPath())
-    return execa('mount', ['-t', 'nfs', '-o', 'vers=3', `${this._remote.host}:${this._remote.path}`, this._getRealPath()])
+    return execa('mount', [
+      '-t',
+      'nfs',
+      '-o',
+      'vers=3',
+      `${this._remote.host}:${this._remote.path}`,
+      this._getRealPath(),
+    ])
   }
 
   async _sync () {

@xen-orchestra/fs/src/smb.js

@@ -1,19 +1,16 @@
 import Smb2 from '@marsaud/smb2-promise'
+import { lastly as pFinally } from 'promise-toolbox'
 
 import RemoteHandlerAbstract from './abstract'
-import {
-  noop,
-  pFinally,
-} from '../utils'
+
+const noop = () => {}
 
 // Normalize the error code for file not found.
 const normalizeError = error => {
   const { code } = error
 
-  return (
-    code === 'STATUS_OBJECT_NAME_NOT_FOUND' ||
+  return code === 'STATUS_OBJECT_NAME_NOT_FOUND' ||
     code === 'STATUS_OBJECT_PATH_NOT_FOUND'
-  )
     ? Object.create(error, {
       code: {
         configurable: true,
@@ -50,9 +47,7 @@ export default class SmbHandler extends RemoteHandlerAbstract {
       file = undefined
     }
 
-    let path = (this._remote.path !== '')
-      ? this._remote.path
-      : ''
+    let path = this._remote.path !== '' ? this._remote.path : ''
 
     // Ensure remote path is a directory.
     if (path !== '' && path[path.length - 1] !== '\\') {
@@ -94,7 +89,9 @@ export default class SmbHandler extends RemoteHandlerAbstract {
       await client.ensureDir(dir)
     }
 
-    return client.writeFile(path, data, options)::pFinally(() => { client.close() })
+    return client.writeFile(path, data, options)::pFinally(() => {
+      client.close()
+    })
   }
 
   async _readFile (file, options = {}) {
@@ -102,7 +99,11 @@ export default class SmbHandler extends RemoteHandlerAbstract {
     let content
 
     try {
-      content = await client.readFile(this._getFilePath(file), options)::pFinally(() => { client.close() })
+      content = await client
+        .readFile(this._getFilePath(file), options)
+        ::pFinally(() => {
+          client.close()
+        })
     } catch (error) {
       throw normalizeError(error)
     }
@@ -114,7 +115,11 @@ export default class SmbHandler extends RemoteHandlerAbstract {
     const client = this._getClient(this._remote)
 
     try {
-      await client.rename(this._getFilePath(oldPath), this._getFilePath(newPath))::pFinally(() => { client.close() })
+      await client
+        .rename(this._getFilePath(oldPath), this._getFilePath(newPath))
+        ::pFinally(() => {
+          client.close()
+        })
     } catch (error) {
       throw normalizeError(error)
     }
@@ -125,7 +130,9 @@ export default class SmbHandler extends RemoteHandlerAbstract {
     let list
 
     try {
-      list = await client.readdir(this._getFilePath(dir))::pFinally(() => { client.close() })
+      list = await client.readdir(this._getFilePath(dir))::pFinally(() => {
+        client.close()
+      })
     } catch (error) {
       throw normalizeError(error)
     }
@@ -134,6 +141,9 @@ export default class SmbHandler extends RemoteHandlerAbstract {
   }
 
   async _createReadStream (file, options = {}) {
+    if (typeof file !== 'string') {
+      file = file.path
+    }
     const client = this._getClient(this._remote)
     let stream
 
@@ -149,6 +159,9 @@ export default class SmbHandler extends RemoteHandlerAbstract {
   }
 
   async _createOutputStream (file, options = {}) {
+    if (typeof file !== 'string') {
+      file = file.path
+    }
     const client = this._getClient(this._remote)
     const path = this._getFilePath(file)
     const dir = this._dirname(path)
@@ -170,7 +183,9 @@ export default class SmbHandler extends RemoteHandlerAbstract {
     const client = this._getClient(this._remote)
 
     try {
-      await client.unlink(this._getFilePath(file))::pFinally(() => { client.close() })
+      await client.unlink(this._getFilePath(file))::pFinally(() => {
+        client.close()
+      })
     } catch (error) {
       throw normalizeError(error)
     }
@@ -181,11 +196,22 @@ export default class SmbHandler extends RemoteHandlerAbstract {
     let size
 
     try {
-      size = await client.getSize(this._getFilePath(file))::pFinally(() => { client.close() })
+      size = await client
+        .getSize(this._getFilePath(typeof file === 'string' ? file : file.path))
+        ::pFinally(() => {
+          client.close()
+        })
     } catch (error) {
       throw normalizeError(error)
     }
 
     return size
   }
+
+  // this is a fake
+  async _openFile (path) {
+    return this._getFilePath(path)
+  }
+
+  async _closeFile (fd) {}
 }

@xen-orchestra/log/.babelrc.js

@@ -0,0 +1,3 @@
+module.exports = require('../../@xen-orchestra/babel-config')(
+  require('./package.json')
+)

@xen-orchestra/log/.npmignore

@@ -0,0 +1,24 @@
+/benchmark/
+/benchmarks/
+*.bench.js
+*.bench.js.map
+
+/examples/
+example.js
+example.js.map
+*.example.js
+*.example.js.map
+
+/fixture/
+/fixtures/
+*.fixture.js
+*.fixture.js.map
+*.fixtures.js
+*.fixtures.js.map
+
+/test/
+/tests/
+*.spec.js
+*.spec.js.map
+
+__snapshots__/

@xen-orchestra/log/README.md

@@ -0,0 +1,149 @@
+# @xen-orchestra/log [![Build Status](https://travis-ci.org/vatesfr/xen-orchestra.png?branch=master)](https://travis-ci.org/vatesfr/xen-orchestra)
+
+> ${pkg.description}
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/log):
+
+```
+> npm install --save @xen-orchestra/log
+```
+
+## Usage
+
+Everywhere something should be logged:
+
+```js
+import { createLogger } from '@xen-orchestra/log'
+
+const log = createLogger('xo-server-api')
+log.warn('foo')
+```
+
+Then at application level you can choose how to handle these logs:
+
+```js
+import configure from '@xen-orchestra/log/configure'
+import createConsoleTransport from '@xen-orchestra/log/transports/console'
+import createEmailTransport from '@xen-orchestra/log/transports/email'
+
+configure([
+  {
+    // if filter is a string, then it is pattern
+    // (https://github.com/visionmedia/debug#wildcards) which is
+    // matched against the namespace of the logs
+    filter: process.env.DEBUG,
+
+    transport: createConsoleTransport()
+  },
+  {
+    // only levels >= warn
+    level: 'warn',
+
+    transport: createEmaileTransport({
+      service: 'gmail',
+      auth: {
+        user: 'jane.smith@gmail.com',
+        pass: 'H&NbECcpXF|pyXe#%ZEb'
+      },
+      from: 'jane.smith@gmail.com',
+      to: [
+        'jane.smith@gmail.com',
+        'sam.doe@yahoo.com'
+      ]
+    })
+  }
+])
+```
+
+### Transports
+
+#### Console
+
+```js
+import createConsoleTransport from '@xen-orchestra/log/transports/console'
+
+configure(createConsoleTransport())
+```
+
+#### Email
+
+Optional dependency:
+
+```
+> yarn add nodemailer pretty-format
+```
+
+Configuration:
+
+```js
+import createEmailTransport from '@xen-orchestra/log/transports/email'
+
+configure(createEmailTransport({
+  service: 'gmail',
+  auth: {
+    user: 'jane.smith@gmail.com',
+    pass: 'H&NbECcpXF|pyXe#%ZEb'
+  },
+  from: 'jane.smith@gmail.com',
+  to: [
+    'jane.smith@gmail.com',
+    'sam.doe@yahoo.com'
+  ]
+}))
+```
+
+#### Syslog
+
+Optional dependency:
+
+```
+> yarn add split-host syslog-client
+```
+
+Configuration:
+
+```js
+import createSyslogTransport from '@xen-orchestra/log/transports/syslog'
+
+// By default, log to udp://localhost:514
+configure(createSyslogTransport())
+
+// But TCP, a different host, or a different port can be used
+configure(createSyslogTransport('tcp://syslog.company.lan'))
+```
+
+## Development
+
+```
+# Install dependencies
+> yarn
+
+# Run the tests
+> yarn test
+
+# Continuously compile
+> yarn dev
+
+# Continuously run the tests
+> yarn dev-test
+
+# Build for production (automatically called by npm install)
+> yarn build
+```
+
+## Contributions
+
+Contributions are *very* welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xo-web/issues/)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+ISC © [Vates SAS](https://vates.fr)

@xen-orchestra/log/configure.js

@@ -0,0 +1 @@
+module.exports = require('./dist/configure')

@xen-orchestra/log/package.json

@@ -0,0 +1,52 @@
+{
+  "private": true,
+  "name": "@xen-orchestra/log",
+  "version": "0.0.0",
+  "license": "ISC",
+  "description": "",
+  "keywords": [],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/@xen-orchestra/log",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Julien Fontanet",
+    "email": "julien.fontanet@vates.fr"
+  },
+  "preferGlobal": false,
+  "main": "dist/",
+  "bin": {},
+  "files": [
+    "dist/"
+  ],
+  "browserslist": [
+    ">2%"
+  ],
+  "engines": {
+    "node": ">=4"
+  },
+  "dependencies": {
+    "@babel/polyfill": "7.0.0-beta.42",
+    "lodash": "^4.17.4",
+    "promise-toolbox": "^0.9.5"
+  },
+  "devDependencies": {
+    "@babel/cli": "7.0.0-beta.42",
+    "@babel/core": "7.0.0-beta.42",
+    "@babel/preset-env": "7.0.0-beta.42",
+    "@babel/preset-flow": "7.0.0-beta.42",
+    "babel-plugin-lodash": "^3.3.2",
+    "cross-env": "^5.1.3",
+    "rimraf": "^2.6.2"
+  },
+  "scripts": {
+    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
+    "clean": "rimraf dist/",
+    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
+    "prebuild": "yarn run clean",
+    "predev": "yarn run prebuild",
+    "prepublishOnly": "yarn run build"
+  }
+}

@xen-orchestra/log/src/configure.js

@@ -0,0 +1,105 @@
+import createConsoleTransport from './transports/console'
+import LEVELS, { resolve } from './levels'
+import { compileGlobPattern } from './utils'
+
+// ===================================================================
+
+const createTransport = config => {
+  if (typeof config === 'function') {
+    return config
+  }
+
+  if (Array.isArray(config)) {
+    const transports = config.map(createTransport)
+    const { length } = transports
+    return function () {
+      for (let i = 0; i < length; ++i) {
+        transports[i].apply(this, arguments)
+      }
+    }
+  }
+
+  let { filter, transport } = config
+  const level = resolve(config.level)
+
+  if (filter !== undefined) {
+    if (typeof filter === 'string') {
+      const re = compileGlobPattern(filter)
+      filter = log => re.test(log.namespace)
+    }
+
+    const orig = transport
+    transport = function (log) {
+      if ((level !== undefined && log.level >= level) || filter(log)) {
+        return orig.apply(this, arguments)
+      }
+    }
+  } else if (level !== undefined) {
+    const orig = transport
+    transport = function (log) {
+      if (log.level >= level) {
+        return orig.apply(this, arguments)
+      }
+    }
+  }
+
+  return transport
+}
+
+let transport = createTransport({
+  // display warnings or above, and all that are enabled via DEBUG or
+  // NODE_DEBUG env
+  filter: process.env.DEBUG || process.env.NODE_DEBUG,
+  level: LEVELS.INFO,
+
+  transport: createConsoleTransport(),
+})
+
+const symbol =
+  typeof Symbol !== 'undefined'
+    ? Symbol.for('@xen-orchestra/log')
+    : '@@@xen-orchestra/log'
+global[symbol] = log => transport(log)
+
+export const configure = config => {
+  transport = createTransport(config)
+}
+
+// -------------------------------------------------------------------
+
+export const catchGlobalErrors = logger => {
+  // patch process
+  const onUncaughtException = error => {
+    logger.error('uncaught exception', { error })
+  }
+  const onUnhandledRejection = error => {
+    logger.warn('possibly unhandled rejection', { error })
+  }
+  const onWarning = error => {
+    logger.warn('Node warning', { error })
+  }
+  process.on('uncaughtException', onUncaughtException)
+  process.on('unhandledRejection', onUnhandledRejection)
+  process.on('warning', onWarning)
+
+  // patch EventEmitter
+  const EventEmitter = require('events')
+  const { prototype } = EventEmitter
+  const { emit } = prototype
+  function patchedEmit (event, error) {
+    event === 'error' && !this.listenerCount(event)
+      ? logger.error('unhandled error event', { error })
+      : emit.apply(this, arguments)
+  }
+  prototype.emit = patchedEmit
+
+  return () => {
+    process.removeListener('uncaughtException', onUncaughtException)
+    process.removeListener('unhandledRejection', onUnhandledRejection)
+    process.removeListener('warning', onWarning)
+
+    if (prototype.emit === patchedEmit) {
+      prototype.emit = emit
+    }
+  }
+}

@xen-orchestra/log/src/index.js

@@ -0,0 +1,65 @@
+import createTransport from './transports/console'
+import LEVELS from './levels'
+
+const symbol =
+  typeof Symbol !== 'undefined'
+    ? Symbol.for('@xen-orchestra/log')
+    : '@@@xen-orchestra/log'
+if (!(symbol in global)) {
+  // the default behavior, without requiring `configure` is to avoid
+  // logging anything unless it's a real error
+  const transport = createTransport()
+  global[symbol] = log => log.level > LEVELS.WARN && transport(log)
+}
+
+// -------------------------------------------------------------------
+
+function Log (data, level, namespace, message, time) {
+  this.data = data
+  this.level = level
+  this.namespace = namespace
+  this.message = message
+  this.time = time
+}
+
+function Logger (namespace) {
+  this._namespace = namespace
+
+  // bind all logging methods
+  for (const name in LEVELS) {
+    const lowerCase = name.toLowerCase()
+    this[lowerCase] = this[lowerCase].bind(this)
+  }
+}
+
+const { prototype } = Logger
+
+for (const name in LEVELS) {
+  const level = LEVELS[name]
+
+  prototype[name.toLowerCase()] = function (message, data) {
+    global[symbol](new Log(data, level, this._namespace, message, new Date()))
+  }
+}
+
+prototype.wrap = function (message, fn) {
+  const logger = this
+  const warnAndRethrow = error => {
+    logger.warn(message, { error })
+    throw error
+  }
+  return function () {
+    try {
+      const result = fn.apply(this, arguments)
+      const then = result != null && result.then
+      return typeof then === 'function'
+        ? then.call(result, warnAndRethrow)
+        : result
+    } catch (error) {
+      warnAndRethrow(error)
+    }
+  }
+}
+
+const createLogger = namespace => new Logger(namespace)
+export { createLogger }

@xen-orchestra/log/src/levels.js

@@ -0,0 +1,24 @@
+const LEVELS = Object.create(null)
+export { LEVELS as default }
+
+// https://github.com/trentm/node-bunyan#levels
+LEVELS.FATAL = 60 // service/app is going to down
+LEVELS.ERROR = 50 // fatal for current action
+LEVELS.WARN = 40 // something went wrong but it's not fatal
+LEVELS.INFO = 30 // detail on unusual but normal operation
+LEVELS.DEBUG = 20
+
+export const NAMES = Object.create(null)
+for (const name in LEVELS) {
+  NAMES[LEVELS[name]] = name
+}
+
+export const resolve = level => {
+  if (typeof level === 'string') {
+    level = LEVELS[level.toUpperCase()]
+  }
+  return level
+}
+
+Object.freeze(LEVELS)
+Object.freeze(NAMES)

@xen-orchestra/log/src/levels.spec.js

@@ -0,0 +1,32 @@
+/* eslint-env jest */
+
+import { forEach, isInteger } from 'lodash'
+
+import LEVELS, { NAMES, resolve } from './levels'
+
+describe('LEVELS', () => {
+  it('maps level names to their integer values', () => {
+    forEach(LEVELS, (value, name) => {
+      expect(isInteger(value)).toBe(true)
+    })
+  })
+})
+
+describe('NAMES', () => {
+  it('maps level values to their names', () => {
+    forEach(LEVELS, (value, name) => {
+      expect(NAMES[value]).toBe(name)
+    })
+  })
+})
+
+describe('resolve()', () => {
+  it('returns level values either from values or names', () => {
+    forEach(LEVELS, value => {
+      expect(resolve(value)).toBe(value)
+    })
+    forEach(NAMES, (name, value) => {
+      expect(resolve(name)).toBe(+value)
+    })
+  })
+})

@xen-orchestra/log/src/transports/console.js

@@ -0,0 +1,20 @@
+import LEVELS, { NAMES } from '../levels'
+
+// Bind console methods (necessary for browsers)
+const debugConsole = console.log.bind(console)
+const infoConsole = console.info.bind(console)
+const warnConsole = console.warn.bind(console)
+const errorConsole = console.error.bind(console)
+
+const { ERROR, INFO, WARN } = LEVELS
+
+const consoleTransport = ({ data, level, namespace, message, time }) => {
+  const fn =
+    level < INFO
+      ? debugConsole
+      : level < WARN ? infoConsole : level < ERROR ? warnConsole : errorConsole
+
+  fn('%s - %s - [%s] %s', time.toISOString(), namespace, NAMES[level], message)
+  data != null && fn(data)
+}
+export default () => consoleTransport

@xen-orchestra/log/src/transports/email.js

@@ -0,0 +1,68 @@
+import prettyFormat from 'pretty-format' // eslint-disable-line node/no-extraneous-import
+import { createTransport } from 'nodemailer' // eslint-disable-line node/no-extraneous-import
+import { fromCallback } from 'promise-toolbox'
+
+import { evalTemplate, required } from '../utils'
+import { NAMES } from '../levels'
+
+export default ({
+  // transport options (https://nodemailer.com/smtp/)
+  auth,
+  authMethod,
+  host,
+  ignoreTLS,
+  port,
+  proxy,
+  requireTLS,
+  secure,
+  service,
+  tls,
+
+  // message options (https://nodemailer.com/message/)
+  bcc,
+  cc,
+  from = required('from'),
+  to = required('to'),
+  subject = '[{{level}} - {{namespace}}] {{time}} {{message}}',
+}) => {
+  const transporter = createTransport(
+    {
+      auth,
+      authMethod,
+      host,
+      ignoreTLS,
+      port,
+      proxy,
+      requireTLS,
+      secure,
+      service,
+      tls,
+
+      disableFileAccess: true,
+      disableUrlAccess: true,
+    },
+    {
+      bcc,
+      cc,
+      from,
+      to,
+    }
+  )
+
+  return log =>
+    fromCallback(cb =>
+      transporter.sendMail(
+        {
+          subject: evalTemplate(
+            subject,
+            key =>
+              key === 'level'
+                ? NAMES[log.level]
+                : key === 'time' ? log.time.toISOString() : log[key]
+          ),
+          text: prettyFormat(log.data),
+        },
+        cb
+      )
+    )
+}

@xen-orchestra/log/src/transports/syslog.js

@@ -0,0 +1,42 @@
+import splitHost from 'split-host' // eslint-disable-line node/no-extraneous-import node/no-missing-import
+import { createClient, Facility, Severity, Transport } from 'syslog-client' // eslint-disable-line node/no-extraneous-import node/no-missing-import
+import { fromCallback } from 'promise-toolbox'
+import { startsWith } from 'lodash'
+
+import LEVELS from '../levels'
+
+// https://github.com/paulgrove/node-syslog-client#syslogseverity
+const LEVEL_TO_SEVERITY = {
+  [LEVELS.FATAL]: Severity.Critical,
+  [LEVELS.ERROR]: Severity.Error,
+  [LEVELS.WARN]: Severity.Warning,
+  [LEVELS.INFO]: Severity.Informational,
+  [LEVELS.DEBUG]: Severity.Debug,
+}
+
+const facility = Facility.User
+
+export default target => {
+  const opts = {}
+  if (target !== undefined) {
+    if (startsWith(target, 'tcp://')) {
+      target = target.slice(6)
+      opts.transport = Transport.Tcp
+    } else if (startsWith(target, 'udp://')) {
+      target = target.slice(6)
+      opts.transport = Transport.Ucp
+    }
+
+    ;({ host: target, port: opts.port } = splitHost(target))
+  }
+
+  const client = createClient(target, opts)
+
+  return log =>
+    fromCallback(cb =>
+      client.log(log.message, {
+        facility,
+        severity: LEVEL_TO_SEVERITY[log.level],
+      })
+    )
+}

@xen-orchestra/log/src/utils.js

@@ -0,0 +1,62 @@
+import escapeRegExp from 'lodash/escapeRegExp'
+
+// ===================================================================
+
+const TPL_RE = /\{\{(.+?)\}\}/g
+export const evalTemplate = (tpl, data) => {
+  const getData =
+    typeof data === 'function' ? (_, key) => data(key) : (_, key) => data[key]
+
+  return tpl.replace(TPL_RE, getData)
+}
+
+// -------------------------------------------------------------------
+
+const compileGlobPatternFragment = pattern =>
+  pattern
+    .split('*')
+    .map(escapeRegExp)
+    .join('.*')
+
+export const compileGlobPattern = pattern => {
+  const no = []
+  const yes = []
+  pattern.split(/[\s,]+/).forEach(pattern => {
+    if (pattern[0] === '-') {
+      no.push(pattern.slice(1))
+    } else {
+      yes.push(pattern)
+    }
+  })
+
+  const raw = ['^']
+
+  if (no.length !== 0) {
+    raw.push('(?!', no.map(compileGlobPatternFragment).join('|'), ')')
+  }
+
+  if (yes.length !== 0) {
+    raw.push('(?:', yes.map(compileGlobPatternFragment).join('|'), ')')
+  } else {
+    raw.push('.*')
+  }
+
+  raw.push('$')
+
+  return new RegExp(raw.join(''))
+}
+
+// -------------------------------------------------------------------
+
+export const required = name => {
+  throw new Error(`missing required arg ${name}`)
+}
+
+// -------------------------------------------------------------------
+
+export const serializeError = error => ({
+  ...error,
+  message: error.message,
+  name: error.name,
+  stack: error.stack,
+})

@xen-orchestra/log/src/utils.spec.js

@@ -0,0 +1,13 @@
+/* eslint-env jest */
+
+import { compileGlobPattern } from './utils'
+
+describe('compileGlobPattern()', () => {
+  it('works', () => {
+    const re = compileGlobPattern('foo, ba*, -bar')
+    expect(re.test('foo')).toBe(true)
+    expect(re.test('bar')).toBe(false)
+    expect(re.test('baz')).toBe(true)
+    expect(re.test('qux')).toBe(false)
+  })
+})

@xen-orchestra/log/transports

@@ -0,0 +1 @@
+dist/transports

CODE_OF_CONDUCT.md

@@ -0,0 +1,46 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at julien.fontanet@vates.fr. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

ISSUE_TEMPLATE.md

@@ -0,0 +1,27 @@
+<!--
+Welcome to the issue section of Xen Orchestra!
+
+Here you can:
+- report an issue
+- propose an enhancement
+- ask a question
+
+Please, respect this template as much as possible, it helps us sort
+the issues :)
+-->
+
+### Context
+
+- **XO origin**: the sources / XO Appliance
+- **Versions**:
+   - Node: **FILL HERE**
+   - xo-web: **FILL HERE**
+   - xo-server: **FILL HERE**
+
+### Expected behavior
+
+<!-- What you expect to happen -->
+
+### Current behavior
+
+<!-- What is actually happening -->

README.md

@@ -1,7 +1,11 @@
-Almost all dev for Xen Orchestra is happening in this repository.
+# Xen Orchestra [![Chat with us](https://storage.crisp.im/plugins/images/936925df-f37b-4ba8-bab0-70cd2edcb0be/badge.svg)](https://go.crisp.im/chat/embed/?website_id=-JzqzzwddSV7bKGtEyAQ) [![Build Status](https://travis-ci.org/vatesfr/xen-orchestra.png?branch=master)](https://travis-ci.org/vatesfr/xen-orchestra)
 
-Because transition is still underway,
-[xo-web](https://github.com/vatesfr/xo-web) is still developed in
-their own repositories.
+![](http://i.imgur.com/tRffA5y.png)
 
-For now, all issues are still to be reported in [xo-web's tracker](https://github.com/vatesfr/xo-web/issues).
+## Installation
+
+XOA or manual install procedure is [available here](https://xen-orchestra.com/docs/installation.html)
+
+## License
+
+AGPL3 © [Vates SAS](http://vates.fr)

flow-typed/lodash.js

@@ -0,0 +1,24 @@
+declare module 'lodash' {
+  declare export function forEach<K, V>(
+    object: { [K]: V },
+    iteratee: (V, K) => void
+  ): void
+  declare export function groupBy<K, V>(
+    object: { [K]: V },
+    iteratee: K | ((V, K) => string)
+  ): { [string]: V[] }
+  declare export function invert<K, V>(object: { [K]: V }): { [V]: K }
+  declare export function isEmpty(mixed): boolean
+  declare export function keyBy<T>(array: T[], iteratee: string): boolean
+  declare export function last<T>(array?: T[]): T | void
+  declare export function map<T1, T2>(
+    collection: T1[],
+    iteratee: (T1) => T2
+  ): T2[]
+  declare export function mapValues<K, V1, V2>(
+    object: { [K]: V1 },
+    iteratee: (V1, K) => V2
+  ): { [K]: V2 }
+  declare export function noop(...args: mixed[]): void
+  declare export function values<K, V>(object: { [K]: V }): V[]
+}

flow-typed/promise-toolbox.js

@@ -0,0 +1,14 @@
+declare module 'promise-toolbox' {
+  declare export function cancelable(Function): Function
+  declare export function defer<T>(): {|
+    promise: Promise<T>,
+    reject: T => void,
+    resolve: T => void,
+  |}
+  declare export function fromCallback<T>(
+    (cb: (error: any, value: T) => void) => void
+  ): Promise<T>
+  declare export function fromEvent(emitter: mixed, string): Promise<mixed>
+  declare export function ignoreErrors(): Promise<void>
+  declare export function timeout<T>(delay: number): Promise<T>
+}

flow-typed/xo.js

@@ -0,0 +1,2 @@
+// eslint-disable-next-line no-undef
+declare type $Dict<T, K = string> = { [K]: T }

package.json

@@ -1,19 +1,24 @@
 {
   "devDependencies": {
+    "@babel/register": "^7.0.0-beta.44",
     "babel-7-jest": "^21.3.2",
     "babel-eslint": "^8.1.2",
+    "benchmark": "^2.1.4",
     "eslint": "^4.14.0",
     "eslint-config-standard": "^11.0.0-beta.0",
+    "eslint-config-standard-jsx": "^5.0.0",
     "eslint-plugin-import": "^2.8.0",
-    "eslint-plugin-node": "^5.2.1",
+    "eslint-plugin-node": "^6.0.0",
     "eslint-plugin-promise": "^3.6.0",
+    "eslint-plugin-react": "^7.6.1",
     "eslint-plugin-standard": "^3.0.1",
     "exec-promise": "^0.7.0",
-    "flow-bin": "^0.64.0",
+    "flow-bin": "^0.69.0",
+    "globby": "^8.0.0",
     "husky": "^0.14.3",
     "jest": "^22.0.4",
-    "lint-staged": "^6.0.0",
     "lodash": "^4.17.4",
+    "prettier": "^1.10.2",
     "promise-toolbox": "^0.9.5",
     "sorted-object": "^2.0.1"
   },
@@ -22,40 +27,40 @@
   },
   "jest": {
     "collectCoverage": true,
+    "projects": [
+      "<rootDir>",
+      "<rootDir>/packages/xo-web"
+    ],
     "testEnvironment": "node",
     "testPathIgnorePatterns": [
       "/dist/",
-      "/xo-vmdk-to-vhd/"
+      "/xo-web/"
     ],
     "testRegex": "\\.spec\\.js$",
     "transform": {
       "/@xen-orchestra/cron/.+\\.jsx?$": "babel-7-jest",
+      "/@xen-orchestra/fs/.+\\.jsx?$": "babel-7-jest",
       "/packages/complex-matcher/.+\\.jsx?$": "babel-7-jest",
       "/packages/value-matcher/.+\\.jsx?$": "babel-7-jest",
+      "/packages/vhd-lib/.+\\.jsx?$": "babel-7-jest",
       "/packages/xo-cli/.+\\.jsx?$": "babel-7-jest",
+      "/packages/xo-server/.+\\.jsx?$": "babel-7-jest",
+      "/packages/xo-vmdk-to-vhd/.+\\.jsx?$": "babel-7-jest",
       "\\.jsx?$": "babel-jest"
     }
   },
-  "lint-staged": {
-    "*.js": [
-      "lint-staged-stash",
-      "eslint --fix",
-      "jest --findRelatedTests --passWithNoTests",
-      "lint-staged-unstash"
-    ]
-  },
   "private": true,
   "scripts": {
     "build": "scripts/run-script --parallel build",
     "clean": "scripts/run-script --parallel clean",
-    "dev-test": "jest --bail --watch",
-    "lint-staged-stash": "touch .lint-staged && git stash save --include-untracked --keep-index && true",
-    "lint-staged-unstash": "git stash pop && rm -f .lint-staged && true",
+    "dev": "scripts/run-script --parallel dev",
+    "dev-test": "jest --bail --watch \"^(?!.*\\.integ\\.spec\\.js$)\"",
     "posttest": "scripts/run-script test",
-    "precommit": "lint-staged",
+    "precommit": "scripts/lint-staged",
     "prepare": "scripts/run-script prepare",
     "pretest": "eslint --ignore-path .gitignore .",
-    "test": "jest && flow status"
+    "test": "jest \"^(?!.*\\.integ\\.spec\\.js$)\"",
+    "test-integration": "jest \".integ\\.spec\\.js$\""
   },
   "workspaces": [
     "@xen-orchestra/*",

packages/complex-matcher/.babelrc.js

@@ -1,28 +1,3 @@
-const dependencies = require('./package').dependencies || {}
-
-const NODE_ENV = process.env.NODE_ENV || 'development'
-const __PROD__ = NODE_ENV === 'production'
-const __TEST__ = NODE_ENV === 'test'
-
-module.exports = {
-  comments: !__PROD__,
-  ignore: __TEST__ ? undefined : [/\.spec\.js$/],
-  plugins: ['lodash'],
-  presets: [
-    [
-      '@babel/env',
-      {
-        debug: !__TEST__,
-        loose: true,
-        shippedProposals: true,
-        targets: __PROD__
-          ? {
-            browsers: '>2%',
-            node: '4',
-          }
-          : { node: 'current' },
-        useBuiltIns: '@babel/polyfill' in dependencies && 'usage',
-      },
-    ],
-  ],
-}
+module.exports = require('../../@xen-orchestra/babel-config')(
+  require('./package.json')
+)

packages/complex-matcher/README.md

@@ -57,7 +57,7 @@ the code.
 
 You may:
 
-- report any [issue](https://github.com/vatesfr/xo-web/issues)
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
   you've encountered;
 - fork and create a pull request.
 

packages/complex-matcher/package.json

@@ -1,11 +1,11 @@
 {
   "name": "complex-matcher",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "license": "ISC",
   "description": "",
   "keywords": [],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/complex-matcher",
-  "bugs": "https://github.com/vatesfr/xo-web/issues",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
@@ -20,6 +20,9 @@
   "files": [
     "dist/"
   ],
+  "browserslist": [
+    ">2%"
+  ],
   "engines": {
     "node": ">=4"
   },
@@ -27,9 +30,9 @@
     "lodash": "^4.17.4"
   },
   "devDependencies": {
-    "@babel/cli": "7.0.0-beta.39",
-    "@babel/core": "7.0.0-beta.39",
-    "@babel/preset-env": "7.0.0-beta.39",
+    "@babel/cli": "7.0.0-beta.44",
+    "@babel/core": "7.0.0-beta.44",
+    "@babel/preset-env": "7.0.0-beta.44",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^5.1.1",
     "rimraf": "^2.6.2"

packages/complex-matcher/src/index.js

@@ -70,6 +70,29 @@ export class And extends Node {
   }
 }
 
+export class Comparison extends Node {
+  constructor (operator, value) {
+    super()
+    this._comparator = Comparison.comparators[operator]
+    this._operator = operator
+    this._value = value
+  }
+
+  match (value) {
+    return typeof value === 'number' && this._comparator(value, this._value)
+  }
+
+  toString () {
+    return this._operator + String(this._value)
+  }
+}
+Comparison.comparators = {
+  '>': (a, b) => a > b,
+  '>=': (a, b) => a >= b,
+  '<': (a, b) => a < b,
+  '<=': (a, b) => a <= b,
+}
+
 export class Or extends Node {
   constructor (children) {
     super()
@@ -325,7 +348,10 @@ class P {
         value.push(result.value)
         pos = result.pos
       }
-      while (i < max && (result = this._parse(input, pos, end)) instanceof Success) {
+      while (
+        i < max &&
+        (result = this._parse(input, pos, end)) instanceof Success
+      ) {
         ++i
         value.push(result.value)
         pos = result.pos
@@ -359,8 +385,9 @@ P.eof = new P(
 
 const parser = P.grammar({
   default: r =>
-    P.seq(r.ws, r.term.repeat(), P.eof)
-      .map(([, terms]) => (terms.length === 0 ? new Null() : new And(terms))),
+    P.seq(r.ws, r.term.repeat(), P.eof).map(
+      ([, terms]) => (terms.length === 0 ? new Null() : new And(terms))
+    ),
   quotedString: new P((input, pos, end) => {
     if (input[pos] !== '"') {
       return new Failure(pos, '"')
@@ -404,6 +431,13 @@ const parser = P.grammar({
         P.text(')')
       ).map(_ => new Or(_[4])),
       P.seq(P.text('!'), r.ws, r.term).map(_ => new Not(_[2])),
+      P.seq(P.regex(/[<>]=?/), r.rawString).map(([op, val]) => {
+        val = +val
+        if (Number.isNaN(val)) {
+          throw new TypeError('value must be a number')
+        }
+        return new Comparison(op, val)
+      }),
       P.seq(r.string, r.ws, P.text(':'), r.ws, r.term).map(
         _ => new Property(_[0], _[4])
       ),
@@ -416,7 +450,7 @@ const parser = P.grammar({
             ? new StringNode(str)
             : new NumberNode(asNum)
         })
-      ),
+      )
     ).skip(r.ws),
   ws: P.regex(/\s*/),
 }).default
@@ -476,17 +510,19 @@ export const getPropertyClausesStrings = node => {
 // -------------------------------------------------------------------
 
 export const setPropertyClause = (node, name, child) => {
-  const property = child && new Property(
-    name,
-    typeof child === 'string' ? new StringNode(child) : child
-  )
+  const property =
+    child &&
+    new Property(
+      name,
+      typeof child === 'string' ? new StringNode(child) : child
+    )
 
   if (node === undefined) {
     return property
   }
 
-  const children = (node instanceof And ? node.children : [node]).filter(child =>
-    !(child instanceof Property && child.name === name)
+  const children = (node instanceof And ? node.children : [node]).filter(
+    child => !(child instanceof Property && child.name === name)
   )
   if (property !== undefined) {
     children.push(property)

packages/complex-matcher/src/index.spec.js

@@ -49,13 +49,15 @@ describe('Number', () => {
 
 describe('setPropertyClause', () => {
   it('creates a node if none passed', () => {
-    expect(setPropertyClause(undefined, 'foo', 'bar').toString()).toBe('foo:bar')
+    expect(setPropertyClause(undefined, 'foo', 'bar').toString()).toBe(
+      'foo:bar'
+    )
   })
 
   it('adds a property clause if there was none', () => {
-    expect(
-      setPropertyClause(parse('baz'), 'foo', 'bar').toString()
-    ).toBe('baz foo:bar')
+    expect(setPropertyClause(parse('baz'), 'foo', 'bar').toString()).toBe(
+      'baz foo:bar'
+    )
   })
 
   it('replaces the property clause if there was one', () => {

packages/value-matcher/.babelrc.js

@@ -1,40 +1,3 @@
-'use strict'
-
-const NODE_ENV = process.env.NODE_ENV || 'development'
-const __PROD__ = NODE_ENV === 'production'
-const __TEST__ = NODE_ENV === 'test'
-
-const pkg = require('./package')
-
-let nodeCompat = (pkg.engines || {}).node
-if (nodeCompat === undefined) {
-  nodeCompat = '6'
-} else {
-  const trimChars = '^=>~'
-  while (trimChars.includes(nodeCompat[0])) {
-    nodeCompat = nodeCompat.slice(1)
-  }
-}
-
-module.exports = {
-  comments: !__PROD__,
-  ignore: __TEST__ ? undefined : [/\.spec\.js$/],
-  presets: [
-    [
-      '@babel/env',
-      {
-        debug: !__TEST__,
-        loose: true,
-        shippedProposals: true,
-        targets: __PROD__
-          ? {
-            browsers: '>2%',
-            node: nodeCompat,
-          }
-          : { node: 'current' },
-        useBuiltIns: '@babel/polyfill' in (pkg.dependencies || {}) && 'usage',
-      },
-    ],
-    '@babel/flow',
-  ],
-}
+module.exports = require('../../@xen-orchestra/babel-config')(
+  require('./package.json')
+)

packages/value-matcher/README.md

@@ -57,7 +57,7 @@ the code.
 
 You may:
 
-- report any [issue](https://github.com/vatesfr/xo-web/issues)
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
   you've encountered;
 - fork and create a pull request.
 

packages/value-matcher/package.json

@@ -1,11 +1,11 @@
 {
   "name": "value-matcher",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "license": "ISC",
   "description": "",
   "keywords": [],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/value-matcher",
-  "bugs": "https://github.com/vatesfr/xo-web/issues",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
@@ -20,15 +20,18 @@
   "files": [
     "dist/"
   ],
+  "browserslist": [
+    ">2%"
+  ],
   "engines": {
     "node": ">=4"
   },
   "dependencies": {},
   "devDependencies": {
-    "@babel/cli": "7.0.0-beta.39",
-    "@babel/core": "7.0.0-beta.39",
-    "@babel/preset-env": "7.0.0-beta.39",
-    "@babel/preset-flow": "7.0.0-beta.39",
+    "@babel/cli": "7.0.0-beta.44",
+    "@babel/core": "7.0.0-beta.44",
+    "@babel/preset-env": "7.0.0-beta.44",
+    "@babel/preset-flow": "7.0.0-beta.44",
     "cross-env": "^5.1.3",
     "rimraf": "^2.6.2"
   },

packages/value-matcher/src/index.js

@@ -1,7 +1,14 @@
 // @flow
 
-// eslint-disable-next-line no-use-before-define
-export type Pattern = OrPattern | NotPattern | ObjectPattern | ArrayPattern | ValuePattern
+/* eslint-disable no-use-before-define */
+export type Pattern =
+  | AndPattern
+  | OrPattern
+  | NotPattern
+  | ObjectPattern
+  | ArrayPattern
+  | ValuePattern
+/* eslint-enable no-use-before-define */
 
 // all patterns must match
 type AndPattern = {| __and: Array<Pattern> |}
@@ -19,13 +26,16 @@ type ObjectPattern = { [string]: Pattern }
 type ArrayPattern = Array<Pattern>
 
 // value equals the pattern
-type ValuePattern = bool | number | string
+type ValuePattern = boolean | number | string
 
 const match = (pattern: Pattern, value: any) => {
   if (Array.isArray(pattern)) {
-    return Array.isArray(value) && pattern.every((subpattern, i) =>
-      // FIXME: subpatterns should match different subvalues
-      value.some(subvalue => match(subpattern, subvalue))
+    return (
+      Array.isArray(value) &&
+      pattern.every((subpattern, i) =>
+        // FIXME: subpatterns should match different subvalues
+        value.some(subvalue => match(subpattern, subvalue))
+      )
     )
   }
 
@@ -34,7 +44,7 @@ const match = (pattern: Pattern, value: any) => {
     const { length } = keys
 
     if (length === 1) {
-      const [ key ] = keys
+      const [key] = keys
       if (key === '__and') {
         const andPattern: AndPattern = (pattern: any)
         return andPattern.__and.every(subpattern => match(subpattern, value))
@@ -67,4 +77,5 @@ const match = (pattern: Pattern, value: any) => {
   return pattern === value
 }
 
-export const createPredicate = (pattern: Pattern) => (value: any) => match(pattern, value)
+export const createPredicate = (pattern: Pattern) => (value: any) =>
+  match(pattern, value)

packages/vhd-cli/README.md

@@ -42,7 +42,7 @@ the code.
 
 You may:
 
-- report any [issue](https://github.com/vatesfr/xo-web/issues)
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
   you've encountered;
 - fork and create a pull request.
 

packages/vhd-cli/package.json

@@ -1,11 +1,11 @@
 {
   "name": "vhd-cli",
-  "version": "0.0.0",
+  "version": "0.0.1",
   "license": "ISC",
   "description": "",
   "keywords": [],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-cli",
-  "bugs": "https://github.com/vatesfr/xo-web/issues",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
@@ -26,10 +26,11 @@
     "node": ">=4"
   },
   "dependencies": {
-    "@nraynaud/struct-fu": "^1.0.1",
-    "@nraynaud/xo-fs": "^0.0.5",
+    "@xen-orchestra/fs": "^0.0.0",
     "babel-runtime": "^6.22.0",
-    "exec-promise": "^0.7.0"
+    "exec-promise": "^0.7.0",
+    "struct-fu": "^1.2.0",
+    "vhd-lib": "^0.0.0"
   },
   "devDependencies": {
     "babel-cli": "^6.24.1",
@@ -38,14 +39,18 @@
     "babel-preset-env": "^1.5.2",
     "babel-preset-stage-3": "^6.24.1",
     "cross-env": "^5.1.3",
-    "rimraf": "^2.6.1"
+    "execa": "^0.10.0",
+    "index-modules": "^0.3.0",
+    "promise-toolbox": "^0.9.5",
+    "rimraf": "^2.6.1",
+    "tmp": "^0.0.33"
   },
   "scripts": {
     "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
     "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
-    "prebuild": "rimraf dist/",
+    "prebuild": "rimraf dist/ && index-modules --cjs-lazy src/commands",
     "predev": "yarn run prebuild",
-    "prepublishOnly": "yarn run build"
+    "prepare": "yarn run build"
   },
   "babel": {
     "plugins": [

packages/vhd-cli/src/commands/check.js

@@ -0,0 +1,15 @@
+import Vhd from 'vhd-lib'
+import { getHandler } from '@xen-orchestra/fs'
+import { resolve } from 'path'
+
+export default async args => {
+  const handler = getHandler({ url: 'file:///' })
+  for (const vhd of args) {
+    try {
+      await new Vhd(handler, resolve(vhd)).readHeaderAndFooter()
+      console.log('ok:', vhd)
+    } catch (error) {
+      console.error('nok:', vhd, error)
+    }
+  }
+}

packages/vhd-cli/src/commands/info.js

@@ -0,0 +1,12 @@
+import Vhd from 'vhd-lib'
+import { getHandler } from '@xen-orchestra/fs'
+import { resolve } from 'path'
+
+export default async args => {
+  const vhd = new Vhd(getHandler({ url: 'file:///' }), resolve(args[0]))
+
+  await vhd.readHeaderAndFooter()
+
+  console.log(vhd.header)
+  console.log(vhd.footer)
+}

packages/vhd-cli/src/commands/synthetize.js

@@ -0,0 +1,21 @@
+import path from 'path'
+import { createSyntheticStream } from 'vhd-lib'
+import { createWriteStream } from 'fs'
+import { getHandler } from '@xen-orchestra/fs'
+
+export default async function main (args) {
+  if (args.length < 2 || args.some(_ => _ === '-h' || _ === '--help')) {
+    return `Usage: ${this.command} <input VHD> <output VHD>`
+  }
+
+  const handler = getHandler({ url: 'file:///' })
+  return new Promise((resolve, reject) => {
+    createSyntheticStream(handler, path.resolve(args[0]))
+      .on('error', reject)
+      .pipe(
+        createWriteStream(args[1])
+          .on('error', reject)
+          .on('finish', resolve)
+      )
+  })
+}

packages/vhd-cli/src/index.js

@@ -1,19 +1,44 @@
 #!/usr/bin/env node
 
 import execPromise from 'exec-promise'
-import { RemoteHandlerLocal } from '@nraynaud/xo-fs'
-import { resolve } from 'path'
 
-import Vhd from './vhd'
+import commands from './commands'
 
-execPromise(async args => {
-  const vhd = new Vhd(
-    new RemoteHandlerLocal({ url: 'file:///' }),
-    resolve(args[0])
+function runCommand (commands, [command, ...args]) {
+  if (command === undefined || command === '-h' || command === '--help') {
+    command = 'help'
+  }
+
+  const fn = commands[command]
+
+  if (fn === undefined) {
+    if (command === 'help') {
+      return `Usage:
+
+${Object.keys(commands)
+        .filter(command => command !== 'help')
+        .map(command => `    ${this.command} ${command}`)
+        .join('\n\n')}`
+    }
+
+    throw `invalid command ${command}` // eslint-disable-line no-throw-literal
+  }
+
+  return fn.call(
+    {
+      __proto__: this,
+      command: `${this.command} ${command}`,
+    },
+    args
   )
+}
 
-  await vhd.readHeaderAndFooter()
-
-  console.log(vhd._header)
-  console.log(vhd._footer)
-})
+execPromise(
+  runCommand.bind(
+    {
+      command: 'vhd-cli',
+      runCommand,
+    },
+    commands
+  )
+)

packages/vhd-cli/src/info.integ.spec.js

@@ -0,0 +1,28 @@
+/* eslint-env jest */
+
+import execa from 'execa'
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { fromCallback as pFromCallback } from 'promise-toolbox'
+
+import command from './commands/info'
+
+const initialDir = process.cwd()
+
+jest.setTimeout(10000)
+
+beforeEach(async () => {
+  const dir = await pFromCallback(cb => tmp.dir(cb))
+  process.chdir(dir)
+})
+
+afterEach(async () => {
+  const tmpDir = process.cwd()
+  process.chdir(initialDir)
+  await pFromCallback(cb => rimraf(tmpDir, cb))
+})
+
+test('can run the command', async () => {
+  await execa('qemu-img', ['create', '-fvpc', 'empty.vhd', '1G'])
+  await command(['empty.vhd'])
+})

packages/vhd-cli/src/vhd.js

@@ -1,441 +0,0 @@
-import assert from 'assert'
-import fu from '@nraynaud/struct-fu'
-import { dirname } from 'path'
-
-// ===================================================================
-//
-// Spec:
-// https://www.microsoft.com/en-us/download/details.aspx?id=23850
-//
-// C implementation:
-// https://github.com/rubiojr/vhd-util-convert
-//
-// ===================================================================
-
-/* eslint-disable no-unused-vars */
-
-const HARD_DISK_TYPE_DIFFERENCING = 4
-const HARD_DISK_TYPE_DYNAMIC = 3
-const HARD_DISK_TYPE_FIXED = 2
-const PLATFORM_CODE_NONE = 0
-export const SECTOR_SIZE = 512
-
-/* eslint-enable no-unused vars */
-
-// ===================================================================
-
-const fuFooter = fu.struct([
-  fu.char('cookie', 8), // 0
-  fu.uint32('features'), // 8
-  fu.uint32('fileFormatVersion'), // 12
-  fu.struct('dataOffset', [
-    fu.uint32('high'), // 16
-    fu.uint32('low'), // 20
-  ]),
-  fu.uint32('timestamp'), // 24
-  fu.char('creatorApplication', 4), // 28
-  fu.uint32('creatorVersion'), // 32
-  fu.uint32('creatorHostOs'), // 36
-  fu.struct('originalSize', [ // At the creation, current size of the hard disk.
-    fu.uint32('high'), // 40
-    fu.uint32('low'), // 44
-  ]),
-  fu.struct('currentSize', [ // Current size of the virtual disk. At the creation: currentSize = originalSize.
-    fu.uint32('high'), // 48
-    fu.uint32('low'), // 52
-  ]),
-  fu.struct('diskGeometry', [
-    fu.uint16('cylinders'), // 56
-    fu.uint8('heads'), // 58
-    fu.uint8('sectorsPerTrackCylinder'), // 59
-  ]),
-  fu.uint32('diskType'), // 60 Disk type, must be equal to HARD_DISK_TYPE_DYNAMIC/HARD_DISK_TYPE_DIFFERENCING.
-  fu.uint32('checksum'), // 64
-  fu.uint8('uuid', 16), // 68
-  fu.char('saved'), // 84
-  fu.char('hidden'), // 85
-  fu.byte('reserved', 426), // 86
-])
-const FOOTER_SIZE = fuFooter.size
-
-const fuHeader = fu.struct([
-  fu.char('cookie', 8),
-  fu.struct('dataOffset', [
-    fu.uint32('high'),
-    fu.uint32('low'),
-  ]),
-  fu.struct('tableOffset', [ // Absolute byte offset of the Block Allocation Table.
-    fu.uint32('high'),
-    fu.uint32('low'),
-  ]),
-  fu.uint32('headerVersion'),
-  fu.uint32('maxTableEntries'), // Max entries in the Block Allocation Table.
-  fu.uint32('blockSize'), // Block size (without bitmap) in bytes.
-  fu.uint32('checksum'),
-  fu.uint8('parentUuid', 16),
-  fu.uint32('parentTimestamp'),
-  fu.byte('reserved1', 4),
-  fu.char16be('parentUnicodeName', 512),
-  fu.struct('parentLocatorEntry', [
-    fu.uint32('platformCode'),
-    fu.uint32('platformDataSpace'),
-    fu.uint32('platformDataLength'),
-    fu.uint32('reserved'),
-    fu.struct('platformDataOffset', [ // Absolute byte offset of the locator data.
-      fu.uint32('high'),
-      fu.uint32('low'),
-    ]),
-  ], 8),
-  fu.byte('reserved2', 256),
-])
-const HEADER_SIZE = fuHeader.size
-
-// ===================================================================
-// Helpers
-// ===================================================================
-
-const SIZE_OF_32_BITS = Math.pow(2, 32)
-const uint32ToUint64 = fu => fu.high * SIZE_OF_32_BITS + fu.low
-
-// Returns a 32 bits integer corresponding to a Vhd version.
-const getVhdVersion = (major, minor) => (major << 16) | (minor & 0x0000FFFF)
-
-// bytes[] bit manipulation
-const testBit = (map, bit) => map[bit >> 3] & 1 << (bit & 7)
-const setBit = (map, bit) => {
-  map[bit >> 3] |= 1 << (bit & 7)
-}
-const unsetBit = (map, bit) => {
-  map[bit >> 3] &= ~(1 << (bit & 7))
-}
-
-const addOffsets = (...offsets) => offsets.reduce(
-  (a, b) => b == null
-    ? a
-    : typeof b === 'object'
-      ? { bytes: a.bytes + b.bytes, bits: a.bits + b.bits }
-      : { bytes: a.bytes + b, bits: a.bits },
-  { bytes: 0, bits: 0 }
-)
-
-const pack = (field, value, buf, offset) => {
-  field.pack(
-    value,
-    buf,
-    addOffsets(field.offset, offset)
-  )
-}
-
-const unpack = (field, buf, offset) =>
-  field.unpack(
-    buf,
-    addOffsets(field.offset, offset)
-  )
-
-// ===================================================================
-
-const streamToNewBuffer = stream => new Promise((resolve, reject) => {
-  const chunks = []
-  let length = 0
-
-  const onData = chunk => {
-    chunks.push(chunk)
-    length += chunk.length
-  }
-  stream.on('data', onData)
-
-  const clean = () => {
-    stream.removeListener('data', onData)
-    stream.removeListener('end', onEnd)
-    stream.removeListener('error', onError)
-  }
-  const onEnd = () => {
-    resolve(Buffer.concat(chunks, length))
-    clean()
-  }
-  stream.on('end', onEnd)
-  const onError = error => {
-    reject(error)
-    clean()
-  }
-  stream.on('error', onError)
-})
-
-const streamToExistingBuffer = (
-  stream,
-  buffer,
-  offset = 0,
-  end = buffer.length
-) => new Promise((resolve, reject) => {
-  assert(offset >= 0)
-  assert(end > offset)
-  assert(end <= buffer.length)
-
-  let i = offset
-
-  const onData = chunk => {
-    const prev = i
-    i += chunk.length
-
-    if (i > end) {
-      return onError(new Error('too much data'))
-    }
-
-    chunk.copy(buffer, prev)
-  }
-  stream.on('data', onData)
-
-  const clean = () => {
-    stream.removeListener('data', onData)
-    stream.removeListener('end', onEnd)
-    stream.removeListener('error', onError)
-  }
-  const onEnd = () => {
-    resolve(i - offset)
-    clean()
-  }
-  stream.on('end', onEnd)
-  const onError = error => {
-    reject(error)
-    clean()
-  }
-  stream.on('error', onError)
-})
-
-// ===================================================================
-
-// Returns the checksum of a raw struct.
-const computeChecksum = (struct, buf, offset = 0) => {
-  let sum = 0
-
-  // Do not use the stored checksum to compute the new checksum.
-  const checksumField = struct.fields.checksum
-  const checksumOffset = offset + checksumField.offset
-  for (let i = offset, n = checksumOffset; i < n; ++i) {
-    sum += buf[i]
-  }
-  for (let i = checksumOffset + checksumField.size, n = offset + struct.size; i < n; ++i) {
-    sum += buf[i]
-  }
-
-  return ~sum >>> 0
-}
-
-const verifyChecksum = (struct, buf, offset) =>
-  unpack(struct.fields.checksum, buf, offset) === computeChecksum(struct, buf, offset)
-
-const getParentLocatorSize = parentLocatorEntry => {
-  const { platformDataSpace } = parentLocatorEntry
-
-  if (platformDataSpace < SECTOR_SIZE) {
-    return platformDataSpace * SECTOR_SIZE
-  }
-
-  return (platformDataSpace % SECTOR_SIZE === 0)
-    ? platformDataSpace
-    : 0
-}
-
-// ===================================================================
-
-// Euclidean division, returns the quotient and the remainder of a / b.
-const div = (a, b) => [ Math.floor(a / b), a % b ]
-
-export default class Vhd {
-  constructor (handler, path) {
-    this._handler = handler
-    this._path = path
-
-    this._blockAllocationTable = null
-    this._blockBitmapSize = null
-    this._footer = null
-    this._header = null
-    this._parent = null
-    this._sectorsPerBlock = null
-  }
-
-  // Read `length` bytes starting from `begin`.
-  //
-  // - if `buffer`: it is filled starting from `offset`, and the
-  //   number of written bytes is returned;
-  // - otherwise: a new buffer is allocated and returned.
-  _read (begin, length, buf, offset) {
-    assert(begin >= 0)
-    assert(length > 0)
-
-    return this._handler.createReadStream(this._path, {
-      end: begin + length - 1,
-      start: begin,
-    }).then(buf
-      ? stream => streamToExistingBuffer(stream, buf, offset, (offset || 0) + length)
-      : streamToNewBuffer
-    )
-  }
-
-  // - if `buffer`: it is filled with 0 starting from `offset`, and
-  //   the number of written bytes is returned;
-  // - otherwise: a new buffer is allocated and returned.
-  _zeroes (length, buf, offset = 0) {
-    if (buf) {
-      assert(offset >= 0)
-      assert(length > 0)
-
-      const end = offset + length
-      assert(end <= buf.length)
-
-      buf.fill(0, offset, end)
-      return Promise.resolve(length)
-    }
-
-    return Promise.resolve(Buffer.alloc(length))
-  }
-
-  // Return the position of a block in the VHD or undefined if not found.
-  _getBlockAddress (block) {
-    assert(block >= 0)
-    assert(block < this._header.maxTableEntries)
-
-    const blockAddr = this._blockAllocationTable[block]
-    if (blockAddr !== 0xFFFFFFFF) {
-      return blockAddr * SECTOR_SIZE
-    }
-  }
-
-  // -----------------------------------------------------------------
-
-  async readHeaderAndFooter () {
-    const buf = await this._read(0, FOOTER_SIZE + HEADER_SIZE)
-
-    if (!verifyChecksum(fuFooter, buf)) {
-      throw new Error('footer checksum does not match')
-    }
-
-    if (!verifyChecksum(fuHeader, buf, FOOTER_SIZE)) {
-      throw new Error('header checksum does not match')
-    }
-
-    return this._initMetadata(
-      unpack(fuHeader, buf, FOOTER_SIZE),
-      unpack(fuFooter, buf)
-    )
-  }
-
-  async _initMetadata (header, footer) {
-    const sectorsPerBlock = header.blockSize / SECTOR_SIZE
-    assert(sectorsPerBlock % 1 === 0)
-
-    // 1 bit per sector, rounded up to full sectors
-    this._blockBitmapSize = Math.ceil(sectorsPerBlock / 8 / SECTOR_SIZE) * SECTOR_SIZE
-    assert(this._blockBitmapSize === SECTOR_SIZE)
-
-    this._footer = footer
-    this._header = header
-    this.size = uint32ToUint64(this._footer.currentSize)
-
-    if (footer.diskType === HARD_DISK_TYPE_DIFFERENCING) {
-      const parent = new Vhd(
-        this._handler,
-        `${dirname(this._path)}/${header.parentUnicodeName}`
-      )
-      await parent.readHeaderAndFooter()
-      await parent.readBlockAllocationTable()
-
-      this._parent = parent
-    }
-  }
-
-  // -----------------------------------------------------------------
-
-  async readBlockAllocationTable () {
-    const { maxTableEntries, tableOffset } = this._header
-    const fuTable = fu.uint32(maxTableEntries)
-
-    this._blockAllocationTable = unpack(
-      fuTable,
-      await this._read(uint32ToUint64(tableOffset), fuTable.size)
-    )
-  }
-
-  // -----------------------------------------------------------------
-
-  // read a single sector in a block
-  async _readBlockSector (block, sector, begin, length, buf, offset) {
-    assert(begin >= 0)
-    assert(length > 0)
-    assert(begin + length <= SECTOR_SIZE)
-
-    const blockAddr = this._getBlockAddress(block)
-    const blockBitmapSize = this._blockBitmapSize
-    const parent = this._parent
-
-    if (blockAddr && (
-      !parent ||
-      testBit(await this._read(blockAddr, blockBitmapSize), sector)
-    )) {
-      return this._read(
-        blockAddr + blockBitmapSize + sector * SECTOR_SIZE + begin,
-        length,
-        buf,
-        offset
-      )
-    }
-
-    return parent
-      ? parent._readBlockSector(block, sector, begin, length, buf, offset)
-      : this._zeroes(length, buf, offset)
-  }
-
-  _readBlock (block, begin, length, buf, offset) {
-    assert(begin >= 0)
-    assert(length > 0)
-
-    const { blockSize } = this._header
-    assert(begin + length <= blockSize)
-
-    const blockAddr = this._getBlockAddress(block)
-    const parent = this._parent
-
-    if (!blockAddr) {
-      return parent
-        ? parent._readBlock(block, begin, length, buf, offset)
-        : this._zeroes(length, buf, offset)
-    }
-
-    if (!parent) {
-      return this._read(blockAddr + this._blockBitmapSize + begin, length, buf, offset)
-    }
-
-    // FIXME: we should read as many sectors in a single pass as
-    // possible for maximum perf.
-    const [ sector, beginInSector ] = div(begin, SECTOR_SIZE)
-    return this._readBlockSector(
-      block,
-      sector,
-      beginInSector,
-      Math.min(length, SECTOR_SIZE - beginInSector),
-      buf,
-      offset
-    )
-  }
-
-  read (buf, begin, length = buf.length, offset) {
-    assert(Buffer.isBuffer(buf))
-    assert(begin >= 0)
-
-    const { size } = this
-    if (begin >= size) {
-      return Promise.resolve(0)
-    }
-
-    const { blockSize } = this._header
-    const [ block, beginInBlock ] = div(begin, blockSize)
-
-    return this._readBlock(
-      block,
-      beginInBlock,
-      Math.min(length, blockSize - beginInBlock, size - begin),
-      buf,
-      offset
-    )
-  }
-}

packages/vhd-lib/.babelrc.js

@@ -0,0 +1,3 @@
+module.exports = require('../../@xen-orchestra/babel-config')(
+  require('./package.json')
+)

packages/vhd-lib/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "vhd-lib",
+  "version": "0.0.0",
+  "license": "AGPL-3.0",
+  "description": "Primitives for VHD file handling",
+  "keywords": [],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-lib",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "preferGlobal": true,
+  "main": "dist/",
+  "bin": {},
+  "files": [
+    "dist/"
+  ],
+  "engines": {
+    "node": ">=6"
+  },
+  "dependencies": {
+    "@babel/runtime": "^7.0.0-beta.44",
+    "@xen-orchestra/fs": "^0.0.0",
+    "async-iterator-to-stream": "^1.0.2",
+    "execa": "^0.10.0",
+    "from2": "^2.3.0",
+    "fs-extra": "^5.0.0",
+    "get-stream": "^3.0.0",
+    "limit-concurrency-decorator": "^0.4.0",
+    "promise-toolbox": "^0.9.5",
+    "struct-fu": "^1.2.0",
+    "uuid": "^3.0.1",
+    "tmp": "^0.0.33"
+  },
+  "devDependencies": {
+    "@babel/cli": "7.0.0-beta.44",
+    "@babel/core": "7.0.0-beta.44",
+    "@babel/plugin-transform-runtime": "^7.0.0-beta.44",
+    "@babel/preset-env": "7.0.0-beta.44",
+    "@babel/preset-flow": "7.0.0-beta.44",
+    "babel-plugin-lodash": "^3.3.2",
+    "cross-env": "^5.1.3",
+    "fs-promise": "^2.0.0",
+    "index-modules": "^0.3.0",
+    "rimraf": "^2.6.2"
+  },
+  "scripts": {
+    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
+    "clean": "rimraf dist/",
+    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
+    "prebuild": "yarn run clean",
+    "predev": "yarn run clean",
+    "prepare": "yarn run build"
+  }
+}

packages/vhd-lib/src/_bitmap.js

@@ -0,0 +1,7 @@
+const MASK = 0x80
+
+export const set = (map, bit) => {
+  map[bit >> 3] |= MASK >> (bit & 7)
+}
+
+export const test = (map, bit) => ((map[bit >> 3] << (bit & 7)) & MASK) !== 0

packages/vhd-lib/src/_computeGeometryForSize.js

@@ -0,0 +1,37 @@
+import { SECTOR_SIZE } from './_constants'
+
+export default function computeGeometryForSize (size) {
+  const totalSectors = Math.ceil(size / 512)
+  let sectorsPerTrackCylinder
+  let heads
+  let cylinderTimesHeads
+  if (totalSectors > 65535 * 16 * 255) {
+    throw Error('disk is too big')
+  }
+  // straight copypasta from the file spec appendix on CHS Calculation
+  if (totalSectors >= 65535 * 16 * 63) {
+    sectorsPerTrackCylinder = 255
+    heads = 16
+    cylinderTimesHeads = totalSectors / sectorsPerTrackCylinder
+  } else {
+    sectorsPerTrackCylinder = 17
+    cylinderTimesHeads = totalSectors / sectorsPerTrackCylinder
+    heads = Math.floor((cylinderTimesHeads + 1023) / 1024)
+    if (heads < 4) {
+      heads = 4
+    }
+    if (cylinderTimesHeads >= heads * 1024 || heads > 16) {
+      sectorsPerTrackCylinder = 31
+      heads = 16
+      cylinderTimesHeads = totalSectors / sectorsPerTrackCylinder
+    }
+    if (cylinderTimesHeads >= heads * 1024) {
+      sectorsPerTrackCylinder = 63
+      heads = 16
+      cylinderTimesHeads = totalSectors / sectorsPerTrackCylinder
+    }
+  }
+  const cylinders = Math.ceil(cylinderTimesHeads / heads)
+  const actualSize = cylinders * heads * sectorsPerTrackCylinder * SECTOR_SIZE
+  return { cylinders, heads, sectorsPerTrackCylinder, actualSize }
+}

packages/vhd-lib/src/_constants.js

@@ -0,0 +1,30 @@
+export const BLOCK_UNUSED = 0xffffffff
+
+// This lib has been extracted from the Xen Orchestra project.
+export const CREATOR_APPLICATION = 'xo  '
+
+// Sizes in bytes.
+export const FOOTER_SIZE = 512
+export const HEADER_SIZE = 1024
+export const SECTOR_SIZE = 512
+export const DEFAULT_BLOCK_SIZE = 0x00200000 // from the spec
+
+export const FOOTER_COOKIE = 'conectix'
+export const HEADER_COOKIE = 'cxsparse'
+
+export const DISK_TYPE_FIXED = 2
+export const DISK_TYPE_DYNAMIC = 3
+export const DISK_TYPE_DIFFERENCING = 4
+
+export const PARENT_LOCATOR_ENTRIES = 8
+
+export const PLATFORM_NONE = 0
+export const PLATFORM_WI2R = 0x57693272
+export const PLATFORM_WI2K = 0x5769326b
+export const PLATFORM_W2RU = 0x57327275
+export const PLATFORM_W2KU = 0x57326b75
+export const PLATFORM_MAC = 0x4d616320
+export const PLATFORM_MACX = 0x4d616358
+
+export const FILE_FORMAT_VERSION = 1 << 16
+export const HEADER_VERSION = 1 << 16

packages/vhd-lib/src/_createFooterHeader.js

@@ -0,0 +1,56 @@
+import { v4 as generateUuid } from 'uuid'
+
+import { checksumStruct, fuFooter, fuHeader } from './_structs'
+import {
+  CREATOR_APPLICATION,
+  DEFAULT_BLOCK_SIZE as VHD_BLOCK_SIZE_BYTES,
+  DISK_TYPE_FIXED,
+  FILE_FORMAT_VERSION,
+  FOOTER_COOKIE,
+  FOOTER_SIZE,
+  HEADER_COOKIE,
+  HEADER_SIZE,
+  HEADER_VERSION,
+  PLATFORM_WI2K,
+} from './_constants'
+
+export function createFooter (
+  size,
+  timestamp,
+  geometry,
+  dataOffset,
+  diskType = DISK_TYPE_FIXED
+) {
+  const footer = fuFooter.pack({
+    cookie: FOOTER_COOKIE,
+    features: 2,
+    fileFormatVersion: FILE_FORMAT_VERSION,
+    dataOffset,
+    timestamp,
+    creatorApplication: CREATOR_APPLICATION,
+    creatorHostOs: PLATFORM_WI2K, // it looks like everybody is using Wi2k
+    originalSize: size,
+    currentSize: size,
+    diskGeometry: geometry,
+    diskType,
+    uuid: generateUuid(null, []),
+  })
+  checksumStruct(footer, fuFooter)
+  return footer
+}
+
+export function createHeader (
+  maxTableEntries,
+  tableOffset = HEADER_SIZE + FOOTER_SIZE,
+  blockSize = VHD_BLOCK_SIZE_BYTES
+) {
+  const header = fuHeader.pack({
+    cookie: HEADER_COOKIE,
+    tableOffset,
+    headerVersion: HEADER_VERSION,
+    maxTableEntries,
+    blockSize,
+  })
+  checksumStruct(header, fuHeader)
+  return header
+}

packages/vhd-lib/src/_structs.js

@@ -0,0 +1,121 @@
+import assert from 'assert'
+import fu from 'struct-fu'
+
+import { FOOTER_SIZE, HEADER_SIZE, PARENT_LOCATOR_ENTRIES } from './_constants'
+
+const SIZE_OF_32_BITS = Math.pow(2, 32)
+
+const uint64 = fu.derive(
+  fu.uint32(2),
+  number => [Math.floor(number / SIZE_OF_32_BITS), number % SIZE_OF_32_BITS],
+  _ => _[0] * SIZE_OF_32_BITS + _[1]
+)
+const uint64Undefinable = fu.derive(
+  fu.uint32(2),
+  number =>
+    number === undefined
+      ? [0xffffffff, 0xffffffff]
+      : [Math.floor(number / SIZE_OF_32_BITS), number % SIZE_OF_32_BITS],
+  _ =>
+    _[0] === 0xffffffff && _[1] === 0xffffffff
+      ? undefined
+      : _[0] * SIZE_OF_32_BITS + _[1]
+)
+
+export const fuFooter = fu.struct([
+  fu.char('cookie', 8), // 0
+  fu.uint32('features'), // 8
+  fu.uint32('fileFormatVersion'), // 12
+  uint64Undefinable('dataOffset'), // offset of the header
+  fu.uint32('timestamp'), // 24
+  fu.char('creatorApplication', 4), // 28
+  fu.uint32('creatorVersion'), // 32
+  fu.uint32('creatorHostOs'), // 36
+  uint64('originalSize'),
+  uint64('currentSize'),
+  fu.struct('diskGeometry', [
+    fu.uint16('cylinders'), // 56
+    fu.uint8('heads'), // 58
+    fu.uint8('sectorsPerTrackCylinder'), // 59
+  ]),
+  fu.uint32('diskType'), // 60 Disk type, must be equal to HARD_DISK_TYPE_DYNAMIC/HARD_DISK_TYPE_DIFFERENCING.
+  fu.uint32('checksum'), // 64
+  fu.uint8('uuid', 16), // 68
+  fu.char('saved'), // 84
+  fu.char('hidden'), // 85 TODO: should probably be merged in reserved
+  fu.char('reserved', 426), // 86
+])
+assert.strictEqual(fuFooter.size, FOOTER_SIZE)
+
+export const fuHeader = fu.struct([
+  fu.char('cookie', 8),
+  uint64Undefinable('dataOffset'),
+  uint64('tableOffset'),
+  fu.uint32('headerVersion'),
+  fu.uint32('maxTableEntries'), // Max entries in the Block Allocation Table.
+  fu.uint32('blockSize'), // Block size in bytes. Default (2097152 => 2MB)
+  fu.uint32('checksum'),
+  fu.uint8('parentUuid', 16),
+  fu.uint32('parentTimestamp'),
+  fu.uint32('reserved1'),
+  fu.char16be('parentUnicodeName', 512),
+  fu.struct(
+    'parentLocatorEntry',
+    [
+      fu.uint32('platformCode'),
+      fu.uint32('platformDataSpace'),
+      fu.uint32('platformDataLength'),
+      fu.uint32('reserved'),
+      uint64('platformDataOffset'), // Absolute byte offset of the locator data.
+    ],
+    PARENT_LOCATOR_ENTRIES
+  ),
+  fu.char('reserved2', 256),
+])
+assert.strictEqual(fuHeader.size, HEADER_SIZE)
+
+export const packField = (field, value, buf) => {
+  const { offset } = field
+
+  field.pack(
+    value,
+    buf,
+    typeof offset !== 'object' ? { bytes: offset, bits: 0 } : offset
+  )
+}
+
+export const unpackField = (field, buf) => {
+  const { offset } = field
+
+  return field.unpack(
+    buf,
+    typeof offset !== 'object' ? { bytes: offset, bits: 0 } : offset
+  )
+}
+
+// Returns the checksum of a raw struct.
+// The raw struct (footer or header) is altered with the new sum.
+export function checksumStruct (buf, struct) {
+  const checksumField = struct.fields.checksum
+  let sum = 0
+
+  // Do not use the stored checksum to compute the new checksum.
+  const checksumOffset = checksumField.offset
+  for (let i = 0, n = checksumOffset; i < n; ++i) {
+    sum += buf[i]
+  }
+  for (
+    let i = checksumOffset + checksumField.size, n = struct.size;
+    i < n;
+    ++i
+  ) {
+    sum += buf[i]
+  }
+
+  sum = ~sum >>> 0
+
+  // Write new sum.
+  packField(checksumField, sum, buf)
+
+  return sum
+}

packages/vhd-lib/src/chain.js

@@ -0,0 +1,37 @@
+import { dirname, relative } from 'path'
+
+import Vhd from './vhd'
+import { DISK_TYPE_DIFFERENCING } from './_constants'
+
+export default async function chain (
+  parentHandler,
+  parentPath,
+  childHandler,
+  childPath,
+  force = false
+) {
+  const parentVhd = new Vhd(parentHandler, parentPath)
+  const childVhd = new Vhd(childHandler, childPath)
+
+  await childVhd.readHeaderAndFooter()
+  const { header, footer } = childVhd
+
+  if (footer.diskType !== DISK_TYPE_DIFFERENCING) {
+    if (!force) {
+      throw new Error('cannot chain disk of type ' + footer.diskType)
+    }
+    footer.diskType = DISK_TYPE_DIFFERENCING
+  }
+
+  await Promise.all([
+    childVhd.readBlockAllocationTable(),
+    parentVhd.readHeaderAndFooter(),
+  ])
+
+  const parentName = relative(dirname(childPath), parentPath)
+  header.parentUuid = parentVhd.footer.uuid
+  header.parentUnicodeName = parentName
+  await childVhd.setUniqueParentLocator(parentName)
+  await childVhd.writeHeader()
+  await childVhd.writeFooter()
+}

packages/vhd-lib/src/createReadableRawStream.js

@@ -0,0 +1,42 @@
+import asyncIteratorToStream from 'async-iterator-to-stream'
+
+import computeGeometryForSize from './_computeGeometryForSize'
+import { createFooter } from './_createFooterHeader'
+
+export default asyncIteratorToStream(async function * (size, blockParser) {
+  const geometry = computeGeometryForSize(size)
+  const actualSize = geometry.actualSize
+  const footer = createFooter(
+    actualSize,
+    Math.floor(Date.now() / 1000),
+    geometry
+  )
+  let position = 0
+
+  function * filePadding (paddingLength) {
+    if (paddingLength > 0) {
+      const chunkSize = 1024 * 1024 // 1Mo
+      for (
+        let paddingPosition = 0;
+        paddingPosition + chunkSize < paddingLength;
+        paddingPosition += chunkSize
+      ) {
+        yield Buffer.alloc(chunkSize)
+      }
+      yield Buffer.alloc(paddingLength % chunkSize)
+    }
+  }
+
+  let next
+  while ((next = await blockParser.next()) !== null) {
+    const paddingLength = next.offsetBytes - position
+    if (paddingLength < 0) {
+      throw new Error('Received out of order blocks')
+    }
+    yield * filePadding(paddingLength)
+    yield next.data
+    position = next.offsetBytes + next.data.length
+  }
+  yield * filePadding(actualSize - position)
+  yield footer
+})

packages/vhd-lib/src/createReadableSparseStream.js

@@ -0,0 +1,143 @@
+import asyncIteratorToStream from 'async-iterator-to-stream'
+
+import computeGeometryForSize from './_computeGeometryForSize'
+import { createFooter, createHeader } from './_createFooterHeader'
+import {
+  BLOCK_UNUSED,
+  DEFAULT_BLOCK_SIZE as VHD_BLOCK_SIZE_BYTES,
+  DISK_TYPE_DYNAMIC,
+  FOOTER_SIZE,
+  HEADER_SIZE,
+  SECTOR_SIZE,
+} from './_constants'
+
+import { set as setBitmap } from './_bitmap'
+
+const VHD_BLOCK_SIZE_SECTORS = VHD_BLOCK_SIZE_BYTES / SECTOR_SIZE
+
+/**
+ * @returns {Array} an array of occupation bitmap, each bit mapping an input block size of bytes
+ */
+function createBAT (
+  firstBlockPosition,
+  blockAddressList,
+  ratio,
+  bat,
+  bitmapSize
+) {
+  const vhdOccupationTable = []
+  let currentVhdPositionSector = firstBlockPosition / SECTOR_SIZE
+  blockAddressList.forEach(blockPosition => {
+    const scaled = blockPosition / VHD_BLOCK_SIZE_BYTES
+    const vhdTableIndex = Math.floor(scaled)
+    if (bat.readUInt32BE(vhdTableIndex * 4) === BLOCK_UNUSED) {
+      bat.writeUInt32BE(currentVhdPositionSector, vhdTableIndex * 4)
+      currentVhdPositionSector +=
+        (bitmapSize + VHD_BLOCK_SIZE_BYTES) / SECTOR_SIZE
+    }
+    // not using bit operators to avoid the int32 coercion, that way we can go to 53 bits
+    vhdOccupationTable[vhdTableIndex] =
+      (vhdOccupationTable[vhdTableIndex] || 0) +
+      Math.pow(2, (scaled % 1) * ratio)
+  })
+  return vhdOccupationTable
+}
+
+function createBitmap (bitmapSize, ratio, vhdOccupationBucket) {
+  const bitmap = Buffer.alloc(bitmapSize)
+  for (let i = 0; i < VHD_BLOCK_SIZE_SECTORS / ratio; i++) {
+    // do not shift to avoid int32 coercion
+    if ((vhdOccupationBucket * Math.pow(2, -i)) & 1) {
+      for (let j = 0; j < ratio; j++) {
+        setBitmap(bitmap, i * ratio + j)
+      }
+    }
+  }
+  return bitmap
+}
+
+function * yieldIfNotEmpty (buffer) {
+  if (buffer.length > 0) {
+    yield buffer
+  }
+}
+
+async function * generateFileContent (
+  blockIterator,
+  bitmapSize,
+  ratio,
+  vhdOccupationTable
+) {
+  let currentVhdBlockIndex = -1
+  let currentBlockBuffer = Buffer.alloc(0)
+  for await (const next of blockIterator) {
+    const batEntry = Math.floor(next.offsetBytes / VHD_BLOCK_SIZE_BYTES)
+    if (batEntry !== currentVhdBlockIndex) {
+      yield * yieldIfNotEmpty(currentBlockBuffer)
+      currentBlockBuffer = Buffer.alloc(VHD_BLOCK_SIZE_BYTES)
+      currentVhdBlockIndex = batEntry
+      yield createBitmap(bitmapSize, ratio, vhdOccupationTable[batEntry])
+    }
+    next.data.copy(currentBlockBuffer, next.offsetBytes % VHD_BLOCK_SIZE_BYTES)
+  }
+  yield * yieldIfNotEmpty(currentBlockBuffer)
+}
+
+export default asyncIteratorToStream(async function * (
+  diskSize,
+  incomingBlockSize,
+  blockAddressList,
+  blockIterator
+) {
+  const ratio = VHD_BLOCK_SIZE_BYTES / incomingBlockSize
+  if (ratio % 1 !== 0) {
+    throw new Error(
+      `Can't import file, grain size (${incomingBlockSize}) is not a divider of VHD block size ${VHD_BLOCK_SIZE_BYTES}`
+    )
+  }
+  if (ratio > 53) {
+    throw new Error(
+      `Can't import file, grain size / block size ratio is > 53 (${ratio})`
+    )
+  }
+
+  const maxTableEntries = Math.ceil(diskSize / VHD_BLOCK_SIZE_BYTES) + 1
+  const tablePhysicalSizeBytes = Math.ceil(maxTableEntries * 4 / 512) * 512
+
+  const batPosition = FOOTER_SIZE + HEADER_SIZE
+  const firstBlockPosition = batPosition + tablePhysicalSizeBytes
+  const geometry = computeGeometryForSize(diskSize)
+  const actualSize = geometry.actualSize
+  const footer = createFooter(
+    actualSize,
+    Math.floor(Date.now() / 1000),
+    geometry,
+    FOOTER_SIZE,
+    DISK_TYPE_DYNAMIC
+  )
+  const header = createHeader(
+    maxTableEntries,
+    batPosition,
+    VHD_BLOCK_SIZE_BYTES
+  )
+  const bitmapSize =
+    Math.ceil(VHD_BLOCK_SIZE_SECTORS / 8 / SECTOR_SIZE) * SECTOR_SIZE
+  const bat = Buffer.alloc(tablePhysicalSizeBytes, 0xff)
+  const vhdOccupationTable = createBAT(
+    firstBlockPosition,
+    blockAddressList,
+    ratio,
+    bat,
+    bitmapSize
+  )
+  yield footer
+  yield header
+  yield bat
+  yield * generateFileContent(
+    blockIterator,
+    bitmapSize,
+    ratio,
+    vhdOccupationTable
+  )
+  yield footer
+})

packages/vhd-lib/src/createSyntheticStream.js

@@ -0,0 +1,153 @@
+import asyncIteratorToStream from 'async-iterator-to-stream'
+import { dirname, resolve } from 'path'
+
+import Vhd from './vhd'
+import {
+  BLOCK_UNUSED,
+  DISK_TYPE_DYNAMIC,
+  FOOTER_SIZE,
+  HEADER_SIZE,
+  SECTOR_SIZE,
+} from './_constants'
+import { fuFooter, fuHeader, checksumStruct } from './_structs'
+import { test as mapTestBit } from './_bitmap'
+
+const resolveRelativeFromFile = (file, path) =>
+  resolve('/', dirname(file), path).slice(1)
+
+export default asyncIteratorToStream(function * (handler, path) {
+  const fds = []
+
+  try {
+    const vhds = []
+    while (true) {
+      const fd = yield handler.openFile(path, 'r')
+      fds.push(fd)
+      const vhd = new Vhd(handler, fd)
+      vhds.push(vhd)
+      yield vhd.readHeaderAndFooter()
+      yield vhd.readBlockAllocationTable()
+
+      if (vhd.footer.diskType === DISK_TYPE_DYNAMIC) {
+        break
+      }
+
+      path = resolveRelativeFromFile(path, vhd.header.parentUnicodeName)
+    }
+    const nVhds = vhds.length
+
+    // this the VHD we want to synthetize
+    const vhd = vhds[0]
+
+    // this is the root VHD
+    const rootVhd = vhds[nVhds - 1]
+
+    // data of our synthetic VHD
+    // TODO: set parentLocatorEntry-s in header
+    let header = {
+      ...vhd.header,
+      tableOffset: FOOTER_SIZE + HEADER_SIZE,
+      parentTimestamp: rootVhd.header.parentTimestamp,
+      parentUnicodeName: rootVhd.header.parentUnicodeName,
+      parentUuid: rootVhd.header.parentUuid,
+    }
+
+    const bat = Buffer.allocUnsafe(vhd.batSize)
+    let footer = {
+      ...vhd.footer,
+      dataOffset: FOOTER_SIZE,
+      diskType: rootVhd.footer.diskType,
+    }
+    const sectorsPerBlockData = vhd.sectorsPerBlock
+    const sectorsPerBlock = sectorsPerBlockData + vhd.bitmapSize / SECTOR_SIZE
+
+    const nBlocks = Math.ceil(footer.currentSize / header.blockSize)
+
+    const blocksOwner = new Array(nBlocks)
+    for (
+      let iBlock = 0,
+        blockOffset = Math.ceil(
+          (header.tableOffset + bat.length) / SECTOR_SIZE
+        );
+      iBlock < nBlocks;
+      ++iBlock
+    ) {
+      let blockSector = BLOCK_UNUSED
+      for (let i = 0; i < nVhds; ++i) {
+        if (vhds[i].containsBlock(iBlock)) {
+          blocksOwner[iBlock] = i
+          blockSector = blockOffset
+          blockOffset += sectorsPerBlock
+          break
+        }
+      }
+      bat.writeUInt32BE(blockSector, iBlock * 4)
+    }
+
+    footer = fuFooter.pack(footer)
+    checksumStruct(footer, fuFooter)
+    yield footer
+
+    header = fuHeader.pack(header)
+    checksumStruct(header, fuHeader)
+    yield header
+
+    yield bat
+
+    // TODO: for generic usage the bitmap needs to be properly computed for each block
+    const bitmap = Buffer.alloc(vhd.bitmapSize, 0xff)
+    for (let iBlock = 0; iBlock < nBlocks; ++iBlock) {
+      const owner = blocksOwner[iBlock]
+      if (owner === undefined) {
+        continue
+      }
+
+      yield bitmap
+
+      const blocksByVhd = new Map()
+      const emitBlockSectors = function * (iVhd, i, n) {
+        const vhd = vhds[iVhd]
+        const isRootVhd = vhd === rootVhd
+        if (!vhd.containsBlock(iBlock)) {
+          if (isRootVhd) {
+            yield Buffer.alloc((n - i) * SECTOR_SIZE)
+          } else {
+            yield * emitBlockSectors(iVhd + 1, i, n)
+          }
+          return
+        }
+        let block = blocksByVhd.get(vhd)
+        if (block === undefined) {
+          block = yield vhd._readBlock(iBlock)
+          blocksByVhd.set(vhd, block)
+        }
+        const { bitmap, data } = block
+        if (isRootVhd) {
+          yield data.slice(i * SECTOR_SIZE, n * SECTOR_SIZE)
+          return
+        }
+        while (i < n) {
+          const hasData = mapTestBit(bitmap, i)
+          const start = i
+          do {
+            ++i
+          } while (i < n && mapTestBit(bitmap, i) === hasData)
+          if (hasData) {
+            yield data.slice(start * SECTOR_SIZE, i * SECTOR_SIZE)
+          } else {
+            yield * emitBlockSectors(iVhd + 1, start, i)
+          }
+        }
+      }
+      yield * emitBlockSectors(owner, 0, sectorsPerBlockData)
+    }
+
+    yield footer
+  } finally {
+    for (let i = 0, n = fds.length; i < n; ++i) {
+      handler.closeFile(fds[i]).catch(error => {
+        console.warn('createReadStream, closeFd', i, error)
+      })
+    }
+  }
+})

packages/vhd-lib/src/index.js

@@ -0,0 +1,8 @@
+export { default } from './vhd'
+export { default as chainVhd } from './chain'
+export { default as createReadableRawStream } from './createReadableRawStream'
+export {
+  default as createReadableSparseStream,
+} from './createReadableSparseStream'
+export { default as createSyntheticStream } from './createSyntheticStream'
+export { default as mergeVhd } from './merge'

packages/vhd-lib/src/merge.integ.spec.js

@@ -0,0 +1,283 @@
+/* eslint-env jest */
+
+import execa from 'execa'
+import fs from 'fs-extra'
+import getStream from 'get-stream'
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { getHandler } from '@xen-orchestra/fs'
+import { randomBytes } from 'crypto'
+import { fromEvent, fromCallback as pFromCallback } from 'promise-toolbox'
+
+import chainVhd from './chain'
+import createReadStream from './createSyntheticStream'
+import Vhd from './vhd'
+import vhdMerge from './merge'
+import { SECTOR_SIZE } from './_constants'
+
+const initialDir = process.cwd()
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  const dir = await pFromCallback(cb => tmp.dir(cb))
+  process.chdir(dir)
+})
+
+afterEach(async () => {
+  const tmpDir = process.cwd()
+  process.chdir(initialDir)
+  await pFromCallback(cb => rimraf(tmpDir, cb))
+})
+
+async function createRandomFile (name, sizeMb) {
+  await execa('bash', [
+    '-c',
+    `< /dev/urandom tr -dc "\\t\\n [:alnum:]" | head -c ${sizeMb}M >${name}`,
+  ])
+}
+
+async function checkFile (vhdName) {
+  await execa('vhd-util', ['check', '-p', '-b', '-t', '-n', vhdName])
+}
+
+async function recoverRawContent (vhdName, rawName, originalSize) {
+  await checkFile(vhdName)
+  await execa('qemu-img', ['convert', '-fvpc', '-Oraw', vhdName, rawName])
+  if (originalSize !== undefined) {
+    await execa('truncate', ['-s', originalSize, rawName])
+  }
+}
+
+async function convertFromRawToVhd (rawName, vhdName) {
+  await execa('qemu-img', ['convert', '-f', 'raw', '-Ovpc', rawName, vhdName])
+}
+
+test('blocks can be moved', async () => {
+  const initalSize = 4
+  await createRandomFile('randomfile', initalSize)
+  await convertFromRawToVhd('randomfile', 'randomfile.vhd')
+  const handler = getHandler({ url: 'file://' + process.cwd() })
+  const originalSize = await handler.getSize('randomfile')
+  const newVhd = new Vhd(handler, 'randomfile.vhd')
+  await newVhd.readHeaderAndFooter()
+  await newVhd.readBlockAllocationTable()
+  await newVhd._freeFirstBlockSpace(8000000)
+  await recoverRawContent('randomfile.vhd', 'recovered', originalSize)
+  expect(await fs.readFile('recovered')).toEqual(
+    await fs.readFile('randomfile')
+  )
+})
+
+test('the BAT MSB is not used for sign', async () => {
+  const randomBuffer = await pFromCallback(cb => randomBytes(SECTOR_SIZE, cb))
+  await execa('qemu-img', ['create', '-fvpc', 'empty.vhd', '1.8T'])
+  const handler = getHandler({ url: 'file://' + process.cwd() })
+  const vhd = new Vhd(handler, 'empty.vhd')
+  await vhd.readHeaderAndFooter()
+  await vhd.readBlockAllocationTable()
+  // we want the bit 31 to be on, to prove it's not been used for sign
+  const hugeWritePositionSectors = Math.pow(2, 31) + 200
+  await vhd.writeData(hugeWritePositionSectors, randomBuffer)
+  await checkFile('empty.vhd')
+  // here we are moving the first sector very far in the VHD to prove the BAT doesn't use signed int32
+  const hugePositionBytes = hugeWritePositionSectors * SECTOR_SIZE
+  await vhd._freeFirstBlockSpace(hugePositionBytes)
+
+  // we recover the data manually for speed reasons.
+  // fs.write() with offset is way faster than qemu-img when there is a 1.5To
+  // hole before the block of data
+  const recoveredFile = await fs.open('recovered', 'w')
+  try {
+    const vhd2 = new Vhd(handler, 'empty.vhd')
+    await vhd2.readHeaderAndFooter()
+    await vhd2.readBlockAllocationTable()
+    for (let i = 0; i < vhd.header.maxTableEntries; i++) {
+      const entry = vhd._getBatEntry(i)
+      if (entry !== 0xffffffff) {
+        const block = (await vhd2._readBlock(i)).data
+        await fs.write(
+          recoveredFile,
+          block,
+          0,
+          block.length,
+          vhd2.header.blockSize * i
+        )
+      }
+    }
+  } finally {
+    fs.close(recoveredFile)
+  }
+  const recovered = await getStream.buffer(
+    await fs.createReadStream('recovered', {
+      start: hugePositionBytes,
+      end: hugePositionBytes + randomBuffer.length - 1,
+    })
+  )
+  expect(recovered).toEqual(randomBuffer)
+})
+
+test('writeData on empty file', async () => {
+  const mbOfRandom = 3
+  await createRandomFile('randomfile', mbOfRandom)
+  await execa('qemu-img', ['create', '-fvpc', 'empty.vhd', mbOfRandom + 'M'])
+  const randomData = await fs.readFile('randomfile')
+  const handler = getHandler({ url: 'file://' + process.cwd() })
+  const originalSize = await handler.getSize('randomfile')
+  const newVhd = new Vhd(handler, 'empty.vhd')
+  await newVhd.readHeaderAndFooter()
+  await newVhd.readBlockAllocationTable()
+  await newVhd.writeData(0, randomData)
+  await recoverRawContent('empty.vhd', 'recovered', originalSize)
+  expect(await fs.readFile('recovered')).toEqual(randomData)
+})
+
+test('writeData in 2 non-overlaping operations', async () => {
+  const mbOfRandom = 3
+  await createRandomFile('randomfile', mbOfRandom)
+  await execa('qemu-img', ['create', '-fvpc', 'empty.vhd', mbOfRandom + 'M'])
+  const randomData = await fs.readFile('randomfile')
+  const handler = getHandler({ url: 'file://' + process.cwd() })
+  const originalSize = await handler.getSize('randomfile')
+  const newVhd = new Vhd(handler, 'empty.vhd')
+  await newVhd.readHeaderAndFooter()
+  await newVhd.readBlockAllocationTable()
+  const splitPointSectors = 2
+  await newVhd.writeData(0, randomData.slice(0, splitPointSectors * 512))
+  await newVhd.writeData(
+    splitPointSectors,
+    randomData.slice(splitPointSectors * 512)
+  )
+  await recoverRawContent('empty.vhd', 'recovered', originalSize)
+  expect(await fs.readFile('recovered')).toEqual(randomData)
+})
+
+test('writeData in 2 overlaping operations', async () => {
+  const mbOfRandom = 3
+  await createRandomFile('randomfile', mbOfRandom)
+  await execa('qemu-img', ['create', '-fvpc', 'empty.vhd', mbOfRandom + 'M'])
+  const randomData = await fs.readFile('randomfile')
+  const handler = getHandler({ url: 'file://' + process.cwd() })
+  const originalSize = await handler.getSize('randomfile')
+  const newVhd = new Vhd(handler, 'empty.vhd')
+  await newVhd.readHeaderAndFooter()
+  await newVhd.readBlockAllocationTable()
+  const endFirstWrite = 3
+  const startSecondWrite = 2
+  await newVhd.writeData(0, randomData.slice(0, endFirstWrite * 512))
+  await newVhd.writeData(
+    startSecondWrite,
+    randomData.slice(startSecondWrite * 512)
+  )
+  await recoverRawContent('empty.vhd', 'recovered', originalSize)
+  expect(await fs.readFile('recovered')).toEqual(randomData)
+})
+
+test('BAT can be extended and blocks moved', async () => {
+  const initalSize = 4
+  await createRandomFile('randomfile', initalSize)
+  await convertFromRawToVhd('randomfile', 'randomfile.vhd')
+  const handler = getHandler({ url: 'file://' + process.cwd() })
+  const originalSize = await handler.getSize('randomfile')
+  const newVhd = new Vhd(handler, 'randomfile.vhd')
+  await newVhd.readHeaderAndFooter()
+  await newVhd.readBlockAllocationTable()
+  await newVhd.ensureBatSize(2000)
+  await recoverRawContent('randomfile.vhd', 'recovered', originalSize)
+  expect(await fs.readFile('recovered')).toEqual(
+    await fs.readFile('randomfile')
+  )
+})
+
+test('coalesce works with empty parent files', async () => {
+  const mbOfRandom = 2
+  await createRandomFile('randomfile', mbOfRandom)
+  await convertFromRawToVhd('randomfile', 'randomfile.vhd')
+  await execa('qemu-img', [
+    'create',
+    '-fvpc',
+    'empty.vhd',
+    mbOfRandom + 1 + 'M',
+  ])
+  await checkFile('randomfile.vhd')
+  await checkFile('empty.vhd')
+  const handler = getHandler({ url: 'file://' + process.cwd() })
+  const originalSize = await handler._getSize('randomfile')
+  await chainVhd(handler, 'empty.vhd', handler, 'randomfile.vhd', true)
+  await checkFile('randomfile.vhd')
+  await checkFile('empty.vhd')
+  await vhdMerge(handler, 'empty.vhd', handler, 'randomfile.vhd')
+  await recoverRawContent('empty.vhd', 'recovered', originalSize)
+  expect(await fs.readFile('recovered')).toEqual(
+    await fs.readFile('randomfile')
+  )
+})
+
+test('coalesce works in normal cases', async () => {
+  const mbOfRandom = 5
+  await createRandomFile('randomfile', mbOfRandom)
+  await createRandomFile('small_randomfile', Math.ceil(mbOfRandom / 2))
+  await execa('qemu-img', [
+    'create',
+    '-fvpc',
+    'parent.vhd',
+    mbOfRandom + 1 + 'M',
+  ])
+  await convertFromRawToVhd('randomfile', 'child1.vhd')
+  const handler = getHandler({ url: 'file://' + process.cwd() })
+  await execa('vhd-util', ['snapshot', '-n', 'child2.vhd', '-p', 'child1.vhd'])
+  const vhd = new Vhd(handler, 'child2.vhd')
+  await vhd.readHeaderAndFooter()
+  await vhd.readBlockAllocationTable()
+  vhd.footer.creatorApplication = 'xoa'
+  await vhd.writeFooter()
+
+  const originalSize = await handler._getSize('randomfile')
+  await chainVhd(handler, 'parent.vhd', handler, 'child1.vhd', true)
+  await execa('vhd-util', ['check', '-t', '-n', 'child1.vhd'])
+  await chainVhd(handler, 'child1.vhd', handler, 'child2.vhd', true)
+  await execa('vhd-util', ['check', '-t', '-n', 'child2.vhd'])
+  const smallRandom = await fs.readFile('small_randomfile')
+  const newVhd = new Vhd(handler, 'child2.vhd')
+  await newVhd.readHeaderAndFooter()
+  await newVhd.readBlockAllocationTable()
+  await newVhd.writeData(5, smallRandom)
+  await checkFile('child2.vhd')
+  await checkFile('child1.vhd')
+  await checkFile('parent.vhd')
+  await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd')
+  await checkFile('parent.vhd')
+  await chainVhd(handler, 'parent.vhd', handler, 'child2.vhd', true)
+  await checkFile('child2.vhd')
+  await vhdMerge(handler, 'parent.vhd', handler, 'child2.vhd')
+  await checkFile('parent.vhd')
+  await recoverRawContent(
+    'parent.vhd',
+    'recovered_from_coalescing',
+    originalSize
+  )
+  await execa('cp', ['randomfile', 'randomfile2'])
+  const fd = await fs.open('randomfile2', 'r+')
+  try {
+    await fs.write(fd, smallRandom, 0, smallRandom.length, 5 * SECTOR_SIZE)
+  } finally {
+    await fs.close(fd)
+  }
+  expect(await fs.readFile('recovered_from_coalescing')).toEqual(
+    await fs.readFile('randomfile2')
+  )
+})
+
+test('createSyntheticStream passes vhd-util check', async () => {
+  const initalSize = 4
+  await createRandomFile('randomfile', initalSize)
+  await convertFromRawToVhd('randomfile', 'randomfile.vhd')
+  const handler = getHandler({ url: 'file://' + process.cwd() })
+  const stream = createReadStream(handler, 'randomfile.vhd')
+  await fromEvent(
+    stream.pipe(await fs.createWriteStream('recovered.vhd')),
+    'finish'
+  )
+  await checkFile('recovered.vhd')
+  await execa('qemu-img', ['compare', 'recovered.vhd', 'randomfile'])
+})

packages/vhd-lib/src/merge.js

@@ -0,0 +1,77 @@
+// TODO: remove once completely merged in vhd.js
+
+import assert from 'assert'
+import concurrency from 'limit-concurrency-decorator'
+
+import Vhd from './vhd'
+import { DISK_TYPE_DIFFERENCING, DISK_TYPE_DYNAMIC } from './_constants'
+
+// Merge vhd child into vhd parent.
+export default concurrency(2)(async function merge (
+  parentHandler,
+  parentPath,
+  childHandler,
+  childPath
+) {
+  const parentFd = await parentHandler.openFile(parentPath, 'r+')
+  try {
+    const parentVhd = new Vhd(parentHandler, parentFd)
+    const childFd = await childHandler.openFile(childPath, 'r')
+    try {
+      const childVhd = new Vhd(childHandler, childFd)
+
+      // Reading footer and header.
+      await Promise.all([
+        parentVhd.readHeaderAndFooter(),
+        childVhd.readHeaderAndFooter(),
+      ])
+
+      assert(childVhd.header.blockSize === parentVhd.header.blockSize)
+
+      const parentDiskType = parentVhd.footer.diskType
+      assert(
+        parentDiskType === DISK_TYPE_DIFFERENCING ||
+          parentDiskType === DISK_TYPE_DYNAMIC
+      )
+      assert.strictEqual(childVhd.footer.diskType, DISK_TYPE_DIFFERENCING)
+
+      // Read allocation table of child/parent.
+      await Promise.all([
+        parentVhd.readBlockAllocationTable(),
+        childVhd.readBlockAllocationTable(),
+      ])
+
+      await parentVhd.ensureBatSize(childVhd.header.maxTableEntries)
+
+      let mergedDataSize = 0
+      for (
+        let blockId = 0;
+        blockId < childVhd.header.maxTableEntries;
+        blockId++
+      ) {
+        if (childVhd.containsBlock(blockId)) {
+          mergedDataSize += await parentVhd.coalesceBlock(childVhd, blockId)
+        }
+      }
+
+      const cFooter = childVhd.footer
+      const pFooter = parentVhd.footer
+
+      pFooter.currentSize = cFooter.currentSize
+      pFooter.diskGeometry = { ...cFooter.diskGeometry }
+      pFooter.originalSize = cFooter.originalSize
+      pFooter.timestamp = cFooter.timestamp
+      pFooter.uuid = cFooter.uuid
+
+      // necessary to update values and to recreate the footer after block
+      // creation
+      await parentVhd.writeFooter()
+
+      return mergedDataSize
+    } finally {
+      await childHandler.closeFile(childFd)
+    }
+  } finally {
+    await parentHandler.closeFile(parentFd)
+  }
+})

packages/vhd-lib/src/vhd.integ.spec.js

@@ -0,0 +1,134 @@
+/* eslint-env jest */
+import execa from 'execa'
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { createWriteStream, readFile } from 'fs-promise'
+import { fromCallback as pFromCallback, fromEvent } from 'promise-toolbox'
+
+import { createFooter } from './_createFooterHeader'
+import createReadableRawVHDStream from './createReadableRawStream'
+import createReadableSparseVHDStream from './createReadableSparseStream'
+
+const initialDir = process.cwd()
+
+beforeEach(async () => {
+  const dir = await pFromCallback(cb => tmp.dir(cb))
+  process.chdir(dir)
+})
+
+afterEach(async () => {
+  const tmpDir = process.cwd()
+  process.chdir(initialDir)
+  await pFromCallback(cb => rimraf(tmpDir, cb))
+})
+
+test('createFooter() does not crash', () => {
+  createFooter(104448, Math.floor(Date.now() / 1000), {
+    cylinders: 3,
+    heads: 4,
+    sectorsPerTrack: 17,
+  })
+})
+
+test('ReadableRawVHDStream does not crash', async () => {
+  const data = [
+    {
+      offsetBytes: 100,
+      data: Buffer.from('azerzaerazeraze', 'ascii'),
+    },
+    {
+      offsetBytes: 700,
+      data: Buffer.from('gdfslkdfguer', 'ascii'),
+    },
+  ]
+  let index = 0
+  const mockParser = {
+    next: () => {
+      if (index < data.length) {
+        const result = data[index]
+        index++
+        return result
+      } else {
+        return null
+      }
+    },
+  }
+  const fileSize = 1000
+  const stream = createReadableRawVHDStream(fileSize, mockParser)
+  const pipe = stream.pipe(createWriteStream('output.vhd'))
+  await fromEvent(pipe, 'finish')
+  await execa('vhd-util', ['check', '-t', '-i', '-n', 'output.vhd'])
+})
+
+test('ReadableRawVHDStream detects when blocks are out of order', async () => {
+  const data = [
+    {
+      offsetBytes: 700,
+      data: Buffer.from('azerzaerazeraze', 'ascii'),
+    },
+    {
+      offsetBytes: 100,
+      data: Buffer.from('gdfslkdfguer', 'ascii'),
+    },
+  ]
+  let index = 0
+  const mockParser = {
+    next: () => {
+      if (index < data.length) {
+        const result = data[index]
+        index++
+        return result
+      } else {
+        return null
+      }
+    },
+  }
+  return expect(
+    new Promise((resolve, reject) => {
+      const stream = createReadableRawVHDStream(100000, mockParser)
+      stream.on('error', reject)
+      const pipe = stream.pipe(createWriteStream('outputStream'))
+      pipe.on('finish', resolve)
+      pipe.on('error', reject)
+    })
+  ).rejects.toThrow('Received out of order blocks')
+})
+
+test('ReadableSparseVHDStream can handle a sparse file', async () => {
+  const blockSize = Math.pow(2, 16)
+  const blocks = [
+    {
+      offsetBytes: blockSize * 3,
+      data: Buffer.alloc(blockSize, 'azerzaerazeraze', 'ascii'),
+    },
+    {
+      offsetBytes: blockSize * 5,
+      data: Buffer.alloc(blockSize, 'gdfslkdfguer', 'ascii'),
+    },
+  ]
+  const fileSize = blockSize * 10
+  const stream = createReadableSparseVHDStream(
+    fileSize,
+    blockSize,
+    [100, 700],
+    blocks
+  )
+  const pipe = stream.pipe(createWriteStream('output.vhd'))
+  await fromEvent(pipe, 'finish')
+  await execa('vhd-util', ['check', '-t', '-i', '-n', 'output.vhd'])
+  await execa('qemu-img', [
+    'convert',
+    '-f',
+    'vpc',
+    '-O',
+    'raw',
+    'output.vhd',
+    'out1.raw',
+  ])
+  const out1 = await readFile('out1.raw')
+  const expected = Buffer.alloc(fileSize)
+  blocks.forEach(b => {
+    b.data.copy(expected, b.offsetBytes)
+  })
+  await expect(out1.slice(0, expected.length)).toEqual(expected)
+})

packages/vhd-lib/src/vhd.js

@@ -0,0 +1,631 @@
+import assert from 'assert'
+import getStream from 'get-stream'
+import { fromEvent } from 'promise-toolbox'
+
+import constantStream from './_constant-stream'
+import { fuFooter, fuHeader, checksumStruct, unpackField } from './_structs'
+import { set as mapSetBit, test as mapTestBit } from './_bitmap'
+import {
+  BLOCK_UNUSED,
+  DISK_TYPE_DIFFERENCING,
+  DISK_TYPE_DYNAMIC,
+  FILE_FORMAT_VERSION,
+  FOOTER_COOKIE,
+  FOOTER_SIZE,
+  HEADER_COOKIE,
+  HEADER_SIZE,
+  HEADER_VERSION,
+  PARENT_LOCATOR_ENTRIES,
+  PLATFORM_NONE,
+  PLATFORM_W2KU,
+  SECTOR_SIZE,
+} from './_constants'
+
+const VHD_UTIL_DEBUG = 0
+const debug = VHD_UTIL_DEBUG
+  ? str => console.log(`[vhd-merge]${str}`)
+  : () => null
+
+// ===================================================================
+//
+// Spec:
+// https://www.microsoft.com/en-us/download/details.aspx?id=23850
+//
+// C implementation:
+// https://github.com/rubiojr/vhd-util-convert
+//
+// ===================================================================
+
+const computeBatSize = entries =>
+  sectorsToBytes(sectorsRoundUpNoZero(entries * 4))
+
+// Sectors conversions.
+const sectorsRoundUpNoZero = bytes => Math.ceil(bytes / SECTOR_SIZE) || 1
+const sectorsToBytes = sectors => sectors * SECTOR_SIZE
+
+const assertChecksum = (name, buf, struct) => {
+  const actual = unpackField(struct.fields.checksum, buf)
+  const expected = checksumStruct(buf, struct)
+  if (actual !== expected) {
+    throw new Error(`invalid ${name} checksum ${actual}, expected ${expected}`)
+  }
+}
+
+// unused block as buffer containing a uint32BE
+const BUF_BLOCK_UNUSED = Buffer.allocUnsafe(4)
+BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
+
+// ===================================================================
+
+// Format:
+//
+// 1. Footer (512)
+// 2. Header (1024)
+// 3. Unordered entries
+//    - BAT (batSize @ header.tableOffset)
+//    - Blocks (@ blockOffset(i))
+//      - bitmap (blockBitmapSize)
+//      - data (header.blockSize)
+//    - Parent locators (parentLocatorSize(i) @ parentLocatorOffset(i))
+// 4. Footer (512 @ vhdSize - 512)
+//
+// Variables:
+//
+// - batSize = min(1, ceil(header.maxTableEntries * 4 / sectorSize)) * sectorSize
+// - blockBitmapSize = ceil(header.blockSize / sectorSize / 8 / sectorSize) * sectorSize
+// - blockOffset(i) = bat[i] * sectorSize
+// - nBlocks = ceil(footer.currentSize / header.blockSize)
+// - parentLocatorOffset(i) = header.parentLocatorEntry[i].platformDataOffset
+// - parentLocatorSize(i) = header.parentLocatorEntry[i].platformDataSpace * sectorSize
+// - sectorSize = 512
+
+export default class Vhd {
+  get batSize () {
+    return computeBatSize(this.header.maxTableEntries)
+  }
+
+  constructor (handler, path) {
+    this._handler = handler
+    this._path = path
+  }
+
+  // =================================================================
+  // Read functions.
+  // =================================================================
+
+  _readStream (start, n) {
+    return this._handler.createReadStream(this._path, {
+      start,
+      end: start + n - 1, // end is inclusive
+    })
+  }
+
+  _read (start, n) {
+    return this._readStream(start, n)
+      .then(getStream.buffer)
+      .then(buf => {
+        assert.equal(buf.length, n)
+        return buf
+      })
+  }
+
+  containsBlock (id) {
+    return this._getBatEntry(id) !== BLOCK_UNUSED
+  }
+
+  // Returns the first address after metadata. (In bytes)
+  getEndOfHeaders () {
+    const { header } = this
+
+    let end = FOOTER_SIZE + HEADER_SIZE
+
+    // Max(end, block allocation table end)
+    end = Math.max(end, header.tableOffset + this.batSize)
+
+    for (let i = 0; i < PARENT_LOCATOR_ENTRIES; i++) {
+      const entry = header.parentLocatorEntry[i]
+
+      if (entry.platformCode !== PLATFORM_NONE) {
+        end = Math.max(
+          end,
+          entry.platformDataOffset + sectorsToBytes(entry.platformDataSpace)
+        )
+      }
+    }
+
+    debug(`End of headers: ${end}.`)
+
+    return end
+  }
+
+  // Returns the first sector after data.
+  getEndOfData () {
+    let end = Math.ceil(this.getEndOfHeaders() / SECTOR_SIZE)
+
+    const fullBlockSize = this.sectorsOfBitmap + this.sectorsPerBlock
+    const { maxTableEntries } = this.header
+    for (let i = 0; i < maxTableEntries; i++) {
+      const blockAddr = this._getBatEntry(i)
+
+      if (blockAddr !== BLOCK_UNUSED) {
+        end = Math.max(end, blockAddr + fullBlockSize)
+      }
+    }
+
+    debug(`End of data: ${end}.`)
+
+    return sectorsToBytes(end)
+  }
+
+  // TODO: extract the checks into reusable functions:
+  // - better human reporting
+  // - auto repair if possible
+  async readHeaderAndFooter (checkSecondFooter = true) {
+    const buf = await this._read(0, FOOTER_SIZE + HEADER_SIZE)
+    const bufFooter = buf.slice(0, FOOTER_SIZE)
+    const bufHeader = buf.slice(FOOTER_SIZE)
+
+    assertChecksum('footer', bufFooter, fuFooter)
+    assertChecksum('header', bufHeader, fuHeader)
+
+    if (checkSecondFooter) {
+      const size = await this._handler.getSize(this._path)
+      assert(
+        bufFooter.equals(await this._read(size - FOOTER_SIZE, FOOTER_SIZE)),
+        'footer1 !== footer2'
+      )
+    }
+
+    const footer = (this.footer = fuFooter.unpack(bufFooter))
+    assert.strictEqual(footer.cookie, FOOTER_COOKIE, 'footer cookie')
+    assert.strictEqual(footer.dataOffset, FOOTER_SIZE)
+    assert.strictEqual(footer.fileFormatVersion, FILE_FORMAT_VERSION)
+    assert(footer.originalSize <= footer.currentSize)
+    assert(
+      footer.diskType === DISK_TYPE_DIFFERENCING ||
+        footer.diskType === DISK_TYPE_DYNAMIC
+    )
+
+    const header = (this.header = fuHeader.unpack(bufHeader))
+    assert.strictEqual(header.cookie, HEADER_COOKIE)
+    assert.strictEqual(header.dataOffset, undefined)
+    assert.strictEqual(header.headerVersion, HEADER_VERSION)
+    assert(header.maxTableEntries >= footer.currentSize / header.blockSize)
+    assert(Number.isInteger(Math.log2(header.blockSize / SECTOR_SIZE)))
+
+    // Compute the number of sectors in one block.
+    // Default: One block contains 4096 sectors of 512 bytes.
+    const sectorsPerBlock = (this.sectorsPerBlock =
+      header.blockSize / SECTOR_SIZE)
+
+    // Compute bitmap size in sectors.
+    // Default: 1.
+    const sectorsOfBitmap = (this.sectorsOfBitmap = sectorsRoundUpNoZero(
+      sectorsPerBlock >> 3
+    ))
+
+    // Full block size => data block size + bitmap size.
+    this.fullBlockSize = sectorsToBytes(sectorsPerBlock + sectorsOfBitmap)
+
+    // In bytes.
+    // Default: 512.
+    this.bitmapSize = sectorsToBytes(sectorsOfBitmap)
+  }
+
+  // Returns a buffer that contains the block allocation table of a vhd file.
+  async readBlockAllocationTable () {
+    const { header } = this
+    this.blockTable = await this._read(
+      header.tableOffset,
+      header.maxTableEntries * 4
+    )
+  }
+
+  // return the first sector (bitmap) of a block
+  _getBatEntry (block) {
+    return this.blockTable.readUInt32BE(block * 4)
+  }
+
+  _readBlock (blockId, onlyBitmap = false) {
+    const blockAddr = this._getBatEntry(blockId)
+    if (blockAddr === BLOCK_UNUSED) {
+      throw new Error(`no such block ${blockId}`)
+    }
+
+    return this._read(
+      sectorsToBytes(blockAddr),
+      onlyBitmap ? this.bitmapSize : this.fullBlockSize
+    ).then(
+      buf =>
+        onlyBitmap
+          ? { id: blockId, bitmap: buf }
+          : {
+              id: blockId,
+              bitmap: buf.slice(0, this.bitmapSize),
+              data: buf.slice(this.bitmapSize),
+              buffer: buf,
+            }
+    )
+  }
+
+  // get the identifiers and first sectors of the first and last block
+  // in the file
+  //
+  _getFirstAndLastBlocks () {
+    const n = this.header.maxTableEntries
+    const bat = this.blockTable
+    let i = 0
+    let j = 0
+    let first, firstSector, last, lastSector
+
+    // get first allocated block for initialization
+    while ((firstSector = bat.readUInt32BE(j)) === BLOCK_UNUSED) {
+      i += 1
+      j += 4
+
+      if (i === n) {
+        const error = new Error('no allocated block found')
+        error.noBlock = true
+        throw error
+      }
+    }
+    lastSector = firstSector
+    first = last = i
+
+    while (i < n) {
+      const sector = bat.readUInt32BE(j)
+      if (sector !== BLOCK_UNUSED) {
+        if (sector < firstSector) {
+          first = i
+          firstSector = sector
+        } else if (sector > lastSector) {
+          last = i
+          lastSector = sector
+        }
+      }
+
+      i += 1
+      j += 4
+    }
+
+    return { first, firstSector, last, lastSector }
+  }
+
+  // =================================================================
+  // Write functions.
+  // =================================================================
+
+  // Write a buffer/stream at a given position in a vhd file.
+  async _write (data, offset) {
+    debug(
+      `_write offset=${offset} size=${
+        Buffer.isBuffer(data) ? data.length : '???'
+      }`
+    )
+    // TODO: could probably be merged in remote handlers.
+    const stream = await this._handler.createOutputStream(this._path, {
+      flags: 'r+',
+      start: offset,
+    })
+    return Buffer.isBuffer(data)
+      ? new Promise((resolve, reject) => {
+          stream.on('error', reject)
+          stream.end(data, resolve)
+        })
+      : fromEvent(data.pipe(stream), 'finish')
+  }
+
+  async _freeFirstBlockSpace (spaceNeededBytes) {
+    try {
+      const { first, firstSector, lastSector } = this._getFirstAndLastBlocks()
+      const tableOffset = this.header.tableOffset
+      const { batSize } = this
+      const newMinSector = Math.ceil(
+        (tableOffset + batSize + spaceNeededBytes) / SECTOR_SIZE
+      )
+      if (
+        tableOffset + batSize + spaceNeededBytes >=
+        sectorsToBytes(firstSector)
+      ) {
+        const { fullBlockSize } = this
+        const newFirstSector = Math.max(
+          lastSector + fullBlockSize / SECTOR_SIZE,
+          newMinSector
+        )
+        debug(
+          `freeFirstBlockSpace: move first block ${firstSector} -> ${newFirstSector}`
+        )
+        // copy the first block at the end
+        const stream = await this._readStream(
+          sectorsToBytes(firstSector),
+          fullBlockSize
+        )
+        await this._write(stream, sectorsToBytes(newFirstSector))
+        await this._setBatEntry(first, newFirstSector)
+        await this.writeFooter(true)
+        spaceNeededBytes -= this.fullBlockSize
+        if (spaceNeededBytes > 0) {
+          return this._freeFirstBlockSpace(spaceNeededBytes)
+        }
+      }
+    } catch (e) {
+      if (!e.noBlock) {
+        throw e
+      }
+    }
+  }
+
+  async ensureBatSize (entries) {
+    const { header } = this
+    const prevMaxTableEntries = header.maxTableEntries
+    if (prevMaxTableEntries >= entries) {
+      return
+    }
+
+    const newBatSize = computeBatSize(entries)
+    await this._freeFirstBlockSpace(newBatSize - this.batSize)
+    const maxTableEntries = (header.maxTableEntries = entries)
+    const prevBat = this.blockTable
+    const bat = (this.blockTable = Buffer.allocUnsafe(newBatSize))
+    prevBat.copy(bat)
+    bat.fill(BUF_BLOCK_UNUSED, prevMaxTableEntries * 4)
+    debug(
+      `ensureBatSize: extend BAT ${prevMaxTableEntries} -> ${maxTableEntries}`
+    )
+    await this._write(
+      constantStream(BUF_BLOCK_UNUSED, maxTableEntries - prevMaxTableEntries),
+      header.tableOffset + prevBat.length
+    )
+    await this.writeHeader()
+  }
+
+  // set the first sector (bitmap) of a block
+  _setBatEntry (block, blockSector) {
+    const i = block * 4
+    const { blockTable } = this
+
+    blockTable.writeUInt32BE(blockSector, i)
+
+    return this._write(blockTable.slice(i, i + 4), this.header.tableOffset + i)
+  }
+
+  // Make a new empty block at vhd end.
+  // Update block allocation table in context and in file.
+  async createBlock (blockId) {
+    const blockAddr = Math.ceil(this.getEndOfData() / SECTOR_SIZE)
+
+    debug(`create block ${blockId} at ${blockAddr}`)
+
+    await Promise.all([
+      // Write an empty block and addr in vhd file.
+      this._write(
+        constantStream([0], this.fullBlockSize),
+        sectorsToBytes(blockAddr)
+      ),
+
+      this._setBatEntry(blockId, blockAddr),
+    ])
+
+    return blockAddr
+  }
+
+  // Write a bitmap at a block address.
+  async writeBlockBitmap (blockAddr, bitmap) {
+    const { bitmapSize } = this
+
+    if (bitmap.length !== bitmapSize) {
+      throw new Error(`Bitmap length is not correct ! ${bitmap.length}`)
+    }
+
+    const offset = sectorsToBytes(blockAddr)
+
+    debug(
+      `Write bitmap at: ${offset}. (size=${bitmapSize}, data=${bitmap.toString(
+        'hex'
+      )})`
+    )
+    await this._write(bitmap, sectorsToBytes(blockAddr))
+  }
+
+  async writeEntireBlock (block) {
+    let blockAddr = this._getBatEntry(block.id)
+
+    if (blockAddr === BLOCK_UNUSED) {
+      blockAddr = await this.createBlock(block.id)
+    }
+    await this._write(block.buffer, sectorsToBytes(blockAddr))
+  }
+
+  async writeBlockSectors (block, beginSectorId, endSectorId, parentBitmap) {
+    let blockAddr = this._getBatEntry(block.id)
+
+    if (blockAddr === BLOCK_UNUSED) {
+      blockAddr = await this.createBlock(block.id)
+      parentBitmap = Buffer.alloc(this.bitmapSize, 0)
+    } else if (parentBitmap === undefined) {
+      parentBitmap = (await this._readBlock(block.id, true)).bitmap
+    }
+
+    const offset = blockAddr + this.sectorsOfBitmap + beginSectorId
+
+    debug(
+      `writeBlockSectors at ${offset} block=${
+        block.id
+      }, sectors=${beginSectorId}...${endSectorId}`
+    )
+
+    for (let i = beginSectorId; i < endSectorId; ++i) {
+      mapSetBit(parentBitmap, i)
+    }
+
+    await this.writeBlockBitmap(blockAddr, parentBitmap)
+    await this._write(
+      block.data.slice(
+        sectorsToBytes(beginSectorId),
+        sectorsToBytes(endSectorId)
+      ),
+      sectorsToBytes(offset)
+    )
+  }
+
+  async coalesceBlock (child, blockId) {
+    const block = await child._readBlock(blockId)
+    const { bitmap, data } = block
+
+    debug(`coalesceBlock block=${blockId}`)
+
+    // For each sector of block data...
+    const { sectorsPerBlock } = child
+    for (let i = 0; i < sectorsPerBlock; i++) {
+      // If no changes on one sector, skip.
+      if (!mapTestBit(bitmap, i)) {
+        continue
+      }
+      let parentBitmap = null
+      let endSector = i + 1
+
+      // Count changed sectors.
+      while (endSector < sectorsPerBlock && mapTestBit(bitmap, endSector)) {
+        ++endSector
+      }
+
+      // Write n sectors into parent.
+      debug(`coalesceBlock: write sectors=${i}...${endSector}`)
+
+      const isFullBlock = i === 0 && endSector === sectorsPerBlock
+      if (isFullBlock) {
+        await this.writeEntireBlock(block)
+      } else {
+        if (parentBitmap === null) {
+          parentBitmap = (await this._readBlock(blockId, true)).bitmap
+        }
+        await this.writeBlockSectors(block, i, endSector, parentBitmap)
+      }
+
+      i = endSector
+    }
+
+    // Return the merged data size
+    return data.length
+  }
+
+  // Write a context footer. (At the end and beginning of a vhd file.)
+  async writeFooter (onlyEndFooter = false) {
+    const { footer } = this
+
+    const rawFooter = fuFooter.pack(footer)
+    const eof = await this._handler.getSize(this._path)
+    // sometimes the file is longer than anticipated, we still need to put the footer at the end
+    const offset = Math.max(this.getEndOfData(), eof - rawFooter.length)
+
+    footer.checksum = checksumStruct(rawFooter, fuFooter)
+    debug(
+      `Write footer at: ${offset} (checksum=${
+        footer.checksum
+      }). (data=${rawFooter.toString('hex')})`
+    )
+    if (!onlyEndFooter) {
+      await this._write(rawFooter, 0)
+    }
+    await this._write(rawFooter, offset)
+  }
+
+  writeHeader () {
+    const { header } = this
+    const rawHeader = fuHeader.pack(header)
+    header.checksum = checksumStruct(rawHeader, fuHeader)
+    const offset = FOOTER_SIZE
+    debug(
+      `Write header at: ${offset} (checksum=${
+        header.checksum
+      }). (data=${rawHeader.toString('hex')})`
+    )
+    return this._write(rawHeader, offset)
+  }
+
+  async writeData (offsetSectors, buffer) {
+    const bufferSizeSectors = Math.ceil(buffer.length / SECTOR_SIZE)
+    const startBlock = Math.floor(offsetSectors / this.sectorsPerBlock)
+    const endBufferSectors = offsetSectors + bufferSizeSectors
+    const lastBlock = Math.ceil(endBufferSectors / this.sectorsPerBlock) - 1
+    await this.ensureBatSize(lastBlock)
+    const blockSizeBytes = this.sectorsPerBlock * SECTOR_SIZE
+    const coversWholeBlock = (offsetInBlockSectors, endInBlockSectors) =>
+      offsetInBlockSectors === 0 && endInBlockSectors === this.sectorsPerBlock
+
+    for (
+      let currentBlock = startBlock;
+      currentBlock <= lastBlock;
+      currentBlock++
+    ) {
+      const offsetInBlockSectors = Math.max(
+        0,
+        offsetSectors - currentBlock * this.sectorsPerBlock
+      )
+      const endInBlockSectors = Math.min(
+        endBufferSectors - currentBlock * this.sectorsPerBlock,
+        this.sectorsPerBlock
+      )
+      const startInBuffer = Math.max(
+        0,
+        (currentBlock * this.sectorsPerBlock - offsetSectors) * SECTOR_SIZE
+      )
+      const endInBuffer = Math.min(
+        ((currentBlock + 1) * this.sectorsPerBlock - offsetSectors) *
+          SECTOR_SIZE,
+        buffer.length
+      )
+      let inputBuffer
+      if (coversWholeBlock(offsetInBlockSectors, endInBlockSectors)) {
+        inputBuffer = buffer.slice(startInBuffer, endInBuffer)
+      } else {
+        inputBuffer = Buffer.alloc(blockSizeBytes, 0)
+        buffer.copy(
+          inputBuffer,
+          offsetInBlockSectors * SECTOR_SIZE,
+          startInBuffer,
+          endInBuffer
+        )
+      }
+      await this.writeBlockSectors(
+        { id: currentBlock, data: inputBuffer },
+        offsetInBlockSectors,
+        endInBlockSectors
+      )
+    }
+    await this.writeFooter()
+  }
+
+  async ensureSpaceForParentLocators (neededSectors) {
+    const firstLocatorOffset = FOOTER_SIZE + HEADER_SIZE
+    const currentSpace =
+      Math.floor(this.header.tableOffset / SECTOR_SIZE) -
+      firstLocatorOffset / SECTOR_SIZE
+    if (currentSpace < neededSectors) {
+      const deltaSectors = neededSectors - currentSpace
+      await this._freeFirstBlockSpace(sectorsToBytes(deltaSectors))
+      this.header.tableOffset += sectorsToBytes(deltaSectors)
+      await this._write(this.blockTable, this.header.tableOffset)
+    }
+    return firstLocatorOffset
+  }
+
+  async setUniqueParentLocator (fileNameString) {
+    const { header } = this
+    header.parentLocatorEntry[0].platformCode = PLATFORM_W2KU
+    const encodedFilename = Buffer.from(fileNameString, 'utf16le')
+    const dataSpaceSectors = Math.ceil(encodedFilename.length / SECTOR_SIZE)
+    const position = await this.ensureSpaceForParentLocators(dataSpaceSectors)
+    await this._write(encodedFilename, position)
+    header.parentLocatorEntry[0].platformDataSpace =
+      dataSpaceSectors * SECTOR_SIZE
+    header.parentLocatorEntry[0].platformDataLength = encodedFilename.length
+    header.parentLocatorEntry[0].platformDataOffset = position
+    for (let i = 1; i < 8; i++) {
+      header.parentLocatorEntry[i].platformCode = PLATFORM_NONE
+      header.parentLocatorEntry[i].platformDataSpace = 0
+      header.parentLocatorEntry[i].platformDataLength = 0
+      header.parentLocatorEntry[i].platformDataOffset = 0
+    }
+  }
+}

packages/xen-api/.gitignore

@@ -1 +0,0 @@
-/plot.dat

packages/xen-api/README.md

@@ -4,6 +4,8 @@
 
 Tested with:
 
+- XenServer 7.3
+- XenServer 7.2
 - XenServer 7.1
 - XenServer 7
 - XenServer 6.5

packages/xen-api/examples/utils.js

@@ -1,7 +1,7 @@
 const { createReadStream, createWriteStream, statSync } = require('fs')
 const { PassThrough } = require('stream')
 
-const { isOpaqueRef } = require('../') // eslint-disable-line node/no-missing-require
+const { isOpaqueRef } = require('../')
 
 exports.createInputStream = path => {
   if (path === undefined || path === '-') {
@@ -29,13 +29,15 @@ exports.createOutputStream = path => {
 exports.resolveRef = (xapi, type, refOrUuidOrNameLabel) =>
   isOpaqueRef(refOrUuidOrNameLabel)
     ? refOrUuidOrNameLabel
-    : xapi.call(`${type}.get_by_uuid`, refOrUuidOrNameLabel).catch(
-      () => xapi.call(`${type}.get_by_name_label`, refOrUuidOrNameLabel).then(
-        refs => {
+    : xapi.call(`${type}.get_by_uuid`, refOrUuidOrNameLabel).catch(() =>
+      xapi
+        .call(`${type}.get_by_name_label`, refOrUuidOrNameLabel)
+        .then(refs => {
           if (refs.length === 1) {
             return refs[0]
           }
-          throw new Error(`no single match for ${type} with name label ${refOrUuidOrNameLabel}`)
-        }
-      )
+          throw new Error(
+            `no single match for ${type} with name label ${refOrUuidOrNameLabel}`
+          )
+        })
     )

packages/xen-api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xen-api",
-  "version": "0.16.4",
+  "version": "0.16.9",
   "license": "ISC",
   "description": "Connector to the Xen API",
   "keywords": [
@@ -11,7 +11,7 @@
     "xapi"
   ],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xen-api",
-  "bugs": "https://github.com/vatesfr/xo-web/issues",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
@@ -66,7 +66,6 @@
     "plot": "gnuplot -p memory-test.gnu",
     "prebuild": "rimraf dist/",
     "predev": "yarn run prebuild",
-    "prepare": "yarn run build",
     "prepublishOnly": "yarn run build"
   },
   "babel": {

packages/xen-api/src/cli.js

@@ -56,7 +56,7 @@ const main = async args => {
 
   let auth
   if (opts._.length > 1) {
-    const [ , user, password = await askPassword() ] = opts._
+    const [, user, password = await askPassword()] = opts._
     auth = { user, password }
   }
 
@@ -86,11 +86,11 @@ const main = async args => {
 
   // Make the REPL waits for promise completion.
   repl.eval = (evaluate => (cmd, context, filename, cb) => {
-    fromCallback(cb => {
+    ;fromCallback(cb => {
       evaluate.call(repl, cmd, context, filename, cb)
-    }).then(value =>
-      isArray(value) ? Promise.all(value) : value
-    )::asCallback(cb)
+    })
+      .then(value => (isArray(value) ? Promise.all(value) : value))
+      ::asCallback(cb)
   })(repl.eval)
 
   await eventToPromise(repl, 'exit')

packages/xen-api/src/index.js

@@ -26,6 +26,8 @@ import {
   delay as pDelay,
   fromEvents,
   lastly,
+  timeout as pTimeout,
+  TimeoutError,
 } from 'promise-toolbox'
 
 import autoTransport from './transports/auto'
@@ -34,6 +36,9 @@ const debug = createDebug('xen-api')
 
 // ===================================================================
 
+// in seconds
+const EVENT_TIMEOUT = 60
+
 // http://www.gnu.org/software/libc/manual/html_node/Error-Codes.html
 const NETWORK_ERRORS = {
   // Connection has been closed outside of our control.
@@ -51,11 +56,14 @@ const NETWORK_ERRORS = {
   // Host is not reachable (does not respond).
   EHOSTUNREACH: true,
 
+  // network is unreachable
+  ENETUNREACH: true,
+
   // Connection configured timed out has been reach.
   ETIMEDOUT: true,
 }
 
-const isNetworkError = ({code}) => NETWORK_ERRORS[code]
+const isNetworkError = ({ code }) => NETWORK_ERRORS[code]
 
 // -------------------------------------------------------------------
 
@@ -64,17 +72,17 @@ const XAPI_NETWORK_ERRORS = {
   HOST_HAS_NO_MANAGEMENT_IP: true,
 }
 
-const isXapiNetworkError = ({code}) => XAPI_NETWORK_ERRORS[code]
+const isXapiNetworkError = ({ code }) => XAPI_NETWORK_ERRORS[code]
 
 // -------------------------------------------------------------------
 
-const areEventsLost = ({code}) => code === 'EVENTS_LOST'
+const areEventsLost = ({ code }) => code === 'EVENTS_LOST'
 
-const isHostSlave = ({code}) => code === 'HOST_IS_SLAVE'
+const isHostSlave = ({ code }) => code === 'HOST_IS_SLAVE'
 
-const isMethodUnknown = ({code}) => code === 'MESSAGE_METHOD_UNKNOWN'
+const isMethodUnknown = ({ code }) => code === 'MESSAGE_METHOD_UNKNOWN'
 
-const isSessionInvalid = ({code}) => code === 'SESSION_INVALID'
+const isSessionInvalid = ({ code }) => code === 'SESSION_INVALID'
 
 // -------------------------------------------------------------------
 
@@ -93,8 +101,9 @@ class XapiError extends BaseError {
 
 export const wrapError = error => {
   let code, params
-  if (isArray(error)) { // < XenServer 7.3
-    [ code, ...params ] = error
+  if (isArray(error)) {
+    // < XenServer 7.3
+    ;[code, ...params] = error
   } else {
     code = error.message
     params = error.data
@@ -111,7 +120,7 @@ const parseUrl = url => {
     throw new Error('invalid URL: ' + url)
   }
 
-  const [ , protocol = 'https:', username, password, hostname, port ] = matches
+  const [, protocol = 'https:', username, password, hostname, port] = matches
   return { protocol, username, password, hostname, port }
 }
 
@@ -128,17 +137,15 @@ const {
 
 const OPAQUE_REF_PREFIX = 'OpaqueRef:'
 export const isOpaqueRef = value =>
-  typeof value === 'string' &&
-  startsWith(value, OPAQUE_REF_PREFIX)
+  typeof value === 'string' && startsWith(value, OPAQUE_REF_PREFIX)
 
 // -------------------------------------------------------------------
 
 const RE_READ_ONLY_METHOD = /^[^.]+\.get_/
-const isReadOnlyCall = (method, args) => (
+const isReadOnlyCall = (method, args) =>
   args.length === 1 &&
-  isOpaqueRef(args[0]) &&
+  typeof args[0] === 'string' &&
   RE_READ_ONLY_METHOD.test(method)
-)
 
 // Prepare values before passing them to the XenAPI:
 //
@@ -175,20 +182,20 @@ const EMPTY_ARRAY = freezeObject([])
 
 // -------------------------------------------------------------------
 
-const getTaskResult = (task, onSuccess, onFailure) => {
+const getTaskResult = task => {
   const { status } = task
   if (status === 'cancelled') {
-    return [ onFailure(new Cancel('task canceled')) ]
+    return Promise.reject(new Cancel('task canceled'))
   }
   if (status === 'failure') {
-    return [ onFailure(wrapError(task.error_info)) ]
+    return Promise.reject(wrapError(task.error_info))
   }
   if (status === 'success') {
     // the result might be:
     // - empty string
     // - an opaque reference
     // - an XML-RPC value
-    return [ onSuccess(task.result) ]
+    return Promise.resolve(task.result)
   }
 }
 
@@ -209,7 +216,7 @@ export class Xapi extends EventEmitter {
     this._pool = null
     this._readOnly = Boolean(opts.readOnly)
     this._sessionId = null
-    const url = this._url = parseUrl(opts.url)
+    const url = (this._url = parseUrl(opts.url))
 
     if (this._auth === undefined) {
       const user = url.username
@@ -224,9 +231,7 @@ export class Xapi extends EventEmitter {
     }
 
     if (opts.watchEvents !== false) {
-      this._debounce = opts.debounce == null
-        ? 200
-        : opts.debounce
+      this._debounce = opts.debounce == null ? 200 : opts.debounce
 
       this._eventWatchers = createObject(null)
 
@@ -237,11 +242,11 @@ export class Xapi extends EventEmitter {
 
       this._nTasks = 0
 
-      const objects = this._objects = new Collection()
+      const objects = (this._objects = new Collection())
       objects.getKey = getKey
 
       this._objectsByRefs = createObject(null)
-      this._objectsByRefs['OpaqueRef:NULL'] = null
+      this._objectsByRefs['OpaqueRef:NULL'] = undefined
 
       this._taskWatchers = Object.create(null)
 
@@ -286,13 +291,7 @@ export class Xapi extends EventEmitter {
   get status () {
     const id = this._sessionId
 
-    return id
-      ? (
-        id === CONNECTING
-          ? CONNECTING
-          : CONNECTED
-      )
-      : DISCONNECTED
+    return id ? (id === CONNECTING ? CONNECTING : CONNECTED) : DISCONNECTED
   }
 
   get _humanId () {
@@ -305,36 +304,46 @@ export class Xapi extends EventEmitter {
   barrier (ref) {
     const eventWatchers = this._eventWatchers
     if (eventWatchers === undefined) {
-      return Promise.reject(new Error('Xapi#barrier() requires events watching'))
+      return Promise.reject(
+        new Error('Xapi#barrier() requires events watching')
+      )
     }
 
-    const key = `xo:barrier:${Math.random().toString(36).slice(2)}`
+    const key = `xo:barrier:${Math.random()
+      .toString(36)
+      .slice(2)}`
     const poolRef = this._pool.$ref
 
     const { promise, resolve } = defer()
     eventWatchers[key] = resolve
 
-    return this._sessionCall(
-      'pool.add_to_other_config',
-      [ poolRef, key, '' ]
-    ).then(() => promise.then(() => {
-      this._sessionCall('pool.remove_from_other_config', [ poolRef, key ]).catch(noop)
+    return this._sessionCall('pool.add_to_other_config', [
+      poolRef,
+      key,
+      '',
+    ]).then(() =>
+      promise.then(() => {
+        this._sessionCall('pool.remove_from_other_config', [
+          poolRef,
+          key,
+        ]).catch(noop)
 
-      if (ref === undefined) {
-        return
-      }
+        if (ref === undefined) {
+          return
+        }
 
-      // support legacy params (type, ref)
-      if (arguments.length === 2) {
-        ref = arguments[1]
-      }
+        // support legacy params (type, ref)
+        if (arguments.length === 2) {
+          ref = arguments[1]
+        }
 
-      return this.getObjectByRef(ref)
-    }))
+        return this.getObjectByRef(ref)
+      })
+    )
   }
 
   connect () {
-    const {status} = this
+    const { status } = this
 
     if (status === CONNECTED) {
       return Promise.reject(new Error('already connected'))
@@ -378,7 +387,7 @@ export class Xapi extends EventEmitter {
         return Promise.reject(new Error('already disconnected'))
       }
 
-      this._transportCall('session.logout', [ this._sessionId ]).catch(noop)
+      this._transportCall('session.logout', [this._sessionId]).catch(noop)
 
       this._sessionId = null
 
@@ -400,22 +409,22 @@ export class Xapi extends EventEmitter {
     return this._readOnly && !isReadOnlyCall(method, args)
       ? Promise.reject(new Error(`cannot call ${method}() in read only mode`))
       : this._sessionCall(`Async.${method}`, args).then(taskRef => {
-        $cancelToken.promise.then(() => {
-          this._sessionCall('task.cancel', [taskRef]).catch(noop)
-        })
+          $cancelToken.promise.then(() => {
+            // TODO: do not trigger if the task is already over
+            this._sessionCall('task.cancel', [taskRef]).catch(noop)
+          })
 
-        return this.watchTask(taskRef)::lastly(() => {
-          this._sessionCall('task.destroy', [taskRef]).catch(noop)
+          return this.watchTask(taskRef)::lastly(() => {
+            this._sessionCall('task.destroy', [taskRef]).catch(noop)
+          })
         })
-      })
   }
 
   // create a task and automatically destroy it when settled
+  //
+  //  allowed even in read-only mode because it does not have impact on the
+  //  XenServer and it's necessary for getResource()
   createTask (nameLabel, nameDescription = '') {
-    if (this._readOnly) {
-      return Promise.reject(new Error('cannot create task in read only mode'))
-    }
-
     const promise = this._sessionCall('task.create', [
       nameLabel,
       nameDescription,
@@ -434,19 +443,18 @@ export class Xapi extends EventEmitter {
   // this lib), UUID (unique identifier that some objects have) or
   // opaque reference (internal to XAPI).
   getObject (idOrUuidOrRef, defaultValue) {
-    const object = typeof idOrUuidOrRef === 'string'
-      ? (
-        // if there is an UUID, it is also the $id.
-        this._objects.all[idOrUuidOrRef] ||
-        this._objectsByRefs[idOrUuidOrRef]
-      )
-      : this._objects.all[idOrUuidOrRef.$id]
+    if (typeof idOrUuidOrRef === 'object') {
+      idOrUuidOrRef = idOrUuidOrRef.$id
+    }
 
-    if (object) return object
+    const object =
+      this._objects.all[idOrUuidOrRef] || this._objectsByRefs[idOrUuidOrRef]
+
+    if (object !== undefined) return object
 
     if (arguments.length > 1) return defaultValue
 
-    throw new Error('there is not object can be matched to ' + idOrUuidOrRef)
+    throw new Error('no object with UUID or opaque ref: ' + idOrUuidOrRef)
   }
 
   // Returns the object for a given opaque reference (internal to
@@ -454,11 +462,11 @@ export class Xapi extends EventEmitter {
   getObjectByRef (ref, defaultValue) {
     const object = this._objectsByRefs[ref]
 
-    if (object) return object
+    if (object !== undefined) return object
 
     if (arguments.length > 1) return defaultValue
 
-    throw new Error('there is no object with the ref ' + ref)
+    throw new Error('no object with opaque ref: ' + ref)
   }
 
   // Returns the object for a given UUID (unique identifier that some
@@ -471,7 +479,7 @@ export class Xapi extends EventEmitter {
 
     if (arguments.length > 1) return defaultValue
 
-    throw new Error('there is no object with the UUID ' + uuid)
+    throw new Error('no object with UUID: ' + uuid)
   }
 
   getRecord (type, ref) {
@@ -479,158 +487,147 @@ export class Xapi extends EventEmitter {
   }
 
   @cancelable
-  getResource ($cancelToken, pathname, {
-    host,
-    query,
-    task,
-  }) {
-    return this._autoTask(
-      task,
-      `Xapi#getResource ${pathname}`
-    ).then(taskRef => {
-      query = { ...query, session_id: this.sessionId }
-      let taskResult
-      if (taskRef !== undefined) {
-        query.task_id = taskRef
-        taskResult = this.watchTask(taskRef)
+  getResource ($cancelToken, pathname, { host, query, task }) {
+    return this._autoTask(task, `Xapi#getResource ${pathname}`).then(
+      taskRef => {
+        query = { ...query, session_id: this.sessionId }
+        let taskResult
+        if (taskRef !== undefined) {
+          query.task_id = taskRef
+          taskResult = this.watchTask(taskRef)
 
-        if (typeof $cancelToken.addHandler === 'function') {
-          $cancelToken.addHandler(() => taskResult)
+          if (typeof $cancelToken.addHandler === 'function') {
+            $cancelToken.addHandler(() => taskResult)
+          }
         }
-      }
 
-      let promise = httpRequest(
-        $cancelToken,
-        this._url,
-        host && {
-          hostname: this.getObject(host).address,
-        },
-        {
-          pathname,
-          query,
-          rejectUnauthorized: !this._allowUnauthorized,
-        }
-      )
-
-      if (taskResult !== undefined) {
-        promise = promise.then(response => {
-          response.task = taskResult
-          return response
-        })
-      }
-
-      return promise
-    })
-  }
-
-  @cancelable
-  putResource ($cancelToken, body, pathname, {
-    host,
-    query,
-    task,
-  } = {}) {
-    if (this._readOnly) {
-      return Promise.reject(new Error(new Error('cannot put resource in read only mode')))
-    }
-
-    return this._autoTask(
-      task,
-      `Xapi#putResource ${pathname}`
-    ).then(taskRef => {
-      query = { ...query, session_id: this.sessionId }
-
-      let taskResult
-      if (taskRef !== undefined) {
-        query.task_id = taskRef
-        taskResult = this.watchTask(taskRef)
-
-        if (typeof $cancelToken.addHandler === 'function') {
-          $cancelToken.addHandler(() => taskResult)
-        }
-      }
-
-      const headers = {}
-
-      // Xen API does not support chunk encoding.
-      const isStream = typeof body.pipe === 'function'
-      const { length } = body
-      if (isStream && length === undefined) {
-        // add a fake huge content length (1 PiB)
-        headers['content-length'] = '1125899906842624'
-      }
-
-      const doRequest = override => httpRequest.put(
-        $cancelToken,
-        this._url,
-        host && {
-          hostname: this.getObject(host).address,
-        },
-        {
-          body,
-          headers,
-          pathname,
-          query,
-          rejectUnauthorized: !this._allowUnauthorized,
-        },
-        override
-      )
-
-      const promise = isStream
-
-        // dummy request to probe for a redirection before consuming body
-        ? doRequest({
-          body: '',
-
-          // omit task_id because this request will fail on purpose
-          query: 'task_id' in query
-            ? omit(query, 'task_id')
-            : query,
-
-          maxRedirects: 0,
-        }).then(
-          response => {
-            response.req.abort()
-            return doRequest()
+        let promise = httpRequest(
+          $cancelToken,
+          this._url,
+          host && {
+            hostname: this.getObject(host).address,
           },
-          error => {
-            let response
-            if (error != null && (response = error.response) != null) {
-              response.req.abort()
-
-              const { headers: { location }, statusCode } = response
-              if (statusCode === 302 && location !== undefined) {
-                return doRequest(location)
-              }
-            }
-
-            throw error
+          {
+            pathname,
+            query,
+            rejectUnauthorized: !this._allowUnauthorized,
           }
         )
 
-        // http-request-plus correctly handle redirects if body is not a stream
-        : doRequest()
-
-      return promise.then(response => {
-        const { req } = response
-
         if (taskResult !== undefined) {
-          taskResult = taskResult.catch(error => {
-            error.url = response.url
-            throw error
+          promise = promise.then(response => {
+            response.task = taskResult
+            return response
           })
         }
 
-        if (req.finished) {
-          req.abort()
-          return taskResult
+        return promise
+      }
+    )
+  }
+
+  @cancelable
+  putResource ($cancelToken, body, pathname, { host, query, task } = {}) {
+    if (this._readOnly) {
+      return Promise.reject(
+        new Error(new Error('cannot put resource in read only mode'))
+      )
+    }
+
+    return this._autoTask(task, `Xapi#putResource ${pathname}`).then(
+      taskRef => {
+        query = { ...query, session_id: this.sessionId }
+
+        let taskResult
+        if (taskRef !== undefined) {
+          query.task_id = taskRef
+          taskResult = this.watchTask(taskRef)
+
+          if (typeof $cancelToken.addHandler === 'function') {
+            $cancelToken.addHandler(() => taskResult)
+          }
         }
 
-        return fromEvents(req, ['close', 'finish']).then(() => {
-          req.abort()
-          return taskResult
+        const headers = {}
+
+        // Xen API does not support chunk encoding.
+        const isStream = typeof body.pipe === 'function'
+        const { length } = body
+        if (isStream && length === undefined) {
+          // add a fake huge content length (1 PiB)
+          headers['content-length'] = '1125899906842624'
+        }
+
+        const doRequest = override =>
+          httpRequest.put(
+            $cancelToken,
+            this._url,
+            host && {
+              hostname: this.getObject(host).address,
+            },
+            {
+              body,
+              headers,
+              pathname,
+              query,
+              rejectUnauthorized: !this._allowUnauthorized,
+            },
+            override
+          )
+
+        // if a stream, sends a dummy request to probe for a
+        // redirection before consuming body
+        const promise = isStream
+          ? doRequest({
+              body: '',
+
+              // omit task_id because this request will fail on purpose
+              query: 'task_id' in query ? omit(query, 'task_id') : query,
+
+              maxRedirects: 0,
+            }).then(
+              response => {
+                response.req.abort()
+                return doRequest()
+              },
+              error => {
+                let response
+                if (error != null && (response = error.response) != null) {
+                  response.req.abort()
+
+                  const { headers: { location }, statusCode } = response
+                  if (statusCode === 302 && location !== undefined) {
+                    return doRequest(location)
+                  }
+                }
+
+                throw error
+              }
+            )
+          : doRequest()
+
+        return promise.then(response => {
+          const { req } = response
+
+          if (taskResult !== undefined) {
+            taskResult = taskResult.catch(error => {
+              error.url = response.url
+              throw error
+            })
+          }
+
+          if (req.finished) {
+            req.abort()
+            return taskResult
+          }
+
+          return fromEvents(req, ['close', 'finish']).then(() => {
+            req.abort()
+            return taskResult
+          })
         })
-      })
-    })
+      }
+    )
   }
 
   watchTask (ref) {
@@ -645,11 +642,11 @@ export class Xapi extends EventEmitter {
     let watcher = watchers[ref]
     if (watcher === undefined) {
       // sync check if the task is already settled
-      const task = this.objects.all[ref]
+      const task = this._objectsByRefs[ref]
       if (task !== undefined) {
-        const result = getTaskResult(task, Promise.resolve, Promise.reject)
-        if (result) {
-          return result[0]
+        const result = getTaskResult(task)
+        if (result !== undefined) {
+          return result
         }
       }
 
@@ -692,22 +689,24 @@ export class Xapi extends EventEmitter {
         newArgs.push.apply(newArgs, args)
       }
 
-      return this._transportCall(method, newArgs)
-        ::pCatch(isSessionInvalid, () => {
+      return this._transportCall(method, newArgs)::pCatch(
+        isSessionInvalid,
+        () => {
           // XAPI is sometimes reinitialized and sessions are lost.
           // Try to login again.
           debug('%s: the session has been reinitialized', this._humanId)
 
           this._sessionId = null
           return this.connect().then(() => this._sessionCall(method, args))
-        })
+        }
+      )
     } catch (error) {
       return Promise.reject(error)
     }
   }
 
   _addObject (type, ref, object) {
-    const {_objectsByRefs: objectsByRefs} = this
+    const { _objectsByRefs: objectsByRefs } = this
 
     const reservedKeys = {
       id: true,
@@ -715,9 +714,8 @@ export class Xapi extends EventEmitter {
       ref: true,
       type: true,
     }
-    const getKey = (key, obj) => reservedKeys[key] && obj === object
-      ? `$$${key}`
-      : `$${key}`
+    const getKey = (key, obj) =>
+      reservedKeys[key] && obj === object ? `$$${key}` : `$${key}`
 
     // Creates resolved properties.
     forEach(object, function resolveObject (value, key, object) {
@@ -736,7 +734,7 @@ export class Xapi extends EventEmitter {
         } else if (isOpaqueRef(value[0])) {
           // This is an array of refs.
           defineProperty(object, getKey(key, object), {
-            get: () => freezeObject(map(value, (ref) => objectsByRefs[ref])),
+            get: () => freezeObject(map(value, ref => objectsByRefs[ref])),
           })
 
           freezeObject(value)
@@ -795,11 +793,12 @@ export class Xapi extends EventEmitter {
 
       const taskWatchers = this._taskWatchers
       const taskWatcher = taskWatchers[ref]
-      if (
-        taskWatcher !== undefined &&
-        getTaskResult(object, taskWatcher.resolve, taskWatcher.reject)
-      ) {
-        delete taskWatchers[ref]
+      if (taskWatcher !== undefined) {
+        const result = getTaskResult(object)
+        if (result !== undefined) {
+          taskWatcher.resolve(result)
+          delete taskWatchers[ref]
+        }
       }
     }
   }
@@ -819,7 +818,10 @@ export class Xapi extends EventEmitter {
     const taskWatchers = this._taskWatchers
     const taskWatcher = taskWatchers[ref]
     if (taskWatcher !== undefined) {
-      taskWatcher.reject(new Error('task has been destroyed before completion'))
+      const error = new Error('task has been destroyed before completion')
+      error.task = object
+      error.taskRef = ref
+      taskWatcher.reject(error)
       delete taskWatchers[ref]
     }
   }
@@ -836,40 +838,48 @@ export class Xapi extends EventEmitter {
   }
 
   _watchEvents () {
-    const loop = () => this.status === CONNECTED && this._sessionCall('event.from', [
-      ['*'],
-      this._fromToken,
-      60 + 0.1, // Force float.
-    ]).then(onSuccess, onFailure)
+    const loop = () =>
+      this.status === CONNECTED &&
+      this._sessionCall('event.from', [
+        ['*'],
+        this._fromToken,
+        EVENT_TIMEOUT + 0.1, // Force float.
+      ])
+        ::pTimeout(EVENT_TIMEOUT * 1.1e3) // 10% longer than the XenAPI timeout
+        .then(onSuccess, onFailure)
 
     const onSuccess = ({ events, token, valid_ref_counts: { task } }) => {
       this._fromToken = token
       this._processEvents(events)
 
       if (task !== this._nTasks) {
-        this._sessionCall('task.get_all_records').then(tasks => {
-          const toRemove = new Set()
-          forEach(this.objects.all, object => {
-            if (object.$type === 'task') {
-              toRemove.add(object.$ref)
-            }
+        this._sessionCall('task.get_all_records')
+          .then(tasks => {
+            const toRemove = new Set()
+            forEach(this.objects.all, object => {
+              if (object.$type === 'task') {
+                toRemove.add(object.$ref)
+              }
+            })
+            forEach(tasks, (task, ref) => {
+              toRemove.delete(ref)
+              this._addObject('task', ref, task)
+            })
+            toRemove.forEach(ref => {
+              this._removeObject('task', ref)
+            })
           })
-          forEach(tasks, (task, ref) => {
-            toRemove.delete(ref)
-            this._addObject('task', ref, task)
-          })
-          toRemove.forEach(ref => {
-            this._removeObject('task', ref)
-          })
-        }).catch(noop)
+          .catch(noop)
       }
 
       const debounce = this._debounce
-      return debounce != null
-        ? pDelay(debounce).then(loop)
-        : loop()
+      return debounce != null ? pDelay(debounce).then(loop) : loop()
     }
     const onFailure = error => {
+      if (error instanceof TimeoutError) {
+        return loop()
+      }
+
       if (areEventsLost(error)) {
         this._fromToken = ''
         this._objects.clear()
@@ -906,41 +916,43 @@ export class Xapi extends EventEmitter {
           ::/\.get_all_records$/.test
         )
 
-        return Promise.all(map(
-          getAllRecordsMethods,
-          method => this._sessionCall(method).then(
-            objects => {
-              const type = method.slice(0, method.indexOf('.')).toLowerCase()
-              forEach(objects, (object, ref) => {
-                this._addObject(type, ref, object)
-              })
-            },
-            error => {
-              if (error.code !== 'MESSAGE_REMOVED') {
-                throw error
+        return Promise.all(
+          map(getAllRecordsMethods, method =>
+            this._sessionCall(method).then(
+              objects => {
+                const type = method.slice(0, method.indexOf('.')).toLowerCase()
+                forEach(objects, (object, ref) => {
+                  this._addObject(type, ref, object)
+                })
+              },
+              error => {
+                if (error.code !== 'MESSAGE_REMOVED') {
+                  throw error
+                }
               }
-            }
+            )
           )
-        ))
+        )
       })
     }
 
-    const watchEvents = () => this._sessionCall('event.register', [ ['*'] ]).then(loop)
+    const watchEvents = () =>
+      this._sessionCall('event.register', [['*']]).then(loop)
 
-    const loop = () => this.status === CONNECTED && this._sessionCall('event.next').then(onSuccess, onFailure)
+    const loop = () =>
+      this.status === CONNECTED &&
+      this._sessionCall('event.next').then(onSuccess, onFailure)
 
     const onSuccess = events => {
       this._processEvents(events)
 
       const debounce = this._debounce
-      return debounce == null
-        ? loop()
-        : pDelay(debounce).then(loop)
+      return debounce == null ? loop() : pDelay(debounce).then(loop)
     }
 
     const onFailure = error => {
       if (areEventsLost(error)) {
-        return this._sessionCall('event.unregister', [ ['*'] ]).then(watchEvents)
+        return this._sessionCall('event.unregister', [['*']]).then(watchEvents)
       }
 
       throw error
@@ -950,85 +962,106 @@ export class Xapi extends EventEmitter {
   }
 }
 
-Xapi.prototype._transportCall = reduce([
-  function (method, args) {
-    return this._call(method, args).catch(error => {
-      if (!(error instanceof Error)) {
-        error = wrapError(error)
-      }
-
-      error.method = method
-      throw error
-    })
-  },
-  call => function () {
-    let iterator // lazily created
-    const loop = () => call.apply(this, arguments)
-      ::pCatch(isNetworkError, isXapiNetworkError, error => {
-        if (iterator === undefined) {
-          iterator = fibonacci().clamp(undefined, 60).take(10).toMs()
+Xapi.prototype._transportCall = reduce(
+  [
+    function (method, args) {
+      return this._call(method, args).catch(error => {
+        if (!(error instanceof Error)) {
+          error = wrapError(error)
         }
 
-        const cursor = iterator.next()
-        if (!cursor.done) {
-          // TODO: ability to cancel the connection
-          // TODO: ability to force immediate reconnection
-
-          const delay = cursor.value
-          debug('%s: network error %s, next try in %s ms', this._humanId, error.code, delay)
-          return pDelay(delay).then(loop)
-        }
-
-        debug('%s: network error %s, aborting', this._humanId, error.code)
-
-        // mark as disconnected
-        this.disconnect()::pCatch(noop)
-
+        error.method = method
         throw error
       })
-    return loop()
-  },
-  call => function loop () {
-    return call.apply(this, arguments)
-      ::pCatch(isHostSlave, ({params: [master]}) => {
-        debug('%s: host is slave, attempting to connect at %s', this._humanId, master)
+    },
+    call =>
+      function () {
+        let iterator // lazily created
+        const loop = () =>
+          call
+            .apply(this, arguments)
+            ::pCatch(isNetworkError, isXapiNetworkError, error => {
+              if (iterator === undefined) {
+                iterator = fibonacci()
+                  .clamp(undefined, 60)
+                  .take(10)
+                  .toMs()
+              }
 
-        const newUrl = {
-          ...this._url,
-          hostname: master,
-        }
-        this.emit('redirect', newUrl)
-        this._url = newUrl
+              const cursor = iterator.next()
+              if (!cursor.done) {
+                // TODO: ability to cancel the connection
+                // TODO: ability to force immediate reconnection
 
-        return loop.apply(this, arguments)
-      })
-  },
-  call => function (method) {
-    const startTime = Date.now()
-    return call.apply(this, arguments).then(
-      result => {
-        debug(
-          '%s: %s(...) [%s] ==> %s',
-          this._humanId,
-          method,
-          ms(Date.now() - startTime),
-          kindOf(result)
-        )
-        return result
+                const delay = cursor.value
+                debug(
+                  '%s: network error %s, next try in %s ms',
+                  this._humanId,
+                  error.code,
+                  delay
+                )
+                return pDelay(delay).then(loop)
+              }
+
+              debug('%s: network error %s, aborting', this._humanId, error.code)
+
+              // mark as disconnected
+              this.disconnect()::pCatch(noop)
+
+              throw error
+            })
+        return loop()
       },
-      error => {
-        debug(
-          '%s: %s(...) [%s] =!> %s',
-          this._humanId,
-          method,
-          ms(Date.now() - startTime),
-          error
+    call =>
+      function loop () {
+        return call
+          .apply(this, arguments)
+          ::pCatch(isHostSlave, ({ params: [master] }) => {
+            debug(
+              '%s: host is slave, attempting to connect at %s',
+              this._humanId,
+              master
+            )
+
+            const newUrl = {
+              ...this._url,
+              hostname: master,
+            }
+            this.emit('redirect', newUrl)
+            this._url = newUrl
+
+            return loop.apply(this, arguments)
+          })
+      },
+    call =>
+      function (method) {
+        const startTime = Date.now()
+        return call.apply(this, arguments).then(
+          result => {
+            debug(
+              '%s: %s(...) [%s] ==> %s',
+              this._humanId,
+              method,
+              ms(Date.now() - startTime),
+              kindOf(result)
+            )
+            return result
+          },
+          error => {
+            debug(
+              '%s: %s(...) [%s] =!> %s',
+              this._humanId,
+              method,
+              ms(Date.now() - startTime),
+              error
+            )
+            throw error
+          }
         )
-        throw error
-      }
-    )
-  },
-], (call, decorator) => decorator(call))
+      },
+  ],
+  (call, decorator) => decorator(call)
+)
 
 // ===================================================================
 

packages/xen-api/src/inject-events.js

@@ -5,7 +5,7 @@ import { delay as pDelay } from 'promise-toolbox'
 import { createClient } from './'
 
 const xapi = (() => {
-  const [ , , url, user, password ] = process.argv
+  const [, , url, user, password] = process.argv
 
   return createClient({
     auth: { user, password },
@@ -14,16 +14,19 @@ const xapi = (() => {
   })
 })()
 
-xapi.connect()
+xapi
+  .connect()
 
   // Get the pool record's ref.
   .then(() => xapi.call('pool.get_all'))
 
   // Injects lots of events.
-  .then(([ poolRef ]) => {
-    const loop = () => xapi.call('event.inject', 'pool', poolRef)
-      ::pDelay(10) // A small delay is required to avoid overloading the Xen API.
-      .then(loop)
+  .then(([poolRef]) => {
+    const loop = () =>
+      xapi
+        .call('event.inject', 'pool', poolRef)
+        ::pDelay(10) // A small delay is required to avoid overloading the Xen API.
+        .then(loop)
 
     return loop()
   })

packages/xen-api/src/memory-test.js

@@ -14,7 +14,7 @@ setInterval(() => {
   )
 }, 1e2)
 
-const [ , , url, user, password ] = process.argv
+const [, , url, user, password] = process.argv
 createClient({
   auth: { user, password },
   readOnly: true,

packages/xen-api/src/transports/auto.js

@@ -3,7 +3,7 @@ import xmlRpc from './xml-rpc'
 import xmlRpcJson from './xml-rpc-json'
 import { UnsupportedTransport } from './_utils'
 
-const factories = [ jsonRpc, xmlRpcJson, xmlRpc ]
+const factories = [jsonRpc, xmlRpcJson, xmlRpc]
 const { length } = factories
 
 export default opts => {
@@ -14,18 +14,18 @@ export default opts => {
     const current = factories[i++](opts)
     if (i < length) {
       const currentI = i
-      call = (method, args) => current(method, args).catch(
-        error => {
+      call = (method, args) =>
+        current(method, args).catch(error => {
           if (error instanceof UnsupportedTransport) {
-            if (currentI === i) { // not changed yet
+            if (currentI === i) {
+              // not changed yet
               create()
             }
             return call(method, args)
           }
 
           throw error
-        }
-      )
+        })
     } else {
       call = current
     }

packages/xen-api/src/transports/json-rpc.js

@@ -4,35 +4,40 @@ import { format, parse } from 'json-rpc-protocol'
 import { UnsupportedTransport } from './_utils'
 
 export default ({ allowUnauthorized, url }) => {
-  return (method, args) => httpRequestPlus.post(url, {
-    rejectUnauthorized: !allowUnauthorized,
-    body: format.request(0, method, args),
-    headers: {
-      'Accept': 'application/json',
-      'Content-Type': 'application/json',
-    },
-    path: '/jsonrpc',
-  }).readAll('utf8').then(
-    text => {
-      let response
-      try {
-        response = parse(text)
-      } catch (error) {
-        throw new UnsupportedTransport()
-      }
+  return (method, args) =>
+    httpRequestPlus
+      .post(url, {
+        rejectUnauthorized: !allowUnauthorized,
+        body: format.request(0, method, args),
+        headers: {
+          Accept: 'application/json',
+          'Content-Type': 'application/json',
+        },
+        path: '/jsonrpc',
+      })
+      .readAll('utf8')
+      .then(
+        text => {
+          let response
+          try {
+            response = parse(text)
+          } catch (error) {
+            throw new UnsupportedTransport()
+          }
 
-      if (response.type === 'response') {
-        return response.result
-      }
+          if (response.type === 'response') {
+            return response.result
+          }
 
-      throw response.error
-    },
-    error => {
-      if (error.response !== undefined) { // HTTP error
-        throw new UnsupportedTransport()
-      }
+          throw response.error
+        },
+        error => {
+          if (error.response !== undefined) {
+            // HTTP error
+            throw new UnsupportedTransport()
+          }
 
-      throw error
-    }
-  )
+          throw error
+        }
+      )
 }

packages/xen-api/src/transports/xml-rpc-json.js

@@ -20,10 +20,7 @@ const SPECIAL_CHARS = {
   '\r': '\\r',
   '\t': '\\t',
 }
-const SPECIAL_CHARS_RE = new RegExp(
-  Object.keys(SPECIAL_CHARS).join('|'),
-  'g'
-)
+const SPECIAL_CHARS_RE = new RegExp(Object.keys(SPECIAL_CHARS).join('|'), 'g')
 
 const parseResult = result => {
   const status = result.Status
@@ -78,11 +75,7 @@ export default ({
   allowUnauthorized,
   url: { hostname, path, port, protocol },
 }) => {
-  const client = (
-    protocol === 'https:'
-      ? createSecureClient
-      : createClient
-  )({
+  const client = (protocol === 'https:' ? createSecureClient : createClient)({
     host: hostname,
     path: '/json',
     port,
@@ -90,8 +83,5 @@ export default ({
   })
   const call = promisify(client.methodCall, client)
 
-  return (method, args) => call(method, args).then(
-    parseResult,
-    logError
-  )
+  return (method, args) => call(method, args).then(parseResult, logError)
 }

packages/xen-api/src/transports/xml-rpc.js

@@ -34,19 +34,12 @@ export default ({
   allowUnauthorized,
   url: { hostname, path, port, protocol },
 }) => {
-  const client = (
-    protocol === 'https:'
-      ? createSecureClient
-      : createClient
-  )({
+  const client = (protocol === 'https:' ? createSecureClient : createClient)({
     host: hostname,
     port,
     rejectUnauthorized: !allowUnauthorized,
   })
   const call = promisify(client.methodCall, client)
 
-  return (method, args) => call(method, args).then(
-    parseResult,
-    logError
-  )
+  return (method, args) => call(method, args).then(parseResult, logError)
 }

packages/xo-acl-resolver/README.md

@@ -65,7 +65,7 @@ the code.
 
 You may:
 
-- report any [issue](https://github.com/vatesfr/xo-web/issues)
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
   you've encountered;
 - fork and create a pull request.
 

packages/xo-acl-resolver/package.json

@@ -5,7 +5,7 @@
   "description": "Xen-Orchestra internal: do ACLs resolution",
   "keywords": [],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-acl-resolver",
-  "bugs": "https://github.com/vatesfr/xo-web/issues",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
@@ -30,7 +30,6 @@
   "scripts": {
     "build": "babel --source-maps --out-dir=dist/ src/",
     "dev": "babel --watch --source-maps --out-dir=dist/ src/",
-    "prepare": "yarn run build",
     "prepublishOnly": "yarn run build"
   }
 }

packages/xo-acl-resolver/src/index.js

@@ -8,7 +8,8 @@ let getObject
 const authorized = () => true // eslint-disable-line no-unused-vars
 const forbiddden = () => false // eslint-disable-line no-unused-vars
 
-const and = (...checkers) => (object, permission) => { // eslint-disable-line no-unused-vars
+// eslint-disable-next-line no-unused-vars
+const and = (...checkers) => (object, permission) => {
   for (const checker of checkers) {
     if (!checker(object, permission)) {
       return false
@@ -17,7 +18,8 @@ const and = (...checkers) => (object, permission) => { // eslint-disable-line no
   return true
 }
 
-const or = (...checkers) => (object, permission) => { // eslint-disable-line no-unused-vars
+// eslint-disable-next-line no-unused-vars
+const or = (...checkers) => (object, permission) => {
   for (const checker of checkers) {
     if (checker(object, permission)) {
       return true
@@ -28,7 +30,7 @@ const or = (...checkers) => (object, permission) => { // eslint-disable-line no-
 
 // -------------------------------------------------------------------
 
-const checkMember = (memberName) => (object, permission) => {
+const checkMember = memberName => (object, permission) => {
   const member = object[memberName]
   return member !== object.id && checkAuthorization(member, permission)
 }
@@ -36,10 +38,7 @@ const checkMember = (memberName) => (object, permission) => {
 const checkSelf = ({ id }, permission) => {
   const permissionsForObject = permissionsByObject[id]
 
-  return (
-    permissionsForObject &&
-    permissionsForObject[permission]
-  )
+  return permissionsForObject && permissionsForObject[permission]
 }
 
 // ===================================================================
@@ -102,12 +101,7 @@ function checkAuthorization (objectId, permission) {
 
 // -------------------------------------------------------------------
 
-export default (
-  permissionsByObject_,
-  getObject_,
-  permissions,
-  permission
-) => {
+export default (permissionsByObject_, getObject_, permissions, permission) => {
   // Assign global variables.
   permissionsByObject = permissionsByObject_
   getObject = getObject_

packages/xo-cli/.babelrc.js

@@ -1,25 +1,3 @@
-const { NODE_ENV = 'development' } = process.env
-const __PROD__ = NODE_ENV === 'production'
-const __TEST__ = NODE_ENV === 'test'
-
-module.exports = {
-  comments: !__PROD__,
-  compact: __PROD__,
-  ignore: __TEST__ ? undefined : [ /\.spec\.js$/ ],
-  plugins: ['lodash'],
-  presets: [
-    [
-      '@babel/env',
-      {
-        debug: !__TEST__,
-        loose: true,
-        shippedProposals: true,
-        targets: __PROD__
-          ? { node: '6' }
-          : { node: 'current' },
-        useBuiltIns: 'usage',
-      },
-    ],
-    '@babel/flow',
-  ],
-}
+module.exports = require('../../@xen-orchestra/babel-config')(
+  require('./package.json')
+)

packages/xo-cli/README.md

@@ -105,6 +105,12 @@ encoding by prefixing with `json:`:
 > xo-cli foo.bar baz='json:[1, 2, 3]'
 ```
 
+##### Configuration export
+
+```
+> xo-cli xo.exportConfig @=config.json
+```
+
 ##### VM export
 
 ```
@@ -143,7 +149,7 @@ the code.
 
 You may:
 
-- report any [issue](https://github.com/vatesfr/xo-web/issues)
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
   you've encountered;
 - fork and create a pull request.
 

packages/xo-cli/package.json

@@ -10,7 +10,7 @@
     "orchestra"
   ],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-cli",
-  "bugs": "https://github.com/vatesfr/xo-web/issues",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
@@ -28,7 +28,7 @@
     "node": ">=6"
   },
   "dependencies": {
-    "@babel/polyfill": "7.0.0-beta.39",
+    "@babel/polyfill": "7.0.0-beta.44",
     "bluebird": "^3.5.1",
     "chalk": "^2.2.0",
     "event-to-promise": "^0.8.0",
@@ -49,10 +49,10 @@
     "xo-lib": "^0.9.0"
   },
   "devDependencies": {
-    "@babel/cli": "7.0.0-beta.39",
-    "@babel/core": "7.0.0-beta.39",
-    "@babel/preset-env": "7.0.0-beta.39",
-    "@babel/preset-flow": "7.0.0-beta.39",
+    "@babel/cli": "7.0.0-beta.44",
+    "@babel/core": "7.0.0-beta.44",
+    "@babel/preset-env": "7.0.0-beta.44",
+    "@babel/preset-flow": "7.0.0-beta.44",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^5.1.3",
     "rimraf": "^2.6.2"
@@ -62,7 +62,6 @@
     "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
     "prebuild": "rimraf dist/",
     "predev": "yarn run prebuild",
-    "prepublishOnly": "yarn run build",
-    "pretest": "flow status"
+    "prepublishOnly": "yarn run build"
   }
 }