feat(@xen-orchestra/fs): v0.7.0

When our tasks cache is desynchronized we re-fetch all tasks.

We must wait before fetching the next events to have fetch the tasks otherwise we risk a race condition.

.editorconfig

@@ -3,63 +3,12 @@
 # Julien Fontanet's configuration
 # https://gist.github.com/julien-f/8096213
 
-# Top-most EditorConfig file.
 root = true
 
-# Common config.
 [*]
 charset = utf-8
 end_of_line = lf
+indent_size = 2
+indent_style = space
 insert_final_newline = true
 trim_trailing_whitespace = true
-
-# CoffeeScript
-#
-# https://github.com/polarmobile/coffeescript-style-guide/blob/master/README.md
-[*.{,lit}coffee]
-indent_size = 2
-indent_style = space
-
-# Markdown
-[*.{md,mdwn,mdown,markdown}]
-indent_size = 4
-indent_style = space
-
-# Package.json
-#
-# This indentation style is the one used by npm.
-[package.json]
-indent_size = 2
-indent_style = space
-
-# Pug (Jade)
-[*.{jade,pug}]
-indent_size = 2
-indent_style = space
-
-# JavaScript
-#
-# Two spaces seems to be the standard most common style, at least in
-# Node.js (http://nodeguide.com/style.html#tabs-vs-spaces).
-[*.{js,jsx,ts,tsx}]
-indent_size = 2
-indent_style = space
-
-# Less
-[*.less]
-indent_size = 2
-indent_style = space
-
-# Sass
-#
-# Style used for http://libsass.com
-[*.s[ac]ss]
-indent_size = 2
-indent_style = space
-
-# YAML
-#
-# Only spaces are allowed.
-[*.yaml]
-indent_size = 2
-indent_style = space

.env.example

@@ -0,0 +1,2 @@
+# xo_fs_nfs=nfs://ip:/folder
+# xo_fs_smb=smb://login:pass@domain\\ip\folder

.eslintrc.js

@@ -1,5 +1,11 @@
 module.exports = {
-  extends: ['standard', 'standard-jsx'],
+  extends: [
+    'standard',
+    'standard-jsx',
+    'prettier',
+    'prettier/standard',
+    'prettier/react',
+  ],
   globals: {
     __DEV__: true,
     $Dict: true,
@@ -17,12 +23,9 @@ module.exports = {
     },
   },
   rules: {
-    'comma-dangle': ['error', 'always-multiline'],
-    indent: 'off',
     'no-var': 'error',
     'node/no-extraneous-import': 'error',
     'node/no-extraneous-require': 'error',
     'prefer-const': 'error',
-    'react/jsx-indent': 'off',
   },
 }

.gitignore

@@ -30,3 +30,4 @@ pnpm-debug.log
 pnpm-debug.log.*
 yarn-error.log
 yarn-error.log.*
+.env

.prettierrc.js

@@ -1,4 +1,5 @@
 module.exports = {
+  jsxSingleQuote: true,
   semi: false,
   singleQuote: true,
   trailingComma: 'es5',

@xen-orchestra/async-map/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/async-map",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/async-map",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

@xen-orchestra/babel-config/index.js

@@ -14,7 +14,7 @@ const configs = {
   '@babel/plugin-proposal-pipeline-operator': {
     proposal: 'minimal',
   },
-  '@babel/preset-env' (pkg) {
+  '@babel/preset-env'(pkg) {
     return {
       debug: !__TEST__,
 
@@ -46,7 +46,7 @@ const getConfig = (key, ...args) => {
     : config
 }
 
-module.exports = function (pkg, plugins, presets) {
+module.exports = function(pkg, plugins, presets) {
   plugins === undefined && (plugins = {})
   presets === undefined && (presets = {})
 

@xen-orchestra/babel-config/package.json

@@ -5,6 +5,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/babel-config",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/babel-config",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   }

@xen-orchestra/cr-seed-cli/index.js

@@ -5,7 +5,7 @@ const { NULL_REF, Xapi } = require('xen-api')
 
 const pkg = require('./package.json')
 
-Xapi.prototype.getVmDisks = async function (vm) {
+Xapi.prototype.getVmDisks = async function(vm) {
   const disks = { __proto__: null }
   await Promise.all([
     ...vm.VBDs.map(async vbdRef => {
@@ -19,7 +19,7 @@ Xapi.prototype.getVmDisks = async function (vm) {
   return disks
 }
 
-defer(async function main ($defer, args) {
+defer(async function main($defer, args) {
   if (args.length === 0 || args.includes('-h') || args.includes('--help')) {
     const cliName = Object.keys(pkg.bin)[0]
     return console.error(
@@ -82,35 +82,26 @@ ${cliName} v${pkg.version}
   )
 
   await Promise.all([
-    srcXapi.setFieldEntries(srcSnapshot, 'other_config', metadata),
-    srcXapi.setFieldEntries(srcSnapshot, 'other_config', {
-      'xo:backup:exported': 'true',
-    }),
-    tgtXapi.setField(
-      tgtVm,
-      'name_label',
-      `${srcVm.name_label} (${srcSnapshot.snapshot_time})`
-    ),
-    tgtXapi.setFieldEntries(tgtVm, 'other_config', metadata),
-    tgtXapi.setFieldEntries(tgtVm, 'other_config', {
+    srcSnapshot.update_other_config(metadata),
+    srcSnapshot.update_other_config('xo:backup:exported', 'true'),
+    tgtVm.set_name_label(`${srcVm.name_label} (${srcSnapshot.snapshot_time})`),
+    tgtVm.update_other_config(metadata),
+    tgtVm.update_other_config({
       'xo:backup:sr': tgtSr.uuid,
       'xo:copy_of': srcSnapshotUuid,
     }),
-    tgtXapi.setFieldEntries(tgtVm, 'blocked_operations', {
-      start:
-        'Start operation for this vm is blocked, clone it if you want to use it.',
-    }),
+    tgtVm.update_blocked_operations(
+      'start',
+      'Start operation for this vm is blocked, clone it if you want to use it.'
+    ),
     Promise.all(
       userDevices.map(userDevice => {
         const srcDisk = srcDisks[userDevice]
         const tgtDisk = tgtDisks[userDevice]
 
-        return tgtXapi.setFieldEntry(
-          tgtDisk,
-          'other_config',
-          'xo:copy_of',
-          srcDisk.uuid
-        )
+        return tgtDisk.update_other_config({
+          'xo:copy_of': srcDisk.uuid,
+        })
       })
     ),
   ])

@xen-orchestra/cr-seed-cli/package.json

@@ -4,6 +4,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/cr-seed-cli",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/cr-seed-cli",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -15,6 +16,6 @@
   },
   "dependencies": {
     "golike-defer": "^0.4.1",
-    "xen-api": "^0.21.0"
+    "xen-api": "^0.24.2"
   }
 }

@xen-orchestra/cron/package.json

@@ -17,6 +17,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/cron",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/cron",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

@xen-orchestra/cron/src/index.js

@@ -6,7 +6,7 @@ import parse from './parse'
 const MAX_DELAY = 2 ** 31 - 1
 
 class Job {
-  constructor (schedule, fn) {
+  constructor(schedule, fn) {
     const wrapper = () => {
       let result
       try {
@@ -33,18 +33,18 @@ class Job {
     this._timeout = undefined
   }
 
-  start () {
+  start() {
     this.stop()
     this._scheduleNext()
   }
 
-  stop () {
+  stop() {
     clearTimeout(this._timeout)
   }
 }
 
 class Schedule {
-  constructor (pattern, zone = 'utc') {
+  constructor(pattern, zone = 'utc') {
     this._schedule = parse(pattern)
     this._createDate =
       zone.toLowerCase() === 'utc'
@@ -54,11 +54,11 @@ class Schedule {
         : () => moment.tz(zone)
   }
 
-  createJob (fn) {
+  createJob(fn) {
     return new Job(this, fn)
   }
 
-  next (n) {
+  next(n) {
     const dates = new Array(n)
     const schedule = this._schedule
     let date = this._createDate()
@@ -68,12 +68,12 @@ class Schedule {
     return dates
   }
 
-  _nextDelay () {
+  _nextDelay() {
     const now = this._createDate()
     return next(this._schedule, now) - now
   }
 
-  startJob (fn) {
+  startJob(fn) {
     const job = this.createJob(fn)
     job.start()
     return job.stop.bind(job)

@xen-orchestra/defined/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/defined",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/defined",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

@xen-orchestra/defined/src/index.js

@@ -13,7 +13,7 @@
 //   process.env.http_proxy
 // ])
 // ```
-export default function defined () {
+export default function defined() {
   let args = arguments
   let n = args.length
   if (n === 1) {

@xen-orchestra/emit-async/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/emit-async",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/emit-async",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

@xen-orchestra/emit-async/src/index.js

@@ -1,4 +1,4 @@
-export default function emitAsync (event) {
+export default function emitAsync(event) {
   let opts
   let i = 1
 

@xen-orchestra/fs/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@xen-orchestra/fs",
-  "version": "0.4.1",
+  "version": "0.7.0",
   "license": "AGPL-3.0",
   "description": "The File System for Xen Orchestra backups.",
   "keywords": [],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/fs",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -20,12 +21,16 @@
     "node": ">=6"
   },
   "dependencies": {
-    "@marsaud/smb2": "^0.9.0",
+    "@marsaud/smb2": "^0.13.0",
+    "@sindresorhus/df": "^2.1.0",
+    "@xen-orchestra/async-map": "^0.0.0",
+    "decorator-synchronized": "^0.3.0",
     "execa": "^1.0.0",
     "fs-extra": "^7.0.0",
     "get-stream": "^4.0.0",
     "lodash": "^4.17.4",
     "promise-toolbox": "^0.11.0",
+    "readable-stream": "^3.0.6",
     "through2": "^3.0.0",
     "tmp": "^0.0.33",
     "xo-remote-parser": "^0.5.0"
@@ -33,11 +38,14 @@
   "devDependencies": {
     "@babel/cli": "^7.0.0",
     "@babel/core": "^7.0.0",
+    "@babel/plugin-proposal-decorators": "^7.1.6",
     "@babel/plugin-proposal-function-bind": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
     "@babel/preset-flow": "^7.0.0",
+    "async-iterator-to-stream": "^1.1.0",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^5.1.3",
+    "dotenv": "^6.1.0",
     "index-modules": "^0.3.0",
     "rimraf": "^2.6.2"
   },

@xen-orchestra/fs/src/_mount.js

@@ -0,0 +1,108 @@
+import execa from 'execa'
+import fs from 'fs-extra'
+import { ignoreErrors } from 'promise-toolbox'
+import { join } from 'path'
+import { tmpdir } from 'os'
+
+import LocalHandler from './local'
+
+const sudoExeca = (command, args, opts) =>
+  execa('sudo', [command, ...args], opts)
+
+export default class MountHandler extends LocalHandler {
+  constructor(
+    remote,
+    {
+      mountsDir = join(tmpdir(), 'xo-fs-mounts'),
+      useSudo = false,
+      ...opts
+    } = {},
+    params
+  ) {
+    super(remote, opts)
+
+    this._execa = useSudo ? sudoExeca : execa
+    this._keeper = undefined
+    this._params = {
+      ...params,
+      options: [params.options, remote.options]
+        .filter(_ => _ !== undefined)
+        .join(','),
+    }
+    this._realPath = join(
+      mountsDir,
+      remote.id ||
+        Math.random()
+          .toString(36)
+          .slice(2)
+    )
+  }
+
+  async _forget() {
+    const keeper = this._keeper
+    if (keeper === undefined) {
+      return
+    }
+    this._keeper = undefined
+    await fs.close(keeper)
+
+    await ignoreErrors.call(
+      this._execa('umount', [this._getRealPath()], {
+        env: {
+          LANG: 'C',
+        },
+      })
+    )
+  }
+
+  _getRealPath() {
+    return this._realPath
+  }
+
+  async _sync() {
+    // in case of multiple `sync`s, ensure we properly close previous keeper
+    {
+      const keeper = this._keeper
+      if (keeper !== undefined) {
+        this._keeper = undefined
+        ignoreErrors.call(fs.close(keeper))
+      }
+    }
+
+    const realPath = this._getRealPath()
+
+    await fs.ensureDir(realPath)
+
+    try {
+      const { type, device, options, env } = this._params
+      await this._execa(
+        'mount',
+        ['-t', type, device, realPath, '-o', options],
+        {
+          env: {
+            LANG: 'C',
+            ...env,
+          },
+        }
+      )
+    } catch (error) {
+      try {
+        // the failure may mean it's already mounted, use `findmnt` to check
+        // that's the case
+        await this._execa('findmnt', ['--target', realPath], {
+          stdio: 'ignore',
+        })
+      } catch (_) {
+        throw error
+      }
+    }
+
+    // keep an open file on the mount to prevent it from being unmounted if used
+    // by another handler/process
+    const keeperPath = `${realPath}/.keeper_${Math.random()
+      .toString(36)
+      .slice(2)}`
+    this._keeper = await fs.open(keeperPath, 'w')
+    ignoreErrors.call(fs.unlink(keeperPath))
+  }
+}

@xen-orchestra/fs/src/_normalizePath.js

@@ -0,0 +1,9 @@
+import path from 'path'
+
+const { resolve } = path.posix
+
+// normalize the path:
+// - does not contains `.` or `..`  (cannot escape root dir)
+// - always starts with `/`
+const normalizePath = path => resolve('/', path)
+export { normalizePath as default }

@xen-orchestra/fs/src/abstract.js

@@ -2,17 +2,25 @@
 
 // $FlowFixMe
 import getStream from 'get-stream'
-import { randomBytes } from 'crypto'
-import { fromCallback, fromEvent, ignoreErrors, timeout } from 'promise-toolbox'
-import { type Readable, type Writable } from 'stream'
-import { parse } from 'xo-remote-parser'
 
+import asyncMap from '@xen-orchestra/async-map'
+import path from 'path'
+import synchronized from 'decorator-synchronized'
+import { fromCallback, fromEvent, ignoreErrors, timeout } from 'promise-toolbox'
+import { parse } from 'xo-remote-parser'
+import { randomBytes } from 'crypto'
+import { type Readable, type Writable } from 'stream'
+
+import normalizePath from './_normalizePath'
 import { createChecksumStream, validChecksumOfReadStream } from './checksum'
 
+const { dirname } = path.posix
+
 type Data = Buffer | Readable | string
 type FileDescriptor = {| fd: mixed, path: string |}
 type LaxReadable = Readable & Object
 type LaxWritable = Writable & Object
+type RemoteInfo = { used?: number, size?: number }
 
 type File = FileDescriptor | string
 
@@ -20,11 +28,48 @@ const checksumFile = file => file + '.checksum'
 
 const DEFAULT_TIMEOUT = 6e5 // 10 min
 
+const ignoreEnoent = error => {
+  if (error == null || error.code !== 'ENOENT') {
+    throw error
+  }
+}
+
+class PrefixWrapper {
+  constructor(handler, prefix) {
+    this._prefix = prefix
+    this._handler = handler
+  }
+
+  get type() {
+    return this._handler.type
+  }
+
+  // necessary to remove the prefix from the path with `prependDir` option
+  async list(dir, opts) {
+    const entries = await this._handler.list(this._resolve(dir), opts)
+    if (opts != null && opts.prependDir) {
+      const n = this._prefix.length
+      entries.forEach((entry, i, entries) => {
+        entries[i] = entry.slice(n)
+      })
+    }
+    return entries
+  }
+
+  rename(oldPath, newPath) {
+    return this._handler.rename(this._resolve(oldPath), this._resolve(newPath))
+  }
+
+  _resolve(path) {
+    return this._prefix + normalizePath(path)
+  }
+}
+
 export default class RemoteHandlerAbstract {
   _remote: Object
   _timeout: number
 
-  constructor (remote: any, options: Object = {}) {
+  constructor(remote: any, options: Object = {}) {
     if (remote.url === 'test://') {
       this._remote = remote
     } else {
@@ -36,236 +81,28 @@ export default class RemoteHandlerAbstract {
     ;({ timeout: this._timeout = DEFAULT_TIMEOUT } = options)
   }
 
-  get type (): string {
+  // Public members
+
+  get type(): string {
     throw new Error('Not implemented')
   }
 
-  /**
-   * Asks the handler to sync the state of the effective remote with its' metadata
-   */
-  async sync (): Promise<mixed> {
-    return this._sync()
+  addPrefix(prefix: string) {
+    prefix = normalizePath(prefix)
+    return prefix === '/' ? this : new PrefixWrapper(this, prefix)
   }
 
-  async _sync (): Promise<mixed> {
-    throw new Error('Not implemented')
-  }
-
-  /**
-   * Free the resources possibly dedicated to put the remote at work, when it is no more needed
-   */
-  async forget (): Promise<void> {
-    await this._forget()
-  }
-
-  async _forget (): Promise<void> {
-    throw new Error('Not implemented')
-  }
-
-  async test (): Promise<Object> {
-    const testFileName = `${Date.now()}.test`
-    const data = await fromCallback(cb => randomBytes(1024 * 1024, cb))
-    let step = 'write'
-    try {
-      await this.outputFile(testFileName, data)
-      step = 'read'
-      const read = await this.readFile(testFileName)
-      if (data.compare(read) !== 0) {
-        throw new Error('output and input did not match')
-      }
-      return {
-        success: true,
-      }
-    } catch (error) {
-      return {
-        success: false,
-        step,
-        file: testFileName,
-        error: error.message || String(error),
-      }
-    } finally {
-      ignoreErrors.call(this.unlink(testFileName))
-    }
-  }
-
-  async outputFile (file: string, data: Data, options?: Object): Promise<void> {
-    return this._outputFile(file, data, {
-      flags: 'wx',
-      ...options,
-    })
-  }
-
-  async _outputFile (file: string, data: Data, options?: Object): Promise<void> {
-    const stream = await this.createOutputStream(file, options)
-    const promise = fromEvent(stream, 'finish')
-    stream.end(data)
-    await promise
-  }
-
-  async read (
-    file: File,
-    buffer: Buffer,
-    position?: number
-  ): Promise<{| bytesRead: number, buffer: Buffer |}> {
-    return this._read(file, buffer, position)
-  }
-
-  _read (
-    file: File,
-    buffer: Buffer,
-    position?: number
-  ): Promise<{| bytesRead: number, buffer: Buffer |}> {
-    throw new Error('Not implemented')
-  }
-
-  async readFile (file: string, options?: Object): Promise<Buffer> {
-    return this._readFile(file, options)
-  }
-
-  _readFile (file: string, options?: Object): Promise<Buffer> {
-    return this.createReadStream(file, options).then(getStream.buffer)
-  }
-
-  async rename (
-    oldPath: string,
-    newPath: string,
-    { checksum = false }: Object = {}
-  ) {
-    let p = timeout.call(this._rename(oldPath, newPath), this._timeout)
-    if (checksum) {
-      p = Promise.all([
-        p,
-        this._rename(checksumFile(oldPath), checksumFile(newPath)),
-      ])
-    }
-    return p
-  }
-
-  async _rename (oldPath: string, newPath: string) {
-    throw new Error('Not implemented')
-  }
-
-  async list (
-    dir: string = '.',
-    {
-      filter,
-      prependDir = false,
-    }: { filter?: (name: string) => boolean, prependDir?: boolean } = {}
-  ): Promise<string[]> {
-    let entries = await timeout.call(this._list(dir), this._timeout)
-    if (filter !== undefined) {
-      entries = entries.filter(filter)
-    }
-
-    if (prependDir) {
-      entries.forEach((entry, i) => {
-        entries[i] = dir + '/' + entry
-      })
-    }
-
-    return entries
-  }
-
-  async _list (dir: string): Promise<string[]> {
-    throw new Error('Not implemented')
-  }
-
-  createReadStream (
-    file: string,
-    { checksum = false, ignoreMissingChecksum = false, ...options }: Object = {}
-  ): Promise<LaxReadable> {
-    const path = typeof file === 'string' ? file : file.path
-    const streamP = timeout
-      .call(this._createReadStream(file, options), this._timeout)
-      .then(stream => {
-        // detect early errors
-        let promise = fromEvent(stream, 'readable')
-
-        // try to add the length prop if missing and not a range stream
-        if (
-          stream.length === undefined &&
-          options.end === undefined &&
-          options.start === undefined
-        ) {
-          promise = Promise.all([
-            promise,
-            ignoreErrors.call(
-              this.getSize(file).then(size => {
-                stream.length = size
-              })
-            ),
-          ])
-        }
-
-        return promise.then(() => stream)
-      })
-
-    if (!checksum) {
-      return streamP
-    }
-
-    // avoid a unhandled rejection warning
-    ignoreErrors.call(streamP)
-
-    return this.readFile(checksumFile(path)).then(
-      checksum =>
-        streamP.then(stream => {
-          const { length } = stream
-          stream = (validChecksumOfReadStream(
-            stream,
-            String(checksum).trim()
-          ): LaxReadable)
-          stream.length = length
-
-          return stream
-        }),
-      error => {
-        if (ignoreMissingChecksum && error && error.code === 'ENOENT') {
-          return streamP
-        }
-        throw error
-      }
-    )
-  }
-
-  async _createReadStream (
-    file: string,
-    options?: Object
-  ): Promise<LaxReadable> {
-    throw new Error('Not implemented')
-  }
-
-  async openFile (path: string, flags?: string): Promise<FileDescriptor> {
-    return {
-      fd: await timeout.call(this._openFile(path, flags), this._timeout),
-      path,
-    }
-  }
-
-  async _openFile (path: string, flags?: string): Promise<mixed> {
-    throw new Error('Not implemented')
-  }
-
-  async closeFile (fd: FileDescriptor): Promise<void> {
+  async closeFile(fd: FileDescriptor): Promise<void> {
     await timeout.call(this._closeFile(fd.fd), this._timeout)
   }
 
-  async _closeFile (fd: mixed): Promise<void> {
-    throw new Error('Not implemented')
-  }
-
-  async refreshChecksum (path: string): Promise<void> {
-    const stream = (await this.createReadStream(path)).pipe(
-      createChecksumStream()
-    )
-    stream.resume() // start reading the whole file
-    await this.outputFile(checksumFile(path), await stream.checksum)
-  }
-
-  async createOutputStream (
+  async createOutputStream(
     file: File,
     { checksum = false, ...options }: Object = {}
   ): Promise<LaxWritable> {
+    if (typeof file === 'string') {
+      file = normalizePath(file)
+    }
     const path = typeof file === 'string' ? file : file.path
     const streamP = timeout.call(
       this._createOutputStream(file, {
@@ -290,36 +127,451 @@ export default class RemoteHandlerAbstract {
 
     // $FlowFixMe
     checksumStream.checksumWritten = checksumStream.checksum
-      .then(value => this.outputFile(checksumFile(path), value))
+      .then(value =>
+        this._outputFile(checksumFile(path), value, { flags: 'wx' })
+      )
       .catch(forwardError)
 
     return checksumStream
   }
 
-  async _createOutputStream (
-    file: mixed,
-    options?: Object
-  ): Promise<LaxWritable> {
-    throw new Error('Not implemented')
+  createReadStream(
+    file: File,
+    { checksum = false, ignoreMissingChecksum = false, ...options }: Object = {}
+  ): Promise<LaxReadable> {
+    if (typeof file === 'string') {
+      file = normalizePath(file)
+    }
+    const path = typeof file === 'string' ? file : file.path
+    const streamP = timeout
+      .call(this._createReadStream(file, options), this._timeout)
+      .then(stream => {
+        // detect early errors
+        let promise = fromEvent(stream, 'readable')
+
+        // try to add the length prop if missing and not a range stream
+        if (
+          stream.length === undefined &&
+          options.end === undefined &&
+          options.start === undefined
+        ) {
+          promise = Promise.all([
+            promise,
+            ignoreErrors.call(
+              this._getSize(file).then(size => {
+                stream.length = size
+              })
+            ),
+          ])
+        }
+
+        return promise.then(() => stream)
+      })
+
+    if (!checksum) {
+      return streamP
+    }
+
+    // avoid a unhandled rejection warning
+    ignoreErrors.call(streamP)
+
+    return this._readFile(checksumFile(path), { flags: 'r' }).then(
+      checksum =>
+        streamP.then(stream => {
+          const { length } = stream
+          stream = (validChecksumOfReadStream(
+            stream,
+            String(checksum).trim()
+          ): LaxReadable)
+          stream.length = length
+
+          return stream
+        }),
+      error => {
+        if (ignoreMissingChecksum && error && error.code === 'ENOENT') {
+          return streamP
+        }
+        throw error
+      }
+    )
   }
 
-  async unlink (file: string, { checksum = true }: Object = {}): Promise<void> {
+  createWriteStream(
+    file: File,
+    options: { end?: number, flags?: string, start?: number } = {}
+  ): Promise<LaxWritable> {
+    return timeout.call(
+      this._createWriteStream(
+        typeof file === 'string' ? normalizePath(file) : file,
+        {
+          flags: 'wx',
+          ...options,
+        }
+      )
+    )
+  }
+
+  // Free the resources possibly dedicated to put the remote at work, when it
+  // is no more needed
+  //
+  // FIXME: Some handlers are implemented based on system-wide mecanisms (such
+  // as mount), forgetting them might breaking other processes using the same
+  // remote.
+  @synchronized()
+  async forget(): Promise<void> {
+    await this._forget()
+  }
+
+  async getInfo(): Promise<RemoteInfo> {
+    return timeout.call(this._getInfo(), this._timeout)
+  }
+
+  async getSize(file: File): Promise<number> {
+    return timeout.call(
+      this._getSize(typeof file === 'string' ? normalizePath(file) : file),
+      this._timeout
+    )
+  }
+
+  async list(
+    dir: string,
+    {
+      filter,
+      prependDir = false,
+    }: { filter?: (name: string) => boolean, prependDir?: boolean } = {}
+  ): Promise<string[]> {
+    const virtualDir = normalizePath(dir)
+    dir = normalizePath(dir)
+
+    let entries = await timeout.call(this._list(dir), this._timeout)
+    if (filter !== undefined) {
+      entries = entries.filter(filter)
+    }
+
+    if (prependDir) {
+      entries.forEach((entry, i) => {
+        entries[i] = virtualDir + '/' + entry
+      })
+    }
+
+    return entries
+  }
+
+  async mkdir(dir: string): Promise<void> {
+    dir = normalizePath(dir)
+    try {
+      await this._mkdir(dir)
+    } catch (error) {
+      if (error == null || error.code !== 'EEXIST') {
+        throw error
+      }
+
+      // this operation will throw if it's not already a directory
+      await this._list(dir)
+    }
+  }
+
+  async mktree(dir: string): Promise<void> {
+    await this._mktree(normalizePath(dir))
+  }
+
+  async openFile(path: string, flags: string): Promise<FileDescriptor> {
+    path = normalizePath(path)
+
+    return {
+      fd: await timeout.call(this._openFile(path, flags), this._timeout),
+      path,
+    }
+  }
+
+  async outputFile(
+    file: string,
+    data: Data,
+    { flags = 'wx' }: { flags?: string } = {}
+  ): Promise<void> {
+    await this._outputFile(normalizePath(file), data, { flags })
+  }
+
+  async read(
+    file: File,
+    buffer: Buffer,
+    position?: number
+  ): Promise<{| bytesRead: number, buffer: Buffer |}> {
+    return this._read(
+      typeof file === 'string' ? normalizePath(file) : file,
+      buffer,
+      position
+    )
+  }
+
+  async readFile(
+    file: string,
+    { flags = 'r' }: { flags?: string } = {}
+  ): Promise<Buffer> {
+    return this._readFile(normalizePath(file), { flags })
+  }
+
+  async refreshChecksum(path: string): Promise<void> {
+    path = normalizePath(path)
+
+    const stream = (await this._createReadStream(path, { flags: 'r' })).pipe(
+      createChecksumStream()
+    )
+    stream.resume() // start reading the whole file
+    await this._outputFile(checksumFile(path), await stream.checksum, {
+      flags: 'wx',
+    })
+  }
+
+  async rename(
+    oldPath: string,
+    newPath: string,
+    { checksum = false }: Object = {}
+  ) {
+    oldPath = normalizePath(oldPath)
+    newPath = normalizePath(newPath)
+
+    let p = timeout.call(this._rename(oldPath, newPath), this._timeout)
+    if (checksum) {
+      p = Promise.all([
+        p,
+        this._rename(checksumFile(oldPath), checksumFile(newPath)),
+      ])
+    }
+    return p
+  }
+
+  async rmdir(dir: string): Promise<void> {
+    await timeout.call(
+      this._rmdir(normalizePath(dir)).catch(ignoreEnoent),
+      this._timeout
+    )
+  }
+
+  async rmtree(dir: string): Promise<void> {
+    await this._rmtree(normalizePath(dir))
+  }
+
+  // Asks the handler to sync the state of the effective remote with its'
+  // metadata
+  //
+  // This method MUST ALWAYS be called before using the handler.
+  @synchronized()
+  async sync(): Promise<void> {
+    await this._sync()
+  }
+
+  async test(): Promise<Object> {
+    const testFileName = normalizePath(`${Date.now()}.test`)
+    const data = await fromCallback(cb => randomBytes(1024 * 1024, cb))
+    let step = 'write'
+    try {
+      await this._outputFile(testFileName, data, { flags: 'wx' })
+      step = 'read'
+      const read = await this._readFile(testFileName, { flags: 'r' })
+      if (!data.equals(read)) {
+        throw new Error('output and input did not match')
+      }
+      return {
+        success: true,
+      }
+    } catch (error) {
+      return {
+        success: false,
+        step,
+        file: testFileName,
+        error: error.message || String(error),
+      }
+    } finally {
+      ignoreErrors.call(this._unlink(testFileName))
+    }
+  }
+
+  async unlink(file: string, { checksum = true }: Object = {}): Promise<void> {
+    file = normalizePath(file)
+
     if (checksum) {
       ignoreErrors.call(this._unlink(checksumFile(file)))
     }
 
-    await timeout.call(this._unlink(file), this._timeout)
+    await this._unlink(file).catch(ignoreEnoent)
   }
 
-  async _unlink (file: mixed): Promise<void> {
+  async writeFile(
+    file: string,
+    data: Data,
+    { flags = 'wx' }: { flags?: string } = {}
+  ): Promise<void> {
+    await this._writeFile(normalizePath(file), data, { flags })
+  }
+
+  // Methods that can be implemented by inheriting classes
+
+  async _closeFile(fd: mixed): Promise<void> {
     throw new Error('Not implemented')
   }
 
-  async getSize (file: mixed): Promise<number> {
-    return timeout.call(this._getSize(file), this._timeout)
+  async _createOutputStream(file: File, options: Object): Promise<LaxWritable> {
+    try {
+      return await this._createWriteStream(file, options)
+    } catch (error) {
+      if (typeof file !== 'string' || error.code !== 'ENOENT') {
+        throw error
+      }
+    }
+
+    await this._mktree(dirname(file))
+    return this._createOutputStream(file, options)
   }
 
-  async _getSize (file: mixed): Promise<number> {
+  async _createReadStream(file: File, options?: Object): Promise<LaxReadable> {
+    throw new Error('Not implemented')
+  }
+
+  async _createWriteStream(file: File, options: Object): Promise<LaxWritable> {
+    throw new Error('Not implemented')
+  }
+
+  // called to finalize the remote
+  async _forget(): Promise<void> {}
+
+  async _getInfo(): Promise<Object> {
+    return {}
+  }
+
+  async _getSize(file: File): Promise<number> {
+    throw new Error('Not implemented')
+  }
+
+  async _list(dir: string): Promise<string[]> {
+    throw new Error('Not implemented')
+  }
+
+  async _mkdir(dir: string): Promise<void> {
+    throw new Error('Not implemented')
+  }
+
+  async _mktree(dir: string): Promise<void> {
+    try {
+      return await this.mkdir(dir)
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        throw error
+      }
+    }
+
+    await this._mktree(dirname(dir))
+    return this._mktree(dir)
+  }
+
+  async _openFile(path: string, flags: string): Promise<mixed> {
+    throw new Error('Not implemented')
+  }
+
+  async _outputFile(
+    file: string,
+    data: Data,
+    options: { flags?: string }
+  ): Promise<void> {
+    try {
+      return await this._writeFile(file, data, options)
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        throw error
+      }
+    }
+
+    await this._mktree(dirname(file))
+    return this._outputFile(file, data, options)
+  }
+
+  _read(
+    file: File,
+    buffer: Buffer,
+    position?: number
+  ): Promise<{| bytesRead: number, buffer: Buffer |}> {
+    throw new Error('Not implemented')
+  }
+
+  _readFile(file: string, options?: Object): Promise<Buffer> {
+    return this._createReadStream(file, options).then(getStream.buffer)
+  }
+
+  async _rename(oldPath: string, newPath: string) {
+    throw new Error('Not implemented')
+  }
+
+  async _rmdir(dir: string) {
+    throw new Error('Not implemented')
+  }
+
+  async _rmtree(dir: string) {
+    try {
+      return await this._rmdir(dir)
+    } catch (error) {
+      if (error.code !== 'ENOTEMPTY') {
+        throw error
+      }
+    }
+
+    const files = await this._list(dir)
+    await asyncMap(files, file =>
+      this._unlink(`${dir}/${file}`).catch(error => {
+        if (error.code === 'EISDIR') {
+          return this._rmtree(`${dir}/${file}`)
+        }
+        throw error
+      })
+    )
+    return this._rmtree(dir)
+  }
+
+  // called to initialize the remote
+  async _sync(): Promise<void> {}
+
+  async _unlink(file: string): Promise<void> {
+    throw new Error('Not implemented')
+  }
+
+  async _writeFile(
+    file: string,
+    data: Data,
+    options: { flags?: string }
+  ): Promise<void> {
     throw new Error('Not implemented')
   }
 }
+
+function createPrefixWrapperMethods() {
+  const pPw = PrefixWrapper.prototype
+  const pRha = RemoteHandlerAbstract.prototype
+
+  const {
+    defineProperty,
+    getOwnPropertyDescriptor,
+    prototype: { hasOwnProperty },
+  } = Object
+
+  Object.getOwnPropertyNames(pRha).forEach(name => {
+    let descriptor, value
+    if (
+      hasOwnProperty.call(pPw, name) ||
+      name[0] === '_' ||
+      typeof (value = (descriptor = getOwnPropertyDescriptor(pRha, name))
+        .value) !== 'function'
+    ) {
+      return
+    }
+
+    descriptor.value = function() {
+      let path
+      if (arguments.length !== 0 && typeof (path = arguments[0]) === 'string') {
+        arguments[0] = this._resolve(path)
+      }
+      return value.apply(this._handler, arguments)
+    }
+
+    defineProperty(pPw, name, descriptor)
+  })
+}
+createPrefixWrapperMethods()

@xen-orchestra/fs/src/abstract.spec.js

@@ -7,7 +7,7 @@ import AbstractHandler from './abstract'
 const TIMEOUT = 10e3
 
 class TestHandler extends AbstractHandler {
-  constructor (impl) {
+  constructor(impl) {
     super({ url: 'test://' }, { timeout: TIMEOUT })
 
     Object.keys(impl).forEach(method => {
@@ -16,53 +16,7 @@ class TestHandler extends AbstractHandler {
   }
 }
 
-describe('rename()', () => {
-  it(`throws in case of timeout`, async () => {
-    const testHandler = new TestHandler({
-      rename: () => new Promise(() => {}),
-    })
-
-    const promise = testHandler.rename('oldPath', 'newPath')
-    jest.advanceTimersByTime(TIMEOUT)
-    await expect(promise).rejects.toThrowError(TimeoutError)
-  })
-})
-
-describe('list()', () => {
-  it(`throws in case of timeout`, async () => {
-    const testHandler = new TestHandler({
-      list: () => new Promise(() => {}),
-    })
-
-    const promise = testHandler.list()
-    jest.advanceTimersByTime(TIMEOUT)
-    await expect(promise).rejects.toThrowError(TimeoutError)
-  })
-})
-
-describe('createReadStream()', () => {
-  it(`throws in case of timeout`, async () => {
-    const testHandler = new TestHandler({
-      createReadStream: () => new Promise(() => {}),
-    })
-
-    const promise = testHandler.createReadStream('file')
-    jest.advanceTimersByTime(TIMEOUT)
-    await expect(promise).rejects.toThrowError(TimeoutError)
-  })
-})
-
-describe('openFile()', () => {
-  it(`throws in case of timeout`, async () => {
-    const testHandler = new TestHandler({
-      openFile: () => new Promise(() => {}),
-    })
-
-    const promise = testHandler.openFile('path')
-    jest.advanceTimersByTime(TIMEOUT)
-    await expect(promise).rejects.toThrowError(TimeoutError)
-  })
-})
+jest.useFakeTimers()
 
 describe('closeFile()', () => {
   it(`throws in case of timeout`, async () => {
@@ -88,13 +42,25 @@ describe('createOutputStream()', () => {
   })
 })
 
-describe('unlink()', () => {
+describe('createReadStream()', () => {
   it(`throws in case of timeout`, async () => {
     const testHandler = new TestHandler({
-      unlink: () => new Promise(() => {}),
+      createReadStream: () => new Promise(() => {}),
     })
 
-    const promise = testHandler.unlink('')
+    const promise = testHandler.createReadStream('file')
+    jest.advanceTimersByTime(TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('getInfo()', () => {
+  it('throws in case of timeout', async () => {
+    const testHandler = new TestHandler({
+      getInfo: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.getInfo()
     jest.advanceTimersByTime(TIMEOUT)
     await expect(promise).rejects.toThrowError(TimeoutError)
   })
@@ -111,3 +77,51 @@ describe('getSize()', () => {
     await expect(promise).rejects.toThrowError(TimeoutError)
   })
 })
+
+describe('list()', () => {
+  it(`throws in case of timeout`, async () => {
+    const testHandler = new TestHandler({
+      list: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.list('.')
+    jest.advanceTimersByTime(TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('openFile()', () => {
+  it(`throws in case of timeout`, async () => {
+    const testHandler = new TestHandler({
+      openFile: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.openFile('path')
+    jest.advanceTimersByTime(TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('rename()', () => {
+  it(`throws in case of timeout`, async () => {
+    const testHandler = new TestHandler({
+      rename: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.rename('oldPath', 'newPath')
+    jest.advanceTimersByTime(TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('rmdir()', () => {
+  it(`throws in case of timeout`, async () => {
+    const testHandler = new TestHandler({
+      rmdir: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.rmdir('dir')
+    jest.advanceTimersByTime(TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})

@xen-orchestra/fs/src/fs.integ.spec.js

@@ -1,26 +0,0 @@
-/* eslint-env jest */
-
-import rimraf from 'rimraf'
-import tmp from 'tmp'
-import { pFromCallback } from 'promise-toolbox'
-
-import { getHandler } from '.'
-
-const initialDir = process.cwd()
-
-beforeEach(async () => {
-  const dir = await pFromCallback(cb => tmp.dir(cb))
-  process.chdir(dir)
-})
-
-afterEach(async () => {
-  const tmpDir = process.cwd()
-  process.chdir(initialDir)
-  await pFromCallback(cb => rimraf(tmpDir, cb))
-})
-
-test("fs test doesn't crash", async () => {
-  const handler = getHandler({ url: 'file://' + process.cwd() })
-  const result = await handler.test()
-  expect(result.success).toBeTruthy()
-})

@xen-orchestra/fs/src/fs.spec.js

@@ -0,0 +1,312 @@
+/* eslint-env jest */
+
+import 'dotenv/config'
+import asyncIteratorToStream from 'async-iterator-to-stream'
+import getStream from 'get-stream'
+import { fromCallback } from 'promise-toolbox'
+import { pipeline } from 'readable-stream'
+import { random } from 'lodash'
+import { tmpdir } from 'os'
+
+import { getHandler } from '.'
+
+// https://gist.github.com/julien-f/3228c3f34fdac01ade09
+const unsecureRandomBytes = n => {
+  const bytes = Buffer.alloc(n)
+
+  const odd = n & 1
+  for (let i = 0, m = n - odd; i < m; i += 2) {
+    bytes.writeUInt16BE((Math.random() * 65536) | 0, i)
+  }
+
+  if (odd) {
+    bytes.writeUInt8((Math.random() * 256) | 0, n - 1)
+  }
+
+  return bytes
+}
+
+const TEST_DATA_LEN = 1024
+const TEST_DATA = unsecureRandomBytes(TEST_DATA_LEN)
+const createTestDataStream = asyncIteratorToStream(function*() {
+  yield TEST_DATA
+})
+
+const rejectionOf = p =>
+  p.then(
+    value => {
+      throw value
+    },
+    reason => reason
+  )
+
+const handlers = [`file://${tmpdir()}`]
+if (process.env.xo_fs_nfs) handlers.push(process.env.xo_fs_nfs)
+if (process.env.xo_fs_smb) handlers.push(process.env.xo_fs_smb)
+
+handlers.forEach(url => {
+  describe(url, () => {
+    let handler
+
+    const testWithFileDescriptor = (path, flags, fn) => {
+      it('with path', () => fn({ file: path, flags }))
+      it('with file descriptor', async () => {
+        const file = await handler.openFile(path, flags)
+        try {
+          await fn({ file })
+        } finally {
+          await handler.closeFile(file)
+        }
+      })
+    }
+
+    beforeAll(async () => {
+      handler = getHandler({ url }).addPrefix(`xo-fs-tests-${Date.now()}`)
+      await handler.sync()
+    })
+    afterAll(async () => {
+      await handler.forget()
+      handler = undefined
+    })
+
+    beforeEach(async () => {
+      // ensure test dir exists
+      await handler.mkdir('.')
+    })
+    afterEach(async () => {
+      await handler.rmtree('.')
+    })
+
+    describe('#type', () => {
+      it('returns the type of the remote', () => {
+        expect(typeof handler.type).toBe('string')
+      })
+    })
+
+    describe('#createOutputStream()', () => {
+      it('creates parent dir if missing', async () => {
+        const stream = await handler.createOutputStream('dir/file')
+        await fromCallback(cb => pipeline(createTestDataStream(), stream, cb))
+        await expect(await handler.readFile('dir/file')).toEqual(TEST_DATA)
+      })
+    })
+
+    describe('#createReadStream()', () => {
+      beforeEach(() => handler.outputFile('file', TEST_DATA))
+
+      testWithFileDescriptor('file', 'r', async ({ file, flags }) => {
+        await expect(
+          await getStream.buffer(
+            await handler.createReadStream(file, { flags })
+          )
+        ).toEqual(TEST_DATA)
+      })
+    })
+
+    describe('#createWriteStream()', () => {
+      testWithFileDescriptor('file', 'wx', async ({ file, flags }) => {
+        const stream = await handler.createWriteStream(file, { flags })
+        await fromCallback(cb => pipeline(createTestDataStream(), stream, cb))
+        await expect(await handler.readFile('file')).toEqual(TEST_DATA)
+      })
+
+      it('fails if parent dir is missing', async () => {
+        const error = await rejectionOf(handler.createWriteStream('dir/file'))
+        expect(error.code).toBe('ENOENT')
+      })
+    })
+
+    describe('#getInfo()', () => {
+      let info
+      beforeAll(async () => {
+        info = await handler.getInfo()
+      })
+
+      it('should return an object with info', async () => {
+        expect(typeof info).toBe('object')
+      })
+
+      it('should return correct type of attribute', async () => {
+        if (info.size !== undefined) {
+          expect(typeof info.size).toBe('number')
+        }
+        if (info.used !== undefined) {
+          expect(typeof info.used).toBe('number')
+        }
+      })
+    })
+
+    describe('#getSize()', () => {
+      beforeEach(() => handler.outputFile('file', TEST_DATA))
+
+      testWithFileDescriptor('file', 'r', async () => {
+        expect(await handler.getSize('file')).toEqual(TEST_DATA_LEN)
+      })
+    })
+
+    describe('#list()', () => {
+      it(`should list the content of folder`, async () => {
+        await handler.outputFile('file', TEST_DATA)
+        await expect(await handler.list('.')).toEqual(['file'])
+      })
+
+      it('can prepend the directory to entries', async () => {
+        await handler.outputFile('dir/file', '')
+        expect(await handler.list('dir', { prependDir: true })).toEqual([
+          '/dir/file',
+        ])
+      })
+
+      it('can prepend the directory to entries', async () => {
+        await handler.outputFile('dir/file', '')
+        expect(await handler.list('dir', { prependDir: true })).toEqual([
+          '/dir/file',
+        ])
+      })
+    })
+
+    describe('#mkdir()', () => {
+      it('creates a directory', async () => {
+        await handler.mkdir('dir')
+        await expect(await handler.list('.')).toEqual(['dir'])
+      })
+
+      it('does not throw on existing directory', async () => {
+        await handler.mkdir('dir')
+        await handler.mkdir('dir')
+      })
+
+      it('throws ENOTDIR on existing file', async () => {
+        await handler.outputFile('file', '')
+        const error = await rejectionOf(handler.mkdir('file'))
+        expect(error.code).toBe('ENOTDIR')
+      })
+    })
+
+    describe('#mktree()', () => {
+      it('creates a tree of directories', async () => {
+        await handler.mktree('dir/dir')
+        await expect(await handler.list('.')).toEqual(['dir'])
+        await expect(await handler.list('dir')).toEqual(['dir'])
+      })
+
+      it('does not throw on existing directory', async () => {
+        await handler.mktree('dir/dir')
+        await handler.mktree('dir/dir')
+      })
+
+      it('throws ENOTDIR on existing file', async () => {
+        await handler.outputFile('dir/file', '')
+        const error = await rejectionOf(handler.mktree('dir/file'))
+        expect(error.code).toBe('ENOTDIR')
+      })
+
+      it('throws ENOTDIR on existing file in path', async () => {
+        await handler.outputFile('file', '')
+        const error = await rejectionOf(handler.mktree('file/dir'))
+        expect(error.code).toBe('ENOTDIR')
+      })
+    })
+
+    describe('#outputFile()', () => {
+      it('writes data to a file', async () => {
+        await handler.outputFile('file', TEST_DATA)
+        expect(await handler.readFile('file')).toEqual(TEST_DATA)
+      })
+
+      it('throws on existing files', async () => {
+        await handler.outputFile('file', '')
+        const error = await rejectionOf(handler.outputFile('file', ''))
+        expect(error.code).toBe('EEXIST')
+      })
+    })
+
+    describe('#read()', () => {
+      beforeEach(() => handler.outputFile('file', TEST_DATA))
+
+      const start = random(TEST_DATA_LEN)
+      const size = random(TEST_DATA_LEN)
+
+      testWithFileDescriptor('file', 'r', async ({ file }) => {
+        const buffer = Buffer.alloc(size)
+        const result = await handler.read(file, buffer, start)
+        expect(result.buffer).toBe(buffer)
+        expect(result).toEqual({
+          buffer,
+          bytesRead: Math.min(size, TEST_DATA_LEN - start),
+        })
+      })
+    })
+
+    describe('#readFile', () => {
+      it('returns a buffer containing the contents of the file', async () => {
+        await handler.outputFile('file', TEST_DATA)
+        expect(await handler.readFile('file')).toEqual(TEST_DATA)
+      })
+
+      it('throws on missing file', async () => {
+        const error = await rejectionOf(handler.readFile('file'))
+        expect(error.code).toBe('ENOENT')
+      })
+    })
+
+    describe('#rename()', () => {
+      it(`should rename the file`, async () => {
+        await handler.outputFile('file', TEST_DATA)
+        await handler.rename('file', `file2`)
+
+        expect(await handler.list('.')).toEqual(['file2'])
+        expect(await handler.readFile(`file2`)).toEqual(TEST_DATA)
+      })
+    })
+
+    describe('#rmdir()', () => {
+      it('should remove an empty directory', async () => {
+        await handler.mkdir('dir')
+        await handler.rmdir('dir')
+        expect(await handler.list('.')).toEqual([])
+      })
+
+      it(`should throw on non-empty directory`, async () => {
+        await handler.outputFile('dir/file', '')
+
+        const error = await rejectionOf(handler.rmdir('.'))
+        await expect(error.code).toEqual('ENOTEMPTY')
+      })
+
+      it('does not throw on missing directory', async () => {
+        await handler.rmdir('dir')
+      })
+    })
+
+    describe('#rmtree', () => {
+      it(`should remove a directory resursively`, async () => {
+        await handler.outputFile('dir/file', '')
+        await handler.rmtree('dir')
+
+        expect(await handler.list('.')).toEqual([])
+      })
+    })
+
+    describe('#test()', () => {
+      it('tests the remote appears to be working', async () => {
+        expect(await handler.test()).toEqual({
+          success: true,
+        })
+      })
+    })
+
+    describe('#unlink()', () => {
+      it(`should remove the file`, async () => {
+        await handler.outputFile('file', TEST_DATA)
+        await handler.unlink('file')
+
+        await expect(await handler.list('.')).toEqual([])
+      })
+
+      it('does not throw on missing file', async () => {
+        await handler.unlink('file')
+      })
+    })
+  })
+})

@xen-orchestra/fs/src/index.js

@@ -1,19 +1,27 @@
 // @flow
+import execa from 'execa'
 
 import type RemoteHandler from './abstract'
 import RemoteHandlerLocal from './local'
 import RemoteHandlerNfs from './nfs'
 import RemoteHandlerSmb from './smb'
+import RemoteHandlerSmbMount from './smb-mount'
 
 export type { default as RemoteHandler } from './abstract'
 export type Remote = { url: string }
 
 const HANDLERS = {
   file: RemoteHandlerLocal,
-  smb: RemoteHandlerSmb,
   nfs: RemoteHandlerNfs,
 }
 
+try {
+  execa.sync('mount.cifs', ['-V'])
+  HANDLERS.smb = RemoteHandlerSmbMount
+} catch (_) {
+  HANDLERS.smb = RemoteHandlerSmb
+}
+
 export const getHandler = (remote: Remote, ...rest: any): RemoteHandler => {
   // FIXME: should be done in xo-remote-parser.
   const type = remote.url.split('://')[0]

@xen-orchestra/fs/src/local.js

@@ -1,50 +1,76 @@
+import df from '@sindresorhus/df'
 import fs from 'fs-extra'
-import { dirname, resolve } from 'path'
-import { noop, startsWith } from 'lodash'
+import { fromEvent } from 'promise-toolbox'
 
 import RemoteHandlerAbstract from './abstract'
 
 export default class LocalHandler extends RemoteHandlerAbstract {
-  get type () {
+  get type() {
     return 'file'
   }
 
-  _getRealPath () {
+  _getRealPath() {
     return this._remote.path
   }
 
-  _getFilePath (file) {
-    const realPath = this._getRealPath()
-    const parts = [realPath]
-    if (file) {
-      parts.push(file)
+  _getFilePath(file) {
+    return this._getRealPath() + file
+  }
+
+  async _closeFile(fd) {
+    return fs.close(fd)
+  }
+
+  async _createReadStream(file, options) {
+    if (typeof file === 'string') {
+      const stream = fs.createReadStream(this._getFilePath(file), options)
+      await fromEvent(stream, 'open')
+      return stream
     }
-    const path = resolve.apply(null, parts)
-    if (!startsWith(path, realPath)) {
-      throw new Error('Remote path is unavailable')
+    return fs.createReadStream('', {
+      autoClose: false,
+      ...options,
+      fd: file.fd,
+    })
+  }
+
+  async _createWriteStream(file, options) {
+    if (typeof file === 'string') {
+      const stream = fs.createWriteStream(this._getFilePath(file), options)
+      await fromEvent(stream, 'open')
+      return stream
     }
-    return path
+    return fs.createWriteStream('', {
+      autoClose: false,
+      ...options,
+      fd: file.fd,
+    })
   }
 
-  async _sync () {
-    const path = this._getRealPath()
-    await fs.ensureDir(path)
-    await fs.access(path, fs.R_OK | fs.W_OK)
-
-    return this._remote
+  _getInfo() {
+    return df.file(this._getFilePath('/'))
   }
 
-  async _forget () {
-    return noop()
+  async _getSize(file) {
+    const stats = await fs.stat(
+      this._getFilePath(typeof file === 'string' ? file : file.path)
+    )
+    return stats.size
   }
 
-  async _outputFile (file, data, options) {
-    const path = this._getFilePath(file)
-    await fs.ensureDir(dirname(path))
-    await fs.writeFile(path, data, options)
+  async _list(dir) {
+    return fs.readdir(this._getFilePath(dir))
   }
 
-  async _read (file, buffer, position) {
+  _mkdir(dir) {
+    return fs.mkdir(this._getFilePath(dir))
+  }
+
+  async _openFile(path, flags) {
+    return fs.open(this._getFilePath(path), flags)
+  }
+
+  async _read(file, buffer, position) {
     const needsClose = typeof file === 'string'
     file = needsClose ? await fs.open(this._getFilePath(file), 'r') : file.fd
     try {
@@ -62,62 +88,29 @@ export default class LocalHandler extends RemoteHandlerAbstract {
     }
   }
 
-  async _readFile (file, options) {
+  async _readFile(file, options) {
     return fs.readFile(this._getFilePath(file), options)
   }
 
-  async _rename (oldPath, newPath) {
+  async _rename(oldPath, newPath) {
     return fs.rename(this._getFilePath(oldPath), this._getFilePath(newPath))
   }
 
-  async _list (dir = '.') {
-    return fs.readdir(this._getFilePath(dir))
+  async _rmdir(dir) {
+    return fs.rmdir(this._getFilePath(dir))
   }
 
-  async _createReadStream (file, options) {
-    return typeof file === 'string'
-      ? fs.createReadStream(this._getFilePath(file), options)
-      : fs.createReadStream('', {
-          autoClose: false,
-          ...options,
-          fd: file.fd,
-        })
+  async _sync() {
+    const path = this._getRealPath('/')
+    await fs.ensureDir(path)
+    await fs.access(path, fs.R_OK | fs.W_OK)
   }
 
-  async _createOutputStream (file, options) {
-    if (typeof file === 'string') {
-      const path = this._getFilePath(file)
-      await fs.ensureDir(dirname(path))
-      return fs.createWriteStream(path, options)
-    }
-    return fs.createWriteStream('', {
-      autoClose: false,
-      ...options,
-      fd: file.fd,
-    })
+  async _unlink(file) {
+    return fs.unlink(this._getFilePath(file))
   }
 
-  async _unlink (file) {
-    return fs.unlink(this._getFilePath(file)).catch(error => {
-      // do not throw if the file did not exist
-      if (error == null || error.code !== 'ENOENT') {
-        throw error
-      }
-    })
-  }
-
-  async _getSize (file) {
-    const stats = await fs.stat(
-      this._getFilePath(typeof file === 'string' ? file : file.path)
-    )
-    return stats.size
-  }
-
-  async _openFile (path, flags) {
-    return fs.open(this._getFilePath(path), flags)
-  }
-
-  async _closeFile (fd) {
-    return fs.close(fd)
+  _writeFile(file, data, { flags }) {
+    return fs.writeFile(this._getFilePath(file), data, { flag: flags })
   }
 }

@xen-orchestra/fs/src/nfs.js

@@ -1,86 +1,20 @@
-import execa from 'execa'
-import fs from 'fs-extra'
-import { join } from 'path'
-import { tmpdir } from 'os'
+import { parse } from 'xo-remote-parser'
 
-import LocalHandler from './local'
+import MountHandler from './_mount'
 
 const DEFAULT_NFS_OPTIONS = 'vers=3'
 
-export default class NfsHandler extends LocalHandler {
-  constructor (
-    remote,
-    { mountsDir = join(tmpdir(), 'xo-fs-mounts'), ...opts } = {}
-  ) {
-    super(remote, opts)
-
-    this._realPath = join(mountsDir, remote.id)
+export default class NfsHandler extends MountHandler {
+  constructor(remote, opts) {
+    const { host, port, path } = parse(remote.url)
+    super(remote, opts, {
+      type: 'nfs',
+      device: `${host}${port !== undefined ? ':' + port : ''}:${path}`,
+      options: DEFAULT_NFS_OPTIONS,
+    })
   }
 
-  get type () {
+  get type() {
     return 'nfs'
   }
-
-  _getRealPath () {
-    return this._realPath
-  }
-
-  async _mount () {
-    await fs.ensureDir(this._getRealPath())
-    const { host, path, port, options } = this._remote
-    return execa(
-      'mount',
-      [
-        '-t',
-        'nfs',
-        '-o',
-        DEFAULT_NFS_OPTIONS + (options !== undefined ? `,${options}` : ''),
-        `${host}${port !== undefined ? ':' + port : ''}:${path}`,
-        this._getRealPath(),
-      ],
-      {
-        env: {
-          LANG: 'C',
-        },
-      }
-    ).catch(error => {
-      if (
-        error == null ||
-        typeof error.stderr !== 'string' ||
-        !error.stderr.includes('already mounted')
-      ) {
-        throw error
-      }
-    })
-  }
-
-  async _sync () {
-    await this._mount()
-
-    return this._remote
-  }
-
-  async _forget () {
-    try {
-      await this._umount(this._remote)
-    } catch (_) {
-      // We have to go on...
-    }
-  }
-
-  async _umount () {
-    await execa('umount', ['--force', this._getRealPath()], {
-      env: {
-        LANG: 'C',
-      },
-    }).catch(error => {
-      if (
-        error == null ||
-        typeof error.stderr !== 'string' ||
-        !error.stderr.includes('not mounted')
-      ) {
-        throw error
-      }
-    })
-  }
 }

@xen-orchestra/fs/src/smb-mount.js

@@ -0,0 +1,25 @@
+import { parse } from 'xo-remote-parser'
+
+import MountHandler from './_mount'
+import normalizePath from './_normalizePath'
+
+export default class SmbMountHandler extends MountHandler {
+  constructor(remote, opts) {
+    const { domain = 'WORKGROUP', host, password, path, username } = parse(
+      remote.url
+    )
+    super(remote, opts, {
+      type: 'cifs',
+      device: '//' + host + normalizePath(path),
+      options: `domain=${domain}`,
+      env: {
+        USER: username,
+        PASSWD: password,
+      },
+    })
+  }
+
+  get type() {
+    return 'smb'
+  }
+}

@xen-orchestra/fs/src/smb.js

@@ -1,245 +1,167 @@
 import Smb2 from '@marsaud/smb2'
-import { pFinally } from 'promise-toolbox'
 
 import RemoteHandlerAbstract from './abstract'
 
-const noop = () => {}
-
 // Normalize the error code for file not found.
-class ErrorWrapper extends Error {
-  constructor (error, newCode) {
-    super(error.message)
-    this.cause = error
-    this.code = newCode
-  }
-}
+const wrapError = (error, code) => ({
+  __proto__: error,
+  cause: error,
+  code,
+})
 const normalizeError = (error, shouldBeDirectory) => {
   const { code } = error
 
-  return code === 'STATUS_OBJECT_NAME_NOT_FOUND' ||
-    code === 'STATUS_OBJECT_PATH_NOT_FOUND'
-    ? new ErrorWrapper(error, 'ENOENT')
+  throw code === 'STATUS_DIRECTORY_NOT_EMPTY'
+    ? wrapError(error, 'ENOTEMPTY')
+    : code === 'STATUS_FILE_IS_A_DIRECTORY'
+    ? wrapError(error, 'EISDIR')
+    : code === 'STATUS_NOT_A_DIRECTORY'
+    ? wrapError(error, 'ENOTDIR')
+    : code === 'STATUS_OBJECT_NAME_NOT_FOUND' ||
+      code === 'STATUS_OBJECT_PATH_NOT_FOUND'
+    ? wrapError(error, 'ENOENT')
+    : code === 'STATUS_OBJECT_NAME_COLLISION'
+    ? wrapError(error, 'EEXIST')
     : code === 'STATUS_NOT_SUPPORTED' || code === 'STATUS_INVALID_PARAMETER'
-    ? new ErrorWrapper(error, shouldBeDirectory ? 'ENOTDIR' : 'EISDIR')
+    ? wrapError(error, shouldBeDirectory ? 'ENOTDIR' : 'EISDIR')
     : error
 }
+const normalizeDirError = error => normalizeError(error, true)
 
 export default class SmbHandler extends RemoteHandlerAbstract {
-  constructor (remote, opts) {
+  constructor(remote, opts) {
     super(remote, opts)
-    this._forget = noop
+
+    // defined in _sync()
+    this._client = undefined
+
+    const prefix = this._remote.path
+    this._prefix = prefix !== '' ? prefix + '\\' : prefix
   }
 
-  get type () {
+  get type() {
     return 'smb'
   }
 
-  _getClient () {
+  _getFilePath(file) {
+    return (
+      this._prefix +
+      (typeof file === 'string' ? file : file.path)
+        .slice(1)
+        .replace(/\//g, '\\')
+    )
+  }
+
+  _dirname(file) {
+    const parts = file.split('\\')
+    parts.pop()
+    return parts.join('\\')
+  }
+
+  _closeFile(file) {
+    return this._client.close(file).catch(normalizeError)
+  }
+
+  _createReadStream(file, options) {
+    if (typeof file === 'string') {
+      file = this._getFilePath(file)
+    } else {
+      options = { autoClose: false, ...options, fd: file.fd }
+      file = ''
+    }
+    return this._client.createReadStream(file, options).catch(normalizeError)
+  }
+
+  _createWriteStream(file, options) {
+    if (typeof file === 'string') {
+      file = this._getFilePath(file)
+    } else {
+      options = { autoClose: false, ...options, fd: file.fd }
+      file = ''
+    }
+    return this._client.createWriteStream(file, options).catch(normalizeError)
+  }
+
+  _forget() {
+    const client = this._client
+    this._client = undefined
+    return client.disconnect()
+  }
+
+  _getSize(file) {
+    return this._client.getSize(this._getFilePath(file)).catch(normalizeError)
+  }
+
+  _list(dir) {
+    return this._client.readdir(this._getFilePath(dir)).catch(normalizeDirError)
+  }
+
+  _mkdir(dir) {
+    return this._client.mkdir(this._getFilePath(dir)).catch(normalizeDirError)
+  }
+
+  // TODO: add flags
+  _openFile(path, flags) {
+    return this._client
+      .open(this._getFilePath(path), flags)
+      .catch(normalizeError)
+  }
+
+  async _read(file, buffer, position) {
+    const client = this._client
+    const needsClose = typeof file === 'string'
+    file = needsClose ? await client.open(this._getFilePath(file)) : file.fd
+    try {
+      return await client.read(file, buffer, 0, buffer.length, position)
+    } catch (error) {
+      normalizeError(error)
+    } finally {
+      if (needsClose) {
+        await client.close(file)
+      }
+    }
+  }
+
+  _readFile(file, options) {
+    return this._client
+      .readFile(this._getFilePath(file), options)
+      .catch(normalizeError)
+  }
+
+  _rename(oldPath, newPath) {
+    return this._client
+      .rename(this._getFilePath(oldPath), this._getFilePath(newPath), {
+        replace: true,
+      })
+      .catch(normalizeError)
+  }
+
+  _rmdir(dir) {
+    return this._client.rmdir(this._getFilePath(dir)).catch(normalizeDirError)
+  }
+
+  _sync() {
     const remote = this._remote
 
-    return new Smb2({
+    this._client = new Smb2({
       share: `\\\\${remote.host}`,
       domain: remote.domain,
       username: remote.username,
       password: remote.password,
       autoCloseTimeout: 0,
     })
-  }
 
-  _getFilePath (file) {
-    if (file === '.') {
-      file = undefined
-    }
-
-    let path = this._remote.path !== '' ? this._remote.path : ''
-
-    // Ensure remote path is a directory.
-    if (path !== '' && path[path.length - 1] !== '\\') {
-      path += '\\'
-    }
-
-    if (file) {
-      path += file.replace(/\//g, '\\')
-    }
-
-    return path
-  }
-
-  _dirname (file) {
-    const parts = file.split('\\')
-    parts.pop()
-    return parts.join('\\')
-  }
-
-  async _sync () {
     // Check access (smb2 does not expose connect in public so far...)
-    await this.list()
-
-    return this._remote
+    return this.list('.')
   }
 
-  async _outputFile (file, data, options = {}) {
-    const client = this._getClient()
-    const path = this._getFilePath(file)
-    const dir = this._dirname(path)
-
-    if (dir) {
-      await client.ensureDir(dir)
-    }
-
-    return client.writeFile(path, data, options)::pFinally(() => {
-      client.disconnect()
-    })
+  _unlink(file) {
+    return this._client.unlink(this._getFilePath(file)).catch(normalizeError)
   }
 
-  async _read (file, buffer, position) {
-    const needsClose = typeof file === 'string'
-
-    let client
-    if (needsClose) {
-      client = this._getClient()
-      file = await client.open(this._getFilePath(file))
-    } else {
-      ;({ client, file } = file.fd)
-    }
-
-    try {
-      return await client.read(file, buffer, 0, buffer.length, position)
-    } finally {
-      if (needsClose) {
-        await client.close(file)
-        client.disconnect()
-      }
-    }
-  }
-
-  async _readFile (file, options = {}) {
-    const client = this._getClient()
-    let content
-
-    try {
-      content = await client
-        .readFile(this._getFilePath(file), options)
-        ::pFinally(() => {
-          client.disconnect()
-        })
-    } catch (error) {
-      throw normalizeError(error)
-    }
-
-    return content
-  }
-
-  async _rename (oldPath, newPath) {
-    const client = this._getClient()
-
-    try {
-      await client
-        .rename(this._getFilePath(oldPath), this._getFilePath(newPath), {
-          replace: true,
-        })
-        ::pFinally(() => {
-          client.disconnect()
-        })
-    } catch (error) {
-      throw normalizeError(error)
-    }
-  }
-
-  async _list (dir = '.') {
-    const client = this._getClient()
-    let list
-
-    try {
-      list = await client.readdir(this._getFilePath(dir))::pFinally(() => {
-        client.disconnect()
-      })
-    } catch (error) {
-      throw normalizeError(error, true)
-    }
-
-    return list
-  }
-
-  async _createReadStream (file, options = {}) {
-    if (typeof file !== 'string') {
-      file = file.path
-    }
-    const client = this._getClient()
-    let stream
-
-    try {
-      // FIXME ensure that options are properly handled by @marsaud/smb2
-      stream = await client.createReadStream(this._getFilePath(file), options)
-      stream.on('end', () => client.disconnect())
-    } catch (error) {
-      throw normalizeError(error)
-    }
-
-    return stream
-  }
-
-  async _createOutputStream (file, options = {}) {
-    if (typeof file !== 'string') {
-      file = file.path
-    }
-    const client = this._getClient()
-    const path = this._getFilePath(file)
-    const dir = this._dirname(path)
-    let stream
-    try {
-      if (dir) {
-        await client.ensureDir(dir)
-      }
-      stream = await client.createWriteStream(path, options) // FIXME ensure that options are properly handled by @marsaud/smb2
-    } catch (err) {
-      client.disconnect()
-      throw err
-    }
-    stream.on('finish', () => client.disconnect())
-    return stream
-  }
-
-  async _unlink (file) {
-    const client = this._getClient()
-
-    try {
-      await client.unlink(this._getFilePath(file))::pFinally(() => {
-        client.disconnect()
-      })
-    } catch (error) {
-      throw normalizeError(error)
-    }
-  }
-
-  async _getSize (file) {
-    const client = await this._getClient()
-    let size
-
-    try {
-      size = await client
-        .getSize(this._getFilePath(typeof file === 'string' ? file : file.path))
-        ::pFinally(() => {
-          client.disconnect()
-        })
-    } catch (error) {
-      throw normalizeError(error)
-    }
-
-    return size
-  }
-
-  // TODO: add flags
-  async _openFile (path) {
-    const client = this._getClient()
-    return {
-      client,
-      file: await client.open(this._getFilePath(path)),
-    }
-  }
-
-  async _closeFile ({ client, file }) {
-    await client.close(file)
-    client.disconnect()
+  _writeFile(file, data, options) {
+    return this._client
+      .writeFile(this._getFilePath(file), data, options)
+      .catch(normalizeError)
   }
 }

@xen-orchestra/log/README.md

@@ -75,7 +75,7 @@ catchGlobalErrors(transport)
 ```js
 import transportConsole from '@xen-orchestra/log/transports/console'
 
-configure(transports.console())
+configure(transportConsole())
 ```
 
 #### Email

@xen-orchestra/log/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/log",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/log",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

@xen-orchestra/log/src/configure.js

@@ -12,7 +12,7 @@ const createTransport = config => {
   if (Array.isArray(config)) {
     const transports = config.map(createTransport)
     const { length } = transports
-    return function () {
+    return function() {
       for (let i = 0; i < length; ++i) {
         transports[i].apply(this, arguments)
       }
@@ -29,14 +29,14 @@ const createTransport = config => {
     }
 
     const orig = transport
-    transport = function (log) {
+    transport = function(log) {
       if ((level !== undefined && log.level >= level) || filter(log)) {
         return orig.apply(this, arguments)
       }
     }
   } else if (level !== undefined) {
     const orig = transport
-    transport = function (log) {
+    transport = function(log) {
       if (log.level >= level) {
         return orig.apply(this, arguments)
       }
@@ -85,7 +85,7 @@ export const catchGlobalErrors = logger => {
   const EventEmitter = require('events')
   const { prototype } = EventEmitter
   const { emit } = prototype
-  function patchedEmit (event, error) {
+  function patchedEmit(event, error) {
     if (event === 'error' && this.listenerCount(event) === 0) {
       logger.error('unhandled error event', { error })
       return false

@xen-orchestra/log/src/index.js

@@ -14,7 +14,7 @@ if (!(symbol in global)) {
 
 // -------------------------------------------------------------------
 
-function Log (data, level, namespace, message, time) {
+function Log(data, level, namespace, message, time) {
   this.data = data
   this.level = level
   this.namespace = namespace
@@ -22,7 +22,7 @@ function Log (data, level, namespace, message, time) {
   this.time = time
 }
 
-function Logger (namespace) {
+function Logger(namespace) {
   this._namespace = namespace
 
   // bind all logging methods
@@ -37,11 +37,11 @@ const { prototype } = Logger
 for (const name in LEVELS) {
   const level = LEVELS[name]
 
-  prototype[name.toLowerCase()] = function (message, data) {
+  prototype[name.toLowerCase()] = function(message, data) {
     if (typeof message !== 'string') {
       if (message instanceof Error) {
         data = { error: message }
-        ;({ message = 'an error has occured' } = message)
+        ;({ message = 'an error has occurred' } = message)
       } else {
         return this.warn('incorrect value passed to logger', {
           level,
@@ -53,13 +53,13 @@ for (const name in LEVELS) {
   }
 }
 
-prototype.wrap = function (message, fn) {
+prototype.wrap = function(message, fn) {
   const logger = this
   const warnAndRethrow = error => {
     logger.warn(message, { error })
     throw error
   }
-  return function () {
+  return function() {
     try {
       const result = fn.apply(this, arguments)
       const then = result != null && result.then

@xen-orchestra/log/src/utils.js

@@ -55,7 +55,8 @@ export const required = name => {
 // -------------------------------------------------------------------
 
 export const serializeError = error => ({
-  ...error,
+  ...error, // Copy enumerable properties.
+  code: error.code,
   message: error.message,
   name: error.name,
   stack: error.stack,

@xen-orchestra/mixin/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/mixin",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/mixin",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

@xen-orchestra/mixin/src/index.js

@@ -61,7 +61,7 @@ const mixin = Mixins => Class => {
 
   const n = Mixins.length
 
-  function DecoratedClass (...args) {
+  function DecoratedClass(...args) {
     const instance = new Class(...args)
 
     for (let i = 0; i < n; ++i) {

CHANGELOG.md

@@ -1,6 +1,151 @@
 # ChangeLog
 
-## *next*
+## **5.31.2** (2019-02-08)
+
+### Enhancements
+
+- [Home] Set description on bulk snapshot [#3925](https://github.com/vatesfr/xen-orchestra/issues/3925) (PR [#3933](https://github.com/vatesfr/xen-orchestra/pull/3933))
+- Work-around the XenServer issue when `VBD#VDI` is an empty string instead of an opaque reference (PR [#3950](https://github.com/vatesfr/xen-orchestra/pull/3950))
+- [VDI migration] Retry when XenServer fails with `TOO_MANY_STORAGE_MIGRATES` (PR [#3940](https://github.com/vatesfr/xen-orchestra/pull/3940))
+- [VM]
+  - [General] The creation date of the VM is now visible [#3932](https://github.com/vatesfr/xen-orchestra/issues/3932) (PR [#3947](https://github.com/vatesfr/xen-orchestra/pull/3947))
+  - [Disks] Display device name [#3902](https://github.com/vatesfr/xen-orchestra/issues/3902) (PR [#3946](https://github.com/vatesfr/xen-orchestra/pull/3946))
+- [VM Snapshotting]
+  - Detect and destroy broken quiesced snapshot left by XenServer [#3936](https://github.com/vatesfr/xen-orchestra/issues/3936) (PR [#3937](https://github.com/vatesfr/xen-orchestra/pull/3937))
+  - Retry twice after a 1 minute delay if quiesce failed [#3938](https://github.com/vatesfr/xen-orchestra/issues/3938) (PR [#3952](https://github.com/vatesfr/xen-orchestra/pull/3952))
+
+### Bug fixes
+
+- [Import] Fix import of big OVA files
+- [Host] Show the host's memory usage instead of the sum of the VMs' memory usage (PR [#3924](https://github.com/vatesfr/xen-orchestra/pull/3924))
+- [SAML] Make `AssertionConsumerServiceURL` matches the callback URL
+- [Backup NG] Correctly delete broken VHD chains [#3875](https://github.com/vatesfr/xen-orchestra/issues/3875) (PR [#3939](https://github.com/vatesfr/xen-orchestra/pull/3939))
+- [Remotes] Don't ignore `mount` options [#3935](https://github.com/vatesfr/xen-orchestra/issues/3935) (PR [#3931](https://github.com/vatesfr/xen-orchestra/pull/3931))
+
+### Released packages
+
+- xen-api v0.24.2
+- @xen-orchestra/fs v0.6.1
+- xo-server-auth-saml v0.5.3
+- xo-server v5.35.0
+- xo-web v5.35.0
+
+## **5.31.0** (2019-01-31)
+
+### Enhancements
+
+- [Backup NG] Restore logs moved to restore tab [#3772](https://github.com/vatesfr/xen-orchestra/issues/3772) (PR [#3802](https://github.com/vatesfr/xen-orchestra/pull/3802))
+- [Remotes] New SMB implementation that provides better stability and performance [#2257](https://github.com/vatesfr/xen-orchestra/issues/2257) (PR [#3708](https://github.com/vatesfr/xen-orchestra/pull/3708))
+- [VM/advanced] ACL management from VM view [#3040](https://github.com/vatesfr/xen-orchestra/issues/3040) (PR [#3774](https://github.com/vatesfr/xen-orchestra/pull/3774))
+- [VM / snapshots] Ability to save the VM memory [#3795](https://github.com/vatesfr/xen-orchestra/issues/3795) (PR [#3812](https://github.com/vatesfr/xen-orchestra/pull/3812))
+- [Backup NG / Health] Show number of lone snapshots in tab label [#3500](https://github.com/vatesfr/xen-orchestra/issues/3500) (PR [#3824](https://github.com/vatesfr/xen-orchestra/pull/3824))
+- [Login] Add autofocus on username input on login page [#3835](https://github.com/vatesfr/xen-orchestra/issues/3835) (PR [#3836](https://github.com/vatesfr/xen-orchestra/pull/3836))
+- [Home/VM] Bulk snapshot: specify snapshots' names [#3778](https://github.com/vatesfr/xen-orchestra/issues/3778) (PR [#3787](https://github.com/vatesfr/xen-orchestra/pull/3787))
+- [Remotes] Show free space and disk usage on remote [#3055](https://github.com/vatesfr/xen-orchestra/issues/3055) (PR [#3767](https://github.com/vatesfr/xen-orchestra/pull/3767))
+- [New SR] Add tooltip for reattach action button [#3845](https://github.com/vatesfr/xen-orchestra/issues/3845) (PR [#3852](https://github.com/vatesfr/xen-orchestra/pull/3852))
+- [VM migration] Display hosts' free memory [#3264](https://github.com/vatesfr/xen-orchestra/issues/3264) (PR [#3832](https://github.com/vatesfr/xen-orchestra/pull/3832))
+- [Plugins] New field to filter displayed plugins (PR [#3832](https://github.com/vatesfr/xen-orchestra/pull/3871))
+- Ability to copy ID of "unknown item"s [#3833](https://github.com/vatesfr/xen-orchestra/issues/3833) (PR [#3856](https://github.com/vatesfr/xen-orchestra/pull/3856))
+- [Cloud-Init] switch config drive type to `nocloud` to prepare for the passing of network config (PR [#3877](https://github.com/vatesfr/xen-orchestra/pull/3877))
+- [UI] Show pool name next to templates' names [#3894](https://github.com/vatesfr/xen-orchestra/issues/3894) (PR [#3896](https://github.com/vatesfr/xen-orchestra/pull/3896))
+- [Backup NG] Support zstd compression for full backups [#3773](https://github.com/vatesfr/xen-orchestra/issues/3773) (PR [#3883](https://github.com/vatesfr/xen-orchestra/pull/3883))
+- [VM] Ability to copy a VM with zstd compression [#3773](https://github.com/vatesfr/xen-orchestra/issues/3773) (PR [#3889](https://github.com/vatesfr/xen-orchestra/pull/3889))
+- [VM & Host] "Pool > Host" breadcrumb at the top of the page (PR [#3898](https://github.com/vatesfr/xen-orchestra/pull/3898))
+- [Hosts] Ability to enable/disable host multipathing [#3659](https://github.com/vatesfr/xen-orchestra/issues/3659) (PR [#3865](https://github.com/vatesfr/xen-orchestra/pull/3865))
+- [Login] Add OTP authentication [#2044](https://github.com/vatesfr/xen-orchestra/issues/2044) (PR [#3879](https://github.com/vatesfr/xen-orchestra/pull/3879))
+- [Notifications] New notification page to provide important information about XOA (PR [#3904](https://github.com/vatesfr/xen-orchestra/pull/3904))
+- [VM] Ability to export a VM with zstd compression [#3773](https://github.com/vatesfr/xen-orchestra/issues/3773) (PR [#3891](https://github.com/vatesfr/xen-orchestra/pull/3891))
+- [Host/network] Display PIF speed [#3887](https://github.com/vatesfr/xen-orchestra/issues/3887) (PR [#3901](https://github.com/vatesfr/xen-orchestra/pull/3901))
+- [SR] Display iscsi paths and mark the SR with a yellow dot if one path is not available. [#3659](https://github.com/vatesfr/xen-orchestra/issues/3659) (PR [#3829](https://github.com/vatesfr/xen-orchestra/pull/3829))
+- [UI] Unifies the Signin buttons (PR [#3913](https://github.com/vatesfr/xen-orchestra/pull/3913))
+- [Settings/remotes] NFS: display default option on placeholder [#3631](https://github.com/vatesfr/xen-orchestra/issues/3631) (PR [#3921](https://github.com/vatesfr/xen-orchestra/pull/3921))
+- [VM/advanced] Ability to pin vCPU to physical cores [#3241](https://github.com/vatesfr/xen-orchestra/issues/3241) (PR [#3254](https://github.com/vatesfr/xen-orchestra/pull/3254))
+
+### Bug fixes
+
+- [Self] Display sorted Resource Sets [#3818](https://github.com/vatesfr/xen-orchestra/issues/3818) (PR [#3823](https://github.com/vatesfr/xen-orchestra/pull/3823))
+- [Servers] Correctly report connecting status (PR [#3838](https://github.com/vatesfr/xen-orchestra/pull/3838))
+- [Servers] Fix cannot reconnect to a server after connection has been lost [#3839](https://github.com/vatesfr/xen-orchestra/issues/3839) (PR [#3841](https://github.com/vatesfr/xen-orchestra/pull/3841))
+- [New VM] Fix `NO_HOSTS_AVAILABLE()` error when creating  a VM on a local SR from template on another local SR [#3084](https://github.com/vatesfr/xen-orchestra/issues/3084) (PR [#3827](https://github.com/vatesfr/xen-orchestra/pull/3827))
+- [Backup NG] Fix typo in the form [#3854](https://github.com/vatesfr/xen-orchestra/issues/3854) (PR [#3855](https://github.com/vatesfr/xen-orchestra/pull/3855))
+- [New SR] No warning when creating a NFS SR on a path that is already used as NFS SR [#3844](https://github.com/vatesfr/xen-orchestra/issues/3844) (PR [#3851](https://github.com/vatesfr/xen-orchestra/pull/3851))
+- [New SR] No redirection if the SR creation failed or canceled [#3843](https://github.com/vatesfr/xen-orchestra/issues/3843) (PR [#3853](https://github.com/vatesfr/xen-orchestra/pull/3853))
+- [Home] Fix two tabs opened by middle click in Firefox [#3450](https://github.com/vatesfr/xen-orchestra/issues/3450) (PR [#3825](https://github.com/vatesfr/xen-orchestra/pull/3825))
+- [XOA] Enable downgrade for ending trial (PR [#3867](https://github.com/vatesfr/xen-orchestra/pull/3867))
+- [OVA import] allow import of big files [#3468](https://github.com/vatesfr/xen-orchestra/issues/3468) (PR [#3504](https://github.com/vatesfr/xen-orchestra/pull/3504))
+- [Backup NG] Smart settings not saved when editing a backup job [#3885](https://github.com/vatesfr/xen-orchestra/issues/3885) (PR [#3886](https://github.com/vatesfr/xen-orchestra/pull/3886))
+- [VM/snapshot] New snapshot with memory: fix "invalid parameters" error (PR [#3903](https://github.com/vatesfr/xen-orchestra/pull/3903))
+- [VM creation] Broken CloudInit config drive when VM created on local SR
+- [Legacy Backup] Fix error when restoring a backup
+- [Home] Fix `user.getAll` error when user is not admin [#3573](https://github.com/vatesfr/xen-orchestra/issues/3573) (PR [#3918](https://github.com/vatesfr/xen-orchestra/pull/3918))
+- [Backup NG] Fix restore issue when a disk has grown [#3910](https://github.com/vatesfr/xen-orchestra/issues/3910)  (PR [#3920](https://github.com/vatesfr/xen-orchestra/pull/3920))
+- [Backup NG] Delete _importing_ VMs due to interrupted CR/DR (PR [#3923](https://github.com/vatesfr/xen-orchestra/pull/3923))
+
+### Released packages
+
+- vhd-cli v0.2.0
+- @xen-orchestra/fs v0.6.0
+- vhd-lib v0.5.1
+- xoa-updater v0.15.0
+- xen-api v0.24.1
+- xo-vmdk-to-vhd v0.1.6
+- xo-server v5.34.0
+- xo-web v5.34.0
+
+## **5.30.0** (2018-12-20)
+
+### Enhancements
+
+- [Users] Display user groups [#3719](https://github.com/vatesfr/xen-orchestra/issues/3719) (PR [#3740](https://github.com/vatesfr/xen-orchestra/pull/3740))
+- [VDI] Display VDI's SR [3021](https://github.com/vatesfr/xen-orchestra/issues/3021) (PR [#3285](https://github.com/vatesfr/xen-orchestra/pull/3285))
+- [Health, VM/disks] Display SR's container [#3021](https://github.com/vatesfr/xen-orchestra/issues/3021) (PRs [#3747](https://github.com/vatesfr/xen-orchestra/pull/3747), [#3751](https://github.com/vatesfr/xen-orchestra/pull/3751))
+- [Servers] Auto-connect to ejected host [#2238](https://github.com/vatesfr/xen-orchestra/issues/2238) (PR [#3738](https://github.com/vatesfr/xen-orchestra/pull/3738))
+- [Backup NG] Add "XOSAN" in excluded tags by default [#2128](https://github.com/vatesfr/xen-orchestra/issues/3563) (PR [#3559](https://github.com/vatesfr/xen-orchestra/pull/3563))
+- [VM] add tooltip for VM status icon [#3749](https://github.com/vatesfr/xen-orchestra/issues/3749) (PR [#3765](https://github.com/vatesfr/xen-orchestra/pull/3765))
+- [New XOSAN] Improve view and possibility to sort SRs by name/size/free space [#2416](https://github.com/vatesfr/xen-orchestra/issues/2416) (PR [#3691](https://github.com/vatesfr/xen-orchestra/pull/3691))
+- [Backup NG] Disable HA on replicated VM (CR, DR) [#2359](https://github.com/vatesfr/xen-orchestra/issues/2359) (PR [#3755](https://github.com/vatesfr/xen-orchestra/pull/3755))
+- [Backup NG] Display the last run status for each schedule with the possibility to show the associated log [#3769](https://github.com/vatesfr/xen-orchestra/issues/3769) (PR [#3779](https://github.com/vatesfr/xen-orchestra/pull/3779))
+- [Backup NG] Add a link to the documentation [#3789](https://github.com/vatesfr/xen-orchestra/issues/3789) (PR [#3790](https://github.com/vatesfr/xen-orchestra/pull/3790))
+- [Backup NG] Ability to copy schedule/job id to the clipboard [#3753](https://github.com/vatesfr/xen-orchestra/issues/3753) (PR [#3791](https://github.com/vatesfr/xen-orchestra/pull/3791))
+- [Backup NG / logs] Merge the job log status with the display details button [#3797](https://github.com/vatesfr/xen-orchestra/issues/3797) (PR [#3800](https://github.com/vatesfr/xen-orchestra/pull/3800))
+- [XOA] Notification banner when XOA is not registered [#3803](https://github.com/vatesfr/xen-orchestra/issues/3803) (PR [#3808](https://github.com/vatesfr/xen-orchestra/pull/3808))
+
+### Bug fixes
+
+- [Home/SRs] Fixed SR status for non admin users [#2204](https://github.com/vatesfr/xen-orchestra/issues/2204) (PR [#3742](https://github.com/vatesfr/xen-orchestra/pull/3742))
+- [Servers] Fix occasional "server's pool already connected" errors when pool is not connected (PR [#3782](https://github.com/vatesfr/xen-orchestra/pull/3782))
+- [Self] Fix missing objects when the self service view is the first one to be loaded when opening XO [#2689](https://github.com/vatesfr/xen-orchestra/issues/2689) (PR [#3096](https://github.com/vatesfr/xen-orchestra/pull/3096))
+
+### Released packages
+
+- @xen-orchestra/fs v0.5.0
+- xen-api v0.23.0
+- xo-acl-resolver v0.4.1
+- xo-server v5.32.0
+- xo-web v5.32.0
+
+## **5.29.0** (2018-11-29)
+
+### Enhancements
+
+- [Perf alert] Ability to trigger an alarm if a host/VM/SR usage value is below the threshold [#3612](https://github.com/vatesfr/xen-orchestra/issues/3612) (PR [#3675](https://github.com/vatesfr/xen-orchestra/pull/3675))
+- [Home/VMs] Display pool's name [#2226](https://github.com/vatesfr/xen-orchestra/issues/2226) (PR [#3709](https://github.com/vatesfr/xen-orchestra/pull/3709))
+- [Servers] Prevent new connection if pool is already connected [#2238](https://github.com/vatesfr/xen-orchestra/issues/2238) (PR [#3724](https://github.com/vatesfr/xen-orchestra/pull/3724))
+- [VM] Pause (like Suspend but doesn't copy RAM on disk) [#3727](https://github.com/vatesfr/xen-orchestra/issues/3727) (PR [#3731](https://github.com/vatesfr/xen-orchestra/pull/3731))
+
+### Bug fixes
+
+- [Servers] Fix deleting server on joining a pool [#2238](https://github.com/vatesfr/xen-orchestra/issues/2238) (PR [#3728](https://github.com/vatesfr/xen-orchestra/pull/3728))
+
+### Released packages
+
+- xen-api v0.22.0
+- xo-server-perf-alert v0.2.0
+- xo-server-usage-report v0.7.1
+- xo-server v5.31.0
+- xo-web v5.31.0
+
+## **5.28.2** (2018-11-16)
 
 ### Enhancements
 
@@ -33,7 +178,7 @@
 - xen-api v0.21.0
 - xo-common v0.2.0
 - xo-acl-resolver v0.4.0
-- xo-server v5.30.0
+- xo-server v5.30.1
 - xo-web v5.30.0
 
 ## **5.28.1** (2018-11-05)

CHANGELOG.unreleased.md

@@ -0,0 +1,28 @@
+> This file contains all changes that have not been released yet.
+
+### Enhancements
+
+- [VM migration] Display same-pool hosts first in the selector [#3262](https://github.com/vatesfr/xen-orchestra/issues/3262) (PR [#3890](https://github.com/vatesfr/xen-orchestra/pull/3890))
+- [Home/VM] Sort VM by start time [#3955](https://github.com/vatesfr/xen-orchestra/issues/3955) (PR [#3970](https://github.com/vatesfr/xen-orchestra/pull/3970))
+- [Editable fields] Unfocusing (clicking outside) submits the change instead of canceling (PR [#3980](https://github.com/vatesfr/xen-orchestra/pull/3980))
+- [Network] Dedicated page for network creation [#3895](https://github.com/vatesfr/xen-orchestra/issues/3895) (PR [#3906](https://github.com/vatesfr/xen-orchestra/pull/3906))
+- [Logs] Add button to download the log [#3957](https://github.com/vatesfr/xen-orchestra/issues/3957) (PR [#3985](https://github.com/vatesfr/xen-orchestra/pull/3985))
+- [Continuous Replication] Share full copy between schedules [#3973](https://github.com/vatesfr/xen-orchestra/issues/3973) (PR [#3995](https://github.com/vatesfr/xen-orchestra/pull/3995))
+- [Backup] Ability to backup XO configuration and pool metadata [#808](https://github.com/vatesfr/xen-orchestra/issues/808) [#3501](https://github.com/vatesfr/xen-orchestra/issues/3501) (PR [#3912](https://github.com/vatesfr/xen-orchestra/pull/3912))
+
+### Bug fixes
+
+- [Host] Fix multipathing status for XenServer < 7.5 [#3956](https://github.com/vatesfr/xen-orchestra/issues/3956) (PR [#3961](https://github.com/vatesfr/xen-orchestra/pull/3961))
+- [Home/VM] Show creation date of the VM on if it available [#3953](https://github.com/vatesfr/xen-orchestra/issues/3953) (PR [#3959](https://github.com/vatesfr/xen-orchestra/pull/3959))
+- [Notifications] Fix invalid notifications when not registered (PR [#3966](https://github.com/vatesfr/xen-orchestra/pull/3966))
+- [Import] Fix import of some OVA files [#3962](https://github.com/vatesfr/xen-orchestra/issues/3962) (PR [#3974](https://github.com/vatesfr/xen-orchestra/pull/3974))
+- [Servers] Fix *already connected error* after a server has been removed during connection [#3976](https://github.com/vatesfr/xen-orchestra/issues/3976) (PR [#3977](https://github.com/vatesfr/xen-orchestra/pull/3977))
+- [Backup] Fix random _mount_ issues with NFS/SMB remotes [#3973](https://github.com/vatesfr/xen-orchestra/issues/3973) (PR [#4003](https://github.com/vatesfr/xen-orchestra/pull/4003))
+
+### Released packages
+
+- @xen-orchestra/fs v0.7.0
+- xen-api v0.24.3
+- xoa-updater v0.15.2
+- xo-server v5.36.0
+- xo-web v5.36.0

PULL_REQUEST_TEMPLATE.md

@@ -4,10 +4,11 @@
 
 - [ ] PR reference the relevant issue (e.g. `Fixes #007`)
 - [ ] if UI changes, a screenshot has been added to the PR
-- [ ] CHANGELOG:
+- [ ] `CHANGELOG.unreleased.md`:
    - enhancement/bug fix entry added
    - list of packages to release updated (`${name} v${new version}`)
 - [ ] documentation updated
+- [ ] **I have tested added/updated features** (and impacted code)
 
 ### Process
 

docs/SUMMARY.md

@@ -51,6 +51,7 @@
    * [Job manager](scheduler.md)
    * [Alerts](alerts.md)
    * [Load balancing](load_balancing.md)
+   * [Emergency Shutdown](emergency_shutdown.md)
    * [Auto scalability](auto_scalability.md)
    * [Forecaster](forecaster.md)
 * [Recipes](recipes.md)

docs/backup_troubleshooting.md

@@ -33,7 +33,7 @@ Just remember this: **coalesce will happen every time a snapshot is removed**.
 
 First check SMlog on the XenServer host for messages relating to VDI corruption or coalesce job failure. For example, by running `cat /var/log/SMlog | grep -i exception` or `cat /var/log/SMlog | grep -i error` on the XenServer host with the affected storage.  
 
-Coalesce jobs can also fail to run if the SR does not have enough free space. Check the problematic SR and make sure it has enough free space, generally 30% or more free is recommended depending on VM size.  
+Coalesce jobs can also fail to run if the SR does not have enough free space. Check the problematic SR and make sure it has enough free space, generally 30% or more free is recommended depending on VM size. You can check if this is the issue by searching `SMlog` with `grep -i coales /var/log/SMlog` (you may have to look at previous logs such as `SMlog.1`).
 
 You can check if a coalesce job is currently active by running `ps axf | grep vhd` on the XenServer host and looking for a VHD process in the results (one of the resulting processes will be the grep command you just ran, ignore that one).  
 

docs/backups.md

@@ -1,5 +1,7 @@
 # Backups
 
+> Watch our [introduction video](https://www.youtube.com/watch?v=FfUqIwT8KzI) (45m) to Backup in Xen Orchestra!
+
 This section is dedicated to all existing methods of rolling back or backing up your VMs in Xen Orchestra.
 
 There are several ways to protect your VMs:
@@ -39,7 +41,7 @@ Each backups' job execution is identified by a `runId`. You can find this `runId
 
 All backup types rely on snapshots. But what about data consistency? By default, Xen Orchestra will try to take a **quiesced snapshot** every time a snapshot is done (and fall back to normal snapshots if it's not possible).
 
-Snapshots of Windows VMs can be quiesced (especially MS SQL or Exchange services) after you have installed Xen Tools in your VMs. However, [there is an extra step to install the VSS provider on windows](quiesce). A quiesced snapshot means the operating system will be notified and the cache will be flushed to disks. This way, your backups will always be consistent.
+Snapshots of Windows VMs can be quiesced (especially MS SQL or Exchange services) after you have installed Xen Tools in your VMs. However, [there is an extra step to install the VSS provider on windows](https://xen-orchestra.com/blog/xenserver-quiesce-snapshots/). A quiesced snapshot means the operating system will be notified and the cache will be flushed to disks. This way, your backups will always be consistent.
 
 To see if you have quiesced snapshots for a VM, just go into its snapshot tab, then the "info" icon means it is a quiesced snapshot:
 
@@ -141,3 +143,12 @@ To make the mount point persistent in XOA, edit the `/etc/fstab` file, and add:
 ```
 
 This way, without modifying your previous scheduled snapshot, they will be written to this new local mountpoint!
+
+## High availability (HA) disabled on replicated VMs
+
+Replicated VMs HA are taken into account by XS/XCP-ng. To avoid the resultant troubles, HA will be disabled from the replicated VMs and a tag indicating this change will be added.
+
+![](./assets/disabled-dr-ha-tag.png)
+![](./assets/disabled-cr-ha-tag.png)
+
+ > The tag won't be automatically removed by XO on the replicated VMs, even if HA is re-enabled.

docs/cloudinit.md

@@ -1,7 +1,5 @@
 # CloudInit
 
-> CloudInit support is available in the 4.11 release and higher
-
 Cloud-init is a program "that handles the early initialization of a cloud instance"[^n]. In other words, you can, on a "cloud-init"-ready template VM, pass a lot of data at first boot:
 
 * setting the hostname
@@ -18,25 +16,27 @@ So it means very easily customizing your VM when you create it from a compatible
 
 You only need to use a template of a VM with CloudInit installed inside it. [Check this blog post to learn how to install CloudInit](https://xen-orchestra.com/blog/centos-cloud-template-for-xenserver/).
 
+**Note:** In XOA 5.31, we changed the cloud-init config drive type from [OpenStack](https://cloudinit.readthedocs.io/en/latest/topics/datasources/configdrive.html) to the [NoCloud](https://cloudinit.readthedocs.io/en/latest/topics/datasources/nocloud.html) type. This will allow us to pass network configuration to VMs in the future. For 99% of users, including default cloud-init installs, this change will have no effect. However if you have previously modified your cloud-init installation in a VM template to only look for `openstack` drive types (for instance with the `datasource_list` setting in `/etc/cloud/cloud.cfg`) you need to modify it to also look for `nocloud`.
+
 ## Usage
 
 First, select your compatible template (CloudInit ready) and name it:
 
-![](https://xen-orchestra.com/blog/content/images/2015/12/template_choice.png)
+![](./assets/cloud-init-1.png)
 
 Then, activate the config drive and insert your SSH key. Or you can also use a custom CloudInit configuration:
 
-![](https://xen-orchestra.com/blog/content/images/2016/02/CloudInit.png)
+![](./assets/cloud-init-2.png)
 
 > CloudInit configuration examples are [available here](http://cloudinit.readthedocs.org/en/latest/topics/examples.html).
 
-You can extend the disk size (**in this case, the template disk was 8 GiB originally**):
+You can extend the disk size (**in this case, the template disk was 8 GiB originally**). We'll extend it to 20GiB:
 
-![](https://xen-orchestra.com/blog/content/images/2015/12/diskedition.png)
+![](./assets/cloud-init-3.png)
 
 Finally, create the VM:
 
-![](https://xen-orchestra.com/blog/content/images/2015/12/recap.png)
+![](./assets/cloud-init-4.png)
 
 Now start the VM and SSH to its IP:
 

docs/configuration.md

@@ -4,42 +4,37 @@ Once Xen Orchestra is installed, you can configure some parameters in the config
 
 ## Configuration
 
-The configuration file is located at `/etc/xo-server/config.yaml`.
-
-**WARNING: YAML is very strict with indentation: use spaces, not tabs.**
+The configuration file is located at `/etc/xo-server/config.toml`.
 
 ### User to run XO-server as
 
 By default, XO-server runs as 'root'. You can change that by uncommenting these lines and choose whatever user/group you want:
 
-```yaml
-user: 'nobody'
-group: 'nogroup'
+```toml
+user = 'nobody'
+group = 'nogroup'
 ```
 
-**Warning!** A non-privileged user:
-
-* can't bind to a port < 1024
-* can't mount NFS shares
+**Warning!** A non-privileged user requires the use of ``sudo`` to mount NFS shares. See [installation from the sources](from_the_sources.md).
 
 ### HTTP listen address and port
 
 By default, XO-server listens on all addresses (0.0.0.0) and runs on port 80. If you need to, you can change this in the `# Basic HTTP` section:
 
-```yaml
-host: '0.0.0.0'
-port: 80
+```toml
+host = '0.0.0.0'
+port = 80
 ```
 
 ### HTTPS
 
 XO-server can also run in HTTPS (you can run HTTP and HTTPS at the same time) - just modify what's needed in the `# Basic HTTPS` section, this time with the certificates/keys you need and their path:
 
-```yaml
-host: '0.0.0.0'
-port: 443
-certificate: './certificate.pem'
-key: './key.pem'
+```toml
+host = '0.0.0.0'
+port = 443
+certificate = './certificate.pem'
+key = './key.pem'
 ```
 
 > If a chain of certificates authorities is needed, you may bundle them directly in the certificate. Note: the order of certificates does matter, your certificate should come first followed by the certificate of the above certificate authority up to the root.
@@ -60,10 +55,9 @@ This should be written just before the `mount` option, inside the `http:` block.
 
 You shouldn't have to change this. It's the path where `xo-web` files are served by `xo-server`.
 
-```yaml
-  mounts:
-    '/':
-      - '../xo-web/dist/'
+```toml
+[http.mounts]
+'/' = '../xo-web/dist/'
 ```
 
 ### Custom certificate authority
@@ -87,8 +81,8 @@ Don't forget to reload `systemd` conf and restart `xo-server`:
 
 By default, XO-server will try to contact Redis server on `localhost`, with the port `6379`. But you can define whatever you want:
 
-```yaml
-  uri: 'tcp://db:password@hostname:port'
+```toml
+uri = 'tcp://db:password@hostname:port'
 ```
 
 ### Proxy for XenServer updates and patches
@@ -101,12 +95,12 @@ To do that behind a corporate proxy, just add the `httpProxy` variable to match
 
 You can add this at the end of your config file:
 
-```yaml
+```toml
 # HTTP proxy configuration used by xo-server to fetch resources on the Internet.
 #
 # See: https://github.com/TooTallNate/node-proxy-agent#maps-proxy-protocols-to-httpagent-implementations
 
-httpProxy: 'http://username:password@proxyAddress:port'
+httpProxy = 'http://username:password@proxyAddress:port'
 ```
 
 ### Log file

docs/continuous_replication.md

@@ -36,38 +36,52 @@ To protect the replication, we removed the possibility to boot your copied VM di
 
 ## Manual initial seed
 
-> This is **only** if you need to make the initial copy without making the whole transfer through your network. Otherwise, **you don't need this**.
+**If you can't transfer the first backup through your network because it's too large**, you can make a seed locally. In order to do this, follow this procedure (until we make it accessible directly in XO).  
 
-**If you can't transfer the first backup through your network**, you can make a seed locally. In order to do this, follow this procedure (until we make it accessible directly in XO):
+> This is **only** if you need to make the initial copy without making the whole transfer through your network. Otherwise, **you don't need this**. These instructions are for Backup-NG jobs, and will not work to seed a legacy backup job. Please migrate any legacy jobs to Backup-NG!
 
-### Preparation
 
-1. create a cont. rep job to a non-distant SR (even the SR where the VM currently is). Do NOT enable the job during creation.
-1. manually start the first replication (only the first)
-1. when finished, export the replicated VM (via XOA or any other means, doesn't matter how you get your XVA file)
-1. import the replicated VM on your distant destination
-1. you can now remove your local replicated copy
+### Job creation
 
-### Modifications
+Create the Continuous Replication backup job, and leave it disabled for now. On the main Backup-NG page, note its identifiers, the main `backupJobId` and the ID of one on the schedules for the job, `backupScheduleId`.
 
-In your source host:
+### Seed creation
 
-1. Get the UUID of the remote destination SR where your VM was imported
-1. On the source host: `xe vm-param-list uuid=<SourceVM_UUID> | grep other-config`.
-  * You should see somewhere in other-config: `xo:base_delta:<SR_UUID>: <VM_snapshot_UUID>;`
-  * Remove this entry with `xe vm-param-remove uuid=<OriginalVM_UUID> param-name=other-config param-key=xo:base_delta:<SR_UUID>`
-  * Recreate the correct param: `xe vm-param-set uuid=<OriginalVM_UUID> other-config:xo:base_delta:<destination_SR_UUID>=<VM_snapshot_UUID>`
+Manually create a snapshot on the VM to backup, and note its UUID as `snapshotUuid` from the snapshot panel for the VM.
 
-In XO:
+> DO NOT ever delete or alter this snapshot, feel free to rename it to make that clear.
 
-1. Edit the replication job and select the new destination SR
+### Seed copy
 
-On the destination host; to avoid data corruption, you need to avoid any VM start:
+Export this snapshot to a file, then import it on the target SR.
+
+Note the UUID of this newly created VM as `targetVmUuid`.
+
+> DO not start this VM or it will break the Continuous Replication job! You can rename this VM to more easily remember this.
+
+### Set up metadata
+
+The XOA backup system requires metadata to correctly associate the source snapshot and the target VM to the backup job. We're going to use the `xo-cr-seed` utility to help us set them up.
+
+First install the tool (all the following is done from the XOA VM CLI):
 
 ```
-xe vm-param-set blocked-operations:start uuid=<DestinationVM_UUID>
+npm i -g xo-cr-seed
 ```
 
-### Enable
+Here is an example of how the utility expects the UUIDs and info passed to it:
 
-Manually run the job the first time to check if everything is OK. Then, enable the job. **Now, only the deltas are sent, your initial seed saved you a LOT of time if you have a slow network.**
+```
+xo-cr-seed
+Usage: xo-cr-seed <source XAPI URL> <source snapshot UUID> <target XAPI URL> <target VM UUID> <backup job id> <backup schedule id>
+
+xo-cr-seed v0.2.0
+```
+Putting it altogether and putting our values and UUID's into the command, it will look like this (it is a long command):
+```
+xo-cr-seed https://root:password@xen1.company.tld 4a21c1cd-e8bd-4466-910a-f7524ecc07b1 https://root:password@xen2.company.tld 5aaf86ca-ae06-4a4e-b6e1-d04f0609e64d 90d11a94-a88f-4a84-b7c1-ed207d3de2f9 369a26f0-da77-41ab-a998-fa6b02c69b9a
+```
+
+### Finished
+
+Your backup job should now be working correctly! Manually run the job the first time to check if everything is OK. Then, enable the job. **Now, only the deltas are sent, your initial seed saved you a LOT of time if you have a slow network.**

docs/contributing.md

@@ -24,10 +24,11 @@ to create a [GitHub pull request](https://help.github.com/articles/using-pull-re
 
 
 1. Create a branch for your work
-2. Create a pull request for this branch against the `master` branch
-3. Push into the branch until the pull request is ready to merge
-4. Avoid unnecessary merges: keep you branch up to date by regularly rebasing `git rebase origin/master`
-5. When ready to merge, clean up the history (reorder commits, squash some of them together, rephrase messages): `git rebase -i origin/master`
+2. Add a summary of your changes to `CHANGELOG.md` under the `next` section, if your changes do not relate to an existing changelog item
+3. Create a pull request for this branch against the `master` branch
+4. Push into the branch until the pull request is ready to merge
+5. Avoid unnecessary merges: keep you branch up to date by regularly rebasing `git rebase origin/master`
+6. When ready to merge, clean up the history (reorder commits, squash some of them together, rephrase messages): `git rebase -i origin/master`
 
 ### Issue triage
 

docs/directpurchase.md

@@ -2,26 +2,50 @@
 
 This is the easiest purchase option: you can buy XOA with your registered email account on `xen-orchestra.com`.
 
+## Choose your edition
+
 You can choose the edition you want in two places:
 
 * [the pricing page](https://xen-orchestra.com/#!/pricing)
-* [your account/member page](https://xen-orchestra.com/#!/member)
+* [your account/purchases page](https://xen-orchestra.com/#!/purchases)
 
 
-> You need to be logged to make a purchase. If you don't have an account, please [register here](https://xen-orchestra.com/#!/signup).
+> You need to be logged in to make a purchase. If you don't have an account, please [register here](https://xen-orchestra.com/#!/signup).
 
 From your account page, click on the purchase menu, then select the edition you need:
 
 ![](./assets/directpurchase.png)
 
-Then you need to fill in your information and select **"buy it for my own use"**:
+## Purchase options
+
+The second step is to select your purchase option: 
+
+- Subscription: only available with a credit card payment. Choose this option for a monthly payment or a yearly payment **renewed automatically** each year.
+
+- Paid period: **check or wire transfer only**. This purchase allows you to subscribe for a one, two or three year period
+
+> A 2 year subscription period grants you 1 month discounted
+> A 3 year subscription period grants you 2 months discounted
+
+Then you need to fill in your information and select **"Buy for my own use"** (direct purchase)
 
 ![](./assets/member_purchase_2.png)
 
-The default payment method is by **credit card**. But you can also choose the "wire transfer" tab (with the "bank" icon):
+## Billing information
+You need to complete all the required information on this page in order to move forward. 
 
-![](./assets/member_purchase3.png)
+> Note: If you are part of the Eurozone, you will need to provide a valid EU VAT number in order to proceed to payment. Transactions between companies inside the Eurozone are VAT free. 
+Transactions outside the Eurozone are VAT free. 
 
-## Wire transfer process
+![](./assets/billing_info.png)
 
-If you select wire transfer, you need to upload a proof of transfer before we can unlock your XOA. If you don't, you'll have to wait for funds to actually be transferred into our account.
+## Select your payment mode
+
+Credit Card, Wire transfer or Bank check are the three payment methods available on our store. Some methods can be unavailable regarding the purchase option you have selected during step one. 
+
+> Wire transfer is not available for monthly and yearly subscription - Credit Card is not available for paid period.
+
+![](./assets/payment_mode.png)
+
+> All required information for wire transfer and Check payment will be available in the last step of the payment AND on your proforma invoice.
+> ⚠ Please, use an explicit reference for your wire transfer in order for us to easily identify your payment.

docs/emergency_shutdown.md

@@ -0,0 +1,27 @@
+# Emergency Shutdown
+
+If you have a UPS for your hosts, and lose power, you may have a limited amount of time to shut down all of your VM infrastructure before the batteries run out. If you find yourself in this situation, or any other situation requiring the fast shutdown of everything, you can use the **Emergency Shutdown** feature.
+
+## How to activate
+On the host view, clicking on this button will trigger the  _Emergency Shutdown_ procedure:
+
+![](./assets/e-shutdown-1.png)
+
+1.  **All running VMs will be suspended**  (think of it like "hibernate" on your laptop: the RAM will be stored in the storage repository).
+2.  Only after this is complete, the host will be halted.
+
+Here, you can see the running VMs are being suspended:
+
+![](./assets/e-shutdown-2.png)
+
+And finally, that's it. They are cleanly shut down with the RAM saved to disk to be resumed later:
+
+![](./assets/e-shutdown-3.png)
+
+Now the host is halted automatically.
+
+## Powering back on
+When the power outage is over, all you need to do is:
+
+1.  Start your host.
+2.  All your VMs can be resumed, your RAM is preserved and therefore your VMs will be in the exact same state as they were before the power outage.

docs/from_the_sources.md

@@ -6,9 +6,9 @@
 
 > Please take time to read this guide carefully.
 
-This installation has been validated against a fresh Debian 8 (Jessie) x64 install. It should be nearly the same on other dpkg systems. For RPM based OS's, it should be close, as most of our dependencies come from NPM and not the OS itself.
+This installation has been validated against a fresh Debian 9 (Stretch) x64 install. It should be nearly the same on other dpkg systems. For RPM based OS's, it should be close, as most of our dependencies come from NPM and not the OS itself.
 
-As you may have seen,in other parts of the documentation, XO is composed of two parts: [xo-server](https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server/) and [xo-web](https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-web/). They can be installed separately, even on different machines, but for the sake of simplicity we will set them up together.
+As you may have seen in other parts of the documentation, XO is composed of two parts: [xo-server](https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server/) and [xo-web](https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-web/). They can be installed separately, even on different machines, but for the sake of simplicity we will set them up together.
 
 ## Packages and Pre-requisites
 
@@ -38,24 +38,25 @@ XO needs the following packages to be installed. Redis is used as a database by
 For example, on Debian:
 
 ```
-apt-get install build-essential redis-server libpng-dev git python-minimal libvhdi-utils lvm2
+apt-get install build-essential redis-server libpng-dev git python-minimal libvhdi-utils lvm2 cifs-utils
 ```
 
 ## Fetching the Code
 
-You need to use the `git` source code manager to fetch the code. Ideally you should run XO as a non-root user, however if you don't run as root you will not be able to mount NFS remotes. As your chosen non-root (or root) user, run the following:
+You need to use the `git` source code manager to fetch the code. Ideally, you should run XO as a non-root user, and if you choose to, you need to set up `sudo` to be able to mount NFS remotes. As your chosen non-root (or root) user, run the following:
 
 ```
 git clone -b master http://github.com/vatesfr/xen-orchestra
 ```
 
-> Note: xo-server and xo-web have been migrated to the [xen-orchestra](https://github.com/vatesfr/xen-orchestra) mono-repository.
+> Note: xo-server and xo-web have been migrated to the [xen-orchestra](https://github.com/vatesfr/xen-orchestra) mono-repository - so you only need the single clone command above
 
 ## Installing dependencies
 
-Once you have it, use `yarn`, as the non-root (or root) user owning the fetched code, to install the other dependencies. Enter the `xen-orchestra` directory and run the following commands:
+Now that you have the code, you can enter the `xen-orchestra` directory and use `yarn` to install other dependencies. Then finally build it using `yarn build`. Be sure to run `yarn` commands as the same user you will be using to run Xen Orchestra:
 
 ```
+$ cd xen-orchestra
 $ yarn
 $ yarn build
 ```
@@ -64,17 +65,15 @@ Now you have to create a config file for `xo-server`:
 
 ```
 $ cd packages/xo-server
-$ cp sample.config.yaml .xo-server.yaml
+$ cp sample.config.toml .xo-server.toml
 ```
 
-Edit and uncomment it to have the right path to serve `xo-web`, because `xo-server` embeds an HTTP server (we assume that `xen-orchestra` and `xo-web` are in the same directory). It's near the end of the file:
+Edit and uncomment it to have the right path to serve `xo-web`, because `xo-server` embeds an HTTP server (we assume that `xen-orchestra` and `xo-web` are in the same directory):
 
-```yaml
-  mounts: '/': '../xo-web/dist/'
+```toml
+[http.mounts]
+'/' = '../xo-web/dist/'
 ```
-> Note this `dist` folder will be created in the next step.
-
-**WARNING: YAML is very strict with indentation: use spaces for it, not tabs**.
 
 In this config file, you can also change default ports (80 and 443) for xo-server. If you are running the server as a non-root user, you will need to set the port to 1024 or higher.
 
@@ -88,7 +87,7 @@ WebServer listening on localhost:80
 
 ## Running XO
 
-The only part you need to launch is xo-server which is quite easy to do. From the `xen-orchestra/packages/xo-server` directory, run the following:
+The only part you need to launch is xo-server, which is quite easy to do. From the `xen-orchestra/packages/xo-server` directory, run the following:
 
 ```
 $ yarn start
@@ -100,6 +99,9 @@ That's it! Use your browser to visit the xo-server IP address, and it works! :)
 If you would like to update your current version, enter your `xen-orchestra` directory and run the following:
 
 ```
+# This will clear any changes you made in the repository!!
+$ git checkout .
+
 $ git pull --ff-only
 $ yarn
 $ yarn build
@@ -140,9 +142,6 @@ If you need to delete the service:
 forever-service delete orchestra
 ```
 
-
-
-
 ## Troubleshooting
 
 If you have problems during the building phase, follow these steps in your `xen-orchestra` directory:
@@ -184,3 +183,17 @@ Don't forget to start redis if you don't reboot now:
 ```
 service redis start
 ```
+
+## SUDO
+
+If you are running `xo-server` as a non-root user, you need to use `sudo` to be able to mount NFS remotes. You can do this by editing `xo-server/.xo-server.toml` and setting `useSudo = true`. It's near the end of the file:
+
+```
+useSudo = true
+```
+
+You need to configure `sudo` to allow the user of your choice to run mount/umount commands without asking for a password. Depending on your operating system / sudo version, the location of this configuration may change. Regardless, you can use:
+
+```
+username ALL=(root)NOPASSWD: /bin/mount, /bin/umount
+```

docs/supported-version.md

@@ -6,10 +6,11 @@ Xen Orchestra is designed to work exclusively on [XCP-ng](https://xcp-ng.org/) a
 
 Backup restore for large VM disks (>1TiB usage) is [broken on all XenServer versions](https://bugs.xenserver.org/browse/XSO-868) until Citrix release a fix.
 
+* XenServer 7.6
 * XenServer 7.5
   * [VDI I/O error](https://bugs.xenserver.org/browse/XSO-873), waiting for Citrix to release our fix
 * XenServer 7.4
-* XenServer 7.3 
+* XenServer 7.3
 * XenServer 7.2
 * XenServer 7.1
 * XenServer 7.0
@@ -26,7 +27,8 @@ Backup restore for large VM disks (>1TiB usage) is [broken on all XenServer vers
 
 All the pending fixes are already integrated in the latest XCP-ng version. We strongly suggest people to keep using the latest XCP-ng version as far as possible.
 
+* XCP-ng 7.6
 * XCP-ng 7.5
 * XCP-ng 7.4.1
 
-![](https://xen-orchestra.com/blog/content/images/2018/02/logo1glossy.png)
+![](https://xen-orchestra.com/blog/content/images/2018/02/logo1glossy.png)

docs/through_purchase_department.md

@@ -4,7 +4,7 @@ If you can't purchase using your own account, usually because you need to go thr
 
 Typically, you will provide two contacts:
 
-* The "billing contact" (in general, the purchaser email). This account will have access to invoices. This is the account which makes the purchase and then binds the XO plan to the second contact account, the technical contact.
+* The "billing contact" (in general, the purchaser email). This account will have access to invoices. This is the account doing the purchase. Once purchased, the license needs to be bound to the second contact account, the technical contact.
 * The "technical contact", the email of the system administrator using the solution and making support requests.
 
 ## As "billing contact"
@@ -17,10 +17,10 @@ Typically, you will provide two contacts:
 
 Now, you just have to pick the edition of Xen Orchestra you want to purchase for your IT team.
 
-2. You will then see the payment screen. If your are not purchasing the edition for yourself, you have to pick the **buy for another account** option. 
+2. On the first payment screen, after you choose the plan and the subscription method. You can select the option "Buy for another account"
 
 
-![](./assets/purchase-for-another.png)
+![](./assets/member_purchase_2.png)
 
 3. Once the payment is completed, you will have to bind the plan with the end-user account (technical contact). If the end-user doesn't have an account yet, the system will create one and send an e-mail to your end user. 
 
@@ -29,4 +29,4 @@ Now, you just have to pick the edition of Xen Orchestra you want to purchase for
 
 That's it, you have now completed the purchase. 
 
-**Once you have bound the plan to your end user account, you cannot change it. Double check the spelling of the e-mail before binding the account.**
+**⚠ Once you have bound the plan to your end user account, you cannot change it. Double check the spelling of the e-mail before binding the account.**

package.json

@@ -7,6 +7,7 @@
     "babel-jest": "^23.0.1",
     "benchmark": "^2.1.4",
     "eslint": "^5.1.0",
+    "eslint-config-prettier": "^3.3.0",
     "eslint-config-standard": "12.0.0",
     "eslint-config-standard-jsx": "^6.0.2",
     "eslint-plugin-import": "^2.8.0",
@@ -15,9 +16,9 @@
     "eslint-plugin-react": "^7.6.1",
     "eslint-plugin-standard": "^4.0.0",
     "exec-promise": "^0.7.0",
-    "flow-bin": "^0.85.0",
-    "globby": "^8.0.0",
-    "husky": "^1.0.0-rc.15",
+    "flow-bin": "^0.90.0",
+    "globby": "^9.0.0",
+    "husky": "^1.2.1",
     "jest": "^23.0.1",
     "lodash": "^4.17.4",
     "prettier": "^1.10.2",
@@ -33,7 +34,6 @@
     }
   },
   "jest": {
-    "timers": "fake",
     "collectCoverage": true,
     "projects": [
       "<rootDir>"

packages/complex-matcher/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/complex-matcher",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/complex-matcher",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/complex-matcher/src/index.js

@@ -33,17 +33,17 @@ const isRawString = string => {
 // -------------------------------------------------------------------
 
 class Node {
-  createPredicate () {
+  createPredicate() {
     return value => this.match(value)
   }
 }
 
 export class Null extends Node {
-  match () {
+  match() {
     return true
   }
 
-  toString () {
+  toString() {
     return ''
   }
 }
@@ -51,7 +51,7 @@ export class Null extends Node {
 const formatTerms = terms => terms.map(term => term.toString(true)).join(' ')
 
 export class And extends Node {
-  constructor (children) {
+  constructor(children) {
     super()
 
     if (children.length === 1) {
@@ -60,29 +60,29 @@ export class And extends Node {
     this.children = children
   }
 
-  match (value) {
+  match(value) {
     return this.children.every(child => child.match(value))
   }
 
-  toString (isNested) {
+  toString(isNested) {
     const terms = formatTerms(this.children)
     return isNested ? `(${terms})` : terms
   }
 }
 
 export class Comparison extends Node {
-  constructor (operator, value) {
+  constructor(operator, value) {
     super()
     this._comparator = Comparison.comparators[operator]
     this._operator = operator
     this._value = value
   }
 
-  match (value) {
+  match(value) {
     return typeof value === 'number' && this._comparator(value, this._value)
   }
 
-  toString () {
+  toString() {
     return this._operator + String(this._value)
   }
 }
@@ -94,7 +94,7 @@ Comparison.comparators = {
 }
 
 export class Or extends Node {
-  constructor (children) {
+  constructor(children) {
     super()
 
     if (children.length === 1) {
@@ -103,33 +103,33 @@ export class Or extends Node {
     this.children = children
   }
 
-  match (value) {
+  match(value) {
     return this.children.some(child => child.match(value))
   }
 
-  toString () {
+  toString() {
     return `|(${formatTerms(this.children)})`
   }
 }
 
 export class Not extends Node {
-  constructor (child) {
+  constructor(child) {
     super()
 
     this.child = child
   }
 
-  match (value) {
+  match(value) {
     return !this.child.match(value)
   }
 
-  toString () {
+  toString() {
     return '!' + this.child.toString(true)
   }
 }
 
 export class NumberNode extends Node {
-  constructor (value) {
+  constructor(value) {
     super()
 
     this.value = value
@@ -140,21 +140,21 @@ export class NumberNode extends Node {
     })
   }
 
-  match (value) {
+  match(value) {
     return (
       value === this.value ||
       (value !== null && typeof value === 'object' && some(value, this.match))
     )
   }
 
-  toString () {
+  toString() {
     return String(this.value)
   }
 }
 export { NumberNode as Number }
 
 export class NumberOrStringNode extends Node {
-  constructor (value) {
+  constructor(value) {
     super()
 
     this.value = value
@@ -165,7 +165,7 @@ export class NumberOrStringNode extends Node {
     })
   }
 
-  match (lcValue, numValue, value) {
+  match(lcValue, numValue, value) {
     return (
       value === numValue ||
       (typeof value === 'string'
@@ -175,25 +175,25 @@ export class NumberOrStringNode extends Node {
     )
   }
 
-  toString () {
+  toString() {
     return this.value
   }
 }
 export { NumberOrStringNode as NumberOrString }
 
 export class Property extends Node {
-  constructor (name, child) {
+  constructor(name, child) {
     super()
 
     this.name = name
     this.child = child
   }
 
-  match (value) {
+  match(value) {
     return value != null && this.child.match(value[this.name])
   }
 
-  toString () {
+  toString() {
     return `${formatString(this.name)}:${this.child.toString(true)}`
   }
 }
@@ -207,7 +207,7 @@ const formatString = value =>
     : `"${value}"`
 
 export class GlobPattern extends Node {
-  constructor (value) {
+  constructor(value) {
     // fallback to string node if no wildcard
     if (value.indexOf('*') === -1) {
       return new StringNode(value)
@@ -232,7 +232,7 @@ export class GlobPattern extends Node {
     })
   }
 
-  match (re, value) {
+  match(re, value) {
     if (typeof value === 'string') {
       return re.test(value)
     }
@@ -244,13 +244,13 @@ export class GlobPattern extends Node {
     return false
   }
 
-  toString () {
+  toString() {
     return this.value
   }
 }
 
 export class RegExpNode extends Node {
-  constructor (pattern, flags) {
+  constructor(pattern, flags) {
     super()
 
     this.re = new RegExp(pattern, flags)
@@ -261,7 +261,7 @@ export class RegExpNode extends Node {
     })
   }
 
-  match (value) {
+  match(value) {
     if (typeof value === 'string') {
       return this.re.test(value)
     }
@@ -273,14 +273,14 @@ export class RegExpNode extends Node {
     return false
   }
 
-  toString () {
+  toString() {
     return this.re.toString()
   }
 }
 export { RegExpNode as RegExp }
 
 export class StringNode extends Node {
-  constructor (value) {
+  constructor(value) {
     super()
 
     this.value = value
@@ -291,7 +291,7 @@ export class StringNode extends Node {
     })
   }
 
-  match (lcValue, value) {
+  match(lcValue, value) {
     if (typeof value === 'string') {
       return value.toLowerCase().indexOf(lcValue) !== -1
     }
@@ -303,24 +303,24 @@ export class StringNode extends Node {
     return false
   }
 
-  toString () {
+  toString() {
     return formatString(this.value)
   }
 }
 export { StringNode as String }
 
 export class TruthyProperty extends Node {
-  constructor (name) {
+  constructor(name) {
     super()
 
     this.name = name
   }
 
-  match (value) {
+  match(value) {
     return value != null && !!value[this.name]
   }
 
-  toString () {
+  toString() {
     return formatString(this.name) + '?'
   }
 }
@@ -330,12 +330,12 @@ export class TruthyProperty extends Node {
 // https://gist.github.com/yelouafi/556e5159e869952335e01f6b473c4ec1
 
 class Failure {
-  constructor (pos, expected) {
+  constructor(pos, expected) {
     this.expected = expected
     this.pos = pos
   }
 
-  get value () {
+  get value() {
     throw new Error(
       `parse error: expected ${this.expected} at position ${this.pos}`
     )
@@ -343,7 +343,7 @@ class Failure {
 }
 
 class Success {
-  constructor (pos, value) {
+  constructor(pos, value) {
     this.pos = pos
     this.value = value
   }
@@ -352,7 +352,7 @@ class Success {
 // -------------------------------------------------------------------
 
 class P {
-  static alt (...parsers) {
+  static alt(...parsers) {
     const { length } = parsers
     return new P((input, pos, end) => {
       for (let i = 0; i < length; ++i) {
@@ -365,7 +365,7 @@ class P {
     })
   }
 
-  static grammar (rules) {
+  static grammar(rules) {
     const grammar = {}
     Object.keys(rules).forEach(k => {
       const rule = rules[k]
@@ -374,14 +374,14 @@ class P {
     return grammar
   }
 
-  static lazy (parserCreator, arg) {
+  static lazy(parserCreator, arg) {
     const parser = new P((input, pos, end) =>
       (parser._parse = parserCreator(arg)._parse)(input, pos, end)
     )
     return parser
   }
 
-  static regex (regex) {
+  static regex(regex) {
     regex = new RegExp(regex.source, 'y')
     return new P((input, pos) => {
       regex.lastIndex = pos
@@ -392,7 +392,7 @@ class P {
     })
   }
 
-  static seq (...parsers) {
+  static seq(...parsers) {
     const { length } = parsers
     return new P((input, pos, end) => {
       const values = new Array(length)
@@ -408,7 +408,7 @@ class P {
     })
   }
 
-  static text (text) {
+  static text(text) {
     const { length } = text
     return new P((input, pos) =>
       input.startsWith(text, pos)
@@ -417,11 +417,11 @@ class P {
     )
   }
 
-  constructor (parse) {
+  constructor(parse) {
     this._parse = parse
   }
 
-  map (fn) {
+  map(fn) {
     return new P((input, pos, end) => {
       const result = this._parse(input, pos, end)
       if (result instanceof Success) {
@@ -431,11 +431,11 @@ class P {
     })
   }
 
-  parse (input, pos = 0, end = input.length) {
+  parse(input, pos = 0, end = input.length) {
     return this._parse(input, pos, end).value
   }
 
-  repeat (min = 0, max = Infinity) {
+  repeat(min = 0, max = Infinity) {
     return new P((input, pos, end) => {
       const value = []
       let result
@@ -461,7 +461,7 @@ class P {
     })
   }
 
-  skip (otherParser) {
+  skip(otherParser) {
     return new P((input, pos, end) => {
       const result = this._parse(input, pos, end)
       if (result instanceof Failure) {

packages/value-matcher/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/value-matcher",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/value-matcher",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/vhd-cli/package.json

@@ -1,12 +1,13 @@
 {
   "name": "vhd-cli",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "license": "ISC",
   "description": "",
   "keywords": [],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-cli",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/vhd-cli",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -26,11 +27,12 @@
     "node": ">=6"
   },
   "dependencies": {
-    "@xen-orchestra/fs": "^0.4.1",
+    "@xen-orchestra/fs": "^0.6.1",
     "cli-progress": "^2.0.0",
     "exec-promise": "^0.7.0",
+    "getopts": "^2.2.3",
     "struct-fu": "^1.2.0",
-    "vhd-lib": "^0.4.0"
+    "vhd-lib": "^0.5.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/vhd-cli/src/commands/check.js

@@ -1,12 +1,24 @@
-import Vhd from 'vhd-lib'
+import Vhd, { checkVhdChain } from 'vhd-lib'
+import getopts from 'getopts'
 import { getHandler } from '@xen-orchestra/fs'
 import { resolve } from 'path'
 
-export default async args => {
+const checkVhd = (handler, path) => new Vhd(handler, path).readHeaderAndFooter()
+
+export default async rawArgs => {
+  const { chain, _: args } = getopts(rawArgs, {
+    boolean: ['chain'],
+    default: {
+      chain: false,
+    },
+  })
+
+  const check = chain ? checkVhdChain : checkVhd
+
   const handler = getHandler({ url: 'file:///' })
   for (const vhd of args) {
     try {
-      await new Vhd(handler, resolve(vhd)).readHeaderAndFooter()
+      await check(handler, resolve(vhd))
       console.log('ok:', vhd)
     } catch (error) {
       console.error('nok:', vhd, error)

packages/vhd-cli/src/commands/merge.js

@@ -3,7 +3,7 @@ import { mergeVhd } from 'vhd-lib'
 import { getHandler } from '@xen-orchestra/fs'
 import { resolve } from 'path'
 
-export default async function main (args) {
+export default async function main(args) {
   if (args.length < 2 || args.some(_ => _ === '-h' || _ === '--help')) {
     return `Usage: ${this.command} <child VHD> <parent VHD>`
   }
@@ -11,7 +11,7 @@ export default async function main (args) {
   const handler = getHandler({ url: 'file:///' })
   let bar
   await mergeVhd(handler, resolve(args[1]), handler, resolve(args[0]), {
-    onProgress ({ done, total }) {
+    onProgress({ done, total }) {
       if (bar === undefined) {
         bar = new Bar({
           format:

packages/vhd-cli/src/commands/synthetize.js

@@ -3,7 +3,7 @@ import { createSyntheticStream } from 'vhd-lib'
 import { createWriteStream } from 'fs'
 import { getHandler } from '@xen-orchestra/fs'
 
-export default async function main (args) {
+export default async function main(args) {
   if (args.length < 2 || args.some(_ => _ === '-h' || _ === '--help')) {
     return `Usage: ${this.command} <input VHD> <output VHD>`
   }

packages/vhd-cli/src/index.js

@@ -2,9 +2,11 @@
 
 import execPromise from 'exec-promise'
 
+import pkg from '../package.json'
+
 import commands from './commands'
 
-function runCommand (commands, [command, ...args]) {
+function runCommand(commands, [command, ...args]) {
   if (command === undefined || command === '-h' || command === '--help') {
     command = 'help'
   }
@@ -16,9 +18,11 @@ function runCommand (commands, [command, ...args]) {
       return `Usage:
 
 ${Object.keys(commands)
-        .filter(command => command !== 'help')
-        .map(command => `    ${this.command} ${command}`)
-        .join('\n\n')}`
+  .filter(command => command !== 'help')
+  .map(command => `    ${this.command} ${command}`)
+  .join('\n\n')}
+
+vhd-cli ${pkg.version}`
     }
 
     throw `invalid command ${command}` // eslint-disable-line no-throw-literal

packages/vhd-lib/merge.integ.spec.js

@@ -28,18 +28,18 @@ afterEach(async () => {
   await pFromCallback(cb => rimraf(tmpDir, cb))
 })
 
-async function createRandomFile (name, sizeMb) {
+async function createRandomFile(name, sizeMb) {
   await execa('bash', [
     '-c',
     `< /dev/urandom tr -dc "\\t\\n [:alnum:]" | head -c ${sizeMb}M >${name}`,
   ])
 }
 
-async function checkFile (vhdName) {
+async function checkFile(vhdName) {
   await execa('vhd-util', ['check', '-p', '-b', '-t', '-n', vhdName])
 }
 
-async function recoverRawContent (vhdName, rawName, originalSize) {
+async function recoverRawContent(vhdName, rawName, originalSize) {
   await checkFile(vhdName)
   await execa('qemu-img', ['convert', '-fvpc', '-Oraw', vhdName, rawName])
   if (originalSize !== undefined) {
@@ -47,7 +47,7 @@ async function recoverRawContent (vhdName, rawName, originalSize) {
   }
 }
 
-async function convertFromRawToVhd (rawName, vhdName) {
+async function convertFromRawToVhd(rawName, vhdName) {
   await execa('qemu-img', ['convert', '-f', 'raw', '-Ovpc', rawName, vhdName])
 }
 

packages/vhd-lib/package.json

@@ -1,12 +1,13 @@
 {
   "name": "vhd-lib",
-  "version": "0.4.0",
+  "version": "0.5.1",
   "license": "AGPL-3.0",
   "description": "Primitives for VHD file handling",
   "keywords": [],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-lib",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/vhd-lib",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -34,7 +35,7 @@
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
     "@babel/preset-flow": "^7.0.0",
-    "@xen-orchestra/fs": "^0.4.1",
+    "@xen-orchestra/fs": "^0.6.1",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^5.1.3",
     "execa": "^1.0.0",

packages/vhd-lib/src/_computeGeometryForSize.js

@@ -1,6 +1,6 @@
 import { SECTOR_SIZE } from './_constants'
 
-export default function computeGeometryForSize (size) {
+export default function computeGeometryForSize(size) {
   const totalSectors = Math.min(Math.ceil(size / 512), 65535 * 16 * 255)
   let sectorsPerTrackCylinder
   let heads

packages/vhd-lib/src/_createFooterHeader.js

@@ -14,7 +14,7 @@ import {
   PLATFORM_WI2K,
 } from './_constants'
 
-export function createFooter (
+export function createFooter(
   size,
   timestamp,
   geometry,
@@ -39,7 +39,7 @@ export function createFooter (
   return footer
 }
 
-export function createHeader (
+export function createHeader(
   maxTableEntries,
   tableOffset = HEADER_SIZE + FOOTER_SIZE,
   blockSize = VHD_BLOCK_SIZE_BYTES

packages/vhd-lib/src/_resolveRelativeFromFile.js

@@ -0,0 +1,6 @@
+import { dirname, resolve } from 'path'
+
+const resolveRelativeFromFile = (file, path) =>
+  resolve('/', dirname(file), path).slice(1)
+
+export { resolveRelativeFromFile as default }

packages/vhd-lib/src/_structs.js

@@ -95,7 +95,7 @@ export const unpackField = (field, buf) => {
 
 // Returns the checksum of a raw struct.
 // The raw struct (footer or header) is altered with the new sum.
-export function checksumStruct (buf, struct) {
+export function checksumStruct(buf, struct) {
   const checksumField = struct.fields.checksum
   let sum = 0
 

packages/vhd-lib/src/chain.js

@@ -3,7 +3,7 @@ import { dirname, relative } from 'path'
 import Vhd from './vhd'
 import { DISK_TYPE_DIFFERENCING } from './_constants'
 
-export default async function chain (
+export default async function chain(
   parentHandler,
   parentPath,
   childHandler,

packages/vhd-lib/src/checkChain.js

@@ -0,0 +1,16 @@
+import Vhd from './vhd'
+import resolveRelativeFromFile from './_resolveRelativeFromFile'
+import { DISK_TYPE_DYNAMIC } from './_constants'
+
+export default async function checkChain(handler, path) {
+  while (true) {
+    const vhd = new Vhd(handler, path)
+    await vhd.readHeaderAndFooter()
+
+    if (vhd.footer.diskType === DISK_TYPE_DYNAMIC) {
+      break
+    }
+
+    path = resolveRelativeFromFile(path, vhd.header.parentUnicodeName)
+  }
+}

packages/vhd-lib/src/createContentStream.js

@@ -2,7 +2,7 @@ import asyncIteratorToStream from 'async-iterator-to-stream'
 
 import Vhd from './vhd'
 
-export default asyncIteratorToStream(async function * (handler, path) {
+export default asyncIteratorToStream(async function*(handler, path) {
   const fd = await handler.openFile(path, 'r')
   try {
     const vhd = new Vhd(handler, fd)

packages/vhd-lib/src/createReadableRawStream.js

@@ -3,7 +3,7 @@ import asyncIteratorToStream from 'async-iterator-to-stream'
 import computeGeometryForSize from './_computeGeometryForSize'
 import { createFooter } from './_createFooterHeader'
 
-export default asyncIteratorToStream(async function * (size, blockParser) {
+export default asyncIteratorToStream(async function*(size, blockParser) {
   const geometry = computeGeometryForSize(size)
   const actualSize = geometry.actualSize
   const footer = createFooter(
@@ -13,7 +13,7 @@ export default asyncIteratorToStream(async function * (size, blockParser) {
   )
   let position = 0
 
-  function * filePadding (paddingLength) {
+  function* filePadding(paddingLength) {
     if (paddingLength > 0) {
       const chunkSize = 1024 * 1024 // 1Mo
       for (
@@ -33,10 +33,10 @@ export default asyncIteratorToStream(async function * (size, blockParser) {
     if (paddingLength < 0) {
       throw new Error('Received out of order blocks')
     }
-    yield * filePadding(paddingLength)
+    yield* filePadding(paddingLength)
     yield next.data
     position = next.offsetBytes + next.data.length
   }
-  yield * filePadding(actualSize - position)
+  yield* filePadding(actualSize - position)
   yield footer
 })

packages/vhd-lib/src/createReadableSparseStream.js

@@ -19,7 +19,7 @@ const VHD_BLOCK_SIZE_SECTORS = VHD_BLOCK_SIZE_BYTES / SECTOR_SIZE
 /**
  * @returns currentVhdPositionSector the first free sector after the data
  */
-function createBAT (
+function createBAT(
   firstBlockPosition,
   blockAddressList,
   ratio,
@@ -39,7 +39,7 @@ function createBAT (
   return currentVhdPositionSector
 }
 
-export default async function createReadableStream (
+export default async function createReadableStream(
   diskSize,
   incomingBlockSize,
   blockAddressList,
@@ -89,7 +89,7 @@ export default async function createReadableStream (
   )
   const fileSize = endOfData * SECTOR_SIZE + FOOTER_SIZE
   let position = 0
-  function * yieldAndTrack (buffer, expectedPosition) {
+  function* yieldAndTrack(buffer, expectedPosition) {
     if (expectedPosition !== undefined) {
       assert.strictEqual(position, expectedPosition)
     }
@@ -98,7 +98,7 @@ export default async function createReadableStream (
       position += buffer.length
     }
   }
-  async function * generateFileContent (blockIterator, bitmapSize, ratio) {
+  async function* generateFileContent(blockIterator, bitmapSize, ratio) {
     let currentBlock = -1
     let currentVhdBlockIndex = -1
     let currentBlockWithBitmap = Buffer.alloc(0)
@@ -108,7 +108,7 @@ export default async function createReadableStream (
       const batIndex = Math.floor(next.offsetBytes / VHD_BLOCK_SIZE_BYTES)
       if (batIndex !== currentVhdBlockIndex) {
         if (currentVhdBlockIndex >= 0) {
-          yield * yieldAndTrack(
+          yield* yieldAndTrack(
             currentBlockWithBitmap,
             bat.readUInt32BE(currentVhdBlockIndex * 4) * SECTOR_SIZE
           )
@@ -126,15 +126,15 @@ export default async function createReadableStream (
         bitmapSize + (next.offsetBytes % VHD_BLOCK_SIZE_BYTES)
       )
     }
-    yield * yieldAndTrack(currentBlockWithBitmap)
+    yield* yieldAndTrack(currentBlockWithBitmap)
   }
 
-  async function * iterator () {
-    yield * yieldAndTrack(footer, 0)
-    yield * yieldAndTrack(header, FOOTER_SIZE)
-    yield * yieldAndTrack(bat, FOOTER_SIZE + HEADER_SIZE)
-    yield * generateFileContent(blockIterator, bitmapSize, ratio)
-    yield * yieldAndTrack(footer)
+  async function* iterator() {
+    yield* yieldAndTrack(footer, 0)
+    yield* yieldAndTrack(header, FOOTER_SIZE)
+    yield* yieldAndTrack(bat, FOOTER_SIZE + HEADER_SIZE)
+    yield* generateFileContent(blockIterator, bitmapSize, ratio)
+    yield* yieldAndTrack(footer)
   }
 
   const stream = asyncIteratorToStream(iterator())

packages/vhd-lib/src/createSyntheticStream.js

@@ -1,5 +1,6 @@
 import asyncIteratorToStream from 'async-iterator-to-stream'
-import { dirname, resolve } from 'path'
+
+import resolveRelativeFromFile from './_resolveRelativeFromFile'
 
 import Vhd from './vhd'
 import {
@@ -12,10 +13,7 @@ import {
 import { fuFooter, fuHeader, checksumStruct } from './_structs'
 import { test as mapTestBit } from './_bitmap'
 
-const resolveRelativeFromFile = (file, path) =>
-  resolve('/', dirname(file), path).slice(1)
-
-export default async function createSyntheticStream (handler, path) {
+export default async function createSyntheticStream(handler, path) {
   const fds = []
   const cleanup = () => {
     for (let i = 0, n = fds.length; i < n; ++i) {
@@ -85,7 +83,7 @@ export default async function createSyntheticStream (handler, path) {
     }
     const fileSize = blockOffset * SECTOR_SIZE + FOOTER_SIZE
 
-    const iterator = function * () {
+    const iterator = function*() {
       try {
         footer = fuFooter.pack(footer)
         checksumStruct(footer, fuFooter)
@@ -108,14 +106,14 @@ export default async function createSyntheticStream (handler, path) {
           yield bitmap
 
           const blocksByVhd = new Map()
-          const emitBlockSectors = function * (iVhd, i, n) {
+          const emitBlockSectors = function*(iVhd, i, n) {
             const vhd = vhds[iVhd]
             const isRootVhd = vhd === rootVhd
             if (!vhd.containsBlock(iBlock)) {
               if (isRootVhd) {
                 yield Buffer.alloc((n - i) * SECTOR_SIZE)
               } else {
-                yield * emitBlockSectors(iVhd + 1, i, n)
+                yield* emitBlockSectors(iVhd + 1, i, n)
               }
               return
             }
@@ -138,11 +136,11 @@ export default async function createSyntheticStream (handler, path) {
               if (hasData) {
                 yield data.slice(start * SECTOR_SIZE, i * SECTOR_SIZE)
               } else {
-                yield * emitBlockSectors(iVhd + 1, start, i)
+                yield* emitBlockSectors(iVhd + 1, start, i)
               }
             }
           }
-          yield * emitBlockSectors(owner, 0, sectorsPerBlockData)
+          yield* emitBlockSectors(owner, 0, sectorsPerBlockData)
         }
         yield footer
       } finally {

packages/vhd-lib/src/index.js

@@ -3,6 +3,7 @@ import 'core-js/features/symbol/async-iterator'
 
 export { default } from './vhd'
 export { default as chainVhd } from './chain'
+export { default as checkVhdChain } from './checkChain'
 export { default as createContentStream } from './createContentStream'
 export { default as createReadableRawStream } from './createReadableRawStream'
 export {

packages/vhd-lib/src/merge.js

@@ -10,7 +10,7 @@ import { DISK_TYPE_DIFFERENCING, DISK_TYPE_DYNAMIC } from './_constants'
 // Merge vhd child into vhd parent.
 //
 // TODO: rename the VHD file during the merge
-export default concurrency(2)(async function merge (
+export default concurrency(2)(async function merge(
   parentHandler,
   parentPath,
   childHandler,

packages/vhd-lib/src/vhd.js

@@ -79,11 +79,11 @@ BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
 // - sectorSize = 512
 
 export default class Vhd {
-  get batSize () {
+  get batSize() {
     return computeBatSize(this.header.maxTableEntries)
   }
 
-  constructor (handler, path) {
+  constructor(handler, path) {
     this._handler = handler
     this._path = path
   }
@@ -92,7 +92,7 @@ export default class Vhd {
   // Read functions.
   // =================================================================
 
-  async _read (start, n) {
+  async _read(start, n) {
     const { bytesRead, buffer } = await this._handler.read(
       this._path,
       Buffer.alloc(n),
@@ -102,12 +102,12 @@ export default class Vhd {
     return buffer
   }
 
-  containsBlock (id) {
+  containsBlock(id) {
     return this._getBatEntry(id) !== BLOCK_UNUSED
   }
 
   // Returns the first address after metadata. (In bytes)
-  getEndOfHeaders () {
+  getEndOfHeaders() {
     const { header } = this
 
     let end = FOOTER_SIZE + HEADER_SIZE
@@ -132,7 +132,7 @@ export default class Vhd {
   }
 
   // Returns the first sector after data.
-  getEndOfData () {
+  getEndOfData() {
     let end = Math.ceil(this.getEndOfHeaders() / SECTOR_SIZE)
 
     const fullBlockSize = this.sectorsOfBitmap + this.sectorsPerBlock
@@ -153,7 +153,7 @@ export default class Vhd {
   // TODO: extract the checks into reusable functions:
   // - better human reporting
   // - auto repair if possible
-  async readHeaderAndFooter (checkSecondFooter = true) {
+  async readHeaderAndFooter(checkSecondFooter = true) {
     const buf = await this._read(0, FOOTER_SIZE + HEADER_SIZE)
     const bufFooter = buf.slice(0, FOOTER_SIZE)
     const bufHeader = buf.slice(FOOTER_SIZE)
@@ -206,7 +206,7 @@ export default class Vhd {
   }
 
   // Returns a buffer that contains the block allocation table of a vhd file.
-  async readBlockAllocationTable () {
+  async readBlockAllocationTable() {
     const { header } = this
     this.blockTable = await this._read(
       header.tableOffset,
@@ -215,11 +215,13 @@ export default class Vhd {
   }
 
   // return the first sector (bitmap) of a block
-  _getBatEntry (block) {
-    return this.blockTable.readUInt32BE(block * 4)
+  _getBatEntry(block) {
+    const i = block * 4
+    const { blockTable } = this
+    return i < blockTable.length ? blockTable.readUInt32BE(i) : BLOCK_UNUSED
   }
 
-  _readBlock (blockId, onlyBitmap = false) {
+  _readBlock(blockId, onlyBitmap = false) {
     const blockAddr = this._getBatEntry(blockId)
     if (blockAddr === BLOCK_UNUSED) {
       throw new Error(`no such block ${blockId}`)
@@ -243,7 +245,7 @@ export default class Vhd {
   // get the identifiers and first sectors of the first and last block
   // in the file
   //
-  _getFirstAndLastBlocks () {
+  _getFirstAndLastBlocks() {
     const n = this.header.maxTableEntries
     const bat = this.blockTable
     let i = 0
@@ -288,7 +290,7 @@ export default class Vhd {
   // =================================================================
 
   // Write a buffer/stream at a given position in a vhd file.
-  async _write (data, offset) {
+  async _write(data, offset) {
     debug(
       `_write offset=${offset} size=${
         Buffer.isBuffer(data) ? data.length : '???'
@@ -307,7 +309,7 @@ export default class Vhd {
       : fromEvent(data.pipe(stream), 'finish')
   }
 
-  async _freeFirstBlockSpace (spaceNeededBytes) {
+  async _freeFirstBlockSpace(spaceNeededBytes) {
     try {
       const { first, firstSector, lastSector } = this._getFirstAndLastBlocks()
       const tableOffset = this.header.tableOffset
@@ -347,7 +349,7 @@ export default class Vhd {
     }
   }
 
-  async ensureBatSize (entries) {
+  async ensureBatSize(entries) {
     const { header } = this
     const prevMaxTableEntries = header.maxTableEntries
     if (prevMaxTableEntries >= entries) {
@@ -372,7 +374,7 @@ export default class Vhd {
   }
 
   // set the first sector (bitmap) of a block
-  _setBatEntry (block, blockSector) {
+  _setBatEntry(block, blockSector) {
     const i = block * 4
     const { blockTable } = this
 
@@ -383,7 +385,7 @@ export default class Vhd {
 
   // Make a new empty block at vhd end.
   // Update block allocation table in context and in file.
-  async createBlock (blockId) {
+  async createBlock(blockId) {
     const blockAddr = Math.ceil(this.getEndOfData() / SECTOR_SIZE)
 
     debug(`create block ${blockId} at ${blockAddr}`)
@@ -402,7 +404,7 @@ export default class Vhd {
   }
 
   // Write a bitmap at a block address.
-  async writeBlockBitmap (blockAddr, bitmap) {
+  async writeBlockBitmap(blockAddr, bitmap) {
     const { bitmapSize } = this
 
     if (bitmap.length !== bitmapSize) {
@@ -419,7 +421,7 @@ export default class Vhd {
     await this._write(bitmap, sectorsToBytes(blockAddr))
   }
 
-  async writeEntireBlock (block) {
+  async writeEntireBlock(block) {
     let blockAddr = this._getBatEntry(block.id)
 
     if (blockAddr === BLOCK_UNUSED) {
@@ -428,7 +430,7 @@ export default class Vhd {
     await this._write(block.buffer, sectorsToBytes(blockAddr))
   }
 
-  async writeBlockSectors (block, beginSectorId, endSectorId, parentBitmap) {
+  async writeBlockSectors(block, beginSectorId, endSectorId, parentBitmap) {
     let blockAddr = this._getBatEntry(block.id)
 
     if (blockAddr === BLOCK_UNUSED) {
@@ -460,7 +462,7 @@ export default class Vhd {
     )
   }
 
-  async coalesceBlock (child, blockId) {
+  async coalesceBlock(child, blockId) {
     const block = await child._readBlock(blockId)
     const { bitmap, data } = block
 
@@ -502,7 +504,7 @@ export default class Vhd {
   }
 
   // Write a context footer. (At the end and beginning of a vhd file.)
-  async writeFooter (onlyEndFooter = false) {
+  async writeFooter(onlyEndFooter = false) {
     const { footer } = this
 
     const rawFooter = fuFooter.pack(footer)
@@ -522,7 +524,7 @@ export default class Vhd {
     await this._write(rawFooter, offset)
   }
 
-  writeHeader () {
+  writeHeader() {
     const { header } = this
     const rawHeader = fuHeader.pack(header)
     header.checksum = checksumStruct(rawHeader, fuHeader)
@@ -535,7 +537,7 @@ export default class Vhd {
     return this._write(rawHeader, offset)
   }
 
-  async writeData (offsetSectors, buffer) {
+  async writeData(offsetSectors, buffer) {
     const bufferSizeSectors = Math.ceil(buffer.length / SECTOR_SIZE)
     const startBlock = Math.floor(offsetSectors / this.sectorsPerBlock)
     const endBufferSectors = offsetSectors + bufferSizeSectors
@@ -588,7 +590,7 @@ export default class Vhd {
     await this.writeFooter()
   }
 
-  async ensureSpaceForParentLocators (neededSectors) {
+  async ensureSpaceForParentLocators(neededSectors) {
     const firstLocatorOffset = FOOTER_SIZE + HEADER_SIZE
     const currentSpace =
       Math.floor(this.header.tableOffset / SECTOR_SIZE) -
@@ -602,7 +604,7 @@ export default class Vhd {
     return firstLocatorOffset
   }
 
-  async setUniqueParentLocator (fileNameString) {
+  async setUniqueParentLocator(fileNameString) {
     const { header } = this
     header.parentLocatorEntry[0].platformCode = PLATFORM_W2KU
     const encodedFilename = Buffer.from(fileNameString, 'utf16le')

packages/xapi-explore-sr/package.json

@@ -15,6 +15,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xapi-explore-sr",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xapi-explore-sr",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -40,7 +41,7 @@
     "human-format": "^0.10.0",
     "lodash": "^4.17.4",
     "pw": "^0.0.4",
-    "xen-api": "^0.21.0"
+    "xen-api": "^0.24.2"
   },
   "devDependencies": {
     "@babel/cli": "^7.1.5",

packages/xapi-explore-sr/src/index.js

@@ -49,7 +49,7 @@ const getStyle = vdi => {
 }
 
 const mapFilter = (collection, iteratee, results = []) => {
-  forEach(collection, function () {
+  forEach(collection, function() {
     const result = iteratee.apply(this, arguments)
     if (result !== undefined) {
       results.push(result)

packages/xen-api/README.md

@@ -47,6 +47,7 @@ Options:
 - `allowUnauthorized`: whether to accept self-signed certificates
 - `auth`: credentials used to sign in (can also be specified in the URL)
 - `readOnly = false`: if true, no methods with side-effects can be called
+- `callTimeout`: number of milliseconds after which a call is considered failed (can also be a map of timeouts by methods)
 
 ```js
 // Force connection.
@@ -94,7 +95,7 @@ root@xen1.company.net> xapi.pool.$master.name_label
 To ease searches, `find()` and `findAll()` functions are available:
 
 ```
-root@xen1.company.net> findAll({ $type: 'vm' }).length
+root@xen1.company.net> findAll({ $type: 'VM' }).length
 183
 ```
 

packages/xen-api/examples/export-vdi

@@ -1,30 +1,42 @@
 #!/usr/bin/env node
 
-process.env.DEBUG = '*'
+process.env.DEBUG = 'xen-api'
 
+const createProgress = require('progress-stream')
+const createTop = require('process-top')
 const defer = require('golike-defer').default
-const pump = require('pump')
-const { CancelToken, fromCallback } = require('promise-toolbox')
+const getopts = require('getopts')
+const { CancelToken } = require('promise-toolbox')
 
 const { createClient } = require('../')
 
-const { createOutputStream, resolveRef } = require('./utils')
+const {
+  createOutputStream,
+  formatProgress,
+  pipeline,
+  resolveRecord,
+  throttle,
+} = require('./utils')
 
-defer(async ($defer, args) => {
-  let raw = false
-  if (args[0] === '--raw') {
-    raw = true
-    args.shift()
-  }
+defer(async ($defer, rawArgs) => {
+  const { raw, throttle: bps, _: args } = getopts(rawArgs, {
+    boolean: 'raw',
+    alias: {
+      raw: 'r',
+      throttle: 't',
+    },
+  })
 
   if (args.length < 2) {
-    return console.log('Usage: export-vdi [--raw] <XS URL> <VDI identifier> [<VHD file>]')
+    return console.log(
+      'Usage: export-vdi [--raw] <XS URL> <VDI identifier> [<VHD file>]'
+    )
   }
 
   const xapi = createClient({
     allowUnauthorized: true,
     url: args[0],
-    watchEvents: false
+    watchEvents: false,
   })
 
   await xapi.connect()
@@ -33,21 +45,32 @@ defer(async ($defer, args) => {
   const { cancel, token } = CancelToken.source()
   process.on('SIGINT', cancel)
 
+  const vdi = await resolveRecord(xapi, 'VDI', args[1])
+
   // https://xapi-project.github.io/xen-api/snapshots.html#downloading-a-disk-or-snapshot
   const exportStream = await xapi.getResource(token, '/export_raw_vdi/', {
     query: {
       format: raw ? 'raw' : 'vhd',
-      vdi: await resolveRef(xapi, 'VDI', args[1])
-    }
+      vdi: vdi.$ref,
+    },
   })
 
   console.warn('Export task:', exportStream.headers['task-id'])
 
-  await fromCallback(cb => pump(
+  const top = createTop()
+  const progressStream = createProgress()
+
+  $defer(
+    clearInterval,
+    setInterval(() => {
+      console.warn('\r %s | %s', top.toString(), formatProgress(progressStream.progress()))
+    }, 1e3)
+  )
+
+  await pipeline(
     exportStream,
-    createOutputStream(args[2]),
-    cb
-  ))
-})(process.argv.slice(2)).catch(
-  console.error.bind(console, 'error')
-)
+    progressStream,
+    throttle(bps),
+    createOutputStream(args[2])
+  )
+})(process.argv.slice(2)).catch(console.error.bind(console, 'error'))

packages/xen-api/examples/export-vm

@@ -2,15 +2,25 @@
 
 process.env.DEBUG = '*'
 
+const createProgress = require('progress-stream')
 const defer = require('golike-defer').default
-const pump = require('pump')
-const { CancelToken, fromCallback } = require('promise-toolbox')
+const getopts = require('getopts')
+const { CancelToken } = require('promise-toolbox')
 
 const { createClient } = require('../')
 
-const { createOutputStream, resolveRef } = require('./utils')
+const {
+  createOutputStream,
+  formatProgress,
+  pipeline,
+  resolveRecord,
+} = require('./utils')
+
+defer(async ($defer, rawArgs) => {
+  const { gzip, zstd, _: args } = getopts(rawArgs, {
+    boolean: ['gzip', 'zstd'],
+  })
 
-defer(async ($defer, args) => {
   if (args.length < 2) {
     return console.log('Usage: export-vm <XS URL> <VM identifier> [<XVA file>]')
   }
@@ -18,7 +28,7 @@ defer(async ($defer, args) => {
   const xapi = createClient({
     allowUnauthorized: true,
     url: args[0],
-    watchEvents: false
+    watchEvents: false,
   })
 
   await xapi.connect()
@@ -30,18 +40,16 @@ defer(async ($defer, args) => {
   // https://xapi-project.github.io/xen-api/importexport.html
   const exportStream = await xapi.getResource(token, '/export/', {
     query: {
-      ref: await resolveRef(xapi, 'VM', args[1]),
-      use_compression: 'true'
-    }
+      ref: (await resolveRecord(xapi, 'VM', args[1])).$ref,
+      use_compression: zstd ? 'zstd' : gzip ? 'true' : 'false',
+    },
   })
 
   console.warn('Export task:', exportStream.headers['task-id'])
 
-  await fromCallback(cb => pump(
+  await pipeline(
     exportStream,
-    createOutputStream(args[2]),
-    cb
-  ))
-})(process.argv.slice(2)).catch(
-  console.error.bind(console, 'error')
-)
+    createProgress({ time: 1e3 }, p => console.warn(formatProgress(p))),
+    createOutputStream(args[2])
+  )
+})(process.argv.slice(2)).catch(console.error.bind(console, 'error'))

packages/xen-api/examples/package.json

@@ -1,6 +1,12 @@
 {
   "dependencies": {
+    "getopts": "^2.2.3",
     "golike-defer": "^0.4.1",
-    "pump": "^3.0.0"
+    "human-format": "^0.10.1",
+    "process-top": "^1.0.0",
+    "progress-stream": "^2.0.0",
+    "promise-toolbox": "^0.11.0",
+    "readable-stream": "^3.1.1",
+    "throttle": "^1.0.3"
   }
 }

packages/xen-api/examples/utils.js

@@ -1,5 +1,8 @@
 const { createReadStream, createWriteStream, statSync } = require('fs')
-const { PassThrough } = require('stream')
+const { fromCallback } = require('promise-toolbox')
+const { PassThrough, pipeline } = require('readable-stream')
+const humanFormat = require('human-format')
+const Throttle = require('throttle')
 
 const { isOpaqueRef } = require('../')
 
@@ -26,7 +29,32 @@ exports.createOutputStream = path => {
   return stream
 }
 
-exports.resolveRef = (xapi, type, refOrUuidOrNameLabel) =>
+const formatSizeOpts = { scale: 'binary', unit: 'B' }
+const formatSize = bytes => humanFormat(bytes, formatSizeOpts)
+
+exports.formatProgress = p =>
+  [
+    formatSize(p.transferred),
+    ' / ',
+    formatSize(p.length),
+    ' | ',
+    p.runtime,
+    's / ',
+    p.eta,
+    's | ',
+    formatSize(p.speed),
+    '/s',
+  ].join('')
+
+exports.pipeline = (...streams) => {
+  return fromCallback(cb => {
+    streams = streams.filter(_ => _ != null)
+    streams.push(cb)
+    pipeline.apply(undefined, streams)
+  })
+}
+
+const resolveRef = (xapi, type, refOrUuidOrNameLabel) =>
   isOpaqueRef(refOrUuidOrNameLabel)
     ? refOrUuidOrNameLabel
     : xapi.call(`${type}.get_by_uuid`, refOrUuidOrNameLabel).catch(() =>
@@ -41,3 +69,10 @@ exports.resolveRef = (xapi, type, refOrUuidOrNameLabel) =>
             )
           })
       )
+
+exports.resolveRecord = async (xapi, type, refOrUuidOrNameLabel) =>
+  xapi.getRecord(type, await resolveRef(xapi, type, refOrUuidOrNameLabel))
+
+exports.resolveRef = resolveRef
+
+exports.throttle = opts => (opts != null ? new Throttle(opts) : undefined)

packages/xen-api/examples/yarn.lock

@@ -2,29 +2,178 @@
 # yarn lockfile v1
 
 
-end-of-stream@^1.1.0:
-  version "1.4.1"
-  resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.1.tgz#ed29634d19baba463b6ce6b80a37213eab71ec43"
+core-util-is@~1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
+  integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=
+
+debug@2:
+  version "2.6.9"
+  resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
+  integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==
   dependencies:
-    once "^1.4.0"
+    ms "2.0.0"
+
+event-loop-delay@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/event-loop-delay/-/event-loop-delay-1.0.0.tgz#5af6282549494fd0d868c499cbdd33e027978b8c"
+  integrity sha512-8YtyeIWHXrvTqlAhv+fmtaGGARmgStbvocERYzrZ3pwhnQULe5PuvMUTjIWw/emxssoaftfHZsJtkeY8xjiXCg==
+  dependencies:
+    napi-macros "^1.8.2"
+    node-gyp-build "^3.7.0"
+
+getopts@^2.2.3:
+  version "2.2.3"
+  resolved "https://registry.yarnpkg.com/getopts/-/getopts-2.2.3.tgz#11d229775e2ec2067ed8be6fcc39d9b4bf39cf7d"
+  integrity sha512-viEcb8TpgeG05+Nqo5EzZ8QR0hxdyrYDp6ZSTZqe2M/h53Bk036NmqG38Vhf5RGirC/Of9Xql+v66B2gp256SQ==
 
 golike-defer@^0.4.1:
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/golike-defer/-/golike-defer-0.4.1.tgz#7a1cd435d61e461305805d980b133a0f3db4e1cc"
 
-once@^1.3.1, once@^1.4.0:
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
-  dependencies:
-    wrappy "1"
+human-format@^0.10.1:
+  version "0.10.1"
+  resolved "https://registry.yarnpkg.com/human-format/-/human-format-0.10.1.tgz#107793f355912e256148d5b5dcf66a0230187ee9"
+  integrity sha512-UzCHToSw3HI9MxH9tYzMr1JbHJbgzr6o0hZCun7sruv59S1leps21bmgpBkkwEvQon5n/2OWKH1iU7BEko02cg==
 
-pump@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64"
-  dependencies:
-    end-of-stream "^1.1.0"
-    once "^1.3.1"
+inherits@^2.0.3, inherits@~2.0.3:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
+  integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=
 
-wrappy@1:
+isarray@~1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
+  integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=
+
+make-error@^1.3.2:
+  version "1.3.5"
+  resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.5.tgz#efe4e81f6db28cadd605c70f29c831b58ef776c8"
+  integrity sha512-c3sIjNUow0+8swNwVpqoH4YCShKNFkMaw6oH1mNS2haDZQqkeZFlHS3dhoeEbKKmJB4vXpJucU6oH75aDYeE9g==
+
+ms@2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
+  integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=
+
+napi-macros@^1.8.2:
+  version "1.8.2"
+  resolved "https://registry.yarnpkg.com/napi-macros/-/napi-macros-1.8.2.tgz#299265c1d8aa401351ad0675107d751228c03eda"
+  integrity sha512-Tr0DNY4RzTaBG2W2m3l7ZtFuJChTH6VZhXVhkGGjF/4cZTt+i8GcM9ozD+30Lmr4mDoZ5Xx34t2o4GJqYWDGcg==
+
+node-gyp-build@^3.7.0:
+  version "3.7.0"
+  resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-3.7.0.tgz#daa77a4f547b9aed3e2aac779eaf151afd60ec8d"
+  integrity sha512-L/Eg02Epx6Si2NXmedx+Okg+4UHqmaf3TNcxd50SF9NQGcJaON3AtU++kax69XV7YWz4tUspqZSAsVofhFKG2w==
+
+prettier-bytes@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/prettier-bytes/-/prettier-bytes-1.0.4.tgz#994b02aa46f699c50b6257b5faaa7fe2557e62d6"
+  integrity sha1-mUsCqkb2mcULYle1+qp/4lV+YtY=
+
+process-nextick-args@~2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa"
+  integrity sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==
+
+process-top@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/process-top/-/process-top-1.0.0.tgz#52892bedb581c5abf0df2d0aa5c429e34275cc7e"
+  integrity sha512-er8iSmBMslOt5cgIHg9m6zilTPsuUqpEb1yfQ4bDmO80zr/e/5hNn+Tay3CJM/FOBnJo8Bt3fFiDDH6GvIgeAg==
+  dependencies:
+    event-loop-delay "^1.0.0"
+    prettier-bytes "^1.0.4"
+
+progress-stream@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/progress-stream/-/progress-stream-2.0.0.tgz#fac63a0b3d11deacbb0969abcc93b214bce19ed5"
+  integrity sha1-+sY6Cz0R3qy7CWmrzJOyFLzhntU=
+  dependencies:
+    speedometer "~1.0.0"
+    through2 "~2.0.3"
+
+promise-toolbox@^0.11.0:
+  version "0.11.0"
+  resolved "https://registry.yarnpkg.com/promise-toolbox/-/promise-toolbox-0.11.0.tgz#9ed928355355395072dace3f879879504e07d1bc"
+  integrity sha512-bjHk0kq+Ke3J3zbkbbJH6kXCyQZbFHwOTrE/Et7vS0uS0tluoV+PLqU/kEyxl8aARM7v04y2wFoDo/wWAEPvjA==
+  dependencies:
+    make-error "^1.3.2"
+
+"readable-stream@>= 0.3.0", readable-stream@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.1.1.tgz#ed6bbc6c5ba58b090039ff18ce670515795aeb06"
+  integrity sha512-DkN66hPyqDhnIQ6Jcsvx9bFjhw214O4poMBcIMgPVpQvNy9a0e0Uhg5SqySyDKAmUlwt8LonTBz1ezOnM8pUdA==
+  dependencies:
+    inherits "^2.0.3"
+    string_decoder "^1.1.1"
+    util-deprecate "^1.0.1"
+
+readable-stream@~2.3.6:
+  version "2.3.6"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf"
+  integrity sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==
+  dependencies:
+    core-util-is "~1.0.0"
+    inherits "~2.0.3"
+    isarray "~1.0.0"
+    process-nextick-args "~2.0.0"
+    safe-buffer "~5.1.1"
+    string_decoder "~1.1.1"
+    util-deprecate "~1.0.1"
+
+safe-buffer@~5.1.0, safe-buffer@~5.1.1:
+  version "5.1.2"
+  resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
+  integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
+
+speedometer@~1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-1.0.0.tgz#cd671cb06752c22bca3370e2f334440be4fc62e2"
+  integrity sha1-zWccsGdSwivKM3Di8zREC+T8YuI=
+
+"stream-parser@>= 0.0.2":
+  version "0.3.1"
+  resolved "https://registry.yarnpkg.com/stream-parser/-/stream-parser-0.3.1.tgz#1618548694420021a1182ff0af1911c129761773"
+  integrity sha1-FhhUhpRCACGhGC/wrxkRwSl2F3M=
+  dependencies:
+    debug "2"
+
+string_decoder@^1.1.1:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.2.0.tgz#fe86e738b19544afe70469243b2a1ee9240eae8d"
+  integrity sha512-6YqyX6ZWEYguAxgZzHGL7SsCeGx3V2TtOTqZz1xSTSWnqsbWwbptafNyvf/ACquZUXV3DANr5BDIwNYe1mN42w==
+  dependencies:
+    safe-buffer "~5.1.0"
+
+string_decoder@~1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"
+  integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==
+  dependencies:
+    safe-buffer "~5.1.0"
+
+throttle@^1.0.3:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/throttle/-/throttle-1.0.3.tgz#8a32e4a15f1763d997948317c5ebe3ad8a41e4b7"
+  integrity sha1-ijLkoV8XY9mXlIMXxevjrYpB5Lc=
+  dependencies:
+    readable-stream ">= 0.3.0"
+    stream-parser ">= 0.0.2"
+
+through2@~2.0.3:
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd"
+  integrity sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==
+  dependencies:
+    readable-stream "~2.3.6"
+    xtend "~4.0.1"
+
+util-deprecate@^1.0.1, util-deprecate@~1.0.1:
   version "1.0.2"
-  resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
+  resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
+  integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=
+
+xtend@~4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"
+  integrity sha1-pcbVMr5lbiPbgg77lDofBJmNY68=

packages/xen-api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xen-api",
-  "version": "0.21.0",
+  "version": "0.24.2",
   "license": "ISC",
   "description": "Connector to the Xen API",
   "keywords": [
@@ -13,6 +13,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xen-api",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xen-api",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -36,7 +37,7 @@
     "debug": "^4.0.1",
     "event-to-promise": "^0.8.0",
     "exec-promise": "^0.7.0",
-    "http-request-plus": "^0.6.0",
+    "http-request-plus": "^0.7.2",
     "iterable-backoff": "^0.0.0",
     "jest-diff": "^23.5.0",
     "json-rpc-protocol": "^0.13.1",

packages/xen-api/src/_replaceSensitiveValues.js

@@ -1,7 +1,7 @@
 import mapValues from 'lodash/mapValues'
 
-export default function replaceSensitiveValues (value, replacement) {
-  function helper (value, name) {
+export default function replaceSensitiveValues(value, replacement) {
+  function helper(value, name) {
     if (name === 'password' && typeof value === 'string') {
       return replacement
     }

packages/xen-api/src/cli.js

@@ -15,7 +15,7 @@ import { createClient } from './'
 
 // ===================================================================
 
-function askPassword (prompt = 'Password: ') {
+function askPassword(prompt = 'Password: ') {
   if (prompt) {
     process.stdout.write(prompt)
   }

packages/xen-api/src/index.js

@@ -7,8 +7,8 @@ import { BaseError } from 'make-error'
 import { EventEmitter } from 'events'
 import { fibonacci } from 'iterable-backoff'
 import {
-  filter,
   forEach,
+  forOwn,
   isArray,
   isInteger,
   map,
@@ -22,6 +22,7 @@ import {
   cancelable,
   defer,
   fromEvents,
+  ignoreErrors,
   pCatch,
   pDelay,
   pFinally,
@@ -87,7 +88,7 @@ const isSessionInvalid = ({ code }) => code === 'SESSION_INVALID'
 // -------------------------------------------------------------------
 
 class XapiError extends BaseError {
-  constructor (code, params) {
+  constructor(code, params) {
     super(`${code}(${params.join(', ')})`)
 
     this.code = code
@@ -108,6 +109,9 @@ export const wrapError = error => {
   } else {
     code = error.message
     params = error.data
+    if (!isArray(params)) {
+      params = []
+    }
   }
   return new XapiError(code, params)
 }
@@ -209,6 +213,34 @@ const getTaskResult = task => {
   }
 }
 
+function defined() {
+  for (let i = 0, n = arguments.length; i < n; ++i) {
+    const arg = arguments[i]
+    if (arg !== undefined) {
+      return arg
+    }
+  }
+}
+
+// TODO: find a better name
+// TODO: merge into promise-toolbox?
+const dontWait = promise => {
+  // https://github.com/JsCommunity/promise-toolbox#promiseignoreerrors
+  ignoreErrors.call(promise)
+
+  // http://bluebirdjs.com/docs/warning-explanations.html#warning-a-promise-was-created-in-a-handler-but-was-not-returned-from-it
+  return null
+}
+
+const makeCallSetting = (setting, defaultValue) =>
+  setting === undefined
+    ? () => defaultValue
+    : typeof setting === 'function'
+    ? setting
+    : typeof setting !== 'object'
+    ? () => setting
+    : method => defined(setting[method], setting['*'], defaultValue)
+
 // -------------------------------------------------------------------
 
 const RESERVED_FIELDS = {
@@ -216,6 +248,11 @@ const RESERVED_FIELDS = {
   pool: true,
   ref: true,
   type: true,
+  xapi: true,
+}
+
+function getPool() {
+  return this.$xapi.pool
 }
 
 // -------------------------------------------------------------------
@@ -224,20 +261,24 @@ const CONNECTED = 'connected'
 const CONNECTING = 'connecting'
 const DISCONNECTED = 'disconnected'
 
+// timeout of XenAPI HTTP connections
+const HTTP_TIMEOUT = 24 * 3600 * 1e3
+
 // -------------------------------------------------------------------
 
 export class Xapi extends EventEmitter {
-  constructor (opts) {
+  constructor(opts) {
     super()
 
     this._allowUnauthorized = opts.allowUnauthorized
-    this._auth = opts.auth
+    this._callTimeout = makeCallSetting(opts.callTimeout, 0)
     this._pool = null
     this._readOnly = Boolean(opts.readOnly)
     this._RecordsByType = createObject(null)
     this._sessionId = null
-    const url = (this._url = parseUrl(opts.url))
 
+    this._auth = opts.auth
+    const url = (this._url = parseUrl(opts.url))
     if (this._auth === undefined) {
       const user = url.username
       if (user !== undefined) {
@@ -250,39 +291,43 @@ export class Xapi extends EventEmitter {
       }
     }
 
-    if (opts.watchEvents !== false) {
-      this._debounce = opts.debounce == null ? 200 : opts.debounce
+    ;(this._objects = new Collection()).getKey = getKey
+    this._debounce = opts.debounce == null ? 200 : opts.debounce
+    this._watchedTypes = undefined
+    this._watching = false
 
-      this._eventWatchers = createObject(null)
+    this.on(DISCONNECTED, this._clearObjects)
+    this._clearObjects()
 
-      this._fromToken = ''
-
-      // Memoize this function _addObject().
-      this._getPool = () => this._pool
-
-      this._nTasks = 0
-
-      const objects = (this._objects = new Collection())
-      objects.getKey = getKey
-
-      this._objectsByRef = createObject(null)
-      this._objectsByRef[NULL_REF] = undefined
-
-      this._taskWatchers = Object.create(null)
-
-      this.on('connected', this._watchEvents)
-      this.on('disconnected', () => {
-        this._fromToken = ''
-        objects.clear()
-      })
+    const { watchEvents } = opts
+    if (watchEvents !== false) {
+      if (Array.isArray(watchEvents)) {
+        this._watchedTypes = watchEvents
+      }
+      this.watchEvents()
     }
   }
 
-  get _url () {
+  watchEvents() {
+    this._eventWatchers = createObject(null)
+
+    this._taskWatchers = Object.create(null)
+
+    if (this.status === CONNECTED) {
+      this._watchEventsWrapper()
+    }
+
+    this.on('connected', this._watchEventsWrapper)
+    this.on('disconnected', () => {
+      this._objects.clear()
+    })
+  }
+
+  get _url() {
     return this.__url
   }
 
-  set _url (url) {
+  set _url(url) {
     this.__url = url
     this._call = autoTransport({
       allowUnauthorized: this._allowUnauthorized,
@@ -290,15 +335,15 @@ export class Xapi extends EventEmitter {
     })
   }
 
-  get readOnly () {
+  get readOnly() {
     return this._readOnly
   }
 
-  set readOnly (ro) {
+  set readOnly(ro) {
     this._readOnly = Boolean(ro)
   }
 
-  get sessionId () {
+  get sessionId() {
     const id = this._sessionId
 
     if (!id || id === CONNECTING) {
@@ -308,20 +353,20 @@ export class Xapi extends EventEmitter {
     return id
   }
 
-  get status () {
+  get status() {
     const id = this._sessionId
 
     return id ? (id === CONNECTING ? CONNECTING : CONNECTED) : DISCONNECTED
   }
 
-  get _humanId () {
+  get _humanId() {
     return `${this._auth.user}@${this._url.hostname}`
   }
 
   // ensure we have received all events up to this call
   //
   // optionally returns the up to date object for the given ref
-  barrier (ref) {
+  barrier(ref) {
     const eventWatchers = this._eventWatchers
     if (eventWatchers === undefined) {
       return Promise.reject(
@@ -362,44 +407,58 @@ export class Xapi extends EventEmitter {
     )
   }
 
-  connect () {
+  async connect() {
     const { status } = this
 
     if (status === CONNECTED) {
-      return Promise.reject(new Error('already connected'))
+      throw new Error('already connected')
     }
 
     if (status === CONNECTING) {
-      return Promise.reject(new Error('already connecting'))
+      throw new Error('already connecting')
     }
 
     const auth = this._auth
     if (auth === undefined) {
-      return Promise.reject(new Error('missing credentials'))
+      throw new Error('missing credentials')
     }
 
     this._sessionId = CONNECTING
 
-    return this._transportCall('session.login_with_password', [
-      auth.user,
-      auth.password,
-    ]).then(
-      sessionId => {
-        this._sessionId = sessionId
+    try {
+      const [methods, sessionId] = await Promise.all([
+        this._transportCall('system.listMethods', []),
+        this._transportCall('session.login_with_password', [
+          auth.user,
+          auth.password,
+        ]),
+      ])
 
-        debug('%s: connected', this._humanId)
+      // Uses introspection to list available types.
+      const types = (this._types = methods
+        .filter(isGetAllRecordsMethod)
+        .map(method => method.slice(0, method.indexOf('.'))))
+      this._lcToTypes = { __proto__: null }
+      types.forEach(type => {
+        const lcType = type.toLowerCase()
+        if (lcType !== type) {
+          this._lcToTypes[lcType] = type
+        }
+      })
 
-        this.emit(CONNECTED)
-      },
-      error => {
-        this._sessionId = null
+      this._sessionId = sessionId
+      this._pool = (await this.getAllRecords('pool'))[0]
 
-        throw error
-      }
-    )
+      debug('%s: connected', this._humanId)
+      this.emit(CONNECTED)
+    } catch (error) {
+      this._sessionId = null
+
+      throw error
+    }
   }
 
-  disconnect () {
+  disconnect() {
     return Promise.resolve().then(() => {
       const { status } = this
 
@@ -418,25 +477,25 @@ export class Xapi extends EventEmitter {
   }
 
   // High level calls.
-  call (method, ...args) {
+  call(method, ...args) {
     return this._readOnly && !isReadOnlyCall(method, args)
       ? Promise.reject(new Error(`cannot call ${method}() in read only mode`))
       : this._sessionCall(method, prepareParam(args))
   }
 
   @cancelable
-  callAsync ($cancelToken, method, ...args) {
+  callAsync($cancelToken, method, ...args) {
     return this._readOnly && !isReadOnlyCall(method, args)
       ? Promise.reject(new Error(`cannot call ${method}() in read only mode`))
       : this._sessionCall(`Async.${method}`, args).then(taskRef => {
-          $cancelToken.promise.then(() => {
+          $cancelToken.promise.then(() =>
             // TODO: do not trigger if the task is already over
-            this._sessionCall('task.cancel', [taskRef]).catch(noop)
-          })
+            dontWait(this._sessionCall('task.cancel', [taskRef]))
+          )
 
-          return pFinally.call(this.watchTask(taskRef), () => {
-            this._sessionCall('task.destroy', [taskRef]).catch(noop)
-          })
+          return pFinally.call(this.watchTask(taskRef), () =>
+            dontWait(this._sessionCall('task.destroy', [taskRef]))
+          )
         })
   }
 
@@ -444,7 +503,7 @@ export class Xapi extends EventEmitter {
   //
   //  allowed even in read-only mode because it does not have impact on the
   //  XenServer and it's necessary for getResource()
-  createTask (nameLabel, nameDescription = '') {
+  createTask(nameLabel, nameDescription = '') {
     const promise = this._sessionCall('task.create', [
       nameLabel,
       nameDescription,
@@ -459,10 +518,14 @@ export class Xapi extends EventEmitter {
     return promise
   }
 
+  getField(type, ref, field) {
+    return this._sessionCall(`${type}.get_${field}`, [ref])
+  }
+
   // Nice getter which returns the object for a given $id (internal to
   // this lib), UUID (unique identifier that some objects have) or
   // opaque reference (internal to XAPI).
-  getObject (idOrUuidOrRef, defaultValue) {
+  getObject(idOrUuidOrRef, defaultValue) {
     if (typeof idOrUuidOrRef === 'object') {
       idOrUuidOrRef = idOrUuidOrRef.$id
     }
@@ -479,7 +542,7 @@ export class Xapi extends EventEmitter {
 
   // Returns the object for a given opaque reference (internal to
   // XAPI).
-  getObjectByRef (ref, defaultValue) {
+  getObjectByRef(ref, defaultValue) {
     const object = this._objectsByRef[ref]
 
     if (object !== undefined) return object
@@ -491,7 +554,7 @@ export class Xapi extends EventEmitter {
 
   // Returns the object for a given UUID (unique identifier that some
   // objects have).
-  getObjectByUuid (uuid, defaultValue) {
+  getObjectByUuid(uuid, defaultValue) {
     // Objects ids are already UUIDs if they have one.
     const object = this._objects.all[uuid]
 
@@ -502,13 +565,26 @@ export class Xapi extends EventEmitter {
     throw new Error('no object with UUID: ' + uuid)
   }
 
-  async getRecord (type, ref) {
+  async getRecord(type, ref) {
     return this._wrapRecord(
+      type,
+      ref,
       await this._sessionCall(`${type}.get_record`, [ref])
     )
   }
 
-  async getRecordByUuid (type, uuid) {
+  getRecords(type, refs) {
+    return Promise.all(refs.map(ref => this.getRecord(type, ref)))
+  }
+
+  async getAllRecords(type) {
+    return map(
+      await this._sessionCall(`${type}.get_all_records`),
+      (record, ref) => this._wrapRecord(type, ref, record)
+    )
+  }
+
+  async getRecordByUuid(type, uuid) {
     return this.getRecord(
       type,
       await this._sessionCall(`${type}.get_by_uuid`, [uuid])
@@ -516,7 +592,7 @@ export class Xapi extends EventEmitter {
   }
 
   @cancelable
-  getResource ($cancelToken, pathname, { host, query, task }) {
+  getResource($cancelToken, pathname, { host, query, task } = {}) {
     return this._autoTask(task, `Xapi#getResource ${pathname}`).then(
       taskRef => {
         query = { ...query, session_id: this.sessionId }
@@ -530,17 +606,20 @@ export class Xapi extends EventEmitter {
           }
         }
 
-        let promise = httpRequest(
-          $cancelToken,
-          this._url,
-          host && {
-            hostname: this.getObject(host).address,
-          },
-          {
-            pathname,
-            query,
-            rejectUnauthorized: !this._allowUnauthorized,
-          }
+        let promise = pTimeout.call(
+          httpRequest(
+            $cancelToken,
+            this._url,
+            host && {
+              hostname: this.getObject(host).address,
+            },
+            {
+              pathname,
+              query,
+              rejectUnauthorized: !this._allowUnauthorized,
+            }
+          ),
+          HTTP_TIMEOUT
         )
 
         if (taskResult !== undefined) {
@@ -556,7 +635,7 @@ export class Xapi extends EventEmitter {
   }
 
   @cancelable
-  putResource ($cancelToken, body, pathname, { host, query, task } = {}) {
+  putResource($cancelToken, body, pathname, { host, query, task } = {}) {
     if (this._readOnly) {
       return Promise.reject(
         new Error(new Error('cannot put resource in read only mode'))
@@ -588,21 +667,23 @@ export class Xapi extends EventEmitter {
         }
 
         const doRequest = (...opts) =>
-          httpRequest.put(
-            $cancelToken,
-            this._url,
-            host && {
-              hostname: this.getObject(host).address,
-            },
-            {
-              body,
-              headers,
-              query,
-              pathname,
-              maxRedirects: 0,
-              rejectUnauthorized: !this._allowUnauthorized,
-            },
-            ...opts
+          pTimeout.call(
+            httpRequest.put(
+              $cancelToken,
+              this._url,
+              host && {
+                hostname: this.getObject(host).address,
+              },
+              {
+                body,
+                headers,
+                query,
+                pathname,
+                rejectUnauthorized: !this._allowUnauthorized,
+              },
+              ...opts
+            ),
+            HTTP_TIMEOUT
           )
 
         // if a stream, sends a dummy request to probe for a
@@ -613,15 +694,17 @@ export class Xapi extends EventEmitter {
 
               // omit task_id because this request will fail on purpose
               query: 'task_id' in query ? omit(query, 'task_id') : query,
+
+              maxRedirects: 0,
             }).then(
               response => {
-                response.req.abort()
+                response.cancel()
                 return doRequest()
               },
               error => {
                 let response
                 if (error != null && (response = error.response) != null) {
-                  response.req.abort()
+                  response.cancel()
 
                   const {
                     headers: { location },
@@ -649,12 +732,12 @@ export class Xapi extends EventEmitter {
           }
 
           if (req.finished) {
-            req.abort()
+            response.cancel()
             return taskResult
           }
 
           return fromEvents(req, ['close', 'finish']).then(() => {
-            req.abort()
+            response.cancel()
             return taskResult
           })
         })
@@ -662,42 +745,39 @@ export class Xapi extends EventEmitter {
     )
   }
 
-  setField ({ $type, $ref }, field, value) {
-    return this.call(`${$type}.set_${field}`, $ref, value).then(noop)
+  setField(type, ref, field, value) {
+    return this.call(`${type}.set_${field}`, ref, value).then(noop)
   }
 
-  setFieldEntries (record, field, entries) {
+  setFieldEntries(type, ref, field, entries) {
     return Promise.all(
       getKeys(entries).map(entry => {
         const value = entries[entry]
         if (value !== undefined) {
-          return value === null
-            ? this.unsetFieldEntry(record, field, entry)
-            : this.setFieldEntry(record, field, entry, value)
+          return this.setFieldEntry(type, ref, field, entry, value)
         }
       })
     ).then(noop)
   }
 
-  async setFieldEntry ({ $type, $ref }, field, entry, value) {
+  async setFieldEntry(type, ref, field, entry, value) {
+    if (value === null) {
+      return this.call(`${type}.remove_from_${field}`, ref, entry).then(noop)
+    }
     while (true) {
       try {
-        await this.call(`${$type}.add_to_${field}`, $ref, entry, value)
+        await this.call(`${type}.add_to_${field}`, ref, entry, value)
         return
       } catch (error) {
         if (error == null || error.code !== 'MAP_DUPLICATE_KEY') {
           throw error
         }
       }
-      await this.unsetFieldEntry({ $type, $ref }, field, entry)
+      await this.call(`${type}.remove_from_${field}`, ref, entry)
     }
   }
 
-  unsetFieldEntry ({ $type, $ref }, field, entry) {
-    return this.call(`${$type}.remove_from_${field}`, $ref, entry)
-  }
-
-  watchTask (ref) {
+  watchTask(ref) {
     const watchers = this._taskWatchers
     if (watchers === undefined) {
       throw new Error('Xapi#watchTask() requires events watching')
@@ -722,16 +802,25 @@ export class Xapi extends EventEmitter {
     return watcher.promise
   }
 
-  get pool () {
+  get pool() {
     return this._pool
   }
 
-  get objects () {
+  get objects() {
     return this._objects
   }
 
+  _clearObjects() {
+    ;(this._objectsByRef = createObject(null))[NULL_REF] = undefined
+    this._nTasks = 0
+    this._objects.clear()
+    this.objectsFetched = new Promise(resolve => {
+      this._resolveObjectsFetched = resolve
+    })
+  }
+
   // return a promise which resolves to a task ref or undefined
-  _autoTask (task = this._taskWatchers !== undefined, name) {
+  _autoTask(task = this._taskWatchers !== undefined, name) {
     if (task === false) {
       return Promise.resolve()
     }
@@ -745,7 +834,7 @@ export class Xapi extends EventEmitter {
   }
 
   // Medium level call: handle session errors.
-  _sessionCall (method, args) {
+  _sessionCall(method, args, timeout = this._callTimeout(method, args)) {
     try {
       if (startsWith(method, 'session.')) {
         throw new Error('session.*() methods are disabled from this interface')
@@ -756,24 +845,27 @@ export class Xapi extends EventEmitter {
         newArgs.push.apply(newArgs, args)
       }
 
-      return pCatch.call(
-        this._transportCall(method, newArgs),
-        isSessionInvalid,
-        () => {
-          // XAPI is sometimes reinitialized and sessions are lost.
-          // Try to login again.
-          debug('%s: the session has been reinitialized', this._humanId)
+      return pTimeout.call(
+        pCatch.call(
+          this._transportCall(method, newArgs),
+          isSessionInvalid,
+          () => {
+            // XAPI is sometimes reinitialized and sessions are lost.
+            // Try to login again.
+            debug('%s: the session has been reinitialized', this._humanId)
 
-          this._sessionId = null
-          return this.connect().then(() => this._sessionCall(method, args))
-        }
+            this._sessionId = null
+            return this.connect().then(() => this._sessionCall(method, args))
+          }
+        ),
+        timeout
       )
     } catch (error) {
       return Promise.reject(error)
     }
   }
 
-  _addObject (type, ref, object) {
+  _addObject(type, ref, object) {
     object = this._wrapRecord(type, ref, object)
 
     // Finally freezes the object.
@@ -820,7 +912,7 @@ export class Xapi extends EventEmitter {
     }
   }
 
-  _removeObject (type, ref) {
+  _removeObject(type, ref) {
     const byRefs = this._objectsByRef
     const object = byRefs[ref]
     if (object !== undefined) {
@@ -843,9 +935,14 @@ export class Xapi extends EventEmitter {
     }
   }
 
-  _processEvents (events) {
+  _processEvents(events) {
     forEach(events, event => {
-      const { class: type, ref } = event
+      let type = event.class
+      const lcToTypes = this._lcToTypes
+      if (type in lcToTypes) {
+        type = lcToTypes[type]
+      }
+      const { ref } = event
       if (event.operation === 'del') {
         this._removeObject(type, ref)
       } else {
@@ -854,34 +951,101 @@ export class Xapi extends EventEmitter {
     })
   }
 
-  _watchEvents () {
-    const loop = () =>
-      this.status === CONNECTED &&
-      pTimeout
-        .call(
-          this._sessionCall('event.from', [
-            ['*'],
-            this._fromToken,
-            EVENT_TIMEOUT + 0.1, // Force float.
-          ]),
-          EVENT_TIMEOUT * 1.1e3 // 10% longer than the XenAPI timeout
+  // - prevent multiple watches
+  // - swallow errors
+  async _watchEventsWrapper() {
+    if (!this._watching) {
+      this._watching = true
+      await ignoreErrors.call(this._watchEvents())
+      this._watching = false
+    }
+  }
+
+  // TODO: cancelation
+  async _watchEvents() {
+    this._clearObjects()
+
+    // compute the initial token for the event loop
+    //
+    // we need to do this before the initial fetch to avoid losing events
+    let fromToken
+    try {
+      fromToken = await this._sessionCall('event.inject', [
+        'pool',
+        this._pool.$ref,
+      ])
+    } catch (error) {
+      if (isMethodUnknown(error)) {
+        return this._watchEventsLegacy()
+      }
+    }
+
+    const types = this._watchedTypes || this._types
+
+    // initial fetch
+    const flush = this.objects.bufferEvents()
+    try {
+      await Promise.all(
+        types.map(async type => {
+          try {
+            // FIXME: use _transportCall to avoid auto-reconnection
+            forOwn(
+              await this._sessionCall(`${type}.get_all_records`),
+              (record, ref) => {
+                // we can bypass _processEvents here because they are all *add*
+                // event and all objects are of the same type
+                this._addObject(type, ref, record)
+              }
+            )
+          } catch (_) {
+            // there is nothing ideal to do here, do not interrupt event
+            // handling
+          }
+        })
+      )
+    } finally {
+      flush()
+    }
+    this._resolveObjectsFetched()
+
+    // event loop
+    const debounce = this._debounce
+    while (true) {
+      if (debounce != null) {
+        await pDelay(debounce)
+      }
+
+      let result
+      try {
+        result = await this._sessionCall(
+          'event.from',
+          [types, fromToken, EVENT_TIMEOUT],
+          EVENT_TIMEOUT * 1.1
         )
-        .then(onSuccess, onFailure)
+      } catch (error) {
+        if (error instanceof TimeoutError) {
+          continue
+        }
+        if (areEventsLost(error)) {
+          return this._watchEvents()
+        }
+        throw error
+      }
 
-    const onSuccess = ({ events, token, valid_ref_counts: { task } }) => {
-      this._fromToken = token
-      this._processEvents(events)
+      fromToken = result.token
+      this._processEvents(result.events)
 
-      if (task !== this._nTasks) {
-        this._sessionCall('task.get_all_records')
-          .then(tasks => {
+      // detect and fix disappearing tasks (e.g. when toolstack restarts)
+      if (result.valid_ref_counts.task !== this._nTasks) {
+        await ignoreErrors.call(
+          this._sessionCall('task.get_all_records').then(tasks => {
             const toRemove = new Set()
-            forEach(this.objects.all, object => {
+            forOwn(this.objects.all, object => {
               if (object.$type === 'task') {
                 toRemove.add(object.$ref)
               }
             })
-            forEach(tasks, (task, ref) => {
+            forOwn(tasks, (task, ref) => {
               toRemove.delete(ref)
               this._addObject('task', ref, task)
             })
@@ -889,56 +1053,23 @@ export class Xapi extends EventEmitter {
               this._removeObject('task', ref)
             })
           })
-          .catch(noop)
+        )
       }
-
-      const debounce = this._debounce
-      return debounce != null ? pDelay(debounce).then(loop) : loop()
     }
-    const onFailure = error => {
-      if (error instanceof TimeoutError) {
-        return loop()
-      }
-
-      if (areEventsLost(error)) {
-        this._fromToken = ''
-        this._objects.clear()
-
-        return loop()
-      }
-
-      throw error
-    }
-
-    return pCatch.call(
-      loop(),
-      isMethodUnknown,
-
-      // If the server failed, it is probably due to an excessively
-      // large response.
-      // Falling back to legacy events watch should be enough.
-      error => error && error.res && error.res.statusCode === 500,
-
-      () => this._watchEventsLegacy()
-    )
   }
 
   // This method watches events using the legacy `event.next` XAPI
   // methods.
   //
   // It also has to manually get all objects first.
-  _watchEventsLegacy () {
-    const getAllObjects = () => {
-      return this._sessionCall('system.listMethods').then(methods => {
-        // Uses introspection to determine the methods to use to get
-        // all objects.
-        const getAllRecordsMethods = filter(methods, isGetAllRecordsMethod)
-
-        return Promise.all(
-          map(getAllRecordsMethods, method =>
-            this._sessionCall(method).then(
+  _watchEventsLegacy() {
+    const getAllObjects = async () => {
+      const flush = this.objects.bufferEvents()
+      try {
+        await Promise.all(
+          this._types.map(type =>
+            this._sessionCall(`${type}.get_all_records`).then(
               objects => {
-                const type = method.slice(0, method.indexOf('.')).toLowerCase()
                 forEach(objects, (object, ref) => {
                   this._addObject(type, ref, object)
                 })
@@ -951,7 +1082,10 @@ export class Xapi extends EventEmitter {
             )
           )
         )
-      })
+      } finally {
+        flush()
+      }
+      this._resolveObjectsFetched()
     }
 
     const watchEvents = () =>
@@ -979,7 +1113,7 @@ export class Xapi extends EventEmitter {
     return getAllObjects().then(watchEvents)
   }
 
-  _wrapRecord (type, ref, data) {
+  _wrapRecord(type, ref, data) {
     const RecordsByType = this._RecordsByType
     let Record = RecordsByType[type]
     if (Record === undefined) {
@@ -987,13 +1121,13 @@ export class Xapi extends EventEmitter {
       const nFields = fields.length
       const xapi = this
 
-      const objectsByRef = this._objectsByRef
-      const getObjectByRef = ref => objectsByRef[ref]
+      const getObjectByRef = ref => this._objectsByRef[ref]
 
-      Record = function (ref, data) {
+      Record = function(ref, data) {
         defineProperties(this, {
           $id: { value: data.uuid || ref },
           $ref: { value: ref },
+          $xapi: { value: xapi },
         })
         for (let i = 0; i < nFields; ++i) {
           const field = fields[i]
@@ -1001,11 +1135,11 @@ export class Xapi extends EventEmitter {
         }
       }
 
-      const getters = { $pool: this._getPool }
+      const getters = { $pool: getPool }
       const props = { $type: type }
       fields.forEach(field => {
-        props[`set_${field}`] = function (value) {
-          return xapi.setField(this, field, value)
+        props[`set_${field}`] = function(value) {
+          return xapi.setField(this.$type, this.$ref, field, value)
         }
 
         const $field = (field in RESERVED_FIELDS ? '$$' : '$') + field
@@ -1013,32 +1147,37 @@ export class Xapi extends EventEmitter {
         const value = data[field]
         if (isArray(value)) {
           if (value.length === 0 || isOpaqueRef(value[0])) {
-            getters[$field] = function () {
+            getters[$field] = function() {
               const value = this[field]
               return value.length === 0 ? value : value.map(getObjectByRef)
             }
           }
 
-          props[`add_to_${field}`] = function (...values) {
+          props[`add_to_${field}`] = function(...values) {
             return xapi
               .call(`${type}.add_${field}`, this.$ref, values)
               .then(noop)
           }
         } else if (value !== null && typeof value === 'object') {
-          getters[$field] = function () {
+          getters[$field] = function() {
             const value = this[field]
             const result = {}
             getKeys(value).forEach(key => {
-              result[key] = objectsByRef[value[key]]
+              result[key] = xapi._objectsByRef[value[key]]
             })
             return result
           }
-          props[`update_${field}`] = function (entries) {
-            return xapi.setFieldEntries(this, field, entries)
+          props[`update_${field}`] = function(entries, value) {
+            return typeof entries === 'string'
+              ? xapi.setFieldEntry(this.$type, this.$ref, field, entries, value)
+              : xapi.setFieldEntries(this.$type, this.$ref, field, entries)
           }
-        } else if (isOpaqueRef(value)) {
-          getters[$field] = function () {
-            return objectsByRef[this[field]]
+        } else if (value === '' || isOpaqueRef(value)) {
+          // 2019-02-07 - JFT: even if `value` should not be an empty string for
+          // a ref property, an user had the case on XenServer 7.0 on the CD VBD
+          // of a VM created by XenCenter
+          getters[$field] = function() {
+            return xapi._objectsByRef[this[field]]
           }
         }
       })
@@ -1066,21 +1205,29 @@ export class Xapi extends EventEmitter {
 
 Xapi.prototype._transportCall = reduce(
   [
-    function (method, args) {
-      return this._call(method, args).catch(error => {
-        if (!(error instanceof Error)) {
-          error = wrapError(error)
-        }
+    function(method, args) {
+      return pTimeout
+        .call(this._call(method, args), HTTP_TIMEOUT)
+        .catch(error => {
+          if (!(error instanceof Error)) {
+            error = wrapError(error)
+          }
 
-        error.call = {
-          method,
-          params: replaceSensitiveValues(args, '* obfuscated *'),
-        }
-        throw error
-      })
+          // do not log the session ID
+          //
+          // TODO: should log at the session level to avoid logging sensitive
+          // values?
+          const params = args[0] === this._sessionId ? args.slice(1) : args
+
+          error.call = {
+            method,
+            params: replaceSensitiveValues(params, '* obfuscated *'),
+          }
+          throw error
+        })
     },
     call =>
-      function () {
+      function() {
         let iterator // lazily created
         const loop = () =>
           pCatch.call(
@@ -1121,7 +1268,7 @@ Xapi.prototype._transportCall = reduce(
         return loop()
       },
     call =>
-      function loop () {
+      function loop() {
         return pCatch.call(
           call.apply(this, arguments),
           isHostSlave,
@@ -1144,7 +1291,7 @@ Xapi.prototype._transportCall = reduce(
         )
       },
     call =>
-      function (method) {
+      function(method) {
         const startTime = Date.now()
         return call.apply(this, arguments).then(
           result => {

packages/xen-api/src/transports/auto.js

@@ -10,7 +10,7 @@ export default opts => {
   let i = 0
 
   let call
-  function create () {
+  function create() {
     const current = factories[i++](opts)
     if (i < length) {
       const currentI = i