feat(xo-acl-resolver): 0.4.0

Related to #3699

This should improve user experience when changing pages.

.eslintrc.js

@@ -4,6 +4,7 @@ module.exports = {
     __DEV__: true,
     $Dict: true,
     $Diff: true,
+    $ElementType: true,
     $Exact: true,
     $Keys: true,
     $PropertyType: true,

.travis.yml

@@ -2,7 +2,6 @@ language: node_js
 node_js:
   #- stable # disable for now due to an issue of indirect dep upath with Node 9
   - 8
-  - 6
 
 # Use containers.
 # http://docs.travis-ci.com/user/workers/container-based-infrastructure/
@@ -10,9 +9,9 @@ sudo: false
 addons:
   apt:
     packages:
-    - qemu-utils
-    - blktap-utils
-    - vmdk-stream-converter
+      - qemu-utils
+      - blktap-utils
+      - vmdk-stream-converter
 
 before_install:
   - curl -o- -L https://yarnpkg.com/install.sh | bash
@@ -22,5 +21,4 @@ cache:
   yarn: true
 
 script:
-  - yarn run test
-  - yarn run test-integration
+  - yarn run travis-tests

@xen-orchestra/babel-config/index.js

@@ -42,8 +42,8 @@ const getConfig = (key, ...args) => {
   return config === undefined
     ? {}
     : typeof config === 'function'
-      ? config(...args)
-      : config
+    ? config(...args)
+    : config
 }
 
 module.exports = function (pkg, plugins, presets) {

@xen-orchestra/cr-seed-cli/package.json

@@ -15,6 +15,6 @@
   },
   "dependencies": {
     "golike-defer": "^0.4.1",
-    "xen-api": "^0.19.0"
+    "xen-api": "^0.21.0"
   }
 }

@xen-orchestra/cron/src/index.js

@@ -8,7 +8,12 @@ const MAX_DELAY = 2 ** 31 - 1
 class Job {
   constructor (schedule, fn) {
     const wrapper = () => {
-      const result = fn()
+      let result
+      try {
+        result = fn()
+      } catch (_) {
+        // catch any thrown value to ensure it does not break the job
+      }
       let then
       if (result != null && typeof (then = result.then) === 'function') {
         then.call(result, scheduleNext, scheduleNext)
@@ -45,8 +50,8 @@ class Schedule {
       zone.toLowerCase() === 'utc'
         ? moment.utc
         : zone === 'local'
-          ? moment
-          : () => moment.tz(zone)
+        ? moment
+        : () => moment.tz(zone)
   }
 
   createJob (fn) {

@xen-orchestra/fs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/fs",
-  "version": "0.3.1",
+  "version": "0.4.1",
   "license": "AGPL-3.0",
   "description": "The File System for Xen Orchestra backups.",
   "keywords": [],
@@ -25,8 +25,8 @@
     "fs-extra": "^7.0.0",
     "get-stream": "^4.0.0",
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.10.1",
-    "through2": "^2.0.3",
+    "promise-toolbox": "^0.11.0",
+    "through2": "^3.0.0",
     "tmp": "^0.0.33",
     "xo-remote-parser": "^0.5.0"
   },

@xen-orchestra/fs/src/abstract.js

@@ -1,8 +1,9 @@
 // @flow
 
+// $FlowFixMe
 import getStream from 'get-stream'
 import { randomBytes } from 'crypto'
-import { fromCallback, fromEvent, ignoreErrors } from 'promise-toolbox'
+import { fromCallback, fromEvent, ignoreErrors, timeout } from 'promise-toolbox'
 import { type Readable, type Writable } from 'stream'
 import { parse } from 'xo-remote-parser'
 
@@ -17,13 +18,22 @@ type File = FileDescriptor | string
 
 const checksumFile = file => file + '.checksum'
 
+const DEFAULT_TIMEOUT = 6e5 // 10 min
+
 export default class RemoteHandlerAbstract {
   _remote: Object
-  constructor (remote: any) {
-    this._remote = { ...remote, ...parse(remote.url) }
-    if (this._remote.type !== this.type) {
-      throw new Error('Incorrect remote type')
+  _timeout: number
+
+  constructor (remote: any, options: Object = {}) {
+    if (remote.url === 'test://') {
+      this._remote = remote
+    } else {
+      this._remote = { ...remote, ...parse(remote.url) }
+      if (this._remote.type !== this.type) {
+        throw new Error('Incorrect remote type')
+      }
     }
+    ;({ timeout: this._timeout = DEFAULT_TIMEOUT } = options)
   }
 
   get type (): string {
@@ -121,7 +131,7 @@ export default class RemoteHandlerAbstract {
     newPath: string,
     { checksum = false }: Object = {}
   ) {
-    let p = this._rename(oldPath, newPath)
+    let p = timeout.call(this._rename(oldPath, newPath), this._timeout)
     if (checksum) {
       p = Promise.all([
         p,
@@ -142,7 +152,7 @@ export default class RemoteHandlerAbstract {
       prependDir = false,
     }: { filter?: (name: string) => boolean, prependDir?: boolean } = {}
   ): Promise<string[]> {
-    let entries = await this._list(dir)
+    let entries = await timeout.call(this._list(dir), this._timeout)
     if (filter !== undefined) {
       entries = entries.filter(filter)
     }
@@ -165,28 +175,30 @@ export default class RemoteHandlerAbstract {
     { checksum = false, ignoreMissingChecksum = false, ...options }: Object = {}
   ): Promise<LaxReadable> {
     const path = typeof file === 'string' ? file : file.path
-    const streamP = this._createReadStream(file, options).then(stream => {
-      // detect early errors
-      let promise = fromEvent(stream, 'readable')
+    const streamP = timeout
+      .call(this._createReadStream(file, options), this._timeout)
+      .then(stream => {
+        // detect early errors
+        let promise = fromEvent(stream, 'readable')
 
-      // try to add the length prop if missing and not a range stream
-      if (
-        stream.length === undefined &&
-        options.end === undefined &&
-        options.start === undefined
-      ) {
-        promise = Promise.all([
-          promise,
-          ignoreErrors.call(
-            this.getSize(file).then(size => {
-              stream.length = size
-            })
-          ),
-        ])
-      }
+        // try to add the length prop if missing and not a range stream
+        if (
+          stream.length === undefined &&
+          options.end === undefined &&
+          options.start === undefined
+        ) {
+          promise = Promise.all([
+            promise,
+            ignoreErrors.call(
+              this.getSize(file).then(size => {
+                stream.length = size
+              })
+            ),
+          ])
+        }
 
-      return promise.then(() => stream)
-    })
+        return promise.then(() => stream)
+      })
 
     if (!checksum) {
       return streamP
@@ -224,7 +236,10 @@ export default class RemoteHandlerAbstract {
   }
 
   async openFile (path: string, flags?: string): Promise<FileDescriptor> {
-    return { fd: await this._openFile(path, flags), path }
+    return {
+      fd: await timeout.call(this._openFile(path, flags), this._timeout),
+      path,
+    }
   }
 
   async _openFile (path: string, flags?: string): Promise<mixed> {
@@ -232,7 +247,7 @@ export default class RemoteHandlerAbstract {
   }
 
   async closeFile (fd: FileDescriptor): Promise<void> {
-    await this._closeFile(fd.fd)
+    await timeout.call(this._closeFile(fd.fd), this._timeout)
   }
 
   async _closeFile (fd: mixed): Promise<void> {
@@ -252,10 +267,13 @@ export default class RemoteHandlerAbstract {
     { checksum = false, ...options }: Object = {}
   ): Promise<LaxWritable> {
     const path = typeof file === 'string' ? file : file.path
-    const streamP = this._createOutputStream(file, {
-      flags: 'wx',
-      ...options,
-    })
+    const streamP = timeout.call(
+      this._createOutputStream(file, {
+        flags: 'wx',
+        ...options,
+      }),
+      this._timeout
+    )
 
     if (!checksum) {
       return streamP
@@ -290,7 +308,7 @@ export default class RemoteHandlerAbstract {
       ignoreErrors.call(this._unlink(checksumFile(file)))
     }
 
-    await this._unlink(file)
+    await timeout.call(this._unlink(file), this._timeout)
   }
 
   async _unlink (file: mixed): Promise<void> {
@@ -298,7 +316,7 @@ export default class RemoteHandlerAbstract {
   }
 
   async getSize (file: mixed): Promise<number> {
-    return this._getSize(file)
+    return timeout.call(this._getSize(file), this._timeout)
   }
 
   async _getSize (file: mixed): Promise<number> {

@xen-orchestra/fs/src/abstract.spec.js

@@ -0,0 +1,113 @@
+/* eslint-env jest */
+
+import { TimeoutError } from 'promise-toolbox'
+
+import AbstractHandler from './abstract'
+
+const TIMEOUT = 10e3
+
+class TestHandler extends AbstractHandler {
+  constructor (impl) {
+    super({ url: 'test://' }, { timeout: TIMEOUT })
+
+    Object.keys(impl).forEach(method => {
+      this[`_${method}`] = impl[method]
+    })
+  }
+}
+
+describe('rename()', () => {
+  it(`throws in case of timeout`, async () => {
+    const testHandler = new TestHandler({
+      rename: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.rename('oldPath', 'newPath')
+    jest.advanceTimersByTime(TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('list()', () => {
+  it(`throws in case of timeout`, async () => {
+    const testHandler = new TestHandler({
+      list: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.list()
+    jest.advanceTimersByTime(TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('createReadStream()', () => {
+  it(`throws in case of timeout`, async () => {
+    const testHandler = new TestHandler({
+      createReadStream: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.createReadStream('file')
+    jest.advanceTimersByTime(TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('openFile()', () => {
+  it(`throws in case of timeout`, async () => {
+    const testHandler = new TestHandler({
+      openFile: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.openFile('path')
+    jest.advanceTimersByTime(TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('closeFile()', () => {
+  it(`throws in case of timeout`, async () => {
+    const testHandler = new TestHandler({
+      closeFile: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.closeFile({ fd: undefined, path: '' })
+    jest.advanceTimersByTime(TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('createOutputStream()', () => {
+  it(`throws in case of timeout`, async () => {
+    const testHandler = new TestHandler({
+      createOutputStream: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.createOutputStream('File')
+    jest.advanceTimersByTime(TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('unlink()', () => {
+  it(`throws in case of timeout`, async () => {
+    const testHandler = new TestHandler({
+      unlink: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.unlink('')
+    jest.advanceTimersByTime(TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('getSize()', () => {
+  it(`throws in case of timeout`, async () => {
+    const testHandler = new TestHandler({
+      getSize: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.getSize('')
+    jest.advanceTimersByTime(TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})

@xen-orchestra/fs/src/checksum.js

@@ -1,6 +1,5 @@
 // @flow
 
-// $FlowFixMe
 import through2 from 'through2'
 import { createHash } from 'crypto'
 import { defer, fromEvent } from 'promise-toolbox'

@xen-orchestra/fs/src/index.js

@@ -14,7 +14,7 @@ const HANDLERS = {
   nfs: RemoteHandlerNfs,
 }
 
-export const getHandler = (remote: Remote): RemoteHandler => {
+export const getHandler = (remote: Remote, ...rest: any): RemoteHandler => {
   // FIXME: should be done in xo-remote-parser.
   const type = remote.url.split('://')[0]
 
@@ -22,5 +22,5 @@ export const getHandler = (remote: Remote): RemoteHandler => {
   if (!Handler) {
     throw new Error('Unhandled remote type')
   }
-  return new Handler(remote)
+  return new Handler(remote, ...rest)
 }

@xen-orchestra/fs/src/local.js

@@ -27,11 +27,10 @@ export default class LocalHandler extends RemoteHandlerAbstract {
   }
 
   async _sync () {
-    if (this._remote.enabled) {
-      const path = this._getRealPath()
-      await fs.ensureDir(path)
-      await fs.access(path, fs.R_OK | fs.W_OK)
-    }
+    const path = this._getRealPath()
+    await fs.ensureDir(path)
+    await fs.access(path, fs.R_OK | fs.W_OK)
+
     return this._remote
   }
 

@xen-orchestra/fs/src/nfs.js

@@ -1,17 +1,28 @@
 import execa from 'execa'
 import fs from 'fs-extra'
+import { join } from 'path'
+import { tmpdir } from 'os'
 
 import LocalHandler from './local'
 
 const DEFAULT_NFS_OPTIONS = 'vers=3'
 
 export default class NfsHandler extends LocalHandler {
+  constructor (
+    remote,
+    { mountsDir = join(tmpdir(), 'xo-fs-mounts'), ...opts } = {}
+  ) {
+    super(remote, opts)
+
+    this._realPath = join(mountsDir, remote.id)
+  }
+
   get type () {
     return 'nfs'
   }
 
   _getRealPath () {
-    return `/run/xo-server/mounts/${this._remote.id}`
+    return this._realPath
   }
 
   async _mount () {
@@ -33,18 +44,18 @@ export default class NfsHandler extends LocalHandler {
         },
       }
     ).catch(error => {
-      if (!error.stderr.includes('already mounted')) {
+      if (
+        error == null ||
+        typeof error.stderr !== 'string' ||
+        !error.stderr.includes('already mounted')
+      ) {
         throw error
       }
     })
   }
 
   async _sync () {
-    if (this._remote.enabled) {
-      await this._mount()
-    } else {
-      await this._umount()
-    }
+    await this._mount()
 
     return this._remote
   }
@@ -63,7 +74,11 @@ export default class NfsHandler extends LocalHandler {
         LANG: 'C',
       },
     }).catch(error => {
-      if (!error.stderr.includes('not mounted')) {
+      if (
+        error == null ||
+        typeof error.stderr !== 'string' ||
+        !error.stderr.includes('not mounted')
+      ) {
         throw error
       }
     })

@xen-orchestra/fs/src/smb.js

@@ -6,25 +6,27 @@ import RemoteHandlerAbstract from './abstract'
 const noop = () => {}
 
 // Normalize the error code for file not found.
-const normalizeError = error => {
+class ErrorWrapper extends Error {
+  constructor (error, newCode) {
+    super(error.message)
+    this.cause = error
+    this.code = newCode
+  }
+}
+const normalizeError = (error, shouldBeDirectory) => {
   const { code } = error
 
   return code === 'STATUS_OBJECT_NAME_NOT_FOUND' ||
     code === 'STATUS_OBJECT_PATH_NOT_FOUND'
-    ? Object.create(error, {
-        code: {
-          configurable: true,
-          readable: true,
-          value: 'ENOENT',
-          writable: true,
-        },
-      })
+    ? new ErrorWrapper(error, 'ENOENT')
+    : code === 'STATUS_NOT_SUPPORTED' || code === 'STATUS_INVALID_PARAMETER'
+    ? new ErrorWrapper(error, shouldBeDirectory ? 'ENOTDIR' : 'EISDIR')
     : error
 }
 
 export default class SmbHandler extends RemoteHandlerAbstract {
-  constructor (remote) {
-    super(remote)
+  constructor (remote, opts) {
+    super(remote, opts)
     this._forget = noop
   }
 
@@ -70,10 +72,9 @@ export default class SmbHandler extends RemoteHandlerAbstract {
   }
 
   async _sync () {
-    if (this._remote.enabled) {
-      // Check access (smb2 does not expose connect in public so far...)
-      await this.list()
-    }
+    // Check access (smb2 does not expose connect in public so far...)
+    await this.list()
+
     return this._remote
   }
 
@@ -154,7 +155,7 @@ export default class SmbHandler extends RemoteHandlerAbstract {
         client.disconnect()
       })
     } catch (error) {
-      throw normalizeError(error)
+      throw normalizeError(error, true)
     }
 
     return list

@xen-orchestra/log/package.json

@@ -1,7 +1,6 @@
 {
-  "private": true,
   "name": "@xen-orchestra/log",
-  "version": "0.0.0",
+  "version": "0.1.4",
   "license": "ISC",
   "description": "",
   "keywords": [],
@@ -19,7 +18,9 @@
   "main": "dist/",
   "bin": {},
   "files": [
-    "dist/"
+    "configure.js",
+    "dist/",
+    "transports/"
   ],
   "browserslist": [
     ">2%"
@@ -29,7 +30,7 @@
   },
   "dependencies": {
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.10.1"
+    "promise-toolbox": "^0.11.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

@xen-orchestra/log/src/configure.js

@@ -86,7 +86,7 @@ export const catchGlobalErrors = logger => {
   const { prototype } = EventEmitter
   const { emit } = prototype
   function patchedEmit (event, error) {
-    if (event === 'error' && !this.listenerCount(event)) {
+    if (event === 'error' && this.listenerCount(event) === 0) {
       logger.error('unhandled error event', { error })
       return false
     }

@xen-orchestra/log/src/index.js

@@ -38,6 +38,17 @@ for (const name in LEVELS) {
   const level = LEVELS[name]
 
   prototype[name.toLowerCase()] = function (message, data) {
+    if (typeof message !== 'string') {
+      if (message instanceof Error) {
+        data = { error: message }
+        ;({ message = 'an error has occured' } = message)
+      } else {
+        return this.warn('incorrect value passed to logger', {
+          level,
+          value: message,
+        })
+      }
+    }
     global[symbol](new Log(data, level, this._namespace, message, new Date()))
   }
 }

@xen-orchestra/log/src/transports/console.js

@@ -13,10 +13,10 @@ const consoleTransport = ({ data, level, namespace, message, time }) => {
     level < INFO
       ? debugConsole
       : level < WARN
-        ? infoConsole
-        : level < ERROR
-          ? warnConsole
-          : errorConsole
+      ? infoConsole
+      : level < ERROR
+      ? warnConsole
+      : errorConsole
 
   fn('%s - %s - [%s] %s', time.toISOString(), namespace, NAMES[level], message)
   data != null && fn(data)

@xen-orchestra/log/src/transports/email.js

@@ -53,14 +53,12 @@ export default ({
     fromCallback(cb =>
       transporter.sendMail(
         {
-          subject: evalTemplate(
-            subject,
-            key =>
-              key === 'level'
-                ? NAMES[log.level]
-                : key === 'time'
-                  ? log.time.toISOString()
-                  : log[key]
+          subject: evalTemplate(subject, key =>
+            key === 'level'
+              ? NAMES[log.level]
+              : key === 'time'
+              ? log.time.toISOString()
+              : log[key]
           ),
           text: prettyFormat(log.data),
         },

@xen-orchestra/log/transports

@@ -1 +0,0 @@
-dist/transports

@xen-orchestra/log/transports/console.js

@@ -0,0 +1 @@
+module.exports = require('../dist/transports/console.js')

@xen-orchestra/log/transports/email.js

@@ -0,0 +1 @@
+module.exports = require('../dist/transports/email.js')

@xen-orchestra/log/transports/memory.js

@@ -0,0 +1 @@
+module.exports = require('../dist/transports/memory.js')

@xen-orchestra/log/transports/syslog.js

@@ -0,0 +1 @@
+module.exports = require('../dist/transports/syslog.js')

CHANGELOG.md

@@ -4,6 +4,115 @@
 
 ### Enhancements
 
+- [VM] Ability to set nested virtualization in settings [#3619](https://github.com/vatesfr/xen-orchestra/issues/3619) (PR [#3625](https://github.com/vatesfr/xen-orchestra/pull/3625))
+- [Legacy Backup] Restore and File restore functionalities moved to the Backup NG view [#3499](https://github.com/vatesfr/xen-orchestra/issues/3499) (PR [#3610](https://github.com/vatesfr/xen-orchestra/pull/3610))
+- [Backup NG logs] Display warning in case of missing VMs instead of a ghosts VMs tasks (PR [#3647](https://github.com/vatesfr/xen-orchestra/pull/3647))
+- [VM] On migration, automatically selects the host and SR when only one is available [#3502](https://github.com/vatesfr/xen-orchestra/issues/3502) (PR [#3654](https://github.com/vatesfr/xen-orchestra/pull/3654))
+- [VM] Display VGA and video RAM for PVHVM guests [#3576](https://github.com/vatesfr/xen-orchestra/issues/3576) (PR [#3664](https://github.com/vatesfr/xen-orchestra/pull/3664))
+- [Backup NG form] Display a warning to let the user know that the Delta Backup and the Continuous Replication are not supported on XenServer < 6.5 [#3540](https://github.com/vatesfr/xen-orchestra/issues/3540) (PR [#3668](https://github.com/vatesfr/xen-orchestra/pull/3668))
+- [Backup NG form] Omit VMs(Simple Backup)/pools(Smart Backup/Resident on) with XenServer < 6.5 from the selection when the Delta Backup mode or the Continuous Replication mode are selected [#3540](https://github.com/vatesfr/xen-orchestra/issues/3540) (PR [#3668](https://github.com/vatesfr/xen-orchestra/pull/3668))
+- [VM] Allow to switch the Virtualization mode [#2372](https://github.com/vatesfr/xen-orchestra/issues/2372) (PR [#3669](https://github.com/vatesfr/xen-orchestra/pull/3669))
+
+### Bug fixes
+
+- [Backup ng logs] Fix restarting VMs with concurrency issue [#3603](https://github.com/vatesfr/xen-orchestra/issues/3603) (PR [#3634](https://github.com/vatesfr/xen-orchestra/pull/3634))
+- Validate modal containing a confirm text input by pressing the Enter key [#2735](https://github.com/vatesfr/xen-orchestra/issues/2735) (PR [#2890](https://github.com/vatesfr/xen-orchestra/pull/2890))
+- [Patches] Bulk install correctly ignores upgrade patches on licensed hosts (PR [#3651](https://github.com/vatesfr/xen-orchestra/pull/3651))
+- [Backup NG logs] Handle failed restores (PR [#3648](https://github.com/vatesfr/xen-orchestra/pull/3648))
+- [Self/New VM] Incorrect limit computation [#3658](https://github.com/vatesfr/xen-orchestra/issues/3658) (PR [#3666](https://github.com/vatesfr/xen-orchestra/pull/3666))
+- [Plugins] Don't expose credentials in config to users (PR [#3671](https://github.com/vatesfr/xen-orchestra/pull/3671))
+- [Self/New VM] `not enough … available in the set …` error in some cases (PR [#3667](https://github.com/vatesfr/xen-orchestra/pull/3667))
+- [XOSAN] Creation stuck at "Configuring VMs" [#3688](https://github.com/vatesfr/xen-orchestra/issues/3688) (PR [#3689](https://github.com/vatesfr/xen-orchestra/pull/3689))
+- [Backup NG] Errors listing backups on SMB remotes with extraneous files (PR [#3685](https://github.com/vatesfr/xen-orchestra/pull/3685))
+- [Remotes] Don't expose credentials to users [#3682](https://github.com/vatesfr/xen-orchestra/issues/3682) (PR [#3687](https://github.com/vatesfr/xen-orchestra/pull/3687))
+- [VM] Correctly display guest metrics updates (tools, network, etc.) [#3533](https://github.com/vatesfr/xen-orchestra/issues/3533) (PR [#3694](https://github.com/vatesfr/xen-orchestra/pull/3694))
+- [VM Templates] Fix deletion [#3498](https://github.com/vatesfr/xen-orchestra/issues/3498) (PR [#3695](https://github.com/vatesfr/xen-orchestra/pull/3695))
+
+### Released packages
+
+- xen-api v0.21.0
+- xo-common v0.2.0
+- xo-acl-resolver v0.4.0
+- xo-server v5.30.0
+- xo-web v5.30.0
+
+## **5.28.1** (2018-11-05)
+
+### Enhancements
+
+### Bug fixes
+
+- [Backup NG] Increase timeout in stale remotes detection to limit false positives (PR [#3632](https://github.com/vatesfr/xen-orchestra/pull/3632))
+- Fix re-registration issue ([4e35b19ac](https://github.com/vatesfr/xen-orchestra/commit/4e35b19ac56c60f61c0e771cde70a50402797b8a))
+- [Backup NG logs] Fix started jobs filter [#3636](https://github.com/vatesfr/xen-orchestra/issues/3636) (PR [#3641](https://github.com/vatesfr/xen-orchestra/pull/3641))
+- [New VM] CPU and memory user inputs were ignored since previous release [#3644](https://github.com/vatesfr/xen-orchestra/issues/3644) (PR [#3646](https://github.com/vatesfr/xen-orchestra/pull/3646))
+
+### Released packages
+
+- @xen-orchestra/fs v0.4.1
+- xo-server v5.29.4
+- xo-web v5.29.3
+
+## **5.28.0** (2018-10-31)
+
+### Enhancements
+
+- [Usage Report] Add IOPS read/write/total per VM [#3309](https://github.com/vatesfr/xen-orchestra/issues/3309) (PR [#3455](https://github.com/vatesfr/xen-orchestra/pull/3455))
+- [Self service] Sort resource sets by name (PR [#3507](https://github.com/vatesfr/xen-orchestra/pull/3507))
+- [Usage Report] Add top 3 SRs which use the most IOPS read/write/total [#3306](https://github.com/vatesfr/xen-orchestra/issues/3306) (PR [#3508](https://github.com/vatesfr/xen-orchestra/pull/3508))
+- [New VM] Display a warning when the memory is below the template memory static min [#3496](https://github.com/vatesfr/xen-orchestra/issues/3496) (PR [#3513](https://github.com/vatesfr/xen-orchestra/pull/3513))
+- [Backup NG form] Add link to plugins setting [#3457](https://github.com/vatesfr/xen-orchestra/issues/3457) (PR [#3514](https://github.com/vatesfr/xen-orchestra/pull/3514))
+- [Backup reports] Add job and run ID [#3488](https://github.com/vatesfr/xen-orchestra/issues/3488) (PR [#3516](https://github.com/vatesfr/xen-orchestra/pull/3516))
+- [Usage Report] Add top 3 VMs which use the most IOPS read/write/total [#3308](https://github.com/vatesfr/xen-orchestra/issues/3308) (PR [#3463](https://github.com/vatesfr/xen-orchestra/pull/3463))
+- [Settings/logs] Homogenize action buttons in table and enable bulk deletion [#3179](https://github.com/vatesfr/xen-orchestra/issues/3179) (PR [#3528](https://github.com/vatesfr/xen-orchestra/pull/3528))
+- [Settings/acls] Add bulk deletion [#3179](https://github.com/vatesfr/xen-orchestra/issues/3179) (PR [#3536](https://github.com/vatesfr/xen-orchestra/pull/3536))
+- [Home] Improve search usage: raw numbers also match in names [#2906](https://github.com/vatesfr/xen-orchestra/issues/2906) (PR [#3552](https://github.com/vatesfr/xen-orchestra/pull/3552))
+- [Backup NG] Timeout of a job is now in hours [#3550](https://github.com/vatesfr/xen-orchestra/issues/3550) (PR [#3553](https://github.com/vatesfr/xen-orchestra/pull/3553))
+- [Backup NG] Explicit error if a VM is missing [#3434](https://github.com/vatesfr/xen-orchestra/issues/3434) (PR [#3522](https://github.com/vatesfr/xen-orchestra/pull/3522))
+- [Backup NG] Show all advanced settings with non-default values in overview [#3549](https://github.com/vatesfr/xen-orchestra/issues/3549) (PR [#3554](https://github.com/vatesfr/xen-orchestra/pull/3554))
+- [Backup NG] Collapse advanced settings by default [#3551](https://github.com/vatesfr/xen-orchestra/issues/3551) (PR [#3559](https://github.com/vatesfr/xen-orchestra/pull/3559))
+- [Scheduling] Merge selection and interval tabs [#1902](https://github.com/vatesfr/xen-orchestra/issues/1902) (PR [#3519](https://github.com/vatesfr/xen-orchestra/pull/3519))
+- [Backup NG/Restore] The backup selector now also shows the job name [#3366](https://github.com/vatesfr/xen-orchestra/issues/3366) (PR [#3564](https://github.com/vatesfr/xen-orchestra/pull/3564))
+- Sort buttons by criticality in tables [#3168](https://github.com/vatesfr/xen-orchestra/issues/3168) (PR [#3545](https://github.com/vatesfr/xen-orchestra/pull/3545))
+- [Usage Report] Ability to send a daily report [#3544](https://github.com/vatesfr/xen-orchestra/issues/3544) (PR [#3582](https://github.com/vatesfr/xen-orchestra/pull/3582))
+- [Backup NG logs] Disable state filters with no entries [#3438](https://github.com/vatesfr/xen-orchestra/issues/3438) (PR [#3442](https://github.com/vatesfr/xen-orchestra/pull/3442))
+- [ACLs] Global performance improvement on UI for non-admin users [#3578](https://github.com/vatesfr/xen-orchestra/issues/3578) (PR [#3584](https://github.com/vatesfr/xen-orchestra/pull/3584))
+- [Backup NG] Improve the Schedule's view (Replace table by list) [#3491](https://github.com/vatesfr/xen-orchestra/issues/3491) (PR [#3586](https://github.com/vatesfr/xen-orchestra/pull/3586))
+- ([Host/Storage], [Sr/hosts]) add bulk deletion [#3179](https://github.com/vatesfr/xen-orchestra/issues/3179) (PR [#3539](https://github.com/vatesfr/xen-orchestra/pull/3539))
+- [xo-server] Use @xen-orchestra/log for basic logging [#3555](https://github.com/vatesfr/xen-orchestra/issues/3555) (PR [#3579](https://github.com/vatesfr/xen-orchestra/pull/3579))
+- [Backup Report] Log error when job failed [#3458](https://github.com/vatesfr/xen-orchestra/issues/3458) (PR [#3593](https://github.com/vatesfr/xen-orchestra/pull/3593))
+- [Backup NG] Display logs for backup restoration [#2511](https://github.com/vatesfr/xen-orchestra/issues/2511) (PR [#3609](https://github.com/vatesfr/xen-orchestra/pull/3609))
+- [XOA] Display product version and list of all installed packages [#3560](https://github.com/vatesfr/xen-orchestra/issues/3560) (PR [#3621](https://github.com/vatesfr/xen-orchestra/pull/3621))
+
+### Bug fixes
+
+- [Remotes] Fix removal of broken remotes [#3327](https://github.com/vatesfr/xen-orchestra/issues/3327) (PR [#3521](https://github.com/vatesfr/xen-orchestra/pull/3521))
+- [Backups] Fix stuck backups due to broken NFS remotes [#3467](https://github.com/vatesfr/xen-orchestra/issues/3467) (PR [#3534](https://github.com/vatesfr/xen-orchestra/pull/3534))
+- [New VM] Fix missing cloud config when creating multiple VMs at once in some cases [#3532](https://github.com/vatesfr/xen-orchestra/issues/3532) (PR [#3535](https://github.com/vatesfr/xen-orchestra/pull/3535))
+- [VM] Fix an error when an admin tried to add a disk on a Self VM whose resource set had been deleted [#2814](https://github.com/vatesfr/xen-orchestra/issues/2814) (PR [#3530](https://github.com/vatesfr/xen-orchestra/pull/3530))
+- [Self/Create VM] Fix some quotas based on the template instead of the user inputs [#2683](https://github.com/vatesfr/xen-orchestra/issues/2683) (PR [#3546](https://github.com/vatesfr/xen-orchestra/pull/3546))
+- [Self] Ignore DR and CR VMs when computing quotas [#3064](https://github.com/vatesfr/xen-orchestra/issues/3064) (PR [#3561](https://github.com/vatesfr/xen-orchestra/pull/3561))
+- [Patches] Wrongly requiring to eject CDs from halted VMs and snapshots before installing patches (PR [#3611](https://github.com/vatesfr/xen-orchestra/pull/3611))
+- [Jobs] Ensure the scheduling is not interrupted in rare cases (PR [#3617](https://github.com/vatesfr/xen-orchestra/pull/3617))
+- [Home] Fix `server.getAll` error at login when user is not admin [#2335](https://github.com/vatesfr/xen-orchestra/issues/2335) (PR [#3613](https://github.com/vatesfr/xen-orchestra/pull/3613))
+
+### Released packages
+
+- xo-server-backup-reports v0.15.0
+- xo-common v0.1.2
+- @xen-orchestra/log v0.1.0
+- @xen-orchestra/fs v0.4.0
+- complex-matcher v0.5.0
+- vhd-lib v0.4.0
+- xen-api v0.20.0
+- xo-server-usage-report v0.7.0
+- xo-server v5.29.0
+- xo-web v5.29.0
+
+## **5.27.2** (2018-10-05)
+
+### Enhancements
+
 - [Host/Networks] Remove "Add network" button [#3386](https://github.com/vatesfr/xen-orchestra/issues/3386) (PR [#3478](https://github.com/vatesfr/xen-orchestra/pull/3478))
 - [Host/networks] Private networks table [#3387](https://github.com/vatesfr/xen-orchestra/issues/3387) (PR [#3481](https://github.com/vatesfr/xen-orchestra/pull/3481))
 - [Home/pool] Patch count pill now shows the number of unique patches in the pool [#3321](https://github.com/vatesfr/xen-orchestra/issues/3321) (PR [#3483](https://github.com/vatesfr/xen-orchestra/pull/3483))

docs/backup_troubleshooting.md

@@ -39,6 +39,10 @@ You can check if a coalesce job is currently active by running `ps axf | grep vh
 
 If you don't see any running coalesce jobs, and can't find any other reason that XenServer has not started one, you can attempt to make it start a coalesce job by rescanning the SR. This is harmless to try, but will not always result in a coalesce. Visit the problematic SR in the XOA UI, then click the "Rescan All Disks" button towards the top right: it looks like a refresh circle icon. This should begin the coalesce process - if you click the Advanced tab in the SR view, the "disks needing to be coalesced" list should become smaller and smaller.
 
+### Parse Error
+
+This is most likely due to running a backup job that uses Delta functionality (eg: delta backups, or continuous replication) on a version of XenServer older than 6.5. To use delta functionality you must run [XenServer 6.5 or later](https://xen-orchestra.com/docs/supported-version.html).
+
 ### SR_BACKEND_FAILURE_44 (insufficient space)
 
 > This message can be triggered by any backup method.
@@ -72,4 +76,4 @@ To check your free space, enter your XOA and run `xoa check` to check free syste
 
 This is happening when you have a *smart backup job* that doesn't match any VMs. For example: you created a job to backup all running VMs. If no VMs are running on backup schedule, you'll have this message. This could also happen if you lost connection with your pool master (the VMs aren't visible anymore from Xen Orchestra).
 
-Edit your job and try to see matching VMs or check if your pool is connected to XOA.
+Edit your job and try to see matching VMs or check if your pool is connected to XOA.

docs/from_the_sources.md

@@ -14,13 +14,13 @@ As you may have seen,in other parts of the documentation, XO is composed of two
 
 ### NodeJS
 
-XO needs Node.js. **Please always use the LTS version of Node**.
+XO needs Node.js. **Please use Node 8**.
 
 We'll consider at this point that you've got a working node on your box. E.g:
 
 ```
 $ node -v
-v8.9.1
+v8.12.0
 ```
 
 If not, see [this page](https://nodejs.org/en/download/package-manager/) for instructions on how to install Node.

docs/xo-cli.md

@@ -1,4 +1,3 @@
-
 # xo-cli
 
 This is another client of `xo-server` - this time in command line form.
@@ -104,5 +103,6 @@ encoding by prefixing with `json:`:
 ##### VM import
 
  ```
-> xo-cli vm.import host=60a6939e-8b0a-4352-9954-5bde44bcdf7d @=vm.xva
+> xo-cli vm.import sr=60a6939e-8b0a-4352-9954-5bde44bcdf7d @=vm.xva
 ```
+> Note: `xo-cli` only supports the import of XVA files. It will not import OVA files. To import OVA images, you must use the XOA web UI.

package.json

@@ -10,18 +10,18 @@
     "eslint-config-standard": "12.0.0",
     "eslint-config-standard-jsx": "^6.0.2",
     "eslint-plugin-import": "^2.8.0",
-    "eslint-plugin-node": "^7.0.1",
+    "eslint-plugin-node": "^8.0.0",
     "eslint-plugin-promise": "^4.0.0",
     "eslint-plugin-react": "^7.6.1",
     "eslint-plugin-standard": "^4.0.0",
     "exec-promise": "^0.7.0",
-    "flow-bin": "^0.82.0",
+    "flow-bin": "^0.85.0",
     "globby": "^8.0.0",
     "husky": "^1.0.0-rc.15",
     "jest": "^23.0.1",
     "lodash": "^4.17.4",
     "prettier": "^1.10.2",
-    "promise-toolbox": "^0.10.1",
+    "promise-toolbox": "^0.11.0",
     "sorted-object": "^2.0.1"
   },
   "engines": {
@@ -33,6 +33,7 @@
     }
   },
   "jest": {
+    "timers": "fake",
     "collectCoverage": true,
     "projects": [
       "<rootDir>"
@@ -57,7 +58,8 @@
     "prepare": "scripts/run-script prepare",
     "pretest": "eslint --ignore-path .gitignore .",
     "test": "jest \"^(?!.*\\.integ\\.spec\\.js$)\"",
-    "test-integration": "jest \".integ\\.spec\\.js$\""
+    "test-integration": "jest \".integ\\.spec\\.js$\"",
+    "travis-tests": "scripts/travis-tests"
   },
   "workspaces": [
     "@xen-orchestra/*",

packages/complex-matcher/package.json

@@ -1,6 +1,6 @@
 {
   "name": "complex-matcher",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "license": "ISC",
   "description": "",
   "keywords": [],

packages/complex-matcher/src/index.fixtures.js

@@ -11,7 +11,7 @@ export const ast = new CM.And([
     new CM.Or([new CM.String('wonderwoman'), new CM.String('batman')])
   ),
   new CM.TruthyProperty('hasCape'),
-  new CM.Property('age', new CM.Number(32)),
+  new CM.Property('age', new CM.NumberOrStringNode('32')),
   new CM.GlobPattern('chi*go'),
   new CM.RegExp('^foo/bar\\.', 'i'),
 ])

packages/complex-matcher/src/index.js

@@ -153,6 +153,34 @@ export class NumberNode extends Node {
 }
 export { NumberNode as Number }
 
+export class NumberOrStringNode extends Node {
+  constructor (value) {
+    super()
+
+    this.value = value
+
+    // should not be enumerable for the tests
+    Object.defineProperty(this, 'match', {
+      value: this.match.bind(this, value.toLowerCase(), +value),
+    })
+  }
+
+  match (lcValue, numValue, value) {
+    return (
+      value === numValue ||
+      (typeof value === 'string'
+        ? value.toLowerCase().indexOf(lcValue) !== -1
+        : (Array.isArray(value) || isPlainObject(value)) &&
+          some(value, this.match))
+    )
+  }
+
+  toString () {
+    return this.value
+  }
+}
+export { NumberOrStringNode as NumberOrString }
+
 export class Property extends Node {
   constructor (name, child) {
     super()
@@ -382,11 +410,10 @@ class P {
 
   static text (text) {
     const { length } = text
-    return new P(
-      (input, pos) =>
-        input.startsWith(text, pos)
-          ? new Success(pos + length, text)
-          : new Failure(pos, `'${text}'`)
+    return new P((input, pos) =>
+      input.startsWith(text, pos)
+        ? new Success(pos + length, text)
+        : new Failure(pos, `'${text}'`)
     )
   }
 
@@ -450,17 +477,16 @@ class P {
   }
 }
 
-P.eof = new P(
-  (input, pos, end) =>
-    pos < end ? new Failure(pos, 'end of input') : new Success(pos)
+P.eof = new P((input, pos, end) =>
+  pos < end ? new Failure(pos, 'end of input') : new Success(pos)
 )
 
 // -------------------------------------------------------------------
 
 const parser = P.grammar({
   default: r =>
-    P.seq(r.ws, r.term.repeat(), P.eof).map(
-      ([, terms]) => (terms.length === 0 ? new Null() : new And(terms))
+    P.seq(r.ws, r.term.repeat(), P.eof).map(([, terms]) =>
+      terms.length === 0 ? new Null() : new And(terms)
     ),
   globPattern: new P((input, pos, end) => {
     let value = ''
@@ -564,7 +590,7 @@ const parser = P.grammar({
         const asNum = +str
         return Number.isNaN(asNum)
           ? new GlobPattern(str)
-          : new NumberNode(asNum)
+          : new NumberOrStringNode(str)
       })
     ),
   ws: P.regex(/\s*/),

packages/complex-matcher/src/index.spec.js

@@ -6,6 +6,7 @@ import {
   GlobPattern,
   Null,
   NumberNode,
+  NumberOrStringNode,
   parse,
   setPropertyClause,
 } from './'
@@ -32,7 +33,7 @@ describe('parse', () => {
 
     node = parse('32')
     expect(node.match(32)).toBe(true)
-    expect(node.match('32')).toBe(false)
+    expect(node.match('32')).toBe(true)
     expect(node.toString()).toBe('32')
 
     node = parse('"32"')
@@ -54,6 +55,12 @@ describe('Number', () => {
   })
 })
 
+describe('NumberOrStringNode', () => {
+  it('match a string', () => {
+    expect(new NumberOrStringNode('123').match([{ foo: '123' }])).toBe(true)
+  })
+})
+
 describe('setPropertyClause', () => {
   it('creates a node if none passed', () => {
     expect(setPropertyClause(undefined, 'foo', 'bar').toString()).toBe(

packages/vhd-cli/package.json

@@ -26,11 +26,11 @@
     "node": ">=6"
   },
   "dependencies": {
-    "@xen-orchestra/fs": "^0.3.0",
+    "@xen-orchestra/fs": "^0.4.1",
     "cli-progress": "^2.0.0",
     "exec-promise": "^0.7.0",
     "struct-fu": "^1.2.0",
-    "vhd-lib": "^0.3.0"
+    "vhd-lib": "^0.4.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",
@@ -40,7 +40,7 @@
     "cross-env": "^5.1.3",
     "execa": "^1.0.0",
     "index-modules": "^0.3.0",
-    "promise-toolbox": "^0.10.1",
+    "promise-toolbox": "^0.11.0",
     "rimraf": "^2.6.1",
     "tmp": "^0.0.33"
   },

packages/vhd-cli/src/commands/synthetize.js

@@ -9,13 +9,12 @@ export default async function main (args) {
   }
 
   const handler = getHandler({ url: 'file:///' })
+  const stream = await createSyntheticStream(handler, path.resolve(args[0]))
   return new Promise((resolve, reject) => {
-    createSyntheticStream(handler, path.resolve(args[0]))
-      .on('error', reject)
-      .pipe(
-        createWriteStream(args[1])
-          .on('error', reject)
-          .on('finish', resolve)
-      )
+    stream.on('error', reject).pipe(
+      createWriteStream(args[1])
+        .on('error', reject)
+        .on('finish', resolve)
+    )
   })
 }

packages/vhd-lib/merge.integ.spec.js

@@ -268,14 +268,18 @@ test('coalesce works in normal cases', async () => {
 
 test('createSyntheticStream passes vhd-util check', async () => {
   const initalSize = 4
+  const expectedVhdSize = 4197888
   await createRandomFile('randomfile', initalSize)
   await convertFromRawToVhd('randomfile', 'randomfile.vhd')
   const handler = getHandler({ url: 'file://' + process.cwd() })
-  const stream = createSyntheticStream(handler, 'randomfile.vhd')
+  const stream = await createSyntheticStream(handler, 'randomfile.vhd')
+  expect(stream.length).toEqual(expectedVhdSize)
   await fromEvent(
     stream.pipe(await fs.createWriteStream('recovered.vhd')),
     'finish'
   )
   await checkFile('recovered.vhd')
+  const stats = await fs.stat('recovered.vhd')
+  expect(stats.size).toEqual(expectedVhdSize)
   await execa('qemu-img', ['compare', 'recovered.vhd', 'randomfile'])
 })

packages/vhd-lib/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vhd-lib",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "license": "AGPL-3.0",
   "description": "Primitives for VHD file handling",
   "keywords": [],
@@ -25,7 +25,7 @@
     "from2": "^2.3.0",
     "fs-extra": "^7.0.0",
     "limit-concurrency-decorator": "^0.4.0",
-    "promise-toolbox": "^0.10.1",
+    "promise-toolbox": "^0.11.0",
     "struct-fu": "^1.2.0",
     "uuid": "^3.0.1"
   },
@@ -34,7 +34,7 @@
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
     "@babel/preset-flow": "^7.0.0",
-    "@xen-orchestra/fs": "^0.3.1",
+    "@xen-orchestra/fs": "^0.4.1",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^5.1.3",
     "execa": "^1.0.0",

packages/vhd-lib/src/createReadableSparseStream.js

@@ -17,7 +17,7 @@ import { set as setBitmap } from './_bitmap'
 const VHD_BLOCK_SIZE_SECTORS = VHD_BLOCK_SIZE_BYTES / SECTOR_SIZE
 
 /**
- * @returns {Array} an array of occupation bitmap, each bit mapping an input block size of bytes
+ * @returns currentVhdPositionSector the first free sector after the data
  */
 function createBAT (
   firstBlockPosition,
@@ -36,9 +36,10 @@ function createBAT (
         (bitmapSize + VHD_BLOCK_SIZE_BYTES) / SECTOR_SIZE
     }
   })
+  return currentVhdPositionSector
 }
 
-export default asyncIteratorToStream(async function * (
+export default async function createReadableStream (
   diskSize,
   incomingBlockSize,
   blockAddressList,
@@ -79,7 +80,14 @@ export default asyncIteratorToStream(async function * (
   const bitmapSize =
     Math.ceil(VHD_BLOCK_SIZE_SECTORS / 8 / SECTOR_SIZE) * SECTOR_SIZE
   const bat = Buffer.alloc(tablePhysicalSizeBytes, 0xff)
-  createBAT(firstBlockPosition, blockAddressList, ratio, bat, bitmapSize)
+  const endOfData = createBAT(
+    firstBlockPosition,
+    blockAddressList,
+    ratio,
+    bat,
+    bitmapSize
+  )
+  const fileSize = endOfData * SECTOR_SIZE + FOOTER_SIZE
   let position = 0
   function * yieldAndTrack (buffer, expectedPosition) {
     if (expectedPosition !== undefined) {
@@ -120,9 +128,16 @@ export default asyncIteratorToStream(async function * (
     }
     yield * yieldAndTrack(currentBlockWithBitmap)
   }
-  yield * yieldAndTrack(footer, 0)
-  yield * yieldAndTrack(header, FOOTER_SIZE)
-  yield * yieldAndTrack(bat, FOOTER_SIZE + HEADER_SIZE)
-  yield * generateFileContent(blockIterator, bitmapSize, ratio)
-  yield * yieldAndTrack(footer)
-})
+
+  async function * iterator () {
+    yield * yieldAndTrack(footer, 0)
+    yield * yieldAndTrack(header, FOOTER_SIZE)
+    yield * yieldAndTrack(bat, FOOTER_SIZE + HEADER_SIZE)
+    yield * generateFileContent(blockIterator, bitmapSize, ratio)
+    yield * yieldAndTrack(footer)
+  }
+
+  const stream = asyncIteratorToStream(iterator())
+  stream.length = fileSize
+  return stream
+}

packages/vhd-lib/src/createSyntheticStream.js

@@ -15,18 +15,24 @@ import { test as mapTestBit } from './_bitmap'
 const resolveRelativeFromFile = (file, path) =>
   resolve('/', dirname(file), path).slice(1)
 
-export default asyncIteratorToStream(function * (handler, path) {
+export default async function createSyntheticStream (handler, path) {
   const fds = []
-
+  const cleanup = () => {
+    for (let i = 0, n = fds.length; i < n; ++i) {
+      handler.closeFile(fds[i]).catch(error => {
+        console.warn('createReadStream, closeFd', i, error)
+      })
+    }
+  }
   try {
     const vhds = []
     while (true) {
-      const fd = yield handler.openFile(path, 'r')
+      const fd = await handler.openFile(path, 'r')
       fds.push(fd)
       const vhd = new Vhd(handler, fd)
       vhds.push(vhd)
-      yield vhd.readHeaderAndFooter()
-      yield vhd.readBlockAllocationTable()
+      await vhd.readHeaderAndFooter()
+      await vhd.readBlockAllocationTable()
 
       if (vhd.footer.diskType === DISK_TYPE_DYNAMIC) {
         break
@@ -64,14 +70,8 @@ export default asyncIteratorToStream(function * (handler, path) {
     const nBlocks = Math.ceil(footer.currentSize / header.blockSize)
 
     const blocksOwner = new Array(nBlocks)
-    for (
-      let iBlock = 0,
-        blockOffset = Math.ceil(
-          (header.tableOffset + bat.length) / SECTOR_SIZE
-        );
-      iBlock < nBlocks;
-      ++iBlock
-    ) {
+    let blockOffset = Math.ceil((header.tableOffset + bat.length) / SECTOR_SIZE)
+    for (let iBlock = 0; iBlock < nBlocks; ++iBlock) {
       let blockSector = BLOCK_UNUSED
       for (let i = 0; i < nVhds; ++i) {
         if (vhds[i].containsBlock(iBlock)) {
@@ -83,71 +83,78 @@ export default asyncIteratorToStream(function * (handler, path) {
       }
       bat.writeUInt32BE(blockSector, iBlock * 4)
     }
+    const fileSize = blockOffset * SECTOR_SIZE + FOOTER_SIZE
 
-    footer = fuFooter.pack(footer)
-    checksumStruct(footer, fuFooter)
-    yield footer
+    const iterator = function * () {
+      try {
+        footer = fuFooter.pack(footer)
+        checksumStruct(footer, fuFooter)
+        yield footer
 
-    header = fuHeader.pack(header)
-    checksumStruct(header, fuHeader)
-    yield header
+        header = fuHeader.pack(header)
+        checksumStruct(header, fuHeader)
+        yield header
 
-    yield bat
+        yield bat
 
-    // TODO: for generic usage the bitmap needs to be properly computed for each block
-    const bitmap = Buffer.alloc(vhd.bitmapSize, 0xff)
-    for (let iBlock = 0; iBlock < nBlocks; ++iBlock) {
-      const owner = blocksOwner[iBlock]
-      if (owner === undefined) {
-        continue
-      }
-
-      yield bitmap
-
-      const blocksByVhd = new Map()
-      const emitBlockSectors = function * (iVhd, i, n) {
-        const vhd = vhds[iVhd]
-        const isRootVhd = vhd === rootVhd
-        if (!vhd.containsBlock(iBlock)) {
-          if (isRootVhd) {
-            yield Buffer.alloc((n - i) * SECTOR_SIZE)
-          } else {
-            yield * emitBlockSectors(iVhd + 1, i, n)
+        // TODO: for generic usage the bitmap needs to be properly computed for each block
+        const bitmap = Buffer.alloc(vhd.bitmapSize, 0xff)
+        for (let iBlock = 0; iBlock < nBlocks; ++iBlock) {
+          const owner = blocksOwner[iBlock]
+          if (owner === undefined) {
+            continue
           }
-          return
-        }
-        let block = blocksByVhd.get(vhd)
-        if (block === undefined) {
-          block = yield vhd._readBlock(iBlock)
-          blocksByVhd.set(vhd, block)
-        }
-        const { bitmap, data } = block
-        if (isRootVhd) {
-          yield data.slice(i * SECTOR_SIZE, n * SECTOR_SIZE)
-          return
-        }
-        while (i < n) {
-          const hasData = mapTestBit(bitmap, i)
-          const start = i
-          do {
-            ++i
-          } while (i < n && mapTestBit(bitmap, i) === hasData)
-          if (hasData) {
-            yield data.slice(start * SECTOR_SIZE, i * SECTOR_SIZE)
-          } else {
-            yield * emitBlockSectors(iVhd + 1, start, i)
+
+          yield bitmap
+
+          const blocksByVhd = new Map()
+          const emitBlockSectors = function * (iVhd, i, n) {
+            const vhd = vhds[iVhd]
+            const isRootVhd = vhd === rootVhd
+            if (!vhd.containsBlock(iBlock)) {
+              if (isRootVhd) {
+                yield Buffer.alloc((n - i) * SECTOR_SIZE)
+              } else {
+                yield * emitBlockSectors(iVhd + 1, i, n)
+              }
+              return
+            }
+            let block = blocksByVhd.get(vhd)
+            if (block === undefined) {
+              block = yield vhd._readBlock(iBlock)
+              blocksByVhd.set(vhd, block)
+            }
+            const { bitmap, data } = block
+            if (isRootVhd) {
+              yield data.slice(i * SECTOR_SIZE, n * SECTOR_SIZE)
+              return
+            }
+            while (i < n) {
+              const hasData = mapTestBit(bitmap, i)
+              const start = i
+              do {
+                ++i
+              } while (i < n && mapTestBit(bitmap, i) === hasData)
+              if (hasData) {
+                yield data.slice(start * SECTOR_SIZE, i * SECTOR_SIZE)
+              } else {
+                yield * emitBlockSectors(iVhd + 1, start, i)
+              }
+            }
           }
+          yield * emitBlockSectors(owner, 0, sectorsPerBlockData)
         }
+        yield footer
+      } finally {
+        cleanup()
       }
-      yield * emitBlockSectors(owner, 0, sectorsPerBlockData)
     }
 
-    yield footer
-  } finally {
-    for (let i = 0, n = fds.length; i < n; ++i) {
-      handler.closeFile(fds[i]).catch(error => {
-        console.warn('createReadStream, closeFd', i, error)
-      })
-    }
+    const stream = asyncIteratorToStream(iterator())
+    stream.length = fileSize
+    return stream
+  } catch (e) {
+    cleanup()
+    throw e
   }
-})
+}

packages/vhd-lib/src/vhd.js

@@ -228,16 +228,15 @@ export default class Vhd {
     return this._read(
       sectorsToBytes(blockAddr),
       onlyBitmap ? this.bitmapSize : this.fullBlockSize
-    ).then(
-      buf =>
-        onlyBitmap
-          ? { id: blockId, bitmap: buf }
-          : {
-              id: blockId,
-              bitmap: buf.slice(0, this.bitmapSize),
-              data: buf.slice(this.bitmapSize),
-              buffer: buf,
-            }
+    ).then(buf =>
+      onlyBitmap
+        ? { id: blockId, bitmap: buf }
+        : {
+            id: blockId,
+            bitmap: buf.slice(0, this.bitmapSize),
+            data: buf.slice(this.bitmapSize),
+            buffer: buf,
+          }
     )
   }
 

packages/vhd-lib/vhd.integ.spec.js

@@ -107,12 +107,13 @@ test('ReadableSparseVHDStream can handle a sparse file', async () => {
     },
   ]
   const fileSize = blockSize * 110
-  const stream = createReadableSparseStream(
+  const stream = await createReadableSparseStream(
     fileSize,
     blockSize,
     blocks.map(b => b.offsetBytes),
     blocks
   )
+  expect(stream.length).toEqual(4197888)
   const pipe = stream.pipe(createWriteStream('output.vhd'))
   await fromEvent(pipe, 'finish')
   await execa('vhd-util', ['check', '-t', '-i', '-n', 'output.vhd'])

packages/xapi-explore-sr/.babelrc.js

@@ -0,0 +1,3 @@
+module.exports = require('../../@xen-orchestra/babel-config')(
+  require('./package.json')
+)

packages/xapi-explore-sr/.npmignore

@@ -0,0 +1,24 @@
+/benchmark/
+/benchmarks/
+*.bench.js
+*.bench.js.map
+
+/examples/
+example.js
+example.js.map
+*.example.js
+*.example.js.map
+
+/fixture/
+/fixtures/
+*.fixture.js
+*.fixture.js.map
+*.fixtures.js
+*.fixtures.js.map
+
+/test/
+/tests/
+*.spec.js
+*.spec.js.map
+
+__snapshots__/

packages/xapi-explore-sr/README.md

@@ -0,0 +1,52 @@
+# xapi-explore-sr [![Build Status](https://travis-ci.org/vatesfr/xen-orchestra.png?branch=master)](https://travis-ci.org/vatesfr/xen-orchestra)
+
+> Display the list of VDIs (unmanaged and snapshots included) of a SR
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/xapi-explore-sr):
+
+```
+> npm install --global xapi-explore-sr
+```
+
+## Usage
+
+```
+> xapi-explore-sr
+Usage: xapi-explore-sr [--full] <SR UUID> <XenServer URL> <XenServer user> [<XenServer password>]
+```
+
+## Development
+
+```
+# Install dependencies
+> npm install
+
+# Run the tests
+> npm test
+
+# Continuously compile
+> npm run dev
+
+# Continuously run the tests
+> npm run dev-test
+
+# Build for production (automatically called by npm install)
+> npm run build
+```
+
+## Contributions
+
+Contributions are *very* welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+ISC © [Vates SAS](https://vates.fr)

packages/xapi-explore-sr/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "xapi-explore-sr",
+  "version": "0.2.1",
+  "license": "ISC",
+  "description": "Display the list of VDIs (unmanaged and snapshots included) of a SR",
+  "keywords": [
+    "api",
+    "sr",
+    "vdi",
+    "vdis",
+    "xen",
+    "xen-api",
+    "xenapi"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xapi-explore-sr",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Julien Fontanet",
+    "email": "julien.fontanet@isonoe.net"
+  },
+  "preferGlobal": true,
+  "main": "dist/",
+  "bin": {
+    "xapi-explore-sr": "dist/index.js"
+  },
+  "files": [
+    "dist/"
+  ],
+  "engines": {
+    "node": ">=8"
+  },
+  "dependencies": {
+    "archy": "^1.0.0",
+    "chalk": "^2.3.2",
+    "exec-promise": "^0.7.0",
+    "human-format": "^0.10.0",
+    "lodash": "^4.17.4",
+    "pw": "^0.0.4",
+    "xen-api": "^0.21.0"
+  },
+  "devDependencies": {
+    "@babel/cli": "^7.1.5",
+    "@babel/core": "^7.1.5",
+    "@babel/preset-env": "^7.1.5",
+    "babel-plugin-lodash": "^3.2.11",
+    "cross-env": "^5.1.4",
+    "rimraf": "^2.6.1"
+  },
+  "scripts": {
+    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
+    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
+    "prebuild": "rimraf dist/",
+    "predev": "yarn run prebuild",
+    "prepublishOnly": "yarn run build"
+  }
+}

packages/xapi-explore-sr/src/index.js

@@ -0,0 +1,161 @@
+#!/usr/bin/env node
+
+import archy from 'archy'
+import chalk from 'chalk'
+import execPromise from 'exec-promise'
+import humanFormat from 'human-format'
+import pw from 'pw'
+import { createClient } from 'xen-api'
+import { forEach, map, orderBy } from 'lodash'
+
+// ===================================================================
+
+const askPassword = prompt =>
+  new Promise(resolve => {
+    prompt && process.stderr.write(`${prompt}: `)
+    pw(resolve)
+  })
+
+const formatSize = bytes =>
+  humanFormat(bytes, {
+    prefix: 'Gi',
+    scale: 'binary',
+  })
+
+const required = name => {
+  const e = `missing required argument <${name}>`
+  throw e
+}
+
+// -------------------------------------------------------------------
+
+const STYLES = [
+  [
+    vdi => !vdi.managed,
+    chalk.enabled ? chalk.red : label => `[unmanaged] ${label}`,
+  ],
+  [
+    vdi => vdi.is_a_snapshot,
+    chalk.enabled ? chalk.yellow : label => `[snapshot] ${label}`,
+  ],
+]
+const getStyle = vdi => {
+  for (let i = 0, n = STYLES.length; i < n; ++i) {
+    const entry = STYLES[i]
+    if (entry[0](vdi)) {
+      return entry[1]
+    }
+  }
+}
+
+const mapFilter = (collection, iteratee, results = []) => {
+  forEach(collection, function () {
+    const result = iteratee.apply(this, arguments)
+    if (result !== undefined) {
+      results.push(result)
+    }
+  })
+  return results
+}
+
+// -------------------------------------------------------------------
+
+execPromise(async args => {
+  if (args.length === 0 || args[0] === '-h' || args[0] === '--help') {
+    return `Usage: xapi-explore-sr [--full] <SR UUID> <XenServer URL> <XenServer user> [<XenServer password>]`
+  }
+
+  const full = args[0] === '--full'
+  if (full) {
+    args.shift()
+  }
+
+  const [
+    srUuid = required('SR UUID'),
+    url = required('XenServer URL'),
+    user = required('XenServer user'),
+    password = await askPassword('XenServer password'),
+  ] = args
+
+  const xapi = createClient({
+    allowUnauthorized: true,
+    auth: { user, password },
+    readOnly: true,
+    url,
+    watchEvents: false,
+  })
+  await xapi.connect()
+
+  const srRef = await xapi.call('SR.get_by_uuid', srUuid)
+  const sr = await xapi.call('SR.get_record', srRef)
+
+  const vdisByRef = {}
+  await Promise.all(
+    map(sr.VDIs, async ref => {
+      const vdi = await xapi.call('VDI.get_record', ref)
+      vdisByRef[ref] = vdi
+    })
+  )
+
+  const hasParents = {}
+  const vhdChildrenByUuid = {}
+  forEach(vdisByRef, vdi => {
+    const vhdParent = vdi.sm_config['vhd-parent']
+    if (vhdParent) {
+      ;(
+        vhdChildrenByUuid[vhdParent] || (vhdChildrenByUuid[vhdParent] = [])
+      ).push(vdi)
+    } else if (!(vdi.snapshot_of in vdisByRef)) {
+      return
+    }
+
+    hasParents[vdi.uuid] = true
+  })
+
+  const makeVdiNode = vdi => {
+    const { uuid } = vdi
+
+    let label = `${vdi.name_label} - ${uuid} - ${formatSize(
+      +vdi.physical_utilisation
+    )}`
+    const nodes = []
+
+    const vhdChildren = vhdChildrenByUuid[uuid]
+    if (vhdChildren) {
+      mapFilter(
+        orderBy(vhdChildren, 'is_a_snapshot', 'desc'),
+        makeVdiNode,
+        nodes
+      )
+    }
+
+    mapFilter(
+      vdi.snapshots,
+      ref => {
+        const vdi = vdisByRef[ref]
+        if (full || !vdi.sm_config['vhd-parent']) {
+          return makeVdiNode(vdi)
+        }
+      },
+      nodes
+    )
+
+    const style = getStyle(vdi)
+    if (style) {
+      label = style(label)
+    }
+
+    return { label, nodes }
+  }
+
+  const nodes = mapFilter(orderBy(vdisByRef, ['name_label', 'uuid']), vdi => {
+    if (!hasParents[vdi.uuid]) {
+      return makeVdiNode(vdi)
+    }
+  })
+
+  return archy({
+    label: `${sr.name_label} (${sr.VDIs.length} VDIs)`,
+    nodes,
+  })
+})

packages/xen-api/README.md

@@ -4,6 +4,9 @@
 
 Tested with:
 
+- XenServer 7.6
+- XenServer 7.5
+- XenServer 7.4
 - XenServer 7.3
 - XenServer 7.2
 - XenServer 7.1

packages/xen-api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xen-api",
-  "version": "0.19.0",
+  "version": "0.21.0",
   "license": "ISC",
   "description": "Connector to the Xen API",
   "keywords": [
@@ -39,13 +39,13 @@
     "http-request-plus": "^0.6.0",
     "iterable-backoff": "^0.0.0",
     "jest-diff": "^23.5.0",
-    "json-rpc-protocol": "^0.12.0",
+    "json-rpc-protocol": "^0.13.1",
     "kindof": "^2.0.0",
     "lodash": "^4.17.4",
     "make-error": "^1.3.0",
     "minimist": "^1.2.0",
     "ms": "^2.1.1",
-    "promise-toolbox": "^0.10.1",
+    "promise-toolbox": "^0.11.0",
     "pw": "0.0.4",
     "xmlrpc": "^1.3.2",
     "xo-collection": "^0.4.1"

packages/xen-api/src/_replaceSensitiveValues.js

@@ -0,0 +1,17 @@
+import mapValues from 'lodash/mapValues'
+
+export default function replaceSensitiveValues (value, replacement) {
+  function helper (value, name) {
+    if (name === 'password' && typeof value === 'string') {
+      return replacement
+    }
+
+    if (typeof value !== 'object' || value === null) {
+      return value
+    }
+
+    return Array.isArray(value) ? value.map(helper) : mapValues(value, helper)
+  }
+
+  return helper(value)
+}

packages/xen-api/src/index.js

@@ -11,7 +11,6 @@ import {
   forEach,
   isArray,
   isInteger,
-  isObject,
   map,
   noop,
   omit,
@@ -31,6 +30,7 @@ import {
 } from 'promise-toolbox'
 
 import autoTransport from './transports/auto'
+import replaceSensitiveValues from './_replaceSensitiveValues'
 
 const debug = createDebug('xen-api')
 
@@ -94,7 +94,7 @@ class XapiError extends BaseError {
     this.params = params
 
     // slots than can be assigned later
-    this.method = undefined
+    this.call = undefined
     this.url = undefined
     this.task = undefined
   }
@@ -137,8 +137,8 @@ const parseUrl = url => {
 const {
   create: createObject,
   defineProperties,
-  defineProperty,
   freeze: freezeObject,
+  keys: getKeys,
 } = Object
 
 // -------------------------------------------------------------------
@@ -190,10 +190,6 @@ const getKey = o => o.$id
 
 // -------------------------------------------------------------------
 
-const EMPTY_ARRAY = freezeObject([])
-
-// -------------------------------------------------------------------
-
 const getTaskResult = task => {
   const { status } = task
   if (status === 'cancelled') {
@@ -215,6 +211,15 @@ const getTaskResult = task => {
 
 // -------------------------------------------------------------------
 
+const RESERVED_FIELDS = {
+  id: true,
+  pool: true,
+  ref: true,
+  type: true,
+}
+
+// -------------------------------------------------------------------
+
 const CONNECTED = 'connected'
 const CONNECTING = 'connecting'
 const DISCONNECTED = 'disconnected'
@@ -229,6 +234,7 @@ export class Xapi extends EventEmitter {
     this._auth = opts.auth
     this._pool = null
     this._readOnly = Boolean(opts.readOnly)
+    this._RecordsByType = createObject(null)
     this._sessionId = null
     const url = (this._url = parseUrl(opts.url))
 
@@ -259,8 +265,8 @@ export class Xapi extends EventEmitter {
       const objects = (this._objects = new Collection())
       objects.getKey = getKey
 
-      this._objectsByRefs = createObject(null)
-      this._objectsByRefs[NULL_REF] = undefined
+      this._objectsByRef = createObject(null)
+      this._objectsByRef[NULL_REF] = undefined
 
       this._taskWatchers = Object.create(null)
 
@@ -462,7 +468,7 @@ export class Xapi extends EventEmitter {
     }
 
     const object =
-      this._objects.all[idOrUuidOrRef] || this._objectsByRefs[idOrUuidOrRef]
+      this._objects.all[idOrUuidOrRef] || this._objectsByRef[idOrUuidOrRef]
 
     if (object !== undefined) return object
 
@@ -474,7 +480,7 @@ export class Xapi extends EventEmitter {
   // Returns the object for a given opaque reference (internal to
   // XAPI).
   getObjectByRef (ref, defaultValue) {
-    const object = this._objectsByRefs[ref]
+    const object = this._objectsByRef[ref]
 
     if (object !== undefined) return object
 
@@ -497,16 +503,9 @@ export class Xapi extends EventEmitter {
   }
 
   async getRecord (type, ref) {
-    const record = await this._sessionCall(`${type}.get_record`, [ref])
-
-    // All custom properties are read-only and non enumerable.
-    defineProperties(record, {
-      $id: { value: record.uuid || ref },
-      $ref: { value: ref },
-      $type: { value: type },
-    })
-
-    return record
+    return this._wrapRecord(
+      await this._sessionCall(`${type}.get_record`, [ref])
+    )
   }
 
   async getRecordByUuid (type, uuid) {
@@ -669,7 +668,7 @@ export class Xapi extends EventEmitter {
 
   setFieldEntries (record, field, entries) {
     return Promise.all(
-      Object.keys(entries).map(entry => {
+      getKeys(entries).map(entry => {
         const value = entries[entry]
         if (value !== undefined) {
           return value === null
@@ -710,7 +709,7 @@ export class Xapi extends EventEmitter {
     let watcher = watchers[ref]
     if (watcher === undefined) {
       // sync check if the task is already settled
-      const task = this._objectsByRefs[ref]
+      const task = this._objectsByRef[ref]
       if (task !== undefined) {
         const result = getTaskResult(task)
         if (result !== undefined) {
@@ -775,78 +774,29 @@ export class Xapi extends EventEmitter {
   }
 
   _addObject (type, ref, object) {
-    const { _objectsByRefs: objectsByRefs } = this
-
-    const reservedKeys = {
-      id: true,
-      pool: true,
-      ref: true,
-      type: true,
-    }
-    const getKey = (key, obj) =>
-      reservedKeys[key] && obj === object ? `$$${key}` : `$${key}`
-
-    // Creates resolved properties.
-    forEach(object, function resolveObject (value, key, object) {
-      if (isArray(value)) {
-        if (!value.length) {
-          // If the array is empty, it isn't possible to be sure that
-          // it is not supposed to contain links, therefore, in
-          // benefice of the doubt, a resolved property is defined.
-          defineProperty(object, getKey(key, object), {
-            value: EMPTY_ARRAY,
-          })
-
-          // Minor memory optimization, use the same empty array for
-          // everyone.
-          object[key] = EMPTY_ARRAY
-        } else if (isOpaqueRef(value[0])) {
-          // This is an array of refs.
-          defineProperty(object, getKey(key, object), {
-            get: () => freezeObject(map(value, ref => objectsByRefs[ref])),
-          })
-
-          freezeObject(value)
-        }
-      } else if (isObject(value)) {
-        forEach(value, resolveObject)
-
-        freezeObject(value)
-      } else if (isOpaqueRef(value)) {
-        defineProperty(object, getKey(key, object), {
-          get: () => objectsByRefs[value],
-        })
-      }
-    })
-
-    // All custom properties are read-only and non enumerable.
-    defineProperties(object, {
-      $id: { value: object.uuid || ref },
-      $pool: { get: this._getPool },
-      $ref: { value: ref },
-      $type: { value: type },
-    })
+    object = this._wrapRecord(type, ref, object)
 
     // Finally freezes the object.
     freezeObject(object)
 
     const objects = this._objects
+    const objectsByRef = this._objectsByRef
 
     // An object's UUID can change during its life.
-    const prev = objectsByRefs[ref]
+    const prev = objectsByRef[ref]
     let prevUuid
     if (prev && (prevUuid = prev.uuid) && prevUuid !== object.uuid) {
       objects.remove(prevUuid)
     }
 
     this._objects.set(object)
-    objectsByRefs[ref] = object
+    objectsByRef[ref] = object
 
     if (type === 'pool') {
       this._pool = object
 
       const eventWatchers = this._eventWatchers
-      Object.keys(object.other_config).forEach(key => {
+      getKeys(object.other_config).forEach(key => {
         const eventWatcher = eventWatchers[key]
         if (eventWatcher !== undefined) {
           delete eventWatchers[key]
@@ -871,7 +821,7 @@ export class Xapi extends EventEmitter {
   }
 
   _removeObject (type, ref) {
-    const byRefs = this._objectsByRefs
+    const byRefs = this._objectsByRef
     const object = byRefs[ref]
     if (object !== undefined) {
       this._objects.unset(object.$id)
@@ -1028,6 +978,90 @@ export class Xapi extends EventEmitter {
 
     return getAllObjects().then(watchEvents)
   }
+
+  _wrapRecord (type, ref, data) {
+    const RecordsByType = this._RecordsByType
+    let Record = RecordsByType[type]
+    if (Record === undefined) {
+      const fields = getKeys(data)
+      const nFields = fields.length
+      const xapi = this
+
+      const objectsByRef = this._objectsByRef
+      const getObjectByRef = ref => objectsByRef[ref]
+
+      Record = function (ref, data) {
+        defineProperties(this, {
+          $id: { value: data.uuid || ref },
+          $ref: { value: ref },
+        })
+        for (let i = 0; i < nFields; ++i) {
+          const field = fields[i]
+          this[field] = data[field]
+        }
+      }
+
+      const getters = { $pool: this._getPool }
+      const props = { $type: type }
+      fields.forEach(field => {
+        props[`set_${field}`] = function (value) {
+          return xapi.setField(this, field, value)
+        }
+
+        const $field = (field in RESERVED_FIELDS ? '$$' : '$') + field
+
+        const value = data[field]
+        if (isArray(value)) {
+          if (value.length === 0 || isOpaqueRef(value[0])) {
+            getters[$field] = function () {
+              const value = this[field]
+              return value.length === 0 ? value : value.map(getObjectByRef)
+            }
+          }
+
+          props[`add_to_${field}`] = function (...values) {
+            return xapi
+              .call(`${type}.add_${field}`, this.$ref, values)
+              .then(noop)
+          }
+        } else if (value !== null && typeof value === 'object') {
+          getters[$field] = function () {
+            const value = this[field]
+            const result = {}
+            getKeys(value).forEach(key => {
+              result[key] = objectsByRef[value[key]]
+            })
+            return result
+          }
+          props[`update_${field}`] = function (entries) {
+            return xapi.setFieldEntries(this, field, entries)
+          }
+        } else if (isOpaqueRef(value)) {
+          getters[$field] = function () {
+            return objectsByRef[this[field]]
+          }
+        }
+      })
+      const descriptors = {}
+      getKeys(getters).forEach(key => {
+        descriptors[key] = {
+          configurable: true,
+          get: getters[key],
+        }
+      })
+      getKeys(props).forEach(key => {
+        descriptors[key] = {
+          configurable: true,
+          value: props[key],
+          writable: true,
+        }
+      })
+      defineProperties(Record.prototype, descriptors)
+
+      RecordsByType[type] = Record
+    }
+    return new Record(ref, data)
+  }
 }
 
 Xapi.prototype._transportCall = reduce(
@@ -1038,7 +1072,10 @@ Xapi.prototype._transportCall = reduce(
           error = wrapError(error)
         }
 
-        error.method = method
+        error.call = {
+          method,
+          params: replaceSensitiveValues(args, '* obfuscated *'),
+        }
         throw error
       })
     },

packages/xo-acl-resolver/index.js

@@ -1,5 +1,9 @@
 'use strict'
 
+const { unauthorized } = require('xo-common/api-errors')
+
+// ===================================================================
+
 // These global variables are not a problem because the algorithm is
 // synchronous.
 let permissionsByObject
@@ -105,23 +109,26 @@ function checkAuthorization (objectId, permission) {
 
 // -------------------------------------------------------------------
 
-module.exports = (
+function assertPermissions (
   permissionsByObject_,
   getObject_,
   permissions,
   permission
-) => {
+) {
   // Assign global variables.
   permissionsByObject = permissionsByObject_
   getObject = getObject_
 
   try {
-    if (permission) {
-      return checkAuthorization(permissions, permission)
+    if (permission !== undefined) {
+      const objectId = permissions
+      if (!checkAuthorization(objectId, permission)) {
+        throw unauthorized(permission, objectId)
+      }
     } else {
       for (const [objectId, permission] of permissions) {
         if (!checkAuthorization(objectId, permission)) {
-          return false
+          throw unauthorized(permission, objectId)
         }
       }
     }
@@ -132,3 +139,16 @@ module.exports = (
     permissionsByObject = getObject = null
   }
 }
+exports.assert = assertPermissions
+
+exports.check = function checkPermissions () {
+  try {
+    assertPermissions.apply(undefined, arguments)
+    return true
+  } catch (error) {
+    if (unauthorized.is(error)) {
+      return false
+    }
+    throw error
+  }
+}

packages/xo-acl-resolver/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-acl-resolver",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "license": "ISC",
   "description": "Xen-Orchestra internal: do ACLs resolution",
   "keywords": [],
@@ -21,5 +21,8 @@
   ],
   "engines": {
     "node": ">=6"
+  },
+  "dependencies": {
+    "xo-common": "^0.2.0"
   }
 }

packages/xo-cli/README.md

@@ -120,7 +120,7 @@ encoding by prefixing with `json:`:
 ##### VM import
 
  ```
-> xo-cli vm.import host=60a6939e-8b0a-4352-9954-5bde44bcdf7d @=vm.xva
+> xo-cli vm.import sr=60a6939e-8b0a-4352-9954-5bde44bcdf7d @=vm.xva
 ```
 
 ## Development

packages/xo-cli/package.json

@@ -42,7 +42,7 @@
     "nice-pipe": "0.0.0",
     "pretty-ms": "^4.0.0",
     "progress-stream": "^2.0.0",
-    "promise-toolbox": "^0.10.1",
+    "promise-toolbox": "^0.11.0",
     "pump": "^3.0.0",
     "pw": "^0.0.4",
     "strip-indent": "^2.0.0",

packages/xo-common/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-common",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "license": "AGPL-3.0",
   "description": "Code shared between [XO](https://xen-orchestra.com) server and clients",
   "keywords": [],

packages/xo-common/src/api-errors.js

@@ -20,7 +20,8 @@ class XoError extends BaseError {
 const create = (code, getProps) => {
   const factory = (...args) => new XoError({ ...getProps(...args), code })
   factory.is = (error, predicate) =>
-    error.code === code && iteratee(predicate)(error)
+    error.code === code &&
+    (predicate === undefined || iteratee(predicate)(error))
 
   return factory
 }
@@ -33,11 +34,18 @@ export const notImplemented = create(0, () => ({
 
 export const noSuchObject = create(1, (id, type) => ({
   data: { id, type },
-  message: 'no such object',
+  message: `no such ${type || 'object'} ${id}`,
 }))
 
-export const unauthorized = create(2, () => ({
-  message: 'not authenticated or not enough permissions',
+export const unauthorized = create(2, (permission, objectId, objectType) => ({
+  data: {
+    permission,
+    object: {
+      id: objectId,
+      type: objectType,
+    },
+  },
+  message: 'not enough permissions',
 }))
 
 export const invalidCredentials = create(3, () => ({

packages/xo-import-servers-csv/.npmignore

@@ -0,0 +1,10 @@
+/examples/
+example.js
+example.js.map
+*.example.js
+*.example.js.map
+
+/test/
+/tests/
+*.spec.js
+*.spec.js.map

packages/xo-import-servers-csv/README.md

@@ -0,0 +1,64 @@
+# xo-import-servers-csv [![Build Status](https://travis-ci.org/vatesfr/xen-orchestra.png?branch=master)](https://travis-ci.org/vatesfr/xen-orchestra)
+
+> CLI to import servers in XO from a CSV file
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/xo-import-servers-csv):
+
+```
+> npm install --global xo-import-servers-csv
+```
+
+## Usage
+
+`servers.csv`:
+
+```csv
+host,username,password
+xs1.company.net,user1,password1
+xs2.company.net:8080,user2,password2
+http://xs3.company.net,user3,password3
+```
+
+> The CSV file can also contains these optional fields: `label`, `autoConnect`, `allowUnauthorized`.
+
+Shell command:
+
+```
+> xo-import-servers-csv 'https://xo.company.tld' admin@admin.net admin < servers.csv
+```
+
+## Development
+
+```
+# Install dependencies
+> npm install
+
+# Run the tests
+> npm test
+
+# Continuously compile
+> npm run dev
+
+# Continuously run the tests
+> npm run dev-test
+
+# Build for production (automatically called by npm install)
+> npm run build
+```
+
+## Contributions
+
+Contributions are *very* welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+ISC © [Vates SAS](http://vates.fr)

packages/xo-import-servers-csv/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "xo-import-servers-csv",
+  "version": "1.1.0",
+  "license": "ISC",
+  "description": "CLI to import servers in XO from a CSV file",
+  "keywords": [
+    "csv",
+    "host",
+    "import",
+    "orchestra",
+    "pool",
+    "server",
+    "xen",
+    "xen-orchestra"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-import-servers-csv",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Julien Fontanet",
+    "email": "julien.fontanet@vates.fr"
+  },
+  "main": "dist/",
+  "bin": {
+    "xo-import-servers-csv": "dist/index.js"
+  },
+  "files": [
+    "dist/"
+  ],
+  "engines": {
+    "node": ">=4"
+  },
+  "dependencies": {
+    "csv-parser": "^2.1.0",
+    "end-of-stream": "^1.1.0",
+    "exec-promise": "^0.7.0",
+    "highland": "^2.10.1",
+    "through2": "^3.0.0",
+    "xo-lib": "^0.9.0"
+  },
+  "devDependencies": {
+    "@types/node": "^10.12.2",
+    "@types/through2": "^2.0.31",
+    "tslint": "^5.9.1",
+    "tslint-config-standard": "^8.0.1",
+    "typescript": "^3.1.6"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc -w",
+    "lint": "tslint 'src/*.ts'",
+    "posttest": "yarn run lint",
+    "prepublishOnly": "yarn run build",
+    "start": "node dist/index.js"
+  }
+}

packages/xo-import-servers-csv/src/index.d.ts

@@ -0,0 +1,23 @@
+declare module 'csv-parser' {
+  function csvParser(opts?: Object): any
+  export = csvParser
+}
+
+declare module 'exec-promise' {
+  function execPromise(cb: (args: string[]) => any): void
+  export = execPromise
+}
+
+declare module 'xo-lib' {
+  export default class Xo {
+    user?: { email: string }
+
+    constructor(opts?: { credentials?: {}; url: string })
+
+    call(method: string, ...params: any[]): Promise<any>
+
+    open(): Promise<void>
+
+    signIn(credentials: {}): Promise<void>
+  }
+}

packages/xo-import-servers-csv/src/index.ts

@@ -0,0 +1,87 @@
+#!/usr/bin/env node
+
+/// <reference path="./index.d.ts" />
+
+import csvParser = require('csv-parser')
+import execPromise = require('exec-promise')
+import through2 = require('through2')
+import Xo from 'xo-lib'
+
+const parseBoolean = (
+  value: string,
+  defaultValue?: boolean
+): boolean | undefined => {
+  if (value === undefined || value === '') {
+    return defaultValue
+  }
+
+  const lcValue = value.toLocaleLowerCase()
+
+  if (value === '0' || lcValue === 'false') {
+    return false
+  }
+
+  if (value === '1' || lcValue === 'true') {
+    return true
+  }
+
+  throw new Error(`invalid boolean value: ${value}`)
+}
+
+const requiredParam = (name: string) => {
+  throw `missing param: ${name}
+
+Usage: xo-import-servers-csv $url $username $password < $csvFile`
+}
+
+execPromise(
+  async ([
+    url = requiredParam('url'),
+    username = requiredParam('username'),
+    password = requiredParam('password'),
+  ]): Promise<void> => {
+    const xo = new Xo({ url })
+
+    await xo.open()
+    await xo.signIn({ username, password })
+    console.log('connected as', xo.user!.email)
+
+    const errors: any[] = []
+
+    const stream = process.stdin.pipe(csvParser()).pipe(
+      through2.obj(
+        (
+          { allowUnauthorized, autoConnect, host, label, password, username },
+          _,
+          next
+        ) => {
+          console.log('server', host)
+
+          xo.call('server.add', {
+            allowUnauthorized: parseBoolean(allowUnauthorized),
+            autoConnect: parseBoolean(autoConnect, false),
+            host,
+            label,
+            password,
+            username,
+          }).then(
+            () => next(),
+            (error: any) => {
+              errors.push({ host, error })
+              return next()
+            }
+          )
+        }
+      )
+    )
+
+    await new Promise((resolve, reject) => {
+      stream.on('error', reject)
+      stream.on('finish', resolve)
+    })
+
+    if (errors.length) {
+      console.error(errors)
+    }
+  }
+)

packages/xo-import-servers-csv/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "moduleResolution": "node",
+    "newLine": "lf",
+    "noImplicitAny": true,
+    "outDir": "dist/",
+    "removeComments": true,
+    "sourceMap": true,
+    "strictNullChecks": true,
+    "target": "es2015"
+  },
+  "includes": "src/**/*"
+}

packages/xo-import-servers-csv/tslint.json

@@ -0,0 +1,3 @@
+{
+  "extends": "tslint-config-standard"
+}

packages/xo-server-auth-ldap/package.json

@@ -38,7 +38,7 @@
     "inquirer": "^6.0.0",
     "ldapjs": "^1.0.1",
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.10.1"
+    "promise-toolbox": "^0.11.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/xo-server-backup-reports/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-server-backup-reports",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "license": "AGPL-3.0",
   "description": "Backup reports plugin for XO-Server",
   "keywords": [

packages/xo-server-backup-reports/src/index.js

@@ -50,6 +50,7 @@ const ICON_FAILURE = '🚨'
 const ICON_INTERRUPTED = '⚠️'
 const ICON_SKIPPED = '⏩'
 const ICON_SUCCESS = '✔'
+const ICON_WARNING = '⚠️'
 
 const STATUS_ICON = {
   failure: ICON_FAILURE,
@@ -99,12 +100,13 @@ const isSkippedError = error =>
   error.message === UNHEALTHY_VDI_CHAIN_ERROR ||
   error.message === NO_SUCH_OBJECT_ERROR
 
+const INDENT = '  '
 const createGetTemporalDataMarkdown = formatDate => (
   start,
   end,
   nbIndent = 0
 ) => {
-  const indent = '  '.repeat(nbIndent)
+  const indent = INDENT.repeat(nbIndent)
 
   const markdown = [`${indent}- **Start time**: ${formatDate(start)}`]
   if (end !== undefined) {
@@ -117,6 +119,17 @@ const createGetTemporalDataMarkdown = formatDate => (
   return markdown
 }
 
+const addWarnings = (text, warnings, nbIndent = 0) => {
+  if (warnings === undefined) {
+    return
+  }
+
+  const indent = INDENT.repeat(nbIndent)
+  warnings.forEach(({ message }) => {
+    text.push(`${indent}- **${ICON_WARNING} ${message}**`)
+  })
+}
+
 class BackupReportsXoPlugin {
   constructor (xo) {
     this._xo = xo
@@ -180,13 +193,14 @@ class BackupReportsXoPlugin {
       let markdown = [
         `##  Global status: ${log.status}`,
         '',
+        `- **Job ID**: ${log.jobId}`,
+        `- **Run ID**: ${runJobId}`,
         `- **mode**: ${mode}`,
         ...getTemporalDataMarkdown(log.start, log.end),
         `- **Error**: ${log.result.message}`,
-        '---',
-        '',
-        `*${pkg.name} v${pkg.version}*`,
       ]
+      addWarnings(markdown, log.warnings)
+      markdown.push('---', '', `*${pkg.name} v${pkg.version}*`)
 
       markdown = markdown.join('\n')
       return this._sendReport({
@@ -228,6 +242,7 @@ class BackupReportsXoPlugin {
         `- **UUID**: ${vm !== undefined ? vm.uuid : vmId}`,
         ...getTemporalDataMarkdown(taskLog.start, taskLog.end),
       ]
+      addWarnings(text, taskLog.warnings)
 
       const failedSubTasks = []
       const snapshotText = []
@@ -262,6 +277,7 @@ class BackupReportsXoPlugin {
             }** (${id}) ${icon}`,
             ...getTemporalDataMarkdown(subTaskLog.start, subTaskLog.end, 2)
           )
+          addWarnings(remotesText, subTaskLog.warnings, 2)
           if (subTaskLog.status === 'failure') {
             failedSubTasks.push(remote !== undefined ? remote.name : id)
             remotesText.push('', errorMessage)
@@ -278,6 +294,7 @@ class BackupReportsXoPlugin {
             `  - **${srName}** (${srUuid}) ${icon}`,
             ...getTemporalDataMarkdown(subTaskLog.start, subTaskLog.end, 2)
           )
+          addWarnings(srsText, subTaskLog.warnings, 2)
           if (subTaskLog.status === 'failure') {
             failedSubTasks.push(sr !== undefined ? sr.name_label : id)
             srsText.push('', errorMessage)
@@ -293,6 +310,7 @@ class BackupReportsXoPlugin {
           }
 
           const operationInfoText = []
+          addWarnings(operationInfoText, operationLog.warnings, 3)
           if (operationLog.status === 'success') {
             const size = operationLog.result.size
             if (operationLog.message === 'merge') {
@@ -395,6 +413,8 @@ class BackupReportsXoPlugin {
     let markdown = [
       `##  Global status: ${log.status}`,
       '',
+      `- **Job ID**: ${log.jobId}`,
+      `- **Run ID**: ${runJobId}`,
       `- **mode**: ${mode}`,
       ...getTemporalDataMarkdown(log.start, log.end),
       `- **Successes**: ${nSuccesses} / ${nVms}`,
@@ -406,6 +426,7 @@ class BackupReportsXoPlugin {
     if (globalMergeSize !== 0) {
       markdown.push(`- **Merge size**: ${formatSize(globalMergeSize)}`)
     }
+    addWarnings(markdown, log.warnings)
     markdown.push('')
 
     if (nFailures !== 0) {
@@ -646,8 +667,8 @@ class BackupReportsXoPlugin {
     const globalStatus = globalSuccess
       ? `Success`
       : nFailures !== 0
-        ? `Failure`
-        : `Skipped`
+      ? `Failure`
+      : `Skipped`
 
     let markdown = [
       `##  Global status: ${globalStatus}`,
@@ -706,8 +727,8 @@ class BackupReportsXoPlugin {
         globalSuccess
           ? ICON_SUCCESS
           : nFailures !== 0
-            ? ICON_FAILURE
-            : ICON_SKIPPED
+          ? ICON_FAILURE
+          : ICON_SKIPPED
       }`,
       nagiosStatus: globalSuccess ? 0 : 2,
       nagiosMarkdown: globalSuccess

packages/xo-server-cloud/src/index.js

@@ -1,9 +1,8 @@
 import Client, { createBackoff } from 'jsonrpc-websocket-client'
 import hrp from 'http-request-plus'
 
-const UPDATER_URL = 'localhost'
-const WS_PORT = 9001
-const HTTP_PORT = 9002
+const WS_URL = 'ws://localhost:9001'
+const HTTP_URL = 'http://localhost:9002'
 
 // ===================================================================
 
@@ -47,7 +46,7 @@ class XoServerCloud {
       this
     )
 
-    const updater = (this._updater = new Client(`${UPDATER_URL}:${WS_PORT}`))
+    const updater = (this._updater = new Client(WS_URL))
     const connect = () =>
       updater.open(createBackoff()).catch(error => {
         console.error('xo-server-cloud: fail to connect to updater', error)
@@ -143,7 +142,7 @@ class XoServerCloud {
       throw new Error('cannot get download token')
     }
 
-    const response = await hrp(`${UPDATER_URL}:${HTTP_PORT}/`, {
+    const response = await hrp(HTTP_URL, {
       headers: {
         Authorization: `Bearer ${downloadToken}`,
       },

packages/xo-server-perf-alert/src/index.js

@@ -463,7 +463,6 @@ ${monitorBodies.join('\n')}`
             try {
               const result = {
                 uuid,
-                name: definition.name,
                 object: this._xo.getXapi(uuid).getObject(uuid),
               }
 

packages/xo-server-transport-email/package.json

@@ -33,7 +33,7 @@
   "dependencies": {
     "nodemailer": "^4.4.1",
     "nodemailer-markdown": "^1.0.1",
-    "promise-toolbox": "^0.10.1"
+    "promise-toolbox": "^0.11.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/xo-server-transport-slack/package.json

@@ -32,7 +32,7 @@
     "node": ">=6"
   },
   "dependencies": {
-    "promise-toolbox": "^0.10.1",
+    "promise-toolbox": "^0.11.0",
     "slack-node": "^0.1.8"
   },
   "devDependencies": {

packages/xo-server-usage-report/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-server-usage-report",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "license": "AGPL-3.0",
   "description": "",
   "keywords": [
@@ -34,11 +34,14 @@
     "node": ">=6"
   },
   "dependencies": {
+    "@xen-orchestra/async-map": "^0.0.0",
     "@xen-orchestra/cron": "^1.0.3",
+    "@xen-orchestra/log": "^0.1.4",
     "handlebars": "^4.0.6",
     "html-minifier": "^3.5.8",
+    "human-format": "^0.10.0",
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.10.1"
+    "promise-toolbox": "^0.11.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/xo-server-usage-report/report.html.tpl

@@ -83,10 +83,6 @@
       border-top: 1px solid #95a5a6;
     }
 
-    .page .global tr:nth-last-child(2) td {
-      border-bottom: 1px solid #95a5a6;
-    }
-
     .top table{
       margin: auto;
       margin-top: 20px;
@@ -149,9 +145,9 @@
 
     <div class="page">
 
-      <table class ="global">
+      <table class="global">
         <tr>
-          <td id="title" rowspan="13">VMs</td>
+          <td id="title" rowspan="8">VMs</td>
         </tr>
         <tr>
           <td>Number:</td>
@@ -160,37 +156,37 @@
         <tr>
           <td>CPU:</td>
           <td>{{normaliseValue global.vms.cpu}} % {{normaliseEvolution global.vmsEvolution.cpu}}</td>
-        <tr>
+        </tr>
         <tr>
           <td>RAM:</td>
           <td>{{normaliseValue global.vms.ram}} GiB {{normaliseEvolution global.vmsEvolution.ram}}</td>
-        <tr>
+        </tr>
         <tr>
           <td>Disk read:</td>
           <td>{{normaliseValue global.vms.diskRead}} MiB {{normaliseEvolution global.vmsEvolution.diskRead}}
           </td>
-        <tr>
+        </tr>
         <tr>
           <td>Disk write:</td>
           <td>{{normaliseValue global.vms.diskWrite}} MiB {{normaliseEvolution global.vmsEvolution.diskWrite}}
           </td>
-        <tr>
+        </tr>
         <tr>
           <td>Network RX:</td>
           <td>{{normaliseValue global.vms.netReception}} KiB {{normaliseEvolution global.vmsEvolution.netReception}}
           </td>
-        <tr>
+        </tr>
         <tr>
           <td>Network TX:</td>
           <td>{{normaliseValue global.vms.netTransmission}} KiB {{normaliseEvolution global.vmsEvolution.netTransmission}}
           </td>
-        <tr>
+        </tr>
       </table>
 
       <div class="top">
 
         <table>
-          <caption>3rd top usages</caption>
+          <caption>Top VMs</caption>
           <tr>
             <th></th>
             <th>UUID</th>
@@ -239,6 +235,9 @@
             <td>{{normaliseValue this.value}} MiB</td>
           </tr>
           {{/each}}
+
+          {{getTopIops topVms}}
+
           <tr>
             <td rowspan='{{math topVms.netReception.length "+" 1}}' class="tableHeader">Network RX</td>
           </tr>
@@ -264,9 +263,9 @@
     </div>
 
     <div class="page">
-      <table class ="global">
+      <table class="global">
         <tr>
-          <td id="title" rowspan="13">Hosts</td>
+          <td id="title" rowspan="7">Hosts</td>
         </tr>
         <tr>
           <td>Number:</td>
@@ -277,34 +276,33 @@
           <td>CPU:</td>
           <td>{{normaliseValue global.hosts.cpu}} % {{normaliseEvolution global.hostsEvolution.cpu}}
           </td>
-        <tr>
+        </tr>
         <tr>
           <td>RAM:</td>
           <td>{{normaliseValue global.hosts.ram}} GiB {{normaliseEvolution global.hostsEvolution.ram}}
           </td>
-          </td>
-        <tr>
+        </tr>
         <tr>
           <td>Load average:</td>
           <td>{{normaliseValue global.hosts.load}}  {{normaliseEvolution global.hostsEvolution.load}}
           </td>
-        <tr>
+        </tr>
         <tr>
           <td>Network RX:</td>
           <td>{{normaliseValue global.hosts.netReception}} KiB {{normaliseEvolution global.hostsEvolution.netReception}}
           </td>
-        <tr>
+        </tr>
         <tr>
           <td>Network TX:</td>
           <td>{{normaliseValue global.hosts.netTransmission}} KiB {{normaliseEvolution global.hostsEvolution.netTransmission}}
           </td>
-        <tr>
+        </tr>
       </table>
 
       <div class="top">
 
         <table>
-          <caption>3rd top usages</caption>
+          <caption>Top hosts</caption>
           <tr>
             <th></th>
             <th>UUID</th>
@@ -368,19 +366,14 @@
     <div class="page">
       <div class="top">
         <table>
-          <caption>Most used storages </caption>
+          <caption>Top SRs</caption>
             <tr>
+              <th />
               <th>UUID</th>
               <th>Name</th>
               <th>value</th>
             </tr>
-            {{#each topSrs}}
-              <tr>
-                <td>{{shortUUID this.uuid}}</td>
-                <td>{{this.name}}</td>
-                <td>{{normaliseValue this.value}} GiB</td>
-              </tr>
-             {{/each}}
+            {{getTopSrs topSrs}}
           </table>
           <table>
              <caption>Hosts missing patches</caption>
@@ -531,6 +524,9 @@
               <th>RAM (GiB)</th>
               <th>Disk read (MiB)</th>
               <th>Disk write (MiB)</th>
+              <th>IOPS read</th>
+              <th>IOPS write</th>
+              <th>IOPS total</th>
               <th>Network RX (KiB)</th>
               <th>Network TX (KiB)</th>
             </tr>
@@ -542,6 +538,9 @@
                 <td>{{normaliseValue this.ram}} {{normaliseEvolution this.evolution.ram}}</td>
                 <td>{{normaliseValue this.diskRead}} {{normaliseEvolution this.evolution.diskRead}}</td>
                 <td>{{normaliseValue this.diskWrite}} {{normaliseEvolution this.evolution.diskWrite}}</td>
+                <td>{{formatIops this.iopsRead}} {{normaliseEvolution this.evolution.iopsRead}}</td>
+                <td>{{formatIops this.iopsWrite}} {{normaliseEvolution this.evolution.iopsWrite}}</td>
+                <td>{{formatIops this.iopsTotal}} {{normaliseEvolution this.evolution.iopsTotal}}</td>
                 <td>{{normaliseValue this.netReception}} {{normaliseEvolution this.evolution.netReception}}</td>
                 <td>{{normaliseValue this.netTransmission}} {{normaliseEvolution this.evolution.netTransmission}}</td>
               </tr>
@@ -584,8 +583,8 @@
                 <td>{{shortUUID this.uuid}}</td>
                 <td>{{this.name}}</td>
                 <td>{{normaliseValue this.total}} {{normaliseEvolution this.evolution.total}}</td>
-                <td>{{normaliseValue this.used}}</td>
-                <td>{{normaliseValue this.free}}</td>
+                <td>{{normaliseValue this.usedSpace}}</td>
+                <td>{{normaliseValue this.freeSpace}}</td>
               </tr>
             {{/each}}
           </table>

packages/xo-server-usage-report/src/index.js

@@ -1,4 +1,7 @@
+import asyncMap from '@xen-orchestra/async-map'
+import createLogger from '@xen-orchestra/log'
 import Handlebars from 'handlebars'
+import humanFormat from 'human-format'
 import { createSchedule } from '@xen-orchestra/cron'
 import { minify } from 'html-minifier'
 import {
@@ -21,6 +24,10 @@ import { readFile, writeFile } from 'fs'
 
 // ===================================================================
 
+const log = createLogger('xo:xo-server-usage-report')
+
+const GRANULARITY = 'days'
+
 const pReadFile = promisify(readFile)
 const pWriteFile = promisify(writeFile)
 
@@ -75,7 +82,7 @@ export const configurationSchema = {
     },
     periodicity: {
       type: 'string',
-      enum: ['monthly', 'weekly'],
+      enum: ['monthly', 'weekly', 'daily'],
       description:
         'If you choose weekly you will receive the report every sunday and if you choose monthly you will receive it every first day of the month.',
     },
@@ -87,6 +94,24 @@ export const configurationSchema = {
 
 // ===================================================================
 
+const shortUuid = uuid => {
+  if (typeof uuid === 'string') {
+    return uuid.split('-')[0]
+  }
+}
+
+const formatIops = value =>
+  isFinite(value)
+    ? humanFormat(value, {
+        unit: 'IOPS',
+        decimals: 2,
+      })
+    : '-'
+
+const normaliseValue = value => (isFinite(value) ? round(value, 2) : '-')
+
+// ===================================================================
+
 Handlebars.registerHelper('compare', function (
   lvalue,
   operator,
@@ -122,29 +147,62 @@ Handlebars.registerHelper('math', function (lvalue, operator, rvalue, options) {
   return mathOperators[operator](+lvalue, +rvalue)
 })
 
-Handlebars.registerHelper('shortUUID', uuid => {
-  if (typeof uuid === 'string') {
-    return uuid.split('-')[0]
-  }
-})
+Handlebars.registerHelper('shortUUID', shortUuid)
 
-Handlebars.registerHelper(
-  'normaliseValue',
-  value => (isFinite(value) ? round(value, 2) : '-')
-)
+Handlebars.registerHelper('normaliseValue', normaliseValue)
 
 Handlebars.registerHelper(
   'normaliseEvolution',
   value =>
     new Handlebars.SafeString(
-      isFinite(+value) && +value !== 0
-        ? (value = round(value, 2)) > 0
+      isFinite((value = round(value, 2))) && value !== 0
+        ? value > 0
           ? `(<b style="color: green;">▲ ${value}%</b>)`
           : `(<b style="color: red;">▼ ${String(value).slice(1)}%</b>)`
         : ''
     )
 )
 
+Handlebars.registerHelper('formatIops', formatIops)
+
+const getHeader = (label, size) => `
+  <tr>
+    <td rowspan='${size + 1}' class="tableHeader">${label}</td>
+  </tr>
+`
+
+const getBody = ({ uuid, name, value }, transformValue, unit) => `
+  <tr>
+    <td>${shortUuid(uuid)}</td>
+    <td>${name}</td>
+    <td>${transformValue(value)}${unit !== undefined ? ` ${unit}` : ''}</td>
+  </tr>
+`
+
+const getTopIops = ({ iopsRead, iopsWrite, iopsTotal }) => `
+  ${getHeader('IOPS read', iopsRead.length)}
+  ${iopsRead.map(obj => getBody(obj, formatIops)).join('')}
+  ${getHeader('IOPS write', iopsWrite.length)}
+  ${iopsWrite.map(obj => getBody(obj, formatIops)).join('')}
+  ${getHeader('IOPS total', iopsTotal.length)}
+  ${iopsTotal.map(obj => getBody(obj, formatIops)).join('')}
+`
+
+Handlebars.registerHelper(
+  'getTopSrs',
+  ({ usedSpace, iopsRead, iopsWrite, iopsTotal }) =>
+    new Handlebars.SafeString(`
+      ${getHeader('Used space', usedSpace.length)}
+      ${usedSpace.map(obj => getBody(obj, normaliseValue, 'GiB')).join('')}
+      ${getTopIops({ iopsRead, iopsWrite, iopsTotal })}
+    `)
+)
+
+Handlebars.registerHelper(
+  'getTopIops',
+  props => new Handlebars.SafeString(getTopIops(props))
+)
+
 // ===================================================================
 
 function computeMean (values) {
@@ -196,12 +254,10 @@ function getTop (objects, options) {
 function computePercentage (curr, prev, options) {
   return zipObject(
     options,
-    map(
-      options,
-      opt =>
-        prev[opt] === 0 || prev[opt] === null
-          ? 'NONE'
-          : `${((curr[opt] - prev[opt]) * 100) / prev[opt]}`
+    map(options, opt =>
+      prev[opt] === 0 || prev[opt] === null
+        ? 'NONE'
+        : `${((curr[opt] - prev[opt]) * 100) / prev[opt]}`
     )
   )
 }
@@ -217,26 +273,47 @@ function getMemoryUsedMetric ({ memory, memoryFree = memory }) {
   return map(memory, (value, key) => value - memoryFree[key])
 }
 
+const METRICS_MEAN = {
+  cpu: computeDoubleMean,
+  disk: value => computeDoubleMean(values(value)) / mibPower,
+  iops: value => computeDoubleMean(values(value)),
+  load: computeMean,
+  net: value => computeDoubleMean(value) / kibPower,
+  ram: stats => computeMean(getMemoryUsedMetric(stats)) / gibPower,
+}
+
 // ===================================================================
 
 async function getVmsStats ({ runningVms, xo }) {
   return orderBy(
     await Promise.all(
       map(runningVms, async vm => {
-        const vmStats = await xo.getXapiVmStats(vm, 'days')
+        const { stats } = await xo
+          .getXapiVmStats(vm, GRANULARITY)
+          .catch(error => {
+            log.warn('Error on fetching VM stats', {
+              error,
+              vmId: vm.id,
+            })
+            return {
+              stats: {},
+            }
+          })
+
+        const iopsRead = METRICS_MEAN.iops(get(stats.iops, 'r'))
+        const iopsWrite = METRICS_MEAN.iops(get(stats.iops, 'w'))
         return {
           uuid: vm.uuid,
           name: vm.name_label,
-          cpu: computeDoubleMean(vmStats.stats.cpus),
-          ram: computeMean(getMemoryUsedMetric(vmStats.stats)) / gibPower,
-          diskRead:
-            computeDoubleMean(values(get(vmStats.stats.xvds, 'r'))) / mibPower,
-          diskWrite:
-            computeDoubleMean(values(get(vmStats.stats.xvds, 'w'))) / mibPower,
-          netReception:
-            computeDoubleMean(get(vmStats.stats.vifs, 'rx')) / kibPower,
-          netTransmission:
-            computeDoubleMean(get(vmStats.stats.vifs, 'tx')) / kibPower,
+          cpu: METRICS_MEAN.cpu(stats.cpus),
+          ram: METRICS_MEAN.ram(stats),
+          diskRead: METRICS_MEAN.disk(get(stats.xvds, 'r')),
+          diskWrite: METRICS_MEAN.disk(get(stats.xvds, 'w')),
+          iopsRead,
+          iopsWrite,
+          iopsTotal: iopsRead + iopsWrite,
+          netReception: METRICS_MEAN.net(get(stats.vifs, 'rx')),
+          netTransmission: METRICS_MEAN.net(get(stats.vifs, 'tx')),
         }
       })
     ),
@@ -249,17 +326,26 @@ async function getHostsStats ({ runningHosts, xo }) {
   return orderBy(
     await Promise.all(
       map(runningHosts, async host => {
-        const hostStats = await xo.getXapiHostStats(host, 'days')
+        const { stats } = await xo
+          .getXapiHostStats(host, GRANULARITY)
+          .catch(error => {
+            log.warn('Error on fetching host stats', {
+              error,
+              hostId: host.id,
+            })
+            return {
+              stats: {},
+            }
+          })
+
         return {
           uuid: host.uuid,
           name: host.name_label,
-          cpu: computeDoubleMean(hostStats.stats.cpus),
-          ram: computeMean(getMemoryUsedMetric(hostStats.stats)) / gibPower,
-          load: computeMean(hostStats.stats.load),
-          netReception:
-            computeDoubleMean(get(hostStats.stats.pifs, 'rx')) / kibPower,
-          netTransmission:
-            computeDoubleMean(get(hostStats.stats.pifs, 'tx')) / kibPower,
+          cpu: METRICS_MEAN.cpu(stats.cpus),
+          ram: METRICS_MEAN.ram(stats),
+          load: METRICS_MEAN.load(stats.load),
+          netReception: METRICS_MEAN.net(get(stats.pifs, 'rx')),
+          netTransmission: METRICS_MEAN.net(get(stats.pifs, 'tx')),
         }
       })
     ),
@@ -268,24 +354,54 @@ async function getHostsStats ({ runningHosts, xo }) {
   )
 }
 
-function getSrsStats (xoObjects) {
+async function getSrsStats ({ xo, xoObjects }) {
   return orderBy(
-    map(filter(xoObjects, obj => obj.type === 'SR' && obj.size > 0), sr => {
-      const total = sr.size / gibPower
-      const used = sr.physical_usage / gibPower
-      let name = sr.name_label
-      if (!sr.shared) {
-        name += ` (${find(xoObjects, { id: sr.$container }).name_label})`
+    await asyncMap(
+      filter(
+        xoObjects,
+        obj => obj.type === 'SR' && obj.size > 0 && obj.$PBDs.length > 0
+      ),
+      async sr => {
+        const totalSpace = sr.size / gibPower
+        const usedSpace = sr.physical_usage / gibPower
+        let name = sr.name_label
+        // [Bug in XO] a SR with not container can be found (SR attached to a PBD with no host attached)
+        let container
+        if (
+          !sr.shared &&
+          (container = find(xoObjects, { id: sr.$container })) !== undefined
+        ) {
+          name += ` (${container.name_label})`
+        }
+
+        const { stats } = await xo
+          .getXapiSrStats(sr.id, GRANULARITY)
+          .catch(error => {
+            log.warn('Error on fetching SR stats', {
+              error,
+              srId: sr.id,
+            })
+            return {
+              stats: {},
+            }
+          })
+
+        const iopsRead = computeMean(get(stats.iops, 'r'))
+        const iopsWrite = computeMean(get(stats.iops, 'w'))
+
+        return {
+          uuid: sr.uuid,
+          name,
+          total: totalSpace,
+          usedSpace,
+          freeSpace: totalSpace - usedSpace,
+          iopsRead,
+          iopsWrite,
+          iopsTotal: iopsRead + iopsWrite,
+        }
       }
-      return {
-        uuid: sr.uuid,
-        name,
-        total,
-        used,
-        free: total - used,
-      }
-    }),
-    'total',
+    ),
+    'name',
     'desc'
   )
 }
@@ -351,6 +467,9 @@ function getTopVms ({ vmsStats, xo }) {
     'ram',
     'diskRead',
     'diskWrite',
+    'iopsRead',
+    'iopsWrite',
+    'iopsTotal',
     'netReception',
     'netTransmission',
   ])
@@ -366,8 +485,8 @@ function getTopHosts ({ hostsStats, xo }) {
   ])
 }
 
-function getTopSrs ({ srsStats, xo }) {
-  return getTop(srsStats, ['used']).used
+function getTopSrs (srsStats) {
+  return getTop(srsStats, ['usedSpace', 'iopsRead', 'iopsWrite', 'iopsTotal'])
 }
 
 async function getHostsMissingPatches ({ runningHosts, xo }) {
@@ -376,6 +495,13 @@ async function getHostsMissingPatches ({ runningHosts, xo }) {
       let hostsPatches = await xo
         .getXapi(host)
         .listMissingPoolPatchesOnHost(host._xapiId)
+        .catch(error => {
+          console.error(
+            '[WARN] error on fetching hosts missing patches:',
+            JSON.stringify(error)
+          )
+          return []
+        })
 
       if (host.license_params.sku_type === 'free') {
         hostsPatches = filter(hostsPatches, { paid: false })
@@ -417,6 +543,9 @@ async function computeEvolution ({ storedStatsPath, ...newStats }) {
         'ram',
         'diskRead',
         'diskWrite',
+        'iopsRead',
+        'iopsWrite',
+        'iopsTotal',
         'netReception',
         'netTransmission',
       ],
@@ -506,7 +635,7 @@ async function dataBuilder ({ xo, storedStatsPath, all }) {
     xo.getAllUsers(),
     getVmsStats({ xo, runningVms }),
     getHostsStats({ xo, runningHosts }),
-    getSrsStats(xoObjects),
+    getSrsStats({ xo, xoObjects }),
     getHostsMissingPatches({ xo, runningHosts }),
   ])
 
@@ -522,7 +651,7 @@ async function dataBuilder ({ xo, storedStatsPath, all }) {
     computeGlobalHostsStats({ xo, hostsStats, haltedHosts }),
     getTopVms({ xo, vmsStats }),
     getTopHosts({ xo, hostsStats }),
-    getTopSrs({ xo, srsStats }),
+    getTopSrs(srsStats),
     getAllUsersEmail(users),
   ])
 
@@ -571,6 +700,12 @@ async function dataBuilder ({ xo, storedStatsPath, all }) {
 
 // ===================================================================
 
+const CRON_BY_PERIODICITY = {
+  monthly: '0 6 1 * *',
+  weekly: '0 6 * * 0',
+  daily: '0 6 * * *',
+}
+
 class UsageReportPlugin {
   constructor ({ xo, getDataDir }) {
     this._xo = xo
@@ -591,7 +726,7 @@ class UsageReportPlugin {
     }
 
     this._job = createSchedule(
-      configuration.periodicity === 'monthly' ? '00 06 1 * *' : '00 06 * * 0'
+      CRON_BY_PERIODICITY[configuration.periodicity]
     ).createJob(async () => {
       try {
         await this._sendReport(true)

packages/xo-server/bin/xo-server

@@ -11,21 +11,8 @@ require('../better-stacks')
 // less memory usage.
 global.Promise = require('bluebird')
 
-// Make unhandled rejected promises visible.
-process.on('unhandledRejection', function (reason) {
-  console.warn('[Warn] Possibly unhandled rejection:', reason && reason.stack || reason)
-})
-
-;(function (EE) {
-  var proto = EE.prototype
-  var emit = proto.emit
-  proto.emit = function patchedError (event, error) {
-    if (event === 'error' && !this.listenerCount(event)) {
-      return console.warn('[Warn] Unhandled error event:', error && error.stack || error)
-    }
-
-    return emit.apply(this, arguments)
-  }
-})(require('events').EventEmitter)
+require('@xen-orchestra/log/configure').catchGlobalErrors(
+  require('@xen-orchestra/log').default('xo:xo-server')
+)
 
 require('exec-promise')(require('../'))

packages/xo-server/config.json

@@ -34,6 +34,12 @@
   // Necessary for external authentication providers.
   "createUserOnFirstSignin": true,
 
+  "remoteOptions": {
+    "mountsDir": "/run/xo-server/mounts",
+
+    "timeout": 600e3
+  },
+
   // Whether API logs should contains the full request/response on
   // errors.
   //

packages/xo-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-server",
-  "version": "5.28.0",
+  "version": "5.29.4",
   "license": "AGPL-3.0",
   "description": "Server part of Xen-Orchestra",
   "keywords": [
@@ -34,7 +34,8 @@
     "@xen-orchestra/async-map": "^0.0.0",
     "@xen-orchestra/cron": "^1.0.3",
     "@xen-orchestra/emit-async": "^0.0.0",
-    "@xen-orchestra/fs": "^0.3.1",
+    "@xen-orchestra/fs": "^0.4.1",
+    "@xen-orchestra/log": "^0.1.4",
     "@xen-orchestra/mixin": "^0.0.0",
     "ajv": "^6.1.1",
     "app-conf": "^0.5.0",
@@ -94,7 +95,7 @@
     "passport": "^0.4.0",
     "passport-local": "^1.0.0",
     "pretty-format": "^23.0.0",
-    "promise-toolbox": "^0.10.1",
+    "promise-toolbox": "^0.11.0",
     "proxy-agent": "^3.0.0",
     "pug": "^2.0.0-rc.4",
     "pump": "^3.0.0",
@@ -108,17 +109,17 @@
     "stoppable": "^1.0.5",
     "struct-fu": "^1.2.0",
     "tar-stream": "^1.5.5",
-    "through2": "^2.0.3",
+    "through2": "^3.0.0",
     "tmp": "^0.0.33",
     "uuid": "^3.0.1",
     "value-matcher": "^0.2.0",
-    "vhd-lib": "^0.3.2",
+    "vhd-lib": "^0.4.0",
     "ws": "^6.0.0",
-    "xen-api": "^0.19.0",
+    "xen-api": "^0.21.0",
     "xml2js": "^0.4.19",
-    "xo-acl-resolver": "^0.3.0",
+    "xo-acl-resolver": "^0.4.0",
     "xo-collection": "^0.4.1",
-    "xo-common": "^0.1.1",
+    "xo-common": "^0.2.0",
     "xo-remote-parser": "^0.5.0",
     "xo-vmdk-to-vhd": "^0.1.5",
     "yazl": "^2.4.3"
@@ -137,6 +138,7 @@
     "@babel/preset-env": "^7.0.0",
     "@babel/preset-flow": "^7.0.0",
     "babel-plugin-lodash": "^3.3.2",
+    "babel-plugin-transform-dev": "^2.0.1",
     "cross-env": "^5.1.3",
     "index-modules": "^0.3.0",
     "rimraf": "^2.6.2"

packages/xo-server/sample.config.yaml

@@ -46,14 +46,12 @@
 
 # Configuration of the embedded HTTP server.
 http:
-
   # Hosts & ports on which to listen.
   #
   # By default, the server listens on [::]:80.
   listen:
     # Basic HTTP.
-    -
-      # Address on which the server is listening on.
+    - # Address on which the server is listening on.
       #
       # Sets it to 'localhost' for IP to listen only on the local host.
       #
@@ -124,23 +122,20 @@ http:
 
 # Connection to the Redis server.
 redis:
-    # Unix sockets can be used
-    #
-    # Default: undefined
-    #socket: /var/run/redis/redis.sock
-
-    # Syntax: redis://[db[:password]@]hostname[:port][/db-number]
-    #
-    # Default: redis://localhost:6379/0
-    #uri: redis://redis.company.lan/42
-
-    # List of aliased commands.
-    #
-    # See http://redis.io/topics/security#disabling-of-specific-commands
-    #renameCommands:
-    #  del: '3dda29ad-3015-44f9-b13b-fa570de92489'
-    #  srem: '3fd758c9-5610-4e9d-a058-dbf4cb6d8bf0'
-
+  # Unix sockets can be used
+  #
+  # Default: undefined
+  #socket: /var/run/redis/redis.sock
+  # Syntax: redis://[db[:password]@]hostname[:port][/db-number]
+  #
+  # Default: redis://localhost:6379/0
+  #uri: redis://redis.company.lan/42
+  # List of aliased commands.
+  #
+  # See http://redis.io/topics/security#disabling-of-specific-commands
+  #renameCommands:
+  #  del: '3dda29ad-3015-44f9-b13b-fa570de92489'
+  #  srem: '3fd758c9-5610-4e9d-a058-dbf4cb6d8bf0'
 
 # Directory containing the database of XO.
 # Currently used for logs.

packages/xo-server/src/api/backup.js

@@ -1,8 +1,11 @@
 import archiver from 'archiver'
+import createLogger from '@xen-orchestra/log'
 import { basename } from 'path'
 import { format } from 'json-rpc-peer'
 import { forEach } from 'lodash'
 
+const log = createLogger('xo:backup')
+
 // ===================================================================
 
 export function list ({ remote }) {
@@ -62,7 +65,7 @@ function handleFetchFiles (
 
       const archive = archiver(archiveFormat)
       archive.on('error', error => {
-        console.error(error)
+        log.error(error)
         res.end(format.error(0, error))
       })
 
@@ -74,7 +77,7 @@ function handleFetchFiles (
       archive.pipe(res)
     })
     .catch(error => {
-      console.error(error)
+      log.error(error)
       res.writeHead(500)
       res.end(format.error(0, error))
     })

packages/xo-server/src/api/disk.js

@@ -1,23 +1,38 @@
+import createLogger from '@xen-orchestra/log'
 import pump from 'pump'
 import { format } from 'json-rpc-peer'
-import { unauthorized } from 'xo-common/api-errors'
+import { noSuchObject } from 'xo-common/api-errors'
 
 import { parseSize } from '../utils'
 
+const log = createLogger('xo:disk')
+
 // ===================================================================
 
 export async function create ({ name, size, sr, vm, bootable, position, mode }) {
   const attach = vm !== undefined
 
-  let resourceSet
-  if (attach && (resourceSet = vm.resourceSet) != null) {
-    await this.checkResourceSetConstraints(resourceSet, this.user.id, [sr.id])
-    await this.allocateLimitsInResourceSet({ disk: size }, resourceSet)
-  } else if (
-    !(await this.hasPermissions(this.user.id, [[sr.id, 'administrate']]))
-  ) {
-    throw unauthorized()
-  }
+  do {
+    let resourceSet
+    if (attach && (resourceSet = vm.resourceSet) != null) {
+      try {
+        await this.checkResourceSetConstraints(resourceSet, this.user.id, [
+          sr.id,
+        ])
+        await this.allocateLimitsInResourceSet({ disk: size }, resourceSet)
+
+        break
+      } catch (error) {
+        if (!noSuchObject.is(error, { data: { id: resourceSet } })) {
+          throw error
+        }
+      }
+
+      // the resource set does not exist, falls back to normal check
+    }
+
+    await this.checkPermissions(this.user.id, [[sr.id, 'administrate']])
+  } while (false)
 
   const xapi = this.getXapi(sr)
   const vdi = await xapi.createVdi({
@@ -72,7 +87,7 @@ async function handleExportContent (req, res, { xapi, id }) {
   )
   pump(stream, res, error => {
     if (error != null) {
-      console.warn('disk.exportContent', error)
+      log.warn('disk.exportContent', { error })
     }
   })
 }
@@ -108,6 +123,7 @@ async function handleImportContent (req, res, { xapi, id }) {
   req.setTimeout(43200000) // 12 hours
 
   try {
+    req.length = +req.headers['content-length']
     await xapi.importVdiContent(id, req)
     res.end(format.response(0, true))
   } catch (e) {

packages/xo-server/src/api/ip-pool.js

@@ -1,5 +1,3 @@
-import { unauthorized } from 'xo-common/api-errors'
-
 export function create (props) {
   return this.createIpPool(props)
 }
@@ -22,15 +20,12 @@ delete_.description = 'Delete an ipPool'
 export function getAll (params) {
   const { user } = this
 
-  if (!user) {
-    throw unauthorized()
-  }
-
   return this.getAllIpPools(
     user.permission === 'admin' ? params && params.userId : user.id
   )
 }
 
+getAll.permission = ''
 getAll.description = 'List all ipPools'
 
 // -------------------------------------------------------------------

packages/xo-server/src/api/resource-set.js

@@ -1,7 +1,3 @@
-import { unauthorized } from 'xo-common/api-errors'
-
-// ===================================================================
-
 export function create ({ name, subjects, objects, limits }) {
   return this.createResourceSet(name, subjects, objects, limits)
 }
@@ -99,14 +95,10 @@ set.params = {
 // -------------------------------------------------------------------
 
 export function get ({ id }) {
-  const { user } = this
-  if (!user) {
-    throw unauthorized()
-  }
-
   return this.getResourceSet(id)
 }
 
+get.permission = ''
 get.params = {
   id: {
     type: 'string',
@@ -116,14 +108,10 @@ get.params = {
 // -------------------------------------------------------------------
 
 export async function getAll () {
-  const { user } = this
-  if (!user) {
-    throw unauthorized()
-  }
-
-  return this.getAllResourceSets(user.id)
+  return this.getAllResourceSets(this.user.id)
 }
 
+getAll.permission = ''
 getAll.description = 'Get the list of all existing resource set'
 
 // -------------------------------------------------------------------

packages/xo-server/src/api/server.js

@@ -4,7 +4,7 @@ export async function add ({ autoConnect = true, ...props }) {
   const server = await this.registerXenServer(props)
 
   if (autoConnect) {
-    ;this.connectXenServer(server.id)::ignoreErrors()
+    this.connectXenServer(server.id)::ignoreErrors()
   }
 
   return server.id
@@ -105,7 +105,7 @@ set.params = {
 // -------------------------------------------------------------------
 
 export async function connect ({ id }) {
-  ;this.updateXenServer(id, { enabled: true })::ignoreErrors()
+  this.updateXenServer(id, { enabled: true })::ignoreErrors()
   await this.connectXenServer(id)
 }
 
@@ -122,7 +122,7 @@ connect.params = {
 // -------------------------------------------------------------------
 
 export async function disconnect ({ id }) {
-  ;this.updateXenServer(id, { enabled: false })::ignoreErrors()
+  this.updateXenServer(id, { enabled: false })::ignoreErrors()
   await this.disconnectXenServer(id)
 }
 

packages/xo-server/src/api/vdi.js

@@ -1,6 +1,6 @@
 // FIXME: rename to disk.*
 
-import { invalidParameters, unauthorized } from 'xo-common/api-errors'
+import { invalidParameters } from 'xo-common/api-errors'
 import { isArray, reduce } from 'lodash'
 
 import { parseSize } from '../utils'
@@ -67,13 +67,8 @@ export async function set (params) {
         { disk: size - vdi.size },
         resourceSetId
       )
-    } else if (
-      !(
-        this.user.permission === 'admin' ||
-        (await this.hasPermissions(this.user.id, [[vdi.$SR, 'operate']]))
-      )
-    ) {
-      throw unauthorized()
+    } else {
+      await this.checkPermissions(this.user.id, [[vdi.$SR, 'operate']])
     }
 
     await xapi.resizeVdi(ref, size)

packages/xo-server/src/api/vif.js

@@ -6,7 +6,7 @@ import { diffItems } from '../utils'
 
 // TODO: move into vm and rename to removeInterface
 async function delete_ ({ vif }) {
-  ;this.allocIpAddresses(
+  this.allocIpAddresses(
     vif.id,
     null,
     vif.allowedIpv4Addresses.concat(vif.allowedIpv6Addresses)

packages/xo-server/src/api/vm.js

@@ -1,7 +1,7 @@
-import concat from 'lodash/concat'
 import defer from 'golike-defer'
 import { format } from 'json-rpc-peer'
 import { ignoreErrors } from 'promise-toolbox'
+import { assignWith, concat } from 'lodash'
 import {
   forbiddenOperation,
   invalidParameters,
@@ -32,13 +32,7 @@ function checkPermissionOnSrs (vm, permission = 'operate') {
     ])
   })
 
-  return this.hasPermissions(this.session.get('user_id'), permissions).then(
-    success => {
-      if (!success) {
-        throw unauthorized()
-      }
-    }
-  )
+  return this.checkPermissions(this.session.get('user_id'), permissions)
 }
 
 // ===================================================================
@@ -54,13 +48,10 @@ export async function create (params) {
   const { user } = this
   const resourceSet = extract(params, 'resourceSet')
   const template = extract(params, 'template')
-  if (
-    resourceSet === undefined &&
-    !(await this.hasPermissions(this.user.id, [
+  if (resourceSet === undefined) {
+    await this.checkPermissions(this.user.id, [
       [template.$pool, 'administrate'],
-    ]))
-  ) {
-    throw unauthorized()
+    ])
   }
 
   params.template = template._xapiId
@@ -151,8 +142,10 @@ export async function create (params) {
   if (resourceSet) {
     await this.checkResourceSetConstraints(resourceSet, user.id, objectIds)
     checkLimits = async limits2 => {
-      await this.allocateLimitsInResourceSet(limits, resourceSet)
-      await this.allocateLimitsInResourceSet(limits2, resourceSet)
+      await this.allocateLimitsInResourceSet(
+        assignWith({}, limits, limits2, (l1 = 0, l2) => l1 + l2),
+        resourceSet
+      )
     }
   }
 
@@ -373,7 +366,7 @@ async function delete_ ({
     vm.type === 'VM' && // only regular VMs
     xapi.xo.getData(vm._xapiId, 'resourceSet') != null
   ) {
-    ;this.setVmResourceSet(vm._xapiId, null)::ignoreErrors()
+    this.setVmResourceSet(vm._xapiId, null)::ignoreErrors()
   }
 
   return xapi.deleteVm(
@@ -424,14 +417,14 @@ ejectCd.resolve = {
 
 // -------------------------------------------------------------------
 
-export async function insertCd ({ vm, vdi, force }) {
+export async function insertCd ({ vm, vdi, force = true }) {
   await this.getXapi(vm).insertCdIntoVm(vdi._xapiId, vm._xapiId, { force })
 }
 
 insertCd.params = {
   id: { type: 'string' },
   cd_id: { type: 'string' },
-  force: { type: 'boolean' },
+  force: { type: 'boolean', optional: true },
 }
 
 insertCd.resolve = {
@@ -475,9 +468,7 @@ export async function migrate ({
     })
   }
 
-  if (!(await this.hasPermissions(this.session.get('user_id'), permissions))) {
-    throw unauthorized()
-  }
+  await this.checkPermissions(this.user.id, permissions)
 
   await this.getXapi(vm).migrateVm(
     vm._xapiId,
@@ -612,6 +603,8 @@ set.params = {
   // Emulate HVM C000 PCI device for Windows Update to fetch or update PV drivers
   hasVendorDevice: { type: 'boolean', optional: true },
 
+  expNestedHvm: { type: 'boolean', optional: true },
+
   // Move the vm In to/Out of Self Service
   resourceSet: { type: ['string', 'null'], optional: true },
 
@@ -627,7 +620,7 @@ set.resolve = {
 
 // -------------------------------------------------------------------
 
-export async function restart ({ vm, force }) {
+export async function restart ({ vm, force = false }) {
   const xapi = this.getXapi(vm)
 
   if (force) {
@@ -639,7 +632,7 @@ export async function restart ({ vm, force }) {
 
 restart.params = {
   id: { type: 'string' },
-  force: { type: 'boolean' },
+  force: { type: 'boolean', optional: true },
 }
 
 restart.resolve = {
@@ -733,13 +726,7 @@ copy.resolve = {
 
 export async function convertToTemplate ({ vm }) {
   // Convert to a template requires pool admin permission.
-  if (
-    !(await this.hasPermissions(this.session.get('user_id'), [
-      [vm.$pool, 'administrate'],
-    ]))
-  ) {
-    throw unauthorized()
-  }
+  await this.checkPermissions(this.user.id, [[vm.$pool, 'administrate']])
 
   await this.getXapi(vm).call('VM.set_is_a_template', vm._xapiRef, true)
 }
@@ -1185,34 +1172,17 @@ async function handleVmImport (req, res, { data, srId, type, xapi }) {
 }
 
 // TODO: "sr_id" can be passed in URL to target a specific SR
-async function import_ ({ data, host, sr, type }) {
-  let xapi
+async function import_ ({ data, sr, type }) {
   if (data && type === 'xva') {
     throw invalidParameters('unsupported field data for the file type xva')
   }
 
-  if (!sr) {
-    if (!host) {
-      throw invalidParameters('you must provide either host or SR')
-    }
-
-    xapi = this.getXapi(host)
-    sr = xapi.pool.$default_SR
-    if (!sr) {
-      throw invalidParameters('there is not default SR in this pool')
-    }
-
-    // FIXME: must have administrate permission on default SR.
-  } else {
-    xapi = this.getXapi(sr)
-  }
-
   return {
     $sendTo: await this.registerHttpRequest(handleVmImport, {
       data,
       srId: sr._xapiId,
       type,
-      xapi,
+      xapi: this.getXapi(sr),
     }),
   }
 }
@@ -1247,13 +1217,11 @@ import_.params = {
       },
     },
   },
-  host: { type: 'string', optional: true },
   type: { type: 'string', optional: true },
-  sr: { type: 'string', optional: true },
+  sr: { type: 'string' },
 }
 
 import_.resolve = {
-  host: ['host', 'host', 'administrate'],
   sr: ['sr', 'SR', 'administrate'],
 }
 
@@ -1305,10 +1273,8 @@ export async function createInterface ({
     await this.checkResourceSetConstraints(resourceSet, this.user.id, [
       network.id,
     ])
-  } else if (
-    !(await this.hasPermissions(this.user.id, [[network.id, 'view']]))
-  ) {
-    throw unauthorized()
+  } else {
+    await this.checkPermissions(this.user.id, [[network.id, 'view']])
   }
 
   let ipAddresses

packages/xo-server/src/api/xosan.js

@@ -1,10 +1,11 @@
 import asyncMap from '@xen-orchestra/async-map'
-import createLogger from 'debug'
+import createLogger from '@xen-orchestra/log'
 import defer from 'golike-defer'
 import execa from 'execa'
 import fs from 'fs-extra'
 import map from 'lodash/map'
 import { tap, delay } from 'promise-toolbox'
+import { NULL_REF } from 'xen-api'
 import { invalidParameters } from 'xo-common/api-errors'
 import { v4 as generateUuid } from 'uuid'
 import { includes, remove, filter, find, range } from 'lodash'
@@ -12,7 +13,7 @@ import { includes, remove, filter, find, range } from 'lodash'
 import { asInteger } from '../xapi/utils'
 import { parseXml, ensureArray } from '../utils'
 
-const debug = createLogger('xo:xosan')
+const log = createLogger('xo:xosan')
 
 const SSH_KEY_FILE = 'id_rsa_xosan'
 const DEFAULT_NETWORK_PREFIX = '172.31.100.'
@@ -73,7 +74,7 @@ async function rateLimitedRetry (action, shouldRetry, retryCount = 20) {
   let result
   while (retryCount > 0 && (result = await action()) && shouldRetry(result)) {
     retryDelay *= 1.1
-    debug('waiting ' + retryDelay + 'ms and retrying')
+    log.debug(`waiting ${retryDelay} ms and retrying`)
     await delay(retryDelay)
     retryCount--
   }
@@ -305,7 +306,7 @@ async function copyVm (xapi, originalVm, sr) {
 }
 
 async function callPlugin (xapi, host, command, params) {
-  debug('calling plugin', host.address, command)
+  log.debug(`calling plugin ${host.address} ${command}`)
   return JSON.parse(
     await xapi.call('host.call_plugin', host.$ref, 'xosan.py', command, params)
   )
@@ -346,15 +347,12 @@ async function remoteSsh (glusterEndpoint, cmd, ignoreError = false) {
         }
       }
     }
-    debug(
-      result.command.join(' '),
-      '\n  =>exit:',
-      result.exit,
-      '\n  =>err :',
-      result.stderr,
-      '\n  =>out (1000 chars) :',
-      result.stdout.substring(0, 1000)
-    )
+
+    log.debug(`result of ${result.command.join(' ')}`, {
+      exit: result.exit,
+      err: result.stderr,
+      out: result.stdout.substring(0, 1000),
+    })
     // 255 seems to be ssh's own error codes.
     if (result.exit !== 255) {
       if (!ignoreError && result.exit !== 0) {
@@ -552,7 +550,7 @@ async function configureGluster (
     creation +
     ' ' +
     brickVms.map(ipAndHost => ipAndHost.brickName).join(' ')
-  debug('creating volume: ', volumeCreation)
+  log.debug(`creating volume: ${volumeCreation}`)
   await glusterCmd(glusterEndpoint, volumeCreation)
   await glusterCmd(
     glusterEndpoint,
@@ -762,7 +760,7 @@ export const createSR = defer(async function (
       glusterType,
       arbiter
     )
-    debug('xosan gluster volume started')
+    log.debug('xosan gluster volume started')
     // We use 10 IPs of the gluster VM range as backup, in the hope that even if the first VM gets destroyed we find at least
     // one VM to give mount the volfile.
     // It is not possible to edit the device_config after the SR is created and this data is only used at mount time when rebooting
@@ -785,7 +783,7 @@ export const createSR = defer(async function (
       true,
       {}
     )
-    debug('sr created')
+    log.debug('sr created')
     // we just forget because the cleanup actions are stacked in the $onFailure system
     $defer.onFailure(() => xapi.forgetSr(xosanSrRef))
     if (arbiter) {
@@ -809,7 +807,7 @@ export const createSR = defer(async function (
       redundancy,
     })
     CURRENT_POOL_OPERATIONS[poolId] = { ...OPERATION_OBJECT, state: 6 }
-    debug('scanning new SR')
+    log.debug('scanning new SR')
     await xapi.call('SR.scan', xosanSrRef)
     await this.rebindLicense({
       licenseId: license.id,
@@ -1139,12 +1137,15 @@ async function _prepareGlusterVm (
     .find(vdi => vdi && vdi.name_label === 'xosan_root')
   const rootDiskSize = rootDisk.virtual_size
   await xapi.startVm(newVM)
-  debug('waiting for boot of ', ip)
+  log.debug(`waiting for boot of ${ip}`)
   // wait until we find the assigned IP in the networks, we are just checking the boot is complete
-  const vmIsUp = vm =>
-    Boolean(vm.$guest_metrics && includes(vm.$guest_metrics.networks, ip))
-  const vm = await xapi._waitObjectState(newVM.$id, vmIsUp)
-  debug('booted ', ip)
+  // fix #3688
+  const vm = await xapi._waitObjectState(
+    newVM.$id,
+    _ => _.guest_metrics !== NULL_REF
+  )
+  await xapi._waitObjectState(vm.guest_metrics, _ => includes(_.networks, ip))
+  log.debug(`booted ${ip}`)
   const localEndpoint = { xapi: xapi, hosts: [host], addresses: [ip] }
   const srFreeSpace = sr.physical_size - sr.physical_utilisation
   // we use a percentage because it looks like the VDI overhead is proportional

packages/xo-server/src/collection/redis.js

@@ -78,19 +78,18 @@ export default class Redis extends Collection {
         .then(keys => keys.length !== 0 && redis.del(keys))
     ).then(() =>
       asyncMap(redis.smembers(idsIndex), id =>
-        redis.hgetall(`${prefix}:${id}`).then(
-          values =>
-            values == null
-              ? redis.srem(idsIndex, id) // entry no longer exists
-              : asyncMap(indexes, index => {
-                  const value = values[index]
-                  if (value !== undefined) {
-                    return redis.sadd(
-                      `${prefix}_${index}:${String(value).toLowerCase()}`,
-                      id
-                    )
-                  }
-                })
+        redis.hgetall(`${prefix}:${id}`).then(values =>
+          values == null
+            ? redis.srem(idsIndex, id) // entry no longer exists
+            : asyncMap(indexes, index => {
+                const value = values[index]
+                if (value !== undefined) {
+                  return redis.sadd(
+                    `${prefix}_${index}:${String(value).toLowerCase()}`,
+                    id
+                  )
+                }
+              })
         )
       )
     )

packages/xo-server/src/index.js

@@ -3,7 +3,7 @@ import assert from 'assert'
 import bind from 'lodash/bind'
 import blocked from 'blocked'
 import createExpress from 'express'
-import createLogger from 'debug'
+import createLogger from '@xen-orchestra/log'
 import has from 'lodash/has'
 import helmet from 'helmet'
 import includes from 'lodash/includes'
@@ -40,13 +40,20 @@ import passport from 'passport'
 import { parse as parseCookies } from 'cookie'
 import { Strategy as LocalStrategy } from 'passport-local'
 
+import transportConsole from '@xen-orchestra/log/transports/console'
+import { configure } from '@xen-orchestra/log/configure'
+
 // ===================================================================
 
-const debug = createLogger('xo:main')
+configure([
+  {
+    filter: process.env.DEBUG,
+    level: 'info',
+    transport: transportConsole(),
+  },
+])
 
-const warn = (...args) => {
-  console.warn('[Warn]', ...args)
-}
+const log = createLogger('xo:main')
 
 // ===================================================================
 
@@ -58,12 +65,12 @@ async function loadConfiguration () {
     ignoreUnknownFormats: true,
   })
 
-  debug('Configuration loaded.')
+  log.info('Configuration loaded.')
 
   // Print a message if deprecated entries are specified.
   forEach(DEPRECATED_ENTRIES, entry => {
     if (has(config, entry)) {
-      warn(`${entry} configuration is deprecated.`)
+      log.warn(`${entry} configuration is deprecated.`)
     }
   })
 
@@ -248,18 +255,18 @@ async function registerPlugin (pluginPath, pluginName) {
   )
 }
 
-const debugPlugin = createLogger('xo:plugin')
+const logPlugin = createLogger('xo:plugin')
 
 function registerPluginWrapper (pluginPath, pluginName) {
-  debugPlugin('register %s', pluginName)
+  logPlugin.info(`register ${pluginName}`)
 
   return registerPlugin.call(this, pluginPath, pluginName).then(
     () => {
-      debugPlugin(`successfully register ${pluginName}`)
+      logPlugin.info(`successfully register ${pluginName}`)
     },
     error => {
-      debugPlugin(`failed register ${pluginName}`)
-      debugPlugin(error)
+      logPlugin.info(`failed register ${pluginName}`)
+      logPlugin.info(error)
     }
   )
 }
@@ -323,20 +330,20 @@ async function makeWebServerListen (
   }
   try {
     const niceAddress = await webServer.listen(opts)
-    debug(`Web server listening on ${niceAddress}`)
+    log.info(`Web server listening on ${niceAddress}`)
   } catch (error) {
     if (error.niceAddress) {
-      warn(`Web server could not listen on ${error.niceAddress}`)
+      log.warn(`Web server could not listen on ${error.niceAddress}`, { error })
 
       const { code } = error
       if (code === 'EACCES') {
-        warn('  Access denied.')
-        warn('  Ports < 1024 are often reserved to privileges users.')
+        log.warn('  Access denied.')
+        log.warn('  Ports < 1024 are often reserved to privileges users.')
       } else if (code === 'EADDRINUSE') {
-        warn('  Address already in use.')
+        log.warn('  Address already in use.')
       }
     } else {
-      warn('Web server could not listen:', error.message)
+      log.warn('Web server could not listen:', { error })
     }
   }
 }
@@ -417,7 +424,7 @@ const setUpStaticFiles = (express, opts) => {
     }
 
     forEach(paths, path => {
-      debug('Setting up %s → %s', url, path)
+      log.info(`Setting up ${url} → ${path}`)
 
       express.use(url, serveStatic(path))
     })
@@ -435,7 +442,7 @@ const setUpApi = (webServer, xo, verboseLogsOnErrors) => {
   const onConnection = (socket, upgradeReq) => {
     const { remoteAddress } = upgradeReq.socket
 
-    debug('+ WebSocket connection (%s)', remoteAddress)
+    log.info(`+ WebSocket connection (${remoteAddress})`)
 
     // Create the abstract XO object for this connection.
     const connection = xo.createUserConnection()
@@ -453,7 +460,7 @@ const setUpApi = (webServer, xo, verboseLogsOnErrors) => {
 
     // Close the XO connection with this WebSocket.
     socket.once('close', () => {
-      debug('- WebSocket connection (%s)', remoteAddress)
+      log.info(`- WebSocket connection (${remoteAddress})`)
 
       connection.close()
     })
@@ -465,7 +472,7 @@ const setUpApi = (webServer, xo, verboseLogsOnErrors) => {
 
     const onSend = error => {
       if (error) {
-        warn('WebSocket send:', error.stack)
+        log.warn('WebSocket send:', { error })
       }
     }
     jsonRpc.on('data', data => {
@@ -513,9 +520,9 @@ const setUpConsoleProxy = (webServer, xo) => {
         }
 
         const { remoteAddress } = socket
-        debug('+ Console proxy (%s - %s)', user.name, remoteAddress)
+        log.info(`+ Console proxy (${user.name} - ${remoteAddress})`)
         socket.on('close', () => {
-          debug('- Console proxy (%s - %s)', user.name, remoteAddress)
+          log.info(`- Console proxy (${user.name} - ${remoteAddress})`)
         })
       }
 
@@ -549,10 +556,10 @@ export default async function main (args) {
   }
 
   {
-    const debug = createLogger('xo:perf')
+    const logPerf = createLogger('xo:perf')
     blocked(
       ms => {
-        debug('blocked for %sms', ms | 0)
+        logPerf.info(`blocked for ${ms | 0}ms`)
       },
       {
         threshold: 500,
@@ -569,14 +576,14 @@ export default async function main (args) {
     const { user, group } = config
     if (group) {
       process.setgid(group)
-      debug('Group changed to', group)
+      log.info(`Group changed to ${group}`)
     }
     if (user) {
       process.setuid(user)
-      debug('User changed to', user)
+      log.info(`User changed to ${user}`)
     }
   } catch (error) {
-    warn('Failed to change user/group:', error)
+    log.warn('Failed to change user/group:', { error })
   }
 
   // Creates main object.
@@ -604,7 +611,7 @@ export default async function main (args) {
     })
 
     if (port === undefined) {
-      warn('Could not setup HTTPs redirection: no HTTPs port found')
+      log.warn('Could not setup HTTPs redirection: no HTTPs port found')
     } else {
       express.use((req, res, next) => {
         if (req.secure) {
@@ -653,17 +660,17 @@ export default async function main (args) {
 
     process.on(signal, () => {
       if (alreadyCalled) {
-        warn('forced exit')
+        log.warn('forced exit')
         process.exit(1)
       }
       alreadyCalled = true
 
-      debug('%s caught, closing…', signal)
+      log.info(`${signal} caught, closing…`)
       xo.stop()
     })
   })
 
   await fromEvent(xo, 'stopped')
 
-  debug('bye :-)')
+  log.info('bye :-)')
 }

packages/xo-server/src/models/plugin-metadata.js

@@ -1,7 +1,10 @@
 import Collection from '../collection/redis'
+import createLogger from '@xen-orchestra/log'
 import Model from '../model'
 import { forEach } from '../utils'
 
+const log = createLogger('xo:plugin-metadata')
+
 // ===================================================================
 
 export default class PluginMetadata extends Model {}
@@ -44,10 +47,7 @@ export class PluginsMetadata extends Collection {
         pluginMetadata.configuration =
           configuration && JSON.parse(configuration)
       } catch (error) {
-        console.warn(
-          'cannot parse pluginMetadata.configuration:',
-          configuration
-        )
+        log.warn(`cannot parse pluginMetadata.configuration: ${configuration}`)
         pluginMetadata.configuration = []
       }
     })

packages/xo-server/src/proxy-console.js

@@ -1,22 +1,22 @@
-import createDebug from 'debug'
+import createLogger from '@xen-orchestra/log'
 import partialStream from 'partial-stream'
 import { connect } from 'tls'
 import { parse } from 'url'
 
-const debug = createDebug('xo:proxy-console')
+const log = createLogger('xo:proxy-console')
 
 export default function proxyConsole (ws, vmConsole, sessionId) {
   const url = parse(vmConsole.location)
   let { hostname } = url
   if (hostname === null || hostname === '') {
-    console.warn(
-      'host is missing in console (%s) URI (%s)',
-      vmConsole.uuid,
-      vmConsole.location
-    )
     const { address } = vmConsole.$VM.$resident_on
-    console.warn('  using host address (%s) as fallback', address)
     hostname = address
+
+    log.warn(
+      `host is missing in console (${vmConsole.uuid}) URI (${
+        vmConsole.location
+      }) using host address (${address}) as fallback`
+    )
   }
 
   let closed = false
@@ -41,10 +41,7 @@ export default function proxyConsole (ws, vmConsole, sessionId) {
 
       const onSend = error => {
         if (error) {
-          debug(
-            'error sending to the XO client: %s',
-            error.stack || error.message || error
-          )
+          log.debug('error sending to the XO client:', { error })
         }
       }
 
@@ -52,7 +49,7 @@ export default function proxyConsole (ws, vmConsole, sessionId) {
         .pipe(
           partialStream('\r\n\r\n', headers => {
             // TODO: check status code 200.
-            debug('connected')
+            log.debug('connected')
           })
         )
         .on('data', data => {
@@ -63,7 +60,7 @@ export default function proxyConsole (ws, vmConsole, sessionId) {
         .on('end', () => {
           if (!closed) {
             closed = true
-            debug('disconnected from the console')
+            log.debug('disconnected from the console')
           }
 
           ws.close()
@@ -71,10 +68,7 @@ export default function proxyConsole (ws, vmConsole, sessionId) {
 
       ws.on('error', error => {
         closed = true
-        debug(
-          'error from the XO client: %s',
-          error.stack || error.message || error
-        )
+        log.debug('error from the XO client:', { error })
 
         socket.end()
       })
@@ -86,7 +80,7 @@ export default function proxyConsole (ws, vmConsole, sessionId) {
         .on('close', () => {
           if (!closed) {
             closed = true
-            debug('disconnected from the XO client')
+            log.debug('disconnected from the XO client')
           }
 
           socket.end()
@@ -94,7 +88,7 @@ export default function proxyConsole (ws, vmConsole, sessionId) {
     }
   ).on('error', error => {
     closed = true
-    debug('error from the console: %s', error.stack || error.message || error)
+    log.debug('error from the console:', { error })
 
     ws.close()
   })

packages/xo-server/src/schemas/log/taskInfo.js

@@ -0,0 +1,15 @@
+export default {
+  $schema: 'http://json-schema.org/draft-04/schema#',
+  type: 'object',
+  properties: {
+    event: {
+      enum: ['task.info'],
+    },
+    taskId: {
+      type: 'string',
+      description: 'identifier of the parent task or job',
+    },
+    data: {},
+  },
+  required: ['event', 'taskId'],
+}

packages/xo-server/src/schemas/log/taskWarning.js

@@ -0,0 +1,15 @@
+export default {
+  $schema: 'http://json-schema.org/draft-04/schema#',
+  type: 'object',
+  properties: {
+    event: {
+      enum: ['task.warning'],
+    },
+    taskId: {
+      type: 'string',
+      description: 'identifier of the parent task or job',
+    },
+    data: {},
+  },
+  required: ['event', 'taskId'],
+}

packages/xo-server/src/sensitive-values.js

@@ -0,0 +1,43 @@
+import mapValues from 'lodash/mapValues'
+
+// this random value is used to obfuscate real data
+const OBFUSCATED_VALUE = 'q3oi6d9X8uenGvdLnHk2'
+
+export const merge = (newValue, oldValue) => {
+  if (newValue === OBFUSCATED_VALUE) {
+    return oldValue
+  }
+
+  let isArray
+
+  if (
+    newValue === null ||
+    oldValue === null ||
+    typeof newValue !== 'object' ||
+    typeof oldValue !== 'object' ||
+    (isArray = Array.isArray(newValue)) !== Array.isArray(oldValue)
+  ) {
+    return newValue
+  }
+
+  const iteratee = (v, k) => merge(v, oldValue[k])
+  return isArray ? newValue.map(iteratee) : mapValues(newValue, iteratee)
+}
+
+export const obfuscate = value => replace(value, OBFUSCATED_VALUE)
+
+export function replace (value, replacement) {
+  function helper (value, name) {
+    if (name === 'password' && typeof value === 'string') {
+      return replacement
+    }
+
+    if (typeof value !== 'object' || value === null) {
+      return value
+    }
+
+    return Array.isArray(value) ? value.map(helper) : mapValues(value, helper)
+  }
+
+  return helper(value)
+}

packages/xo-server/src/size-stream.js

@@ -1,6 +1,5 @@
 // @flow
 
-// $FlowFixMe
 import through2 from 'through2'
 import { type Readable } from 'stream'
 

packages/xo-server/src/utils.js

@@ -6,7 +6,6 @@ import humanFormat from 'human-format'
 import isArray from 'lodash/isArray'
 import isString from 'lodash/isString'
 import keys from 'lodash/keys'
-import kindOf from 'kindof'
 import multiKeyHashInt from 'multikey-hash'
 import pick from 'lodash/pick'
 import tmp from 'tmp'
@@ -192,35 +191,6 @@ export const noop = () => {}
 
 // -------------------------------------------------------------------
 
-// Usage: pDebug(promise, name) or promise::pDebug(name)
-export function pDebug (promise, name) {
-  if (arguments.length === 1) {
-    name = promise
-    promise = this
-  }
-
-  Promise.resolve(promise).then(
-    value => {
-      console.log(
-        '%s',
-        `Promise ${name} resolved${
-          value !== undefined ? ` with ${kindOf(value)}` : ''
-        }`
-      )
-    },
-    reason => {
-      console.log(
-        '%s',
-        `Promise ${name} rejected${
-          reason !== undefined ? ` with ${kindOf(reason)}` : ''
-        }`
-      )
-    }
-  )
-
-  return promise
-}
-
 // Given a collection (array or object) which contains promises,
 // return a promise that is fulfilled when all the items in the
 // collection are either fulfilled or rejected.

packages/xo-server/src/xapi-object-to-xo.js

@@ -10,7 +10,12 @@ import {
   mapToArray,
   parseXml,
 } from './utils'
-import { isHostRunning, isVmHvm, isVmRunning, parseDateTime } from './xapi'
+import {
+  getVmDomainType,
+  isHostRunning,
+  isVmRunning,
+  parseDateTime,
+} from './xapi'
 import { useUpdateSystem } from './xapi/utils'
 
 // ===================================================================
@@ -218,14 +223,18 @@ const TRANSFORMS = {
 
   // -----------------------------------------------------------------
 
-  vm (obj) {
+  vm (obj, dependents) {
+    dependents[obj.guest_metrics] = obj.$id
+    dependents[obj.metrics] = obj.$id
+
     const {
       $guest_metrics: guestMetrics,
       $metrics: metrics,
       other_config: otherConfig,
     } = obj
 
-    const isHvm = isVmHvm(obj)
+    const domainType = getVmDomainType(obj)
+    const isHvm = domainType === 'hvm'
     const isRunning = isVmRunning(obj)
     const xenTools = (() => {
       if (!isRunning || !metrics) {
@@ -302,7 +311,7 @@ const TRANSFORMS = {
           version: version && parseXml(version).docker_version,
         }
       })(),
-
+      expNestedHvm: obj.platform['exp-nested-hvm'] === 'true',
       high_availability: obj.ha_restart_priority,
 
       memory: (function () {
@@ -343,11 +352,7 @@ const TRANSFORMS = {
       startTime: metrics && toTimestamp(metrics.start_time),
       tags: obj.tags,
       VIFs: link(obj, 'VIFs'),
-      virtualizationMode: isHvm
-        ? guestMetrics !== undefined && guestMetrics.PV_drivers_detected
-          ? 'pvhvm'
-          : 'hvm'
-        : 'pv',
+      virtualizationMode: domainType,
 
       // <=> Are the Xen Server tools installed?
       //
@@ -739,13 +744,13 @@ const TRANSFORMS = {
 
 // ===================================================================
 
-export default xapiObj => {
+export default function xapiObjectToXo (xapiObj, dependents) {
   const transform = TRANSFORMS[xapiObj.$type.toLowerCase()]
   if (!transform) {
     return
   }
 
-  const xoObj = transform(xapiObj)
+  const xoObj = transform(xapiObj, dependents)
   if (!xoObj) {
     return
   }

packages/xo-server/src/xapi-stats.js

@@ -99,8 +99,8 @@ const testMetric = (test, type) =>
   typeof test === 'string'
     ? test === type
     : typeof test === 'function'
-      ? test(type)
-      : test.exec(type)
+    ? test(type)
+    : test.exec(type)
 
 const findMetric = (metrics, metricType) => {
   let testResult
@@ -321,8 +321,12 @@ export default class XapiStats {
     const hostUuid = host.uuid
 
     if (
+      !(
+        vmUuid !== undefined &&
+        get(this._statsByObject, [vmUuid, step]) === undefined
+      ) &&
       get(this._statsByObject, [hostUuid, step, 'localTimestamp']) + step >
-      getCurrentTimestamp()
+        getCurrentTimestamp()
     ) {
       return this._getStats(hostUuid, step, vmUuid)
     }
@@ -414,7 +418,7 @@ export default class XapiStats {
     })
   }
 
-  getVmStats (xapi, vmId, granularity) {
+  async getVmStats (xapi, vmId, granularity) {
     const vm = xapi.getObject(vmId)
     const host = vm.$resident_on
     if (!host) {

packages/xo-server/src/xapi/index.js

@@ -1,9 +1,11 @@
 /* eslint-disable camelcase */
 import asyncMap from '@xen-orchestra/async-map'
 import concurrency from 'limit-concurrency-decorator'
+import createLogger from '@xen-orchestra/log'
 import deferrable from 'golike-defer'
 import fatfs from 'fatfs'
 import mixin from '@xen-orchestra/mixin'
+import ms from 'ms'
 import synchronized from 'decorator-synchronized'
 import tarStream from 'tar-stream'
 import vmdkToVhd from 'xo-vmdk-to-vhd'
@@ -55,7 +57,6 @@ import { type DeltaVmExport } from './'
 import {
   asBoolean,
   asInteger,
-  debug,
   extractOpaqueRef,
   filterUndefineds,
   getNamespaceForType,
@@ -68,6 +69,8 @@ import {
   prepareXapiParam,
 } from './utils'
 
+const log = createLogger('xo:xapi')
+
 // ===================================================================
 
 const TAG_BASE_DELTA = 'xo:base_delta'
@@ -205,8 +208,8 @@ export default class Xapi extends XapiBase {
     }
 
     const loop = () =>
-      this._waitObject(idOrUuidOrRef).then(
-        object => (predicate(object) ? object : loop())
+      this._waitObject(idOrUuidOrRef).then(object =>
+        predicate(object) ? object : loop()
       )
 
     return loop()
@@ -364,7 +367,7 @@ export default class Xapi extends XapiBase {
   async emergencyShutdownHost (hostId) {
     const host = this.getObject(hostId)
     const vms = host.$resident_VMs
-    debug(`Emergency shutdown: ${host.name_label}`)
+    log.debug(`Emergency shutdown: ${host.name_label}`)
     await pSettle(
       mapToArray(vms, vm => {
         if (!vm.is_control_domain) {
@@ -447,7 +450,7 @@ export default class Xapi extends XapiBase {
   // Clone a VM: make a fast copy by fast copying each of its VDIs
   // (using snapshots where possible) on the same SRs.
   _cloneVm (vm, nameLabel = vm.name_label) {
-    debug(
+    log.debug(
       `Cloning VM ${vm.name_label}${
         nameLabel !== vm.name_label ? ` as ${nameLabel}` : ''
       }`
@@ -466,7 +469,7 @@ export default class Xapi extends XapiBase {
       snapshot = await this._snapshotVm(vm)
     }
 
-    debug(
+    log.debug(
       `Copying VM ${vm.name_label}${
         nameLabel !== vm.name_label ? ` as ${nameLabel}` : ''
       }${sr ? ` on ${sr.name_label}` : ''}`
@@ -587,7 +590,7 @@ export default class Xapi extends XapiBase {
     version,
     xenstore_data,
   }) {
-    debug(`Creating VM ${name_label}`)
+    log.debug(`Creating VM ${name_label}`)
 
     return this.call(
       'VM.create',
@@ -648,30 +651,39 @@ export default class Xapi extends XapiBase {
     force = false,
     forceDeleteDefaultTemplate = false
   ) {
-    debug(`Deleting VM ${vm.name_label}`)
+    log.debug(`Deleting VM ${vm.name_label}`)
 
     const { $ref } = vm
 
     // ensure the vm record is up-to-date
     vm = await this.barrier($ref)
 
+    if (!force && 'destroy' in vm.blocked_operations) {
+      throw forbiddenOperation('destroy', vm.blocked_operations.destroy.reason)
+    }
+
+    if (
+      !forceDeleteDefaultTemplate &&
+      vm.other_config.default_template === 'true'
+    ) {
+      throw forbiddenOperation('destroy', 'VM is default template')
+    }
+
     // It is necessary for suspended VMs to be shut down
     // to be able to delete their VDIs.
     if (vm.power_state !== 'Halted') {
       await this.call('VM.hard_shutdown', $ref)
     }
 
-    if (force) {
-      await this._updateObjectMapProperty(vm, 'blocked_operations', {
+    await Promise.all(
+      this.call('VM.set_is_a_template', vm.$ref, false),
+      this._updateObjectMapProperty(vm, 'blocked_operations', {
         destroy: null,
-      })
-    }
-
-    if (forceDeleteDefaultTemplate) {
-      await this._updateObjectMapProperty(vm, 'other_config', {
+      }),
+      this._updateObjectMapProperty(vm, 'other_config', {
         default_template: null,
       })
-    }
+    )
 
     // must be done before destroying the VM
     const disks = getVmDisks(vm)
@@ -689,13 +701,13 @@ export default class Xapi extends XapiBase {
         asyncMap(disks, ({ $ref: vdiRef }) => {
           let onFailure = () => {
             onFailure = vdi => {
-              console.error(
+              log.error(
                 `cannot delete VDI ${vdi.name_label} (from VM ${vm.name_label})`
               )
               forEach(vdi.$VBDs, vbd => {
                 if (vbd.VM !== $ref) {
                   const vm = vbd.$VM
-                  console.error('- %s (%s)', vm.name_label, vm.uuid)
+                  log.error(`- ${vm.name_label} (${vm.uuid})`)
                 }
               })
             }
@@ -874,7 +886,7 @@ export default class Xapi extends XapiBase {
         //
         // The snapshot must not exist otherwise it could break the
         // next export.
-        ;this._deleteVdi(vdi)::ignoreErrors()
+        this._deleteVdi(vdi)::ignoreErrors()
         return
       }
 
@@ -1092,6 +1104,7 @@ export default class Xapi extends XapiBase {
               transferSize += sizeStream.size
             })
           sizeStream.task = stream.task
+          sizeStream.length = stream.length
           await this._importVdiContent(vdi, sizeStream, VDI_FORMAT_VHD)
         }
       }),
@@ -1124,7 +1137,7 @@ export default class Xapi extends XapiBase {
     ])
 
     if (deleteBase && baseVm) {
-      ;this._deleteVm(baseVm)::ignoreErrors()
+      this._deleteVm(baseVm)::ignoreErrors()
     }
 
     await Promise.all([
@@ -1164,8 +1177,8 @@ export default class Xapi extends XapiBase {
           mapVdisSrs && mapVdisSrs[vdi.$id]
             ? hostXapi.getObject(mapVdisSrs[vdi.$id]).$ref
             : sr !== undefined
-              ? hostXapi.getObject(sr).$ref
-              : defaultSr.$ref // Will error if there are no default SR.
+            ? hostXapi.getObject(sr).$ref
+            : defaultSr.$ref // Will error if there are no default SR.
       }
     }
 
@@ -1217,7 +1230,7 @@ export default class Xapi extends XapiBase {
       { vdi }
     ).catch(error => {
       if (error.code !== 'XENAPI_PLUGIN_FAILURE') {
-        console.warn('_callInstallationPlugin', error)
+        log.warn('_callInstallationPlugin', { error })
         throw error
       }
     })
@@ -1312,7 +1325,7 @@ export default class Xapi extends XapiBase {
     }
 
     if (onVmCreation != null) {
-      ;this._waitObject(
+      this._waitObject(
         obj =>
           obj != null &&
           obj.current_operations != null &&
@@ -1491,7 +1504,7 @@ export default class Xapi extends XapiBase {
   @concurrency(2)
   @cancelable
   async _snapshotVm ($cancelToken, vm, nameLabel = vm.name_label) {
-    debug(
+    log.debug(
       `Snapshotting VM ${vm.name_label}${
         nameLabel !== vm.name_label ? ` as ${nameLabel}` : ''
       }`
@@ -1553,7 +1566,7 @@ export default class Xapi extends XapiBase {
   }
 
   async _startVm (vm, host, force) {
-    debug(`Starting VM ${vm.name_label}`)
+    log.debug(`Starting VM ${vm.name_label}`)
 
     if (force) {
       await this._updateObjectMapProperty(vm, 'blocked_operations', {
@@ -1650,12 +1663,12 @@ export default class Xapi extends XapiBase {
 
         await this._startVm(vm)
       } finally {
-        ;this._setObjectProperties(vm, {
+        this._setObjectProperties(vm, {
           PV_bootloader: bootloader,
         })::ignoreErrors()
 
         forEach(bootables, ([vbd, bootable]) => {
-          ;this._setObjectProperties(vbd, { bootable })::ignoreErrors()
+          this._setObjectProperties(vbd, { bootable })::ignoreErrors()
         })
       }
     }
@@ -1701,7 +1714,7 @@ export default class Xapi extends XapiBase {
     vdi = this.getObject(vdi)
     vm = this.getObject(vm)
 
-    debug(`Creating VBD for VDI ${vdi.name_label} on VM ${vm.name_label}`)
+    log.debug(`Creating VBD for VDI ${vdi.name_label} on VM ${vm.name_label}`)
 
     if (userdevice == null) {
       const allowed = await this.call('VM.get_allowed_VBD_devices', vm.$ref)
@@ -1744,7 +1757,7 @@ export default class Xapi extends XapiBase {
   }
 
   _cloneVdi (vdi) {
-    debug(`Cloning VDI ${vdi.name_label}`)
+    log.debug(`Cloning VDI ${vdi.name_label}`)
 
     return this.call('VDI.clone', vdi.$ref)
   }
@@ -1766,7 +1779,7 @@ export default class Xapi extends XapiBase {
     sr = SR !== undefined && SR !== NULL_REF ? SR : this.pool.default_SR,
   }) {
     sr = this.getObject(sr)
-    debug(`Creating VDI ${name_label} on ${sr.name_label}`)
+    log.debug(`Creating VDI ${name_label} on ${sr.name_label}`)
 
     return this._getOrWaitObject(
       await this.call('VDI.create', {
@@ -1793,7 +1806,7 @@ export default class Xapi extends XapiBase {
       return // nothing to do
     }
 
-    debug(
+    log.debug(
       `Moving VDI ${vdi.name_label} from ${vdi.$SR.name_label} to ${
         sr.name_label
       }`
@@ -1826,13 +1839,15 @@ export default class Xapi extends XapiBase {
 
   // TODO: check whether the VDI is attached.
   async _deleteVdi (vdi) {
-    debug(`Deleting VDI ${vdi.name_label}`)
+    log.debug(`Deleting VDI ${vdi.name_label}`)
 
     await this.call('VDI.destroy', vdi.$ref)
   }
 
   _resizeVdi (vdi, size) {
-    debug(`Resizing VDI ${vdi.name_label} from ${vdi.virtual_size} to ${size}`)
+    log.debug(
+      `Resizing VDI ${vdi.name_label} from ${vdi.virtual_size} to ${size}`
+    )
 
     return this.call('VDI.resize', vdi.$ref, size)
   }
@@ -1963,7 +1978,7 @@ export default class Xapi extends XapiBase {
       query.base = base.$ref
     }
 
-    debug(
+    log.debug(
       `exporting VDI ${vdi.name_label}${
         base ? ` (from base ${vdi.name_label})` : ''
       }`
@@ -1990,6 +2005,11 @@ export default class Xapi extends XapiBase {
   // -----------------------------------------------------------------
 
   async _importVdiContent (vdi, body, format = VDI_FORMAT_VHD) {
+    if (__DEV__ && body.length == null) {
+      throw new Error(
+        'Trying to import a VDI without a length field. Please report this error to Xen Orchestra.'
+      )
+    }
     await Promise.all([
       body.task,
       body.checksumVerified,
@@ -2034,7 +2054,7 @@ export default class Xapi extends XapiBase {
       qos_algorithm_type = '',
     } = {}
   ) {
-    debug(
+    log.debug(
       `Creating VIF for VM ${vm.name_label} on network ${network.name_label}`
     )
 
@@ -2297,9 +2317,9 @@ export default class Xapi extends XapiBase {
 
     // ignore errors, I (JFT) don't understand why they are emitted
     // because it works
-    await this._importVdiContent(vdi, buffer, VDI_FORMAT_RAW).catch(
-      console.warn
-    )
+    await this._importVdiContent(vdi, buffer, VDI_FORMAT_RAW).catch(error => {
+      log.warn('importVdiContent: ', { error })
+    })
 
     await this.createVbd({ vdi, vm })
   }
@@ -2352,15 +2372,14 @@ export default class Xapi extends XapiBase {
   }
 
   async _assertConsistentHostServerTime (hostRef) {
-    if (
-      Math.abs(
-        parseDateTime(
-          await this.call('host.get_servertime', hostRef)
-        ).getTime() - Date.now()
-      ) > 2e3
-    ) {
+    const delta =
+      parseDateTime(await this.call('host.get_servertime', hostRef)).getTime() -
+      Date.now()
+    if (Math.abs(delta) > 30e3) {
       throw new Error(
-        'host server time and XOA date are not consistent with each other'
+        `host server time and XOA date are not consistent with each other (${ms(
+          delta
+        )})`
       )
     }
   }

packages/xo-server/src/xapi/mixins/patching.js

@@ -1,4 +1,5 @@
 import asyncMap from '@xen-orchestra/async-map'
+import createLogger from '@xen-orchestra/log'
 import deferrable from 'golike-defer'
 import every from 'lodash/every'
 import filter from 'lodash/filter'
@@ -20,7 +21,9 @@ import {
   parseXml,
 } from '../../utils'
 
-import { debug, extractOpaqueRef, useUpdateSystem } from '../utils'
+import { extractOpaqueRef, useUpdateSystem } from '../utils'
+
+const log = createLogger('xo:xapi')
 
 export default {
   // FIXME: should be static
@@ -227,7 +230,7 @@ export default {
       return this.getObjectByUuid(uuid)
     } catch (error) {}
 
-    debug('downloading patch %s', uuid)
+    log.debug(`downloading patch ${uuid}`)
 
     const patchInfo = (await this._getXenUpdates()).patches[uuid]
     if (!patchInfo) {
@@ -255,7 +258,7 @@ export default {
 
   // patform_version >= 2.1.1 ----------------------------------------
   async _getUpdateVdi ($defer, patchUuid, hostId) {
-    debug('downloading patch %s', patchUuid)
+    log.debug(`downloading patch ${patchUuid}`)
 
     const patchInfo = (await this._getXenUpdates()).patches[patchUuid]
     if (!patchInfo) {
@@ -339,7 +342,7 @@ export default {
   // -----------------------------------------------------------------
 
   async installPoolPatchOnHost (patchUuid, host) {
-    debug('installing patch %s', patchUuid)
+    log.debug(`installing patch ${patchUuid}`)
     if (!isObject(host)) {
       host = this.getObject(host)
     }
@@ -380,7 +383,7 @@ export default {
   }),
 
   async installPoolPatchOnAllHosts (patchUuid) {
-    debug('installing patch %s on all hosts', patchUuid)
+    log.debug(`installing patch ${patchUuid} on all hosts`)
 
     return useUpdateSystem(this.pool.$master)
       ? this._installPatchUpdateOnAllHosts(patchUuid)
@@ -459,7 +462,9 @@ export default {
   async _installAllPoolPatchesOnHost (host) {
     const installableByUuid =
       host.license_params.sku_type !== 'free'
-        ? await this._listMissingPoolPatchesOnHost(host)
+        ? pickBy(await this._listMissingPoolPatchesOnHost(host), {
+            upgrade: false,
+          })
         : pickBy(await this._listMissingPoolPatchesOnHost(host), {
             paid: false,
             upgrade: false,
@@ -506,11 +511,10 @@ export default {
       ...(await Promise.all(
         mapFilter(this.objects.all, host => {
           if (host.$type === 'host') {
-            return this._listMissingPoolPatchesOnHost(host).then(
-              patches =>
-                host.license_params.sku_type !== 'free'
-                  ? patches
-                  : pickBy(patches, { paid: false, upgrade: false })
+            return this._listMissingPoolPatchesOnHost(host).then(patches =>
+              host.license_params.sku_type !== 'free'
+                ? pickBy(patches, { upgrade: false })
+                : pickBy(patches, { paid: false, upgrade: false })
             )
           }
         })

packages/xo-server/src/xapi/mixins/storage.js

@@ -1,7 +1,10 @@
+import createLogger from '@xen-orchestra/log'
 import { forEach, groupBy } from 'lodash'
 
 import { mapToArray } from '../../utils'
 
+const log = createLogger('xo:storage')
+
 export default {
   _connectAllSrPbds (sr) {
     return Promise.all(mapToArray(sr.$PBDs, pbd => this._plugPbd(pbd)))
@@ -58,7 +61,7 @@ export default {
           length += this._getUnhealthyVdiChainLength(parent, childrenMap, cache)
         }
       } catch (error) {
-        console.warn('Xapi#_getUnhealthyVdiChainLength(%s)', uuid, error)
+        log.warn(`Xapi#_getUnhealthyVdiChainLength(${uuid})`, { error })
       }
       cache[uuid] = length
     }

packages/xo-server/src/xapi/mixins/vm.js

@@ -257,6 +257,18 @@ export default {
       },
     },
 
+    virtualizationMode: {
+      set (virtualizationMode, vm) {
+        if (virtualizationMode !== 'pv' && virtualizationMode !== 'hvm') {
+          throw new Error(`The virtualization mode must be 'pv' or 'hvm'`)
+        }
+        return this._set(
+          'HVM_boot_policy',
+          virtualizationMode === 'hvm' ? 'Boot order' : ''
+        )
+      },
+    },
+
     coresPerSocket: {
       set (coresPerSocket, vm) {
         return this._updateObjectMapProperty(vm, 'platform', {
@@ -383,6 +395,14 @@ export default {
 
     hasVendorDevice: true,
 
+    expNestedHvm: {
+      set (expNestedHvm, vm) {
+        return this._updateObjectMapProperty(vm, 'platform', {
+          'exp-nested-hvm': expNestedHvm ? 'true' : null,
+        })
+      },
+    },
+
     nicType: {
       set (nicType, vm) {
         return this._updateObjectMapProperty(vm, 'platform', {
@@ -427,9 +447,9 @@ export default {
     if (snapshot.snapshot_info['power-state-at-snapshot'] === 'Running') {
       const vm = snapshot.$snapshot_of
       if (vm.power_state === 'Halted') {
-        ;this.startVm(vm.$id)::ignoreErrors()
+        this.startVm(vm.$id)::ignoreErrors()
       } else if (vm.power_state === 'Suspended') {
-        ;this.resumeVm(vm.$id)::ignoreErrors()
+        this.resumeVm(vm.$id)::ignoreErrors()
       }
     }
   },