feat(xo-web): 5.29.1

Fixes #2708

.travis.yml

@@ -2,7 +2,6 @@ language: node_js
 node_js:
   #- stable # disable for now due to an issue of indirect dep upath with Node 9
   - 8
-  - 6
 
 # Use containers.
 # http://docs.travis-ci.com/user/workers/container-based-infrastructure/
@@ -10,9 +9,9 @@ sudo: false
 addons:
   apt:
     packages:
-    - qemu-utils
-    - blktap-utils
-    - vmdk-stream-converter
+      - qemu-utils
+      - blktap-utils
+      - vmdk-stream-converter
 
 before_install:
   - curl -o- -L https://yarnpkg.com/install.sh | bash
@@ -22,5 +21,4 @@ cache:
   yarn: true
 
 script:
-  - yarn run test
-  - yarn run test-integration
+  - yarn run travis-tests

@xen-orchestra/async-map/package.json

@@ -44,6 +44,7 @@
     "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
     "prebuild": "yarn run clean",
     "predev": "yarn run prebuild",
+    "prepare": "yarn run build",
     "prepublishOnly": "yarn run build"
   }
 }

@xen-orchestra/cr-seed-cli/index.js

@@ -83,6 +83,9 @@ ${cliName} v${pkg.version}
 
   await Promise.all([
     srcXapi.setFieldEntries(srcSnapshot, 'other_config', metadata),
+    srcXapi.setFieldEntries(srcSnapshot, 'other_config', {
+      'xo:backup:exported': 'true',
+    }),
     tgtXapi.setField(
       tgtVm,
       'name_label',

@xen-orchestra/cr-seed-cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/cr-seed-cli",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/cr-seed-cli",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
@@ -15,6 +15,6 @@
   },
   "dependencies": {
     "golike-defer": "^0.4.1",
-    "xen-api": "^0.19.0"
+    "xen-api": "^0.20.0"
   }
 }

@xen-orchestra/cron/src/index.js

@@ -8,7 +8,12 @@ const MAX_DELAY = 2 ** 31 - 1
 class Job {
   constructor (schedule, fn) {
     const wrapper = () => {
-      const result = fn()
+      let result
+      try {
+        result = fn()
+      } catch (_) {
+        // catch any thrown value to ensure it does not break the job
+      }
       let then
       if (result != null && typeof (then = result.then) === 'function') {
         then.call(result, scheduleNext, scheduleNext)

@xen-orchestra/fs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/fs",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "license": "AGPL-3.0",
   "description": "The File System for Xen Orchestra backups.",
   "keywords": [],

@xen-orchestra/fs/src/abstract.js

@@ -2,7 +2,7 @@
 
 import getStream from 'get-stream'
 import { randomBytes } from 'crypto'
-import { fromCallback, fromEvent, ignoreErrors } from 'promise-toolbox'
+import { fromCallback, fromEvent, ignoreErrors, timeout } from 'promise-toolbox'
 import { type Readable, type Writable } from 'stream'
 import { parse } from 'xo-remote-parser'
 
@@ -17,12 +17,18 @@ type File = FileDescriptor | string
 
 const checksumFile = file => file + '.checksum'
 
+export const DEFAULT_TIMEOUT = 10000
+
 export default class RemoteHandlerAbstract {
   _remote: Object
   constructor (remote: any) {
-    this._remote = { ...remote, ...parse(remote.url) }
-    if (this._remote.type !== this.type) {
-      throw new Error('Incorrect remote type')
+    if (remote.url === 'test://') {
+      this._remote = remote
+    } else {
+      this._remote = { ...remote, ...parse(remote.url) }
+      if (this._remote.type !== this.type) {
+        throw new Error('Incorrect remote type')
+      }
     }
   }
 
@@ -121,7 +127,7 @@ export default class RemoteHandlerAbstract {
     newPath: string,
     { checksum = false }: Object = {}
   ) {
-    let p = this._rename(oldPath, newPath)
+    let p = timeout.call(this._rename(oldPath, newPath), DEFAULT_TIMEOUT)
     if (checksum) {
       p = Promise.all([
         p,
@@ -142,7 +148,7 @@ export default class RemoteHandlerAbstract {
       prependDir = false,
     }: { filter?: (name: string) => boolean, prependDir?: boolean } = {}
   ): Promise<string[]> {
-    let entries = await this._list(dir)
+    let entries = await timeout.call(this._list(dir), DEFAULT_TIMEOUT)
     if (filter !== undefined) {
       entries = entries.filter(filter)
     }
@@ -165,28 +171,30 @@ export default class RemoteHandlerAbstract {
     { checksum = false, ignoreMissingChecksum = false, ...options }: Object = {}
   ): Promise<LaxReadable> {
     const path = typeof file === 'string' ? file : file.path
-    const streamP = this._createReadStream(file, options).then(stream => {
-      // detect early errors
-      let promise = fromEvent(stream, 'readable')
+    const streamP = timeout
+      .call(this._createReadStream(file, options), DEFAULT_TIMEOUT)
+      .then(stream => {
+        // detect early errors
+        let promise = fromEvent(stream, 'readable')
 
-      // try to add the length prop if missing and not a range stream
-      if (
-        stream.length === undefined &&
-        options.end === undefined &&
-        options.start === undefined
-      ) {
-        promise = Promise.all([
-          promise,
-          ignoreErrors.call(
-            this.getSize(file).then(size => {
-              stream.length = size
-            })
-          ),
-        ])
-      }
+        // try to add the length prop if missing and not a range stream
+        if (
+          stream.length === undefined &&
+          options.end === undefined &&
+          options.start === undefined
+        ) {
+          promise = Promise.all([
+            promise,
+            ignoreErrors.call(
+              this.getSize(file).then(size => {
+                stream.length = size
+              })
+            ),
+          ])
+        }
 
-      return promise.then(() => stream)
-    })
+        return promise.then(() => stream)
+      })
 
     if (!checksum) {
       return streamP
@@ -224,7 +232,10 @@ export default class RemoteHandlerAbstract {
   }
 
   async openFile (path: string, flags?: string): Promise<FileDescriptor> {
-    return { fd: await this._openFile(path, flags), path }
+    return {
+      fd: await timeout.call(this._openFile(path, flags), DEFAULT_TIMEOUT),
+      path,
+    }
   }
 
   async _openFile (path: string, flags?: string): Promise<mixed> {
@@ -232,7 +243,7 @@ export default class RemoteHandlerAbstract {
   }
 
   async closeFile (fd: FileDescriptor): Promise<void> {
-    await this._closeFile(fd.fd)
+    await timeout.call(this._closeFile(fd.fd), DEFAULT_TIMEOUT)
   }
 
   async _closeFile (fd: mixed): Promise<void> {
@@ -252,10 +263,13 @@ export default class RemoteHandlerAbstract {
     { checksum = false, ...options }: Object = {}
   ): Promise<LaxWritable> {
     const path = typeof file === 'string' ? file : file.path
-    const streamP = this._createOutputStream(file, {
-      flags: 'wx',
-      ...options,
-    })
+    const streamP = timeout.call(
+      this._createOutputStream(file, {
+        flags: 'wx',
+        ...options,
+      }),
+      DEFAULT_TIMEOUT
+    )
 
     if (!checksum) {
       return streamP
@@ -290,7 +304,7 @@ export default class RemoteHandlerAbstract {
       ignoreErrors.call(this._unlink(checksumFile(file)))
     }
 
-    await this._unlink(file)
+    await timeout.call(this._unlink(file), DEFAULT_TIMEOUT)
   }
 
   async _unlink (file: mixed): Promise<void> {
@@ -298,7 +312,7 @@ export default class RemoteHandlerAbstract {
   }
 
   async getSize (file: mixed): Promise<number> {
-    return this._getSize(file)
+    return timeout.call(this._getSize(file), DEFAULT_TIMEOUT)
   }
 
   async _getSize (file: mixed): Promise<number> {

@xen-orchestra/fs/src/abstract.spec.js

@@ -0,0 +1,111 @@
+/* eslint-env jest */
+
+import { TimeoutError } from 'promise-toolbox'
+
+import AbstractHandler, { DEFAULT_TIMEOUT } from './abstract'
+
+class TestHandler extends AbstractHandler {
+  constructor (impl) {
+    super({ url: 'test://' })
+
+    Object.keys(impl).forEach(method => {
+      this[`_${method}`] = impl[method]
+    })
+  }
+}
+
+describe('rename()', () => {
+  it(`return TimeoutError after ${DEFAULT_TIMEOUT} ms`, async () => {
+    const testHandler = new TestHandler({
+      rename: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.rename('oldPath', 'newPath')
+    jest.advanceTimersByTime(DEFAULT_TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('list()', () => {
+  it(`return TimeoutError after ${DEFAULT_TIMEOUT} ms`, async () => {
+    const testHandler = new TestHandler({
+      list: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.list()
+    jest.advanceTimersByTime(DEFAULT_TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('createReadStream()', () => {
+  it(`return TimeoutError after ${DEFAULT_TIMEOUT} ms`, async () => {
+    const testHandler = new TestHandler({
+      createReadStream: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.createReadStream('file')
+    jest.advanceTimersByTime(DEFAULT_TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('openFile()', () => {
+  it(`return TimeoutError after ${DEFAULT_TIMEOUT} ms`, async () => {
+    const testHandler = new TestHandler({
+      openFile: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.openFile('path')
+    jest.advanceTimersByTime(DEFAULT_TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('closeFile()', () => {
+  it(`return TimeoutError after ${DEFAULT_TIMEOUT} ms`, async () => {
+    const testHandler = new TestHandler({
+      closeFile: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.closeFile({ fd: undefined, path: '' })
+    jest.advanceTimersByTime(DEFAULT_TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('createOutputStream()', () => {
+  it(`return TimeoutError after ${DEFAULT_TIMEOUT} ms`, async () => {
+    const testHandler = new TestHandler({
+      createOutputStream: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.createOutputStream('File')
+    jest.advanceTimersByTime(DEFAULT_TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('unlink()', () => {
+  it(`return TimeoutError after ${DEFAULT_TIMEOUT} ms`, async () => {
+    const testHandler = new TestHandler({
+      unlink: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.unlink('')
+    jest.advanceTimersByTime(DEFAULT_TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
+describe('getSize()', () => {
+  it(`return TimeoutError after ${DEFAULT_TIMEOUT} ms`, async () => {
+    const testHandler = new TestHandler({
+      getSize: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.getSize('')
+    jest.advanceTimersByTime(DEFAULT_TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})

@xen-orchestra/log/package.json

@@ -1,7 +1,6 @@
 {
-  "private": true,
   "name": "@xen-orchestra/log",
-  "version": "0.0.0",
+  "version": "0.1.4",
   "license": "ISC",
   "description": "",
   "keywords": [],
@@ -19,7 +18,9 @@
   "main": "dist/",
   "bin": {},
   "files": [
-    "dist/"
+    "configure.js",
+    "dist/",
+    "transports/"
   ],
   "browserslist": [
     ">2%"

@xen-orchestra/log/src/configure.js

@@ -86,7 +86,7 @@ export const catchGlobalErrors = logger => {
   const { prototype } = EventEmitter
   const { emit } = prototype
   function patchedEmit (event, error) {
-    if (event === 'error' && !this.listenerCount(event)) {
+    if (event === 'error' && this.listenerCount(event) === 0) {
       logger.error('unhandled error event', { error })
       return false
     }

@xen-orchestra/log/src/index.js

@@ -38,6 +38,17 @@ for (const name in LEVELS) {
   const level = LEVELS[name]
 
   prototype[name.toLowerCase()] = function (message, data) {
+    if (typeof message !== 'string') {
+      if (message instanceof Error) {
+        data = { error: message }
+        ;({ message = 'an error has occured' } = message)
+      } else {
+        return this.warn('incorrect value passed to logger', {
+          level,
+          value: message,
+        })
+      }
+    }
     global[symbol](new Log(data, level, this._namespace, message, new Date()))
   }
 }

@xen-orchestra/log/transports

@@ -1 +0,0 @@
-dist/transports

@xen-orchestra/log/transports/console.js

@@ -0,0 +1 @@
+module.exports = require('../dist/transports/console.js')

@xen-orchestra/log/transports/email.js

@@ -0,0 +1 @@
+module.exports = require('../dist/transports/email.js')

@xen-orchestra/log/transports/memory.js

@@ -0,0 +1 @@
+module.exports = require('../dist/transports/memory.js')

@xen-orchestra/log/transports/syslog.js

@@ -0,0 +1 @@
+module.exports = require('../dist/transports/syslog.js')

CHANGELOG.md

@@ -6,6 +6,99 @@
 
 ### Bug fixes
 
+### Released packages
+
+- xo-server v5.30.0
+- xo-web v5.30.0
+
+## **5.28.0** (2018-10-31)
+
+### Enhancements
+
+- [Usage Report] Add IOPS read/write/total per VM [#3309](https://github.com/vatesfr/xen-orchestra/issues/3309) (PR [#3455](https://github.com/vatesfr/xen-orchestra/pull/3455))
+- [Self service] Sort resource sets by name (PR [#3507](https://github.com/vatesfr/xen-orchestra/pull/3507))
+- [Usage Report] Add top 3 SRs which use the most IOPS read/write/total [#3306](https://github.com/vatesfr/xen-orchestra/issues/3306) (PR [#3508](https://github.com/vatesfr/xen-orchestra/pull/3508))
+- [New VM] Display a warning when the memory is below the template memory static min [#3496](https://github.com/vatesfr/xen-orchestra/issues/3496) (PR [#3513](https://github.com/vatesfr/xen-orchestra/pull/3513))
+- [Backup NG form] Add link to plugins setting [#3457](https://github.com/vatesfr/xen-orchestra/issues/3457) (PR [#3514](https://github.com/vatesfr/xen-orchestra/pull/3514))
+- [Backup reports] Add job and run ID [#3488](https://github.com/vatesfr/xen-orchestra/issues/3488) (PR [#3516](https://github.com/vatesfr/xen-orchestra/pull/3516))
+- [Usage Report] Add top 3 VMs which use the most IOPS read/write/total [#3308](https://github.com/vatesfr/xen-orchestra/issues/3308) (PR [#3463](https://github.com/vatesfr/xen-orchestra/pull/3463))
+- [Settings/logs] Homogenize action buttons in table and enable bulk deletion [#3179](https://github.com/vatesfr/xen-orchestra/issues/3179) (PR [#3528](https://github.com/vatesfr/xen-orchestra/pull/3528))
+- [Settings/acls] Add bulk deletion [#3179](https://github.com/vatesfr/xen-orchestra/issues/3179) (PR [#3536](https://github.com/vatesfr/xen-orchestra/pull/3536))
+- [Home] Improve search usage: raw numbers also match in names [#2906](https://github.com/vatesfr/xen-orchestra/issues/2906) (PR [#3552](https://github.com/vatesfr/xen-orchestra/pull/3552))
+- [Backup NG] Timeout of a job is now in hours [#3550](https://github.com/vatesfr/xen-orchestra/issues/3550) (PR [#3553](https://github.com/vatesfr/xen-orchestra/pull/3553))
+- [Backup NG] Explicit error if a VM is missing [#3434](https://github.com/vatesfr/xen-orchestra/issues/3434) (PR [#3522](https://github.com/vatesfr/xen-orchestra/pull/3522))
+- [Backup NG] Show all advanced settings with non-default values in overview [#3549](https://github.com/vatesfr/xen-orchestra/issues/3549) (PR [#3554](https://github.com/vatesfr/xen-orchestra/pull/3554))
+- [Backup NG] Collapse advanced settings by default [#3551](https://github.com/vatesfr/xen-orchestra/issues/3551) (PR [#3559](https://github.com/vatesfr/xen-orchestra/pull/3559))
+- [Scheduling] Merge selection and interval tabs [#1902](https://github.com/vatesfr/xen-orchestra/issues/1902) (PR [#3519](https://github.com/vatesfr/xen-orchestra/pull/3519))
+- [Backup NG/Restore] The backup selector now also shows the job name [#3366](https://github.com/vatesfr/xen-orchestra/issues/3366) (PR [#3564](https://github.com/vatesfr/xen-orchestra/pull/3564))
+- Sort buttons by criticality in tables [#3168](https://github.com/vatesfr/xen-orchestra/issues/3168) (PR [#3545](https://github.com/vatesfr/xen-orchestra/pull/3545))
+- [Usage Report] Ability to send a daily report [#3544](https://github.com/vatesfr/xen-orchestra/issues/3544) (PR [#3582](https://github.com/vatesfr/xen-orchestra/pull/3582))
+- [Backup NG logs] Disable state filters with no entries [#3438](https://github.com/vatesfr/xen-orchestra/issues/3438) (PR [#3442](https://github.com/vatesfr/xen-orchestra/pull/3442))
+- [ACLs] Global performance improvement on UI for non-admin users [#3578](https://github.com/vatesfr/xen-orchestra/issues/3578) (PR [#3584](https://github.com/vatesfr/xen-orchestra/pull/3584))
+- [Backup NG] Improve the Schedule's view (Replace table by list) [#3491](https://github.com/vatesfr/xen-orchestra/issues/3491) (PR [#3586](https://github.com/vatesfr/xen-orchestra/pull/3586))
+- ([Host/Storage], [Sr/hosts]) add bulk deletion [#3179](https://github.com/vatesfr/xen-orchestra/issues/3179) (PR [#3539](https://github.com/vatesfr/xen-orchestra/pull/3539))
+- [xo-server] Use @xen-orchestra/log for basic logging [#3555](https://github.com/vatesfr/xen-orchestra/issues/3555) (PR [#3579](https://github.com/vatesfr/xen-orchestra/pull/3579))
+- [Backup Report] Log error when job failed [#3458](https://github.com/vatesfr/xen-orchestra/issues/3458) (PR [#3593](https://github.com/vatesfr/xen-orchestra/pull/3593))
+- [Backup NG] Display logs for backup restoration [#2511](https://github.com/vatesfr/xen-orchestra/issues/2511) (PR [#3609](https://github.com/vatesfr/xen-orchestra/pull/3609))
+- [XOA] Display product version and list of all installed packages [#3560](https://github.com/vatesfr/xen-orchestra/issues/3560) (PR [#3621](https://github.com/vatesfr/xen-orchestra/pull/3621))
+
+### Bug fixes
+
+- [Remotes] Fix removal of broken remotes [#3327](https://github.com/vatesfr/xen-orchestra/issues/3327) (PR [#3521](https://github.com/vatesfr/xen-orchestra/pull/3521))
+- [Backups] Fix stuck backups due to broken NFS remotes [#3467](https://github.com/vatesfr/xen-orchestra/issues/3467) (PR [#3534](https://github.com/vatesfr/xen-orchestra/pull/3534))
+- [New VM] Fix missing cloud config when creating multiple VMs at once in some cases [#3532](https://github.com/vatesfr/xen-orchestra/issues/3532) (PR [#3535](https://github.com/vatesfr/xen-orchestra/pull/3535))
+- [VM] Fix an error when an admin tried to add a disk on a Self VM whose resource set had been deleted [#2814](https://github.com/vatesfr/xen-orchestra/issues/2814) (PR [#3530](https://github.com/vatesfr/xen-orchestra/pull/3530))
+- [Self/Create VM] Fix some quotas based on the template instead of the user inputs [#2683](https://github.com/vatesfr/xen-orchestra/issues/2683) (PR [#3546](https://github.com/vatesfr/xen-orchestra/pull/3546))
+- [Self] Ignore DR and CR VMs when computing quotas [#3064](https://github.com/vatesfr/xen-orchestra/issues/3064) (PR [#3561](https://github.com/vatesfr/xen-orchestra/pull/3561))
+- [Patches] Wrongly requiring to eject CDs from halted VMs and snapshots before installing patches (PR [#3611](https://github.com/vatesfr/xen-orchestra/pull/3611))
+- [Jobs] Ensure the scheduling is not interrupted in rare cases (PR [#3617](https://github.com/vatesfr/xen-orchestra/pull/3617))
+- [Home] Fix `server.getAll` error at login when user is not admin [#2335](https://github.com/vatesfr/xen-orchestra/issues/2335) (PR [#3613](https://github.com/vatesfr/xen-orchestra/pull/3613))
+
+### Released packages
+
+- xo-server-backup-reports v0.15.0
+- xo-common v0.1.2
+- @xen-orchestra/log v0.1.0
+- @xen-orchestra/fs v0.4.0
+- complex-matcher v0.5.0
+- vhd-lib v0.4.0
+- xen-api v0.20.0
+- xo-server-usage-report v0.7.0
+- xo-server v5.29.0
+- xo-web v5.29.0
+
+## **5.27.2** (2018-10-05)
+
+### Enhancements
+
+- [Host/Networks] Remove "Add network" button [#3386](https://github.com/vatesfr/xen-orchestra/issues/3386) (PR [#3478](https://github.com/vatesfr/xen-orchestra/pull/3478))
+- [Host/networks] Private networks table [#3387](https://github.com/vatesfr/xen-orchestra/issues/3387) (PR [#3481](https://github.com/vatesfr/xen-orchestra/pull/3481))
+- [Home/pool] Patch count pill now shows the number of unique patches in the pool [#3321](https://github.com/vatesfr/xen-orchestra/issues/3321) (PR [#3483](https://github.com/vatesfr/xen-orchestra/pull/3483))
+- [Patches] Pre-install checks to avoid errors [#3252](https://github.com/vatesfr/xen-orchestra/issues/3252) (PR [#3484](https://github.com/vatesfr/xen-orchestra/pull/3484))
+- [Vm/Snapshots] Allow VM operators to create snapshots and delete those they created [#3443](https://github.com/vatesfr/xen-orchestra/issues/3443) (PR [#3482](https://github.com/vatesfr/xen-orchestra/pull/3482))
+- [VM/clone] Handle ACLs and Self Service [#3139](https://github.com/vatesfr/xen-orchestra/issues/3139) (PR [#3493](https://github.com/vatesfr/xen-orchestra/pull/3493))
+
+### Bug fixes
+
+- [Backup NG] Fix `Cannot read property 'uuid' of undefined` when a disk is removed from a VM to backup (PR [#3479](https://github.com/vatesfr/xen-orchestra/pull/3479))
+- [Backup NG] Fix unexpected full after failure, interruption or basic rolling snapshot (PR [#3485](https://github.com/vatesfr/xen-orchestra/pull/3485))
+- [Usage report] Display top 3 used SRs instead of top 3 biggest SRs [#3307](https://github.com/vatesfr/xen-orchestra/issues/3307) (PR [#3475](https://github.com/vatesfr/xen-orchestra/pull/3475))
+
+### Released packages
+
+- vhd-lib v0.3.2
+- xo-vmdk-to-vhd v0.1.5
+- xo-server-usage-report v0.6.0
+- xo-acl-resolver v0.3.0
+- xo-server v5.28.0
+- xo-web v5.28.0
+
+## **5.27.1** (2018-09-28)
+
+### Enhancements
+
+### Bug fixes
+
 - [OVA Import] Allow import of files bigger than 127GB (PR [#3451](https://github.com/vatesfr/xen-orchestra/pull/3451))
 - [File restore] Fix a path issue when going back to the parent folder (PR [#3446](https://github.com/vatesfr/xen-orchestra/pull/3446))
 - [File restore] Fix a minor issue when showing which selected files are redundant (PR [#3447](https://github.com/vatesfr/xen-orchestra/pull/3447))
@@ -18,8 +111,8 @@
 - @xen-orchestra/fs v0.3.1
 - vhd-lib v0.3.1
 - xo-vmdk-to-vhd v0.1.4
-- xo-server v5.28.0
-- xo-web v5.28.0
+- xo-server v5.27.2
+- xo-web v5.27.1
 
 ## **5.27.0** (2018-09-24)
 

docs/backup_troubleshooting.md

@@ -39,6 +39,10 @@ You can check if a coalesce job is currently active by running `ps axf | grep vh
 
 If you don't see any running coalesce jobs, and can't find any other reason that XenServer has not started one, you can attempt to make it start a coalesce job by rescanning the SR. This is harmless to try, but will not always result in a coalesce. Visit the problematic SR in the XOA UI, then click the "Rescan All Disks" button towards the top right: it looks like a refresh circle icon. This should begin the coalesce process - if you click the Advanced tab in the SR view, the "disks needing to be coalesced" list should become smaller and smaller.
 
+### Parse Error
+
+This is most likely due to running a backup job that uses Delta functionality (eg: delta backups, or continuous replication) on a version of XenServer older than 6.5. To use delta functionality you must run [XenServer 6.5 or later](https://xen-orchestra.com/docs/supported-version.html).
+
 ### SR_BACKEND_FAILURE_44 (insufficient space)
 
 > This message can be triggered by any backup method.
@@ -72,4 +76,4 @@ To check your free space, enter your XOA and run `xoa check` to check free syste
 
 This is happening when you have a *smart backup job* that doesn't match any VMs. For example: you created a job to backup all running VMs. If no VMs are running on backup schedule, you'll have this message. This could also happen if you lost connection with your pool master (the VMs aren't visible anymore from Xen Orchestra).
 
-Edit your job and try to see matching VMs or check if your pool is connected to XOA.
+Edit your job and try to see matching VMs or check if your pool is connected to XOA.

docs/xo-cli.md

@@ -1,4 +1,3 @@
-
 # xo-cli
 
 This is another client of `xo-server` - this time in command line form.
@@ -106,3 +105,4 @@ encoding by prefixing with `json:`:
  ```
 > xo-cli vm.import host=60a6939e-8b0a-4352-9954-5bde44bcdf7d @=vm.xva
 ```
+> Note: `xo-cli` only supports the import of XVA files. It will not import OVA files. To import OVA images, you must use the XOA web UI.

package.json

@@ -3,7 +3,7 @@
     "@babel/core": "^7.0.0",
     "@babel/register": "^7.0.0",
     "babel-core": "^7.0.0-0",
-    "babel-eslint": "^9.0.0",
+    "babel-eslint": "^10.0.1",
     "babel-jest": "^23.0.1",
     "benchmark": "^2.1.4",
     "eslint": "^5.1.0",
@@ -15,7 +15,7 @@
     "eslint-plugin-react": "^7.6.1",
     "eslint-plugin-standard": "^4.0.0",
     "exec-promise": "^0.7.0",
-    "flow-bin": "^0.81.0",
+    "flow-bin": "^0.82.0",
     "globby": "^8.0.0",
     "husky": "^1.0.0-rc.15",
     "jest": "^23.0.1",
@@ -33,6 +33,7 @@
     }
   },
   "jest": {
+    "timers": "fake",
     "collectCoverage": true,
     "projects": [
       "<rootDir>"
@@ -57,7 +58,8 @@
     "prepare": "scripts/run-script prepare",
     "pretest": "eslint --ignore-path .gitignore .",
     "test": "jest \"^(?!.*\\.integ\\.spec\\.js$)\"",
-    "test-integration": "jest \".integ\\.spec\\.js$\""
+    "test-integration": "jest \".integ\\.spec\\.js$\"",
+    "travis-tests": "scripts/travis-tests"
   },
   "workspaces": [
     "@xen-orchestra/*",

packages/complex-matcher/package.json

@@ -1,6 +1,6 @@
 {
   "name": "complex-matcher",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "license": "ISC",
   "description": "",
   "keywords": [],

packages/complex-matcher/src/index.fixtures.js

@@ -11,7 +11,7 @@ export const ast = new CM.And([
     new CM.Or([new CM.String('wonderwoman'), new CM.String('batman')])
   ),
   new CM.TruthyProperty('hasCape'),
-  new CM.Property('age', new CM.Number(32)),
+  new CM.Property('age', new CM.NumberOrStringNode('32')),
   new CM.GlobPattern('chi*go'),
   new CM.RegExp('^foo/bar\\.', 'i'),
 ])

packages/complex-matcher/src/index.js

@@ -153,6 +153,34 @@ export class NumberNode extends Node {
 }
 export { NumberNode as Number }
 
+export class NumberOrStringNode extends Node {
+  constructor (value) {
+    super()
+
+    this.value = value
+
+    // should not be enumerable for the tests
+    Object.defineProperty(this, 'match', {
+      value: this.match.bind(this, value.toLowerCase(), +value),
+    })
+  }
+
+  match (lcValue, numValue, value) {
+    return (
+      value === numValue ||
+      (typeof value === 'string'
+        ? value.toLowerCase().indexOf(lcValue) !== -1
+        : (Array.isArray(value) || isPlainObject(value)) &&
+          some(value, this.match))
+    )
+  }
+
+  toString () {
+    return this.value
+  }
+}
+export { NumberOrStringNode as NumberOrString }
+
 export class Property extends Node {
   constructor (name, child) {
     super()
@@ -564,7 +592,7 @@ const parser = P.grammar({
         const asNum = +str
         return Number.isNaN(asNum)
           ? new GlobPattern(str)
-          : new NumberNode(asNum)
+          : new NumberOrStringNode(str)
       })
     ),
   ws: P.regex(/\s*/),

packages/complex-matcher/src/index.spec.js

@@ -6,6 +6,7 @@ import {
   GlobPattern,
   Null,
   NumberNode,
+  NumberOrStringNode,
   parse,
   setPropertyClause,
 } from './'
@@ -32,7 +33,7 @@ describe('parse', () => {
 
     node = parse('32')
     expect(node.match(32)).toBe(true)
-    expect(node.match('32')).toBe(false)
+    expect(node.match('32')).toBe(true)
     expect(node.toString()).toBe('32')
 
     node = parse('"32"')
@@ -54,6 +55,12 @@ describe('Number', () => {
   })
 })
 
+describe('NumberOrStringNode', () => {
+  it('match a string', () => {
+    expect(new NumberOrStringNode('123').match([{ foo: '123' }])).toBe(true)
+  })
+})
+
 describe('setPropertyClause', () => {
   it('creates a node if none passed', () => {
     expect(setPropertyClause(undefined, 'foo', 'bar').toString()).toBe(

packages/vhd-cli/package.json

@@ -26,11 +26,11 @@
     "node": ">=6"
   },
   "dependencies": {
-    "@xen-orchestra/fs": "^0.3.0",
+    "@xen-orchestra/fs": "^0.4.0",
     "cli-progress": "^2.0.0",
     "exec-promise": "^0.7.0",
     "struct-fu": "^1.2.0",
-    "vhd-lib": "^0.3.0"
+    "vhd-lib": "^0.4.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/vhd-cli/src/commands/synthetize.js

@@ -9,13 +9,12 @@ export default async function main (args) {
   }
 
   const handler = getHandler({ url: 'file:///' })
+  const stream = await createSyntheticStream(handler, path.resolve(args[0]))
   return new Promise((resolve, reject) => {
-    createSyntheticStream(handler, path.resolve(args[0]))
-      .on('error', reject)
-      .pipe(
-        createWriteStream(args[1])
-          .on('error', reject)
-          .on('finish', resolve)
-      )
+    stream.on('error', reject).pipe(
+      createWriteStream(args[1])
+        .on('error', reject)
+        .on('finish', resolve)
+    )
   })
 }

packages/vhd-lib/merge.integ.spec.js

@@ -9,11 +9,9 @@ import { fromEvent, pFromCallback } from 'promise-toolbox'
 import { getHandler } from '@xen-orchestra/fs'
 import { randomBytes } from 'crypto'
 
-import chainVhd from './chain'
-import createReadStream from './createSyntheticStream'
-import Vhd from './vhd'
-import vhdMerge from './merge'
-import { SECTOR_SIZE } from './_constants'
+import Vhd, { chainVhd, createSyntheticStream, mergeVhd as vhdMerge } from './'
+
+import { SECTOR_SIZE } from './src/_constants'
 
 const initialDir = process.cwd()
 
@@ -270,14 +268,18 @@ test('coalesce works in normal cases', async () => {
 
 test('createSyntheticStream passes vhd-util check', async () => {
   const initalSize = 4
+  const expectedVhdSize = 4197888
   await createRandomFile('randomfile', initalSize)
   await convertFromRawToVhd('randomfile', 'randomfile.vhd')
   const handler = getHandler({ url: 'file://' + process.cwd() })
-  const stream = createReadStream(handler, 'randomfile.vhd')
+  const stream = await createSyntheticStream(handler, 'randomfile.vhd')
+  expect(stream.length).toEqual(expectedVhdSize)
   await fromEvent(
     stream.pipe(await fs.createWriteStream('recovered.vhd')),
     'finish'
   )
   await checkFile('recovered.vhd')
+  const stats = await fs.stat('recovered.vhd')
+  expect(stats.size).toEqual(expectedVhdSize)
   await execa('qemu-img', ['compare', 'recovered.vhd', 'randomfile'])
 })

packages/vhd-lib/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vhd-lib",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "license": "AGPL-3.0",
   "description": "Primitives for VHD file handling",
   "keywords": [],
@@ -21,6 +21,7 @@
   },
   "dependencies": {
     "async-iterator-to-stream": "^1.0.2",
+    "core-js": "3.0.0-beta.3",
     "from2": "^2.3.0",
     "fs-extra": "^7.0.0",
     "limit-concurrency-decorator": "^0.4.0",
@@ -33,7 +34,7 @@
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
     "@babel/preset-flow": "^7.0.0",
-    "@xen-orchestra/fs": "^0.3.1",
+    "@xen-orchestra/fs": "^0.4.0",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^5.1.3",
     "execa": "^1.0.0",

packages/vhd-lib/src/createReadableSparseStream.js

@@ -17,7 +17,7 @@ import { set as setBitmap } from './_bitmap'
 const VHD_BLOCK_SIZE_SECTORS = VHD_BLOCK_SIZE_BYTES / SECTOR_SIZE
 
 /**
- * @returns {Array} an array of occupation bitmap, each bit mapping an input block size of bytes
+ * @returns currentVhdPositionSector the first free sector after the data
  */
 function createBAT (
   firstBlockPosition,
@@ -36,9 +36,10 @@ function createBAT (
         (bitmapSize + VHD_BLOCK_SIZE_BYTES) / SECTOR_SIZE
     }
   })
+  return currentVhdPositionSector
 }
 
-export default asyncIteratorToStream(async function * (
+export default async function createReadableStream (
   diskSize,
   incomingBlockSize,
   blockAddressList,
@@ -79,7 +80,14 @@ export default asyncIteratorToStream(async function * (
   const bitmapSize =
     Math.ceil(VHD_BLOCK_SIZE_SECTORS / 8 / SECTOR_SIZE) * SECTOR_SIZE
   const bat = Buffer.alloc(tablePhysicalSizeBytes, 0xff)
-  createBAT(firstBlockPosition, blockAddressList, ratio, bat, bitmapSize)
+  const endOfData = createBAT(
+    firstBlockPosition,
+    blockAddressList,
+    ratio,
+    bat,
+    bitmapSize
+  )
+  const fileSize = endOfData * SECTOR_SIZE + FOOTER_SIZE
   let position = 0
   function * yieldAndTrack (buffer, expectedPosition) {
     if (expectedPosition !== undefined) {
@@ -120,9 +128,16 @@ export default asyncIteratorToStream(async function * (
     }
     yield * yieldAndTrack(currentBlockWithBitmap)
   }
-  yield * yieldAndTrack(footer, 0)
-  yield * yieldAndTrack(header, FOOTER_SIZE)
-  yield * yieldAndTrack(bat, FOOTER_SIZE + HEADER_SIZE)
-  yield * generateFileContent(blockIterator, bitmapSize, ratio)
-  yield * yieldAndTrack(footer)
-})
+
+  async function * iterator () {
+    yield * yieldAndTrack(footer, 0)
+    yield * yieldAndTrack(header, FOOTER_SIZE)
+    yield * yieldAndTrack(bat, FOOTER_SIZE + HEADER_SIZE)
+    yield * generateFileContent(blockIterator, bitmapSize, ratio)
+    yield * yieldAndTrack(footer)
+  }
+
+  const stream = asyncIteratorToStream(iterator())
+  stream.length = fileSize
+  return stream
+}

packages/vhd-lib/src/createSyntheticStream.js

@@ -15,18 +15,24 @@ import { test as mapTestBit } from './_bitmap'
 const resolveRelativeFromFile = (file, path) =>
   resolve('/', dirname(file), path).slice(1)
 
-export default asyncIteratorToStream(function * (handler, path) {
+export default async function createSyntheticStream (handler, path) {
   const fds = []
-
+  const cleanup = () => {
+    for (let i = 0, n = fds.length; i < n; ++i) {
+      handler.closeFile(fds[i]).catch(error => {
+        console.warn('createReadStream, closeFd', i, error)
+      })
+    }
+  }
   try {
     const vhds = []
     while (true) {
-      const fd = yield handler.openFile(path, 'r')
+      const fd = await handler.openFile(path, 'r')
       fds.push(fd)
       const vhd = new Vhd(handler, fd)
       vhds.push(vhd)
-      yield vhd.readHeaderAndFooter()
-      yield vhd.readBlockAllocationTable()
+      await vhd.readHeaderAndFooter()
+      await vhd.readBlockAllocationTable()
 
       if (vhd.footer.diskType === DISK_TYPE_DYNAMIC) {
         break
@@ -64,14 +70,8 @@ export default asyncIteratorToStream(function * (handler, path) {
     const nBlocks = Math.ceil(footer.currentSize / header.blockSize)
 
     const blocksOwner = new Array(nBlocks)
-    for (
-      let iBlock = 0,
-        blockOffset = Math.ceil(
-          (header.tableOffset + bat.length) / SECTOR_SIZE
-        );
-      iBlock < nBlocks;
-      ++iBlock
-    ) {
+    let blockOffset = Math.ceil((header.tableOffset + bat.length) / SECTOR_SIZE)
+    for (let iBlock = 0; iBlock < nBlocks; ++iBlock) {
       let blockSector = BLOCK_UNUSED
       for (let i = 0; i < nVhds; ++i) {
         if (vhds[i].containsBlock(iBlock)) {
@@ -83,71 +83,78 @@ export default asyncIteratorToStream(function * (handler, path) {
       }
       bat.writeUInt32BE(blockSector, iBlock * 4)
     }
+    const fileSize = blockOffset * SECTOR_SIZE + FOOTER_SIZE
 
-    footer = fuFooter.pack(footer)
-    checksumStruct(footer, fuFooter)
-    yield footer
+    const iterator = function * () {
+      try {
+        footer = fuFooter.pack(footer)
+        checksumStruct(footer, fuFooter)
+        yield footer
 
-    header = fuHeader.pack(header)
-    checksumStruct(header, fuHeader)
-    yield header
+        header = fuHeader.pack(header)
+        checksumStruct(header, fuHeader)
+        yield header
 
-    yield bat
+        yield bat
 
-    // TODO: for generic usage the bitmap needs to be properly computed for each block
-    const bitmap = Buffer.alloc(vhd.bitmapSize, 0xff)
-    for (let iBlock = 0; iBlock < nBlocks; ++iBlock) {
-      const owner = blocksOwner[iBlock]
-      if (owner === undefined) {
-        continue
-      }
-
-      yield bitmap
-
-      const blocksByVhd = new Map()
-      const emitBlockSectors = function * (iVhd, i, n) {
-        const vhd = vhds[iVhd]
-        const isRootVhd = vhd === rootVhd
-        if (!vhd.containsBlock(iBlock)) {
-          if (isRootVhd) {
-            yield Buffer.alloc((n - i) * SECTOR_SIZE)
-          } else {
-            yield * emitBlockSectors(iVhd + 1, i, n)
+        // TODO: for generic usage the bitmap needs to be properly computed for each block
+        const bitmap = Buffer.alloc(vhd.bitmapSize, 0xff)
+        for (let iBlock = 0; iBlock < nBlocks; ++iBlock) {
+          const owner = blocksOwner[iBlock]
+          if (owner === undefined) {
+            continue
           }
-          return
-        }
-        let block = blocksByVhd.get(vhd)
-        if (block === undefined) {
-          block = yield vhd._readBlock(iBlock)
-          blocksByVhd.set(vhd, block)
-        }
-        const { bitmap, data } = block
-        if (isRootVhd) {
-          yield data.slice(i * SECTOR_SIZE, n * SECTOR_SIZE)
-          return
-        }
-        while (i < n) {
-          const hasData = mapTestBit(bitmap, i)
-          const start = i
-          do {
-            ++i
-          } while (i < n && mapTestBit(bitmap, i) === hasData)
-          if (hasData) {
-            yield data.slice(start * SECTOR_SIZE, i * SECTOR_SIZE)
-          } else {
-            yield * emitBlockSectors(iVhd + 1, start, i)
+
+          yield bitmap
+
+          const blocksByVhd = new Map()
+          const emitBlockSectors = function * (iVhd, i, n) {
+            const vhd = vhds[iVhd]
+            const isRootVhd = vhd === rootVhd
+            if (!vhd.containsBlock(iBlock)) {
+              if (isRootVhd) {
+                yield Buffer.alloc((n - i) * SECTOR_SIZE)
+              } else {
+                yield * emitBlockSectors(iVhd + 1, i, n)
+              }
+              return
+            }
+            let block = blocksByVhd.get(vhd)
+            if (block === undefined) {
+              block = yield vhd._readBlock(iBlock)
+              blocksByVhd.set(vhd, block)
+            }
+            const { bitmap, data } = block
+            if (isRootVhd) {
+              yield data.slice(i * SECTOR_SIZE, n * SECTOR_SIZE)
+              return
+            }
+            while (i < n) {
+              const hasData = mapTestBit(bitmap, i)
+              const start = i
+              do {
+                ++i
+              } while (i < n && mapTestBit(bitmap, i) === hasData)
+              if (hasData) {
+                yield data.slice(start * SECTOR_SIZE, i * SECTOR_SIZE)
+              } else {
+                yield * emitBlockSectors(iVhd + 1, start, i)
+              }
+            }
           }
+          yield * emitBlockSectors(owner, 0, sectorsPerBlockData)
         }
+        yield footer
+      } finally {
+        cleanup()
       }
-      yield * emitBlockSectors(owner, 0, sectorsPerBlockData)
     }
 
-    yield footer
-  } finally {
-    for (let i = 0, n = fds.length; i < n; ++i) {
-      handler.closeFile(fds[i]).catch(error => {
-        console.warn('createReadStream, closeFd', i, error)
-      })
-    }
+    const stream = asyncIteratorToStream(iterator())
+    stream.length = fileSize
+    return stream
+  } catch (e) {
+    cleanup()
+    throw e
   }
-})
+}

packages/vhd-lib/src/index.js

@@ -1,3 +1,6 @@
+// see https://github.com/babel/babel/issues/8450
+import 'core-js/features/symbol/async-iterator'
+
 export { default } from './vhd'
 export { default as chainVhd } from './chain'
 export { default as createContentStream } from './createContentStream'

packages/vhd-lib/vhd.integ.spec.js

@@ -5,9 +5,9 @@ import tmp from 'tmp'
 import { createWriteStream, readFile } from 'fs-promise'
 import { fromEvent, pFromCallback } from 'promise-toolbox'
 
-import { createFooter } from './_createFooterHeader'
-import createReadableRawVHDStream from './createReadableRawStream'
-import createReadableSparseVHDStream from './createReadableSparseStream'
+import { createReadableRawStream, createReadableSparseStream } from './'
+
+import { createFooter } from './src/_createFooterHeader'
 
 const initialDir = process.cwd()
 
@@ -54,7 +54,7 @@ test('ReadableRawVHDStream does not crash', async () => {
     },
   }
   const fileSize = 1000
-  const stream = createReadableRawVHDStream(fileSize, mockParser)
+  const stream = createReadableRawStream(fileSize, mockParser)
   const pipe = stream.pipe(createWriteStream('output.vhd'))
   await fromEvent(pipe, 'finish')
   await execa('vhd-util', ['check', '-t', '-i', '-n', 'output.vhd'])
@@ -85,7 +85,7 @@ test('ReadableRawVHDStream detects when blocks are out of order', async () => {
   }
   return expect(
     new Promise((resolve, reject) => {
-      const stream = createReadableRawVHDStream(100000, mockParser)
+      const stream = createReadableRawStream(100000, mockParser)
       stream.on('error', reject)
       const pipe = stream.pipe(createWriteStream('outputStream'))
       pipe.on('finish', resolve)
@@ -107,12 +107,13 @@ test('ReadableSparseVHDStream can handle a sparse file', async () => {
     },
   ]
   const fileSize = blockSize * 110
-  const stream = createReadableSparseVHDStream(
+  const stream = await createReadableSparseStream(
     fileSize,
     blockSize,
     blocks.map(b => b.offsetBytes),
     blocks
   )
+  expect(stream.length).toEqual(4197888)
   const pipe = stream.pipe(createWriteStream('output.vhd'))
   await fromEvent(pipe, 'finish')
   await execa('vhd-util', ['check', '-t', '-i', '-n', 'output.vhd'])

packages/xen-api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xen-api",
-  "version": "0.19.0",
+  "version": "0.20.0",
   "license": "ISC",
   "description": "Connector to the Xen API",
   "keywords": [

packages/xen-api/src/index.js

@@ -11,7 +11,6 @@ import {
   forEach,
   isArray,
   isInteger,
-  isObject,
   map,
   noop,
   omit,
@@ -137,8 +136,8 @@ const parseUrl = url => {
 const {
   create: createObject,
   defineProperties,
-  defineProperty,
   freeze: freezeObject,
+  keys: getKeys,
 } = Object
 
 // -------------------------------------------------------------------
@@ -190,10 +189,6 @@ const getKey = o => o.$id
 
 // -------------------------------------------------------------------
 
-const EMPTY_ARRAY = freezeObject([])
-
-// -------------------------------------------------------------------
-
 const getTaskResult = task => {
   const { status } = task
   if (status === 'cancelled') {
@@ -215,6 +210,15 @@ const getTaskResult = task => {
 
 // -------------------------------------------------------------------
 
+const RESERVED_FIELDS = {
+  id: true,
+  pool: true,
+  ref: true,
+  type: true,
+}
+
+// -------------------------------------------------------------------
+
 const CONNECTED = 'connected'
 const CONNECTING = 'connecting'
 const DISCONNECTED = 'disconnected'
@@ -229,6 +233,7 @@ export class Xapi extends EventEmitter {
     this._auth = opts.auth
     this._pool = null
     this._readOnly = Boolean(opts.readOnly)
+    this._RecordsByType = createObject(null)
     this._sessionId = null
     const url = (this._url = parseUrl(opts.url))
 
@@ -259,8 +264,8 @@ export class Xapi extends EventEmitter {
       const objects = (this._objects = new Collection())
       objects.getKey = getKey
 
-      this._objectsByRefs = createObject(null)
-      this._objectsByRefs[NULL_REF] = undefined
+      this._objectsByRef = createObject(null)
+      this._objectsByRef[NULL_REF] = undefined
 
       this._taskWatchers = Object.create(null)
 
@@ -462,7 +467,7 @@ export class Xapi extends EventEmitter {
     }
 
     const object =
-      this._objects.all[idOrUuidOrRef] || this._objectsByRefs[idOrUuidOrRef]
+      this._objects.all[idOrUuidOrRef] || this._objectsByRef[idOrUuidOrRef]
 
     if (object !== undefined) return object
 
@@ -474,7 +479,7 @@ export class Xapi extends EventEmitter {
   // Returns the object for a given opaque reference (internal to
   // XAPI).
   getObjectByRef (ref, defaultValue) {
-    const object = this._objectsByRefs[ref]
+    const object = this._objectsByRef[ref]
 
     if (object !== undefined) return object
 
@@ -497,16 +502,9 @@ export class Xapi extends EventEmitter {
   }
 
   async getRecord (type, ref) {
-    const record = await this._sessionCall(`${type}.get_record`, [ref])
-
-    // All custom properties are read-only and non enumerable.
-    defineProperties(record, {
-      $id: { value: record.uuid || ref },
-      $ref: { value: ref },
-      $type: { value: type },
-    })
-
-    return record
+    return this._wrapRecord(
+      await this._sessionCall(`${type}.get_record`, [ref])
+    )
   }
 
   async getRecordByUuid (type, uuid) {
@@ -669,7 +667,7 @@ export class Xapi extends EventEmitter {
 
   setFieldEntries (record, field, entries) {
     return Promise.all(
-      Object.keys(entries).map(entry => {
+      getKeys(entries).map(entry => {
         const value = entries[entry]
         if (value !== undefined) {
           return value === null
@@ -710,7 +708,7 @@ export class Xapi extends EventEmitter {
     let watcher = watchers[ref]
     if (watcher === undefined) {
       // sync check if the task is already settled
-      const task = this._objectsByRefs[ref]
+      const task = this._objectsByRef[ref]
       if (task !== undefined) {
         const result = getTaskResult(task)
         if (result !== undefined) {
@@ -775,78 +773,29 @@ export class Xapi extends EventEmitter {
   }
 
   _addObject (type, ref, object) {
-    const { _objectsByRefs: objectsByRefs } = this
-
-    const reservedKeys = {
-      id: true,
-      pool: true,
-      ref: true,
-      type: true,
-    }
-    const getKey = (key, obj) =>
-      reservedKeys[key] && obj === object ? `$$${key}` : `$${key}`
-
-    // Creates resolved properties.
-    forEach(object, function resolveObject (value, key, object) {
-      if (isArray(value)) {
-        if (!value.length) {
-          // If the array is empty, it isn't possible to be sure that
-          // it is not supposed to contain links, therefore, in
-          // benefice of the doubt, a resolved property is defined.
-          defineProperty(object, getKey(key, object), {
-            value: EMPTY_ARRAY,
-          })
-
-          // Minor memory optimization, use the same empty array for
-          // everyone.
-          object[key] = EMPTY_ARRAY
-        } else if (isOpaqueRef(value[0])) {
-          // This is an array of refs.
-          defineProperty(object, getKey(key, object), {
-            get: () => freezeObject(map(value, ref => objectsByRefs[ref])),
-          })
-
-          freezeObject(value)
-        }
-      } else if (isObject(value)) {
-        forEach(value, resolveObject)
-
-        freezeObject(value)
-      } else if (isOpaqueRef(value)) {
-        defineProperty(object, getKey(key, object), {
-          get: () => objectsByRefs[value],
-        })
-      }
-    })
-
-    // All custom properties are read-only and non enumerable.
-    defineProperties(object, {
-      $id: { value: object.uuid || ref },
-      $pool: { get: this._getPool },
-      $ref: { value: ref },
-      $type: { value: type },
-    })
+    object = this._wrapRecord(type, ref, object)
 
     // Finally freezes the object.
     freezeObject(object)
 
     const objects = this._objects
+    const objectsByRef = this._objectsByRef
 
     // An object's UUID can change during its life.
-    const prev = objectsByRefs[ref]
+    const prev = objectsByRef[ref]
     let prevUuid
     if (prev && (prevUuid = prev.uuid) && prevUuid !== object.uuid) {
       objects.remove(prevUuid)
     }
 
     this._objects.set(object)
-    objectsByRefs[ref] = object
+    objectsByRef[ref] = object
 
     if (type === 'pool') {
       this._pool = object
 
       const eventWatchers = this._eventWatchers
-      Object.keys(object.other_config).forEach(key => {
+      getKeys(object.other_config).forEach(key => {
         const eventWatcher = eventWatchers[key]
         if (eventWatcher !== undefined) {
           delete eventWatchers[key]
@@ -871,7 +820,7 @@ export class Xapi extends EventEmitter {
   }
 
   _removeObject (type, ref) {
-    const byRefs = this._objectsByRefs
+    const byRefs = this._objectsByRef
     const object = byRefs[ref]
     if (object !== undefined) {
       this._objects.unset(object.$id)
@@ -1028,6 +977,90 @@ export class Xapi extends EventEmitter {
 
     return getAllObjects().then(watchEvents)
   }
+
+  _wrapRecord (type, ref, data) {
+    const RecordsByType = this._RecordsByType
+    let Record = RecordsByType[type]
+    if (Record === undefined) {
+      const fields = getKeys(data)
+      const nFields = fields.length
+      const xapi = this
+
+      const objectsByRef = this._objectsByRef
+      const getObjectByRef = ref => objectsByRef[ref]
+
+      Record = function (ref, data) {
+        defineProperties(this, {
+          $id: { value: data.uuid || ref },
+          $ref: { value: ref },
+        })
+        for (let i = 0; i < nFields; ++i) {
+          const field = fields[i]
+          this[field] = data[field]
+        }
+      }
+
+      const getters = { $pool: this._getPool }
+      const props = { $type: type }
+      fields.forEach(field => {
+        props[`set_${field}`] = function (value) {
+          return xapi.setField(this, field, value)
+        }
+
+        const $field = (field in RESERVED_FIELDS ? '$$' : '$') + field
+
+        const value = data[field]
+        if (isArray(value)) {
+          if (value.length === 0 || isOpaqueRef(value[0])) {
+            getters[$field] = function () {
+              const value = this[field]
+              return value.length === 0 ? value : value.map(getObjectByRef)
+            }
+          }
+
+          props[`add_to_${field}`] = function (...values) {
+            return xapi
+              .call(`${type}.add_${field}`, this.$ref, values)
+              .then(noop)
+          }
+        } else if (value !== null && typeof value === 'object') {
+          getters[$field] = function () {
+            const value = this[field]
+            const result = {}
+            getKeys(value).forEach(key => {
+              result[key] = objectsByRef[value[key]]
+            })
+            return result
+          }
+          props[`update_${field}`] = function (entries) {
+            return xapi.setFieldEntries(this, field, entries)
+          }
+        } else if (isOpaqueRef(value)) {
+          getters[$field] = function () {
+            return objectsByRef[this[field]]
+          }
+        }
+      })
+      const descriptors = {}
+      getKeys(getters).forEach(key => {
+        descriptors[key] = {
+          configurable: true,
+          get: getters[key],
+        }
+      })
+      getKeys(props).forEach(key => {
+        descriptors[key] = {
+          configurable: true,
+          value: props[key],
+          writable: true,
+        }
+      })
+      defineProperties(Record.prototype, descriptors)
+
+      RecordsByType[type] = Record
+    }
+    return new Record(ref, data)
+  }
 }
 
 Xapi.prototype._transportCall = reduce(

packages/xo-acl-resolver/index.js

@@ -86,7 +86,7 @@ const checkAuthorizationByTypes = {
 
   'VM-controller': checkMember('$container'),
 
-  'VM-snapshot': checkMember('$snapshot_of'),
+  'VM-snapshot': or(checkSelf, checkMember('$snapshot_of')),
 
   'VM-template': or(checkSelf, checkMember('$pool')),
 }

packages/xo-acl-resolver/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-acl-resolver",
-  "version": "0.2.4",
+  "version": "0.3.0",
   "license": "ISC",
   "description": "Xen-Orchestra internal: do ACLs resolution",
   "keywords": [],

packages/xo-cli/package.json

@@ -31,10 +31,9 @@
     "@babel/polyfill": "^7.0.0",
     "bluebird": "^3.5.1",
     "chalk": "^2.2.0",
-    "event-to-promise": "^0.8.0",
     "exec-promise": "^0.7.0",
     "fs-promise": "^2.0.3",
-    "got": "^8.0.1",
+    "http-request-plus": "^0.6.0",
     "human-format": "^0.10.0",
     "l33teral": "^3.0.3",
     "lodash": "^4.17.4",
@@ -43,6 +42,8 @@
     "nice-pipe": "0.0.0",
     "pretty-ms": "^4.0.0",
     "progress-stream": "^2.0.0",
+    "promise-toolbox": "^0.10.1",
+    "pump": "^3.0.0",
     "pw": "^0.0.4",
     "strip-indent": "^2.0.0",
     "xdg-basedir": "^3.0.0",

packages/xo-cli/src/index.js

@@ -11,10 +11,10 @@ const resolveUrl = require('url').resolve
 const stat = require('fs-promise').stat
 
 const chalk = require('chalk')
-const eventToPromise = require('event-to-promise')
 const forEach = require('lodash/forEach')
+const fromCallback = require('promise-toolbox/fromCallback')
 const getKeys = require('lodash/keys')
-const got = require('got')
+const hrp = require('http-request-plus').default
 const humanFormat = require('human-format')
 const identity = require('lodash/identity')
 const isArray = require('lodash/isArray')
@@ -23,6 +23,7 @@ const micromatch = require('micromatch')
 const nicePipe = require('nice-pipe')
 const pairs = require('lodash/toPairs')
 const pick = require('lodash/pick')
+const pump = require('pump')
 const startsWith = require('lodash/startsWith')
 const prettyMs = require('pretty-ms')
 const progressStream = require('progress-stream')
@@ -362,50 +363,43 @@ async function call (args) {
       ensurePathParam(method, file)
       url = resolveUrl(baseUrl, result[key])
       const output = createWriteStream(file)
+      const response = await hrp(url)
 
-      const progress = progressStream({ time: 1e3 }, printProgress)
-
-      return eventToPromise(
-        nicePipe([
-          got.stream(url).on('response', function (response) {
-            const length = response.headers['content-length']
-            if (length !== undefined) {
-              progress.length(length)
-            }
-          }),
-          progress,
-          output,
-        ]),
-        'finish'
+      const progress = progressStream(
+        {
+          length: response.headers['content-length'],
+          time: 1e3,
+        },
+        printProgress
       )
+
+      return fromCallback(cb => pump(response, progress, output, cb))
     }
 
     if (key === '$sendTo') {
       ensurePathParam(method, file)
       url = resolveUrl(baseUrl, result[key])
 
-      const stats = await stat(file)
-      const length = stats.size
-
+      const { size: length } = await stat(file)
       const input = nicePipe([
         createReadStream(file),
         progressStream(
           {
-            length: length,
+            length,
             time: 1e3,
           },
           printProgress
         ),
       ])
 
-      const response = await got.post(url, {
-        body: input,
-        headers: {
-          'content-length': length,
-        },
-        method: 'POST',
-      })
-      return response.body
+      return hrp
+        .post(url, {
+          body: input,
+          headers: {
+            'content-length': length,
+          },
+        })
+        .readAll('utf-8')
     }
   }
 

packages/xo-common/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-common",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "license": "AGPL-3.0",
   "description": "Code shared between [XO](https://xen-orchestra.com) server and clients",
   "keywords": [],

packages/xo-common/src/api-errors.js

@@ -20,7 +20,8 @@ class XoError extends BaseError {
 const create = (code, getProps) => {
   const factory = (...args) => new XoError({ ...getProps(...args), code })
   factory.is = (error, predicate) =>
-    error.code === code && iteratee(predicate)(error)
+    error.code === code &&
+    (predicate === undefined || iteratee(predicate)(error))
 
   return factory
 }
@@ -33,7 +34,7 @@ export const notImplemented = create(0, () => ({
 
 export const noSuchObject = create(1, (id, type) => ({
   data: { id, type },
-  message: 'no such object',
+  message: `no such ${type || 'object'} ${id}`,
 }))
 
 export const unauthorized = create(2, () => ({

packages/xo-server-backup-reports/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-server-backup-reports",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "license": "AGPL-3.0",
   "description": "Backup reports plugin for XO-Server",
   "keywords": [

packages/xo-server-backup-reports/src/index.js

@@ -50,6 +50,7 @@ const ICON_FAILURE = '🚨'
 const ICON_INTERRUPTED = '⚠️'
 const ICON_SKIPPED = '⏩'
 const ICON_SUCCESS = '✔'
+const ICON_WARNING = '⚠️'
 
 const STATUS_ICON = {
   failure: ICON_FAILURE,
@@ -99,12 +100,13 @@ const isSkippedError = error =>
   error.message === UNHEALTHY_VDI_CHAIN_ERROR ||
   error.message === NO_SUCH_OBJECT_ERROR
 
+const INDENT = '  '
 const createGetTemporalDataMarkdown = formatDate => (
   start,
   end,
   nbIndent = 0
 ) => {
-  const indent = '  '.repeat(nbIndent)
+  const indent = INDENT.repeat(nbIndent)
 
   const markdown = [`${indent}- **Start time**: ${formatDate(start)}`]
   if (end !== undefined) {
@@ -117,6 +119,17 @@ const createGetTemporalDataMarkdown = formatDate => (
   return markdown
 }
 
+const addWarnings = (text, warnings, nbIndent = 0) => {
+  if (warnings === undefined) {
+    return
+  }
+
+  const indent = INDENT.repeat(nbIndent)
+  warnings.forEach(({ message }) => {
+    text.push(`${indent}- **${ICON_WARNING} ${message}**`)
+  })
+}
+
 class BackupReportsXoPlugin {
   constructor (xo) {
     this._xo = xo
@@ -180,13 +193,14 @@ class BackupReportsXoPlugin {
       let markdown = [
         `##  Global status: ${log.status}`,
         '',
+        `- **Job ID**: ${log.jobId}`,
+        `- **Run ID**: ${runJobId}`,
         `- **mode**: ${mode}`,
         ...getTemporalDataMarkdown(log.start, log.end),
         `- **Error**: ${log.result.message}`,
-        '---',
-        '',
-        `*${pkg.name} v${pkg.version}*`,
       ]
+      addWarnings(markdown, log.warnings)
+      markdown.push('---', '', `*${pkg.name} v${pkg.version}*`)
 
       markdown = markdown.join('\n')
       return this._sendReport({
@@ -228,6 +242,7 @@ class BackupReportsXoPlugin {
         `- **UUID**: ${vm !== undefined ? vm.uuid : vmId}`,
         ...getTemporalDataMarkdown(taskLog.start, taskLog.end),
       ]
+      addWarnings(text, taskLog.warnings)
 
       const failedSubTasks = []
       const snapshotText = []
@@ -262,6 +277,7 @@ class BackupReportsXoPlugin {
             }** (${id}) ${icon}`,
             ...getTemporalDataMarkdown(subTaskLog.start, subTaskLog.end, 2)
           )
+          addWarnings(remotesText, subTaskLog.warnings, 2)
           if (subTaskLog.status === 'failure') {
             failedSubTasks.push(remote !== undefined ? remote.name : id)
             remotesText.push('', errorMessage)
@@ -278,6 +294,7 @@ class BackupReportsXoPlugin {
             `  - **${srName}** (${srUuid}) ${icon}`,
             ...getTemporalDataMarkdown(subTaskLog.start, subTaskLog.end, 2)
           )
+          addWarnings(srsText, subTaskLog.warnings, 2)
           if (subTaskLog.status === 'failure') {
             failedSubTasks.push(sr !== undefined ? sr.name_label : id)
             srsText.push('', errorMessage)
@@ -293,6 +310,7 @@ class BackupReportsXoPlugin {
           }
 
           const operationInfoText = []
+          addWarnings(operationInfoText, operationLog.warnings, 3)
           if (operationLog.status === 'success') {
             const size = operationLog.result.size
             if (operationLog.message === 'merge') {
@@ -395,6 +413,8 @@ class BackupReportsXoPlugin {
     let markdown = [
       `##  Global status: ${log.status}`,
       '',
+      `- **Job ID**: ${log.jobId}`,
+      `- **Run ID**: ${runJobId}`,
       `- **mode**: ${mode}`,
       ...getTemporalDataMarkdown(log.start, log.end),
       `- **Successes**: ${nSuccesses} / ${nVms}`,
@@ -406,6 +426,7 @@ class BackupReportsXoPlugin {
     if (globalMergeSize !== 0) {
       markdown.push(`- **Merge size**: ${formatSize(globalMergeSize)}`)
     }
+    addWarnings(markdown, log.warnings)
     markdown.push('')
 
     if (nFailures !== 0) {

packages/xo-server-cloud/package.json

@@ -31,9 +31,8 @@
     "node": ">=6"
   },
   "dependencies": {
-    "event-to-promise": "^0.8.0",
-    "jsonrpc-websocket-client": "^0.4.1",
-    "superagent": "^3.8.2"
+    "http-request-plus": "^0.6.0",
+    "jsonrpc-websocket-client": "^0.4.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/xo-server-cloud/src/index.js

@@ -1,7 +1,5 @@
 import Client, { createBackoff } from 'jsonrpc-websocket-client'
-import eventToPromise from 'event-to-promise'
-import request from 'superagent'
-import { PassThrough } from 'stream'
+import hrp from 'http-request-plus'
 
 const UPDATER_URL = 'localhost'
 const WS_PORT = 9001
@@ -145,17 +143,16 @@ class XoServerCloud {
       throw new Error('cannot get download token')
     }
 
-    const req = request
-      .get(`${UPDATER_URL}:${HTTP_PORT}/`)
-      .set('Authorization', `Bearer ${downloadToken}`)
+    const response = await hrp(`${UPDATER_URL}:${HTTP_PORT}/`, {
+      headers: {
+        Authorization: `Bearer ${downloadToken}`,
+      },
+    })
 
-    // Impossible to pipe the response directly: https://github.com/visionmedia/superagent/issues/1187
-    const pt = new PassThrough()
-    req.pipe(pt)
-    const { headers } = await eventToPromise(req, 'response')
-    pt.length = headers['content-length']
+    // currently needed for XenApi#putResource()
+    response.length = response.headers['content-length']
 
-    return pt
+    return response
   }
 }
 

packages/xo-server-usage-report/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-server-usage-report",
-  "version": "0.5.0",
+  "version": "0.7.0",
   "license": "AGPL-3.0",
   "description": "",
   "keywords": [
@@ -34,9 +34,11 @@
     "node": ">=6"
   },
   "dependencies": {
+    "@xen-orchestra/async-map": "^0.0.0",
     "@xen-orchestra/cron": "^1.0.3",
     "handlebars": "^4.0.6",
     "html-minifier": "^3.5.8",
+    "human-format": "^0.10.0",
     "lodash": "^4.17.4",
     "promise-toolbox": "^0.10.1"
   },

packages/xo-server-usage-report/report.html.tpl

@@ -83,10 +83,6 @@
       border-top: 1px solid #95a5a6;
     }
 
-    .page .global tr:nth-last-child(2) td {
-      border-bottom: 1px solid #95a5a6;
-    }
-
     .top table{
       margin: auto;
       margin-top: 20px;
@@ -149,9 +145,9 @@
 
     <div class="page">
 
-      <table class ="global">
+      <table class="global">
         <tr>
-          <td id="title" rowspan="13">VMs</td>
+          <td id="title" rowspan="8">VMs</td>
         </tr>
         <tr>
           <td>Number:</td>
@@ -160,37 +156,37 @@
         <tr>
           <td>CPU:</td>
           <td>{{normaliseValue global.vms.cpu}} % {{normaliseEvolution global.vmsEvolution.cpu}}</td>
-        <tr>
+        </tr>
         <tr>
           <td>RAM:</td>
           <td>{{normaliseValue global.vms.ram}} GiB {{normaliseEvolution global.vmsEvolution.ram}}</td>
-        <tr>
+        </tr>
         <tr>
           <td>Disk read:</td>
           <td>{{normaliseValue global.vms.diskRead}} MiB {{normaliseEvolution global.vmsEvolution.diskRead}}
           </td>
-        <tr>
+        </tr>
         <tr>
           <td>Disk write:</td>
           <td>{{normaliseValue global.vms.diskWrite}} MiB {{normaliseEvolution global.vmsEvolution.diskWrite}}
           </td>
-        <tr>
+        </tr>
         <tr>
           <td>Network RX:</td>
           <td>{{normaliseValue global.vms.netReception}} KiB {{normaliseEvolution global.vmsEvolution.netReception}}
           </td>
-        <tr>
+        </tr>
         <tr>
           <td>Network TX:</td>
           <td>{{normaliseValue global.vms.netTransmission}} KiB {{normaliseEvolution global.vmsEvolution.netTransmission}}
           </td>
-        <tr>
+        </tr>
       </table>
 
       <div class="top">
 
         <table>
-          <caption>3rd top usages</caption>
+          <caption>Top VMs</caption>
           <tr>
             <th></th>
             <th>UUID</th>
@@ -239,6 +235,9 @@
             <td>{{normaliseValue this.value}} MiB</td>
           </tr>
           {{/each}}
+
+          {{getTopIops topVms}}
+
           <tr>
             <td rowspan='{{math topVms.netReception.length "+" 1}}' class="tableHeader">Network RX</td>
           </tr>
@@ -264,9 +263,9 @@
     </div>
 
     <div class="page">
-      <table class ="global">
+      <table class="global">
         <tr>
-          <td id="title" rowspan="13">Hosts</td>
+          <td id="title" rowspan="7">Hosts</td>
         </tr>
         <tr>
           <td>Number:</td>
@@ -277,34 +276,33 @@
           <td>CPU:</td>
           <td>{{normaliseValue global.hosts.cpu}} % {{normaliseEvolution global.hostsEvolution.cpu}}
           </td>
-        <tr>
+        </tr>
         <tr>
           <td>RAM:</td>
           <td>{{normaliseValue global.hosts.ram}} GiB {{normaliseEvolution global.hostsEvolution.ram}}
           </td>
-          </td>
-        <tr>
+        </tr>
         <tr>
           <td>Load average:</td>
           <td>{{normaliseValue global.hosts.load}}  {{normaliseEvolution global.hostsEvolution.load}}
           </td>
-        <tr>
+        </tr>
         <tr>
           <td>Network RX:</td>
           <td>{{normaliseValue global.hosts.netReception}} KiB {{normaliseEvolution global.hostsEvolution.netReception}}
           </td>
-        <tr>
+        </tr>
         <tr>
           <td>Network TX:</td>
           <td>{{normaliseValue global.hosts.netTransmission}} KiB {{normaliseEvolution global.hostsEvolution.netTransmission}}
           </td>
-        <tr>
+        </tr>
       </table>
 
       <div class="top">
 
         <table>
-          <caption>3rd top usages</caption>
+          <caption>Top hosts</caption>
           <tr>
             <th></th>
             <th>UUID</th>
@@ -368,19 +366,14 @@
     <div class="page">
       <div class="top">
         <table>
-          <caption>Most used storages </caption>
+          <caption>Top SRs</caption>
             <tr>
+              <th />
               <th>UUID</th>
               <th>Name</th>
               <th>value</th>
             </tr>
-            {{#each topSrs}}
-              <tr>
-                <td>{{shortUUID this.uuid}}</td>
-                <td>{{this.name}}</td>
-                <td>{{normaliseValue this.value}} GiB</td>
-              </tr>
-             {{/each}}
+            {{getTopSrs topSrs}}
           </table>
           <table>
              <caption>Hosts missing patches</caption>
@@ -531,6 +524,9 @@
               <th>RAM (GiB)</th>
               <th>Disk read (MiB)</th>
               <th>Disk write (MiB)</th>
+              <th>IOPS read</th>
+              <th>IOPS write</th>
+              <th>IOPS total</th>
               <th>Network RX (KiB)</th>
               <th>Network TX (KiB)</th>
             </tr>
@@ -542,6 +538,9 @@
                 <td>{{normaliseValue this.ram}} {{normaliseEvolution this.evolution.ram}}</td>
                 <td>{{normaliseValue this.diskRead}} {{normaliseEvolution this.evolution.diskRead}}</td>
                 <td>{{normaliseValue this.diskWrite}} {{normaliseEvolution this.evolution.diskWrite}}</td>
+                <td>{{formatIops this.iopsRead}} {{normaliseEvolution this.evolution.iopsRead}}</td>
+                <td>{{formatIops this.iopsWrite}} {{normaliseEvolution this.evolution.iopsWrite}}</td>
+                <td>{{formatIops this.iopsTotal}} {{normaliseEvolution this.evolution.iopsTotal}}</td>
                 <td>{{normaliseValue this.netReception}} {{normaliseEvolution this.evolution.netReception}}</td>
                 <td>{{normaliseValue this.netTransmission}} {{normaliseEvolution this.evolution.netTransmission}}</td>
               </tr>
@@ -584,8 +583,8 @@
                 <td>{{shortUUID this.uuid}}</td>
                 <td>{{this.name}}</td>
                 <td>{{normaliseValue this.total}} {{normaliseEvolution this.evolution.total}}</td>
-                <td>{{normaliseValue this.used}}</td>
-                <td>{{normaliseValue this.free}}</td>
+                <td>{{normaliseValue this.usedSpace}}</td>
+                <td>{{normaliseValue this.freeSpace}}</td>
               </tr>
             {{/each}}
           </table>

packages/xo-server-usage-report/src/index.js

@@ -1,4 +1,6 @@
+import asyncMap from '@xen-orchestra/async-map'
 import Handlebars from 'handlebars'
+import humanFormat from 'human-format'
 import { createSchedule } from '@xen-orchestra/cron'
 import { minify } from 'html-minifier'
 import {
@@ -21,6 +23,8 @@ import { readFile, writeFile } from 'fs'
 
 // ===================================================================
 
+const GRANULARITY = 'days'
+
 const pReadFile = promisify(readFile)
 const pWriteFile = promisify(writeFile)
 
@@ -75,7 +79,7 @@ export const configurationSchema = {
     },
     periodicity: {
       type: 'string',
-      enum: ['monthly', 'weekly'],
+      enum: ['monthly', 'weekly', 'daily'],
       description:
         'If you choose weekly you will receive the report every sunday and if you choose monthly you will receive it every first day of the month.',
     },
@@ -87,6 +91,24 @@ export const configurationSchema = {
 
 // ===================================================================
 
+const shortUuid = uuid => {
+  if (typeof uuid === 'string') {
+    return uuid.split('-')[0]
+  }
+}
+
+const formatIops = value =>
+  isFinite(value)
+    ? humanFormat(value, {
+        unit: 'IOPS',
+        decimals: 2,
+      })
+    : '-'
+
+const normaliseValue = value => (isFinite(value) ? round(value, 2) : '-')
+
+// ===================================================================
+
 Handlebars.registerHelper('compare', function (
   lvalue,
   operator,
@@ -122,29 +144,62 @@ Handlebars.registerHelper('math', function (lvalue, operator, rvalue, options) {
   return mathOperators[operator](+lvalue, +rvalue)
 })
 
-Handlebars.registerHelper('shortUUID', uuid => {
-  if (typeof uuid === 'string') {
-    return uuid.split('-')[0]
-  }
-})
+Handlebars.registerHelper('shortUUID', shortUuid)
 
-Handlebars.registerHelper(
-  'normaliseValue',
-  value => (isFinite(value) ? round(value, 2) : '-')
-)
+Handlebars.registerHelper('normaliseValue', normaliseValue)
 
 Handlebars.registerHelper(
   'normaliseEvolution',
   value =>
     new Handlebars.SafeString(
-      isFinite(+value) && +value !== 0
-        ? (value = round(value, 2)) > 0
+      isFinite((value = round(value, 2))) && value !== 0
+        ? value > 0
           ? `(<b style="color: green;">▲ ${value}%</b>)`
           : `(<b style="color: red;">▼ ${String(value).slice(1)}%</b>)`
         : ''
     )
 )
 
+Handlebars.registerHelper('formatIops', formatIops)
+
+const getHeader = (label, size) => `
+  <tr>
+    <td rowspan='${size + 1}' class="tableHeader">${label}</td>
+  </tr>
+`
+
+const getBody = ({ uuid, name, value }, transformValue, unit) => `
+  <tr>
+    <td>${shortUuid(uuid)}</td>
+    <td>${name}</td>
+    <td>${transformValue(value)}${unit !== undefined ? ` ${unit}` : ''}</td>
+  </tr>
+`
+
+const getTopIops = ({ iopsRead, iopsWrite, iopsTotal }) => `
+  ${getHeader('IOPS read', iopsRead.length)}
+  ${iopsRead.map(obj => getBody(obj, formatIops)).join('')}
+  ${getHeader('IOPS write', iopsWrite.length)}
+  ${iopsWrite.map(obj => getBody(obj, formatIops)).join('')}
+  ${getHeader('IOPS total', iopsTotal.length)}
+  ${iopsTotal.map(obj => getBody(obj, formatIops)).join('')}
+`
+
+Handlebars.registerHelper(
+  'getTopSrs',
+  ({ usedSpace, iopsRead, iopsWrite, iopsTotal }) =>
+    new Handlebars.SafeString(`
+      ${getHeader('Used space', usedSpace.length)}
+      ${usedSpace.map(obj => getBody(obj, normaliseValue, 'GiB')).join('')}
+      ${getTopIops({ iopsRead, iopsWrite, iopsTotal })}
+    `)
+)
+
+Handlebars.registerHelper(
+  'getTopIops',
+  props => new Handlebars.SafeString(getTopIops(props))
+)
+
 // ===================================================================
 
 function computeMean (values) {
@@ -217,26 +272,36 @@ function getMemoryUsedMetric ({ memory, memoryFree = memory }) {
   return map(memory, (value, key) => value - memoryFree[key])
 }
 
+const METRICS_MEAN = {
+  cpu: computeDoubleMean,
+  disk: value => computeDoubleMean(values(value)) / mibPower,
+  iops: value => computeDoubleMean(values(value)),
+  load: computeMean,
+  net: value => computeDoubleMean(value) / kibPower,
+  ram: stats => computeMean(getMemoryUsedMetric(stats)) / gibPower,
+}
+
 // ===================================================================
 
 async function getVmsStats ({ runningVms, xo }) {
   return orderBy(
     await Promise.all(
       map(runningVms, async vm => {
-        const vmStats = await xo.getXapiVmStats(vm, 'days')
+        const { stats } = await xo.getXapiVmStats(vm, GRANULARITY)
+        const iopsRead = METRICS_MEAN.iops(get(stats.iops, 'r'))
+        const iopsWrite = METRICS_MEAN.iops(get(stats.iops, 'w'))
         return {
           uuid: vm.uuid,
           name: vm.name_label,
-          cpu: computeDoubleMean(vmStats.stats.cpus),
-          ram: computeMean(getMemoryUsedMetric(vmStats.stats)) / gibPower,
-          diskRead:
-            computeDoubleMean(values(get(vmStats.stats.xvds, 'r'))) / mibPower,
-          diskWrite:
-            computeDoubleMean(values(get(vmStats.stats.xvds, 'w'))) / mibPower,
-          netReception:
-            computeDoubleMean(get(vmStats.stats.vifs, 'rx')) / kibPower,
-          netTransmission:
-            computeDoubleMean(get(vmStats.stats.vifs, 'tx')) / kibPower,
+          cpu: METRICS_MEAN.cpu(stats.cpus),
+          ram: METRICS_MEAN.ram(stats),
+          diskRead: METRICS_MEAN.disk(get(stats.xvds, 'r')),
+          diskWrite: METRICS_MEAN.disk(get(stats.xvds, 'w')),
+          iopsRead,
+          iopsWrite,
+          iopsTotal: iopsRead + iopsWrite,
+          netReception: METRICS_MEAN.net(get(stats.vifs, 'rx')),
+          netTransmission: METRICS_MEAN.net(get(stats.vifs, 'tx')),
         }
       })
     ),
@@ -249,17 +314,15 @@ async function getHostsStats ({ runningHosts, xo }) {
   return orderBy(
     await Promise.all(
       map(runningHosts, async host => {
-        const hostStats = await xo.getXapiHostStats(host, 'days')
+        const { stats } = await xo.getXapiHostStats(host, GRANULARITY)
         return {
           uuid: host.uuid,
           name: host.name_label,
-          cpu: computeDoubleMean(hostStats.stats.cpus),
-          ram: computeMean(getMemoryUsedMetric(hostStats.stats)) / gibPower,
-          load: computeMean(hostStats.stats.load),
-          netReception:
-            computeDoubleMean(get(hostStats.stats.pifs, 'rx')) / kibPower,
-          netTransmission:
-            computeDoubleMean(get(hostStats.stats.pifs, 'tx')) / kibPower,
+          cpu: METRICS_MEAN.cpu(stats.cpus),
+          ram: METRICS_MEAN.ram(stats),
+          load: METRICS_MEAN.load(stats.load),
+          netReception: METRICS_MEAN.net(get(stats.pifs, 'rx')),
+          netTransmission: METRICS_MEAN.net(get(stats.pifs, 'tx')),
         }
       })
     ),
@@ -268,24 +331,43 @@ async function getHostsStats ({ runningHosts, xo }) {
   )
 }
 
-function getSrsStats (xoObjects) {
+async function getSrsStats ({ xo, xoObjects }) {
   return orderBy(
-    map(filter(xoObjects, obj => obj.type === 'SR' && obj.size > 0), sr => {
-      const total = sr.size / gibPower
-      const used = sr.physical_usage / gibPower
-      let name = sr.name_label
-      if (!sr.shared) {
-        name += ` (${find(xoObjects, { id: sr.$container }).name_label})`
+    await asyncMap(
+      filter(
+        xoObjects,
+        obj => obj.type === 'SR' && obj.size > 0 && obj.$PBDs.length > 0
+      ),
+      async sr => {
+        const totalSpace = sr.size / gibPower
+        const usedSpace = sr.physical_usage / gibPower
+        let name = sr.name_label
+        // [Bug in XO] a SR with not container can be found (SR attached to a PBD with no host attached)
+        let container
+        if (
+          !sr.shared &&
+          (container = find(xoObjects, { id: sr.$container })) !== undefined
+        ) {
+          name += ` (${container.name_label})`
+        }
+
+        const { stats } = await xo.getXapiSrStats(sr.id, GRANULARITY)
+        const iopsRead = computeMean(get(stats.iops, 'r'))
+        const iopsWrite = computeMean(get(stats.iops, 'w'))
+
+        return {
+          uuid: sr.uuid,
+          name,
+          total: totalSpace,
+          usedSpace,
+          freeSpace: totalSpace - usedSpace,
+          iopsRead,
+          iopsWrite,
+          iopsTotal: iopsRead + iopsWrite,
+        }
       }
-      return {
-        uuid: sr.uuid,
-        name,
-        total,
-        used,
-        free: total - used,
-      }
-    }),
-    'total',
+    ),
+    'name',
     'desc'
   )
 }
@@ -351,6 +433,9 @@ function getTopVms ({ vmsStats, xo }) {
     'ram',
     'diskRead',
     'diskWrite',
+    'iopsRead',
+    'iopsWrite',
+    'iopsTotal',
     'netReception',
     'netTransmission',
   ])
@@ -366,8 +451,8 @@ function getTopHosts ({ hostsStats, xo }) {
   ])
 }
 
-function getTopSrs ({ srsStats, xo }) {
-  return getTop(srsStats, ['total']).total
+function getTopSrs (srsStats) {
+  return getTop(srsStats, ['usedSpace', 'iopsRead', 'iopsWrite', 'iopsTotal'])
 }
 
 async function getHostsMissingPatches ({ runningHosts, xo }) {
@@ -376,6 +461,13 @@ async function getHostsMissingPatches ({ runningHosts, xo }) {
       let hostsPatches = await xo
         .getXapi(host)
         .listMissingPoolPatchesOnHost(host._xapiId)
+        .catch(error => {
+          console.error(
+            '[WARN] error on fetching hosts missing patches:',
+            JSON.stringify(error)
+          )
+          return []
+        })
 
       if (host.license_params.sku_type === 'free') {
         hostsPatches = filter(hostsPatches, { paid: false })
@@ -417,6 +509,9 @@ async function computeEvolution ({ storedStatsPath, ...newStats }) {
         'ram',
         'diskRead',
         'diskWrite',
+        'iopsRead',
+        'iopsWrite',
+        'iopsTotal',
         'netReception',
         'netTransmission',
       ],
@@ -506,7 +601,7 @@ async function dataBuilder ({ xo, storedStatsPath, all }) {
     xo.getAllUsers(),
     getVmsStats({ xo, runningVms }),
     getHostsStats({ xo, runningHosts }),
-    getSrsStats(xoObjects),
+    getSrsStats({ xo, xoObjects }),
     getHostsMissingPatches({ xo, runningHosts }),
   ])
 
@@ -522,7 +617,7 @@ async function dataBuilder ({ xo, storedStatsPath, all }) {
     computeGlobalHostsStats({ xo, hostsStats, haltedHosts }),
     getTopVms({ xo, vmsStats }),
     getTopHosts({ xo, hostsStats }),
-    getTopSrs({ xo, srsStats }),
+    getTopSrs(srsStats),
     getAllUsersEmail(users),
   ])
 
@@ -571,6 +666,12 @@ async function dataBuilder ({ xo, storedStatsPath, all }) {
 
 // ===================================================================
 
+const CRON_BY_PERIODICITY = {
+  monthly: '0 6 1 * *',
+  weekly: '0 6 * * 0',
+  daily: '0 6 * * *',
+}
+
 class UsageReportPlugin {
   constructor ({ xo, getDataDir }) {
     this._xo = xo
@@ -591,7 +692,7 @@ class UsageReportPlugin {
     }
 
     this._job = createSchedule(
-      configuration.periodicity === 'monthly' ? '00 06 1 * *' : '00 06 * * 0'
+      CRON_BY_PERIODICITY[configuration.periodicity]
     ).createJob(async () => {
       try {
         await this._sendReport(true)

packages/xo-server/bin/xo-server

@@ -11,21 +11,8 @@ require('../better-stacks')
 // less memory usage.
 global.Promise = require('bluebird')
 
-// Make unhandled rejected promises visible.
-process.on('unhandledRejection', function (reason) {
-  console.warn('[Warn] Possibly unhandled rejection:', reason && reason.stack || reason)
-})
-
-;(function (EE) {
-  var proto = EE.prototype
-  var emit = proto.emit
-  proto.emit = function patchedError (event, error) {
-    if (event === 'error' && !this.listenerCount(event)) {
-      return console.warn('[Warn] Unhandled error event:', error && error.stack || error)
-    }
-
-    return emit.apply(this, arguments)
-  }
-})(require('events').EventEmitter)
+require('@xen-orchestra/log/configure').catchGlobalErrors(
+  require('@xen-orchestra/log').default('xo:xo-server')
+)
 
 require('exec-promise')(require('../'))

packages/xo-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-server",
-  "version": "5.27.1",
+  "version": "5.29.2",
   "license": "AGPL-3.0",
   "description": "Server part of Xen-Orchestra",
   "keywords": [
@@ -34,7 +34,8 @@
     "@xen-orchestra/async-map": "^0.0.0",
     "@xen-orchestra/cron": "^1.0.3",
     "@xen-orchestra/emit-async": "^0.0.0",
-    "@xen-orchestra/fs": "^0.3.1",
+    "@xen-orchestra/fs": "^0.4.0",
+    "@xen-orchestra/log": "^0.1.4",
     "@xen-orchestra/mixin": "^0.0.0",
     "ajv": "^6.1.1",
     "app-conf": "^0.5.0",
@@ -112,15 +113,15 @@
     "tmp": "^0.0.33",
     "uuid": "^3.0.1",
     "value-matcher": "^0.2.0",
-    "vhd-lib": "^0.3.0",
+    "vhd-lib": "^0.4.0",
     "ws": "^6.0.0",
-    "xen-api": "^0.19.0",
+    "xen-api": "^0.20.0",
     "xml2js": "^0.4.19",
-    "xo-acl-resolver": "^0.2.4",
+    "xo-acl-resolver": "^0.3.0",
     "xo-collection": "^0.4.1",
-    "xo-common": "^0.1.1",
+    "xo-common": "^0.1.2",
     "xo-remote-parser": "^0.5.0",
-    "xo-vmdk-to-vhd": "^0.1.3",
+    "xo-vmdk-to-vhd": "^0.1.5",
     "yazl": "^2.4.3"
   },
   "devDependencies": {
@@ -130,6 +131,7 @@
     "@babel/plugin-proposal-export-default-from": "^7.0.0",
     "@babel/plugin-proposal-export-namespace-from": "^7.0.0",
     "@babel/plugin-proposal-function-bind": "^7.0.0",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.0.0",
     "@babel/plugin-proposal-optional-chaining": "^7.0.0",
     "@babel/plugin-proposal-pipeline-operator": "^7.0.0",
     "@babel/plugin-proposal-throw-expressions": "^7.0.0",

packages/xo-server/sample.config.yaml

@@ -46,14 +46,12 @@
 
 # Configuration of the embedded HTTP server.
 http:
-
   # Hosts & ports on which to listen.
   #
   # By default, the server listens on [::]:80.
   listen:
     # Basic HTTP.
-    -
-      # Address on which the server is listening on.
+    - # Address on which the server is listening on.
       #
       # Sets it to 'localhost' for IP to listen only on the local host.
       #
@@ -124,23 +122,20 @@ http:
 
 # Connection to the Redis server.
 redis:
-    # Unix sockets can be used
-    #
-    # Default: undefined
-    #socket: /var/run/redis/redis.sock
-
-    # Syntax: redis://[db[:password]@]hostname[:port][/db-number]
-    #
-    # Default: redis://localhost:6379/0
-    #uri: redis://redis.company.lan/42
-
-    # List of aliased commands.
-    #
-    # See http://redis.io/topics/security#disabling-of-specific-commands
-    #renameCommands:
-    #  del: '3dda29ad-3015-44f9-b13b-fa570de92489'
-    #  srem: '3fd758c9-5610-4e9d-a058-dbf4cb6d8bf0'
-
+  # Unix sockets can be used
+  #
+  # Default: undefined
+  #socket: /var/run/redis/redis.sock
+  # Syntax: redis://[db[:password]@]hostname[:port][/db-number]
+  #
+  # Default: redis://localhost:6379/0
+  #uri: redis://redis.company.lan/42
+  # List of aliased commands.
+  #
+  # See http://redis.io/topics/security#disabling-of-specific-commands
+  #renameCommands:
+  #  del: '3dda29ad-3015-44f9-b13b-fa570de92489'
+  #  srem: '3fd758c9-5610-4e9d-a058-dbf4cb6d8bf0'
 
 # Directory containing the database of XO.
 # Currently used for logs.

packages/xo-server/src/api/backup.js

@@ -1,8 +1,11 @@
 import archiver from 'archiver'
+import createLogger from '@xen-orchestra/log'
 import { basename } from 'path'
 import { format } from 'json-rpc-peer'
 import { forEach } from 'lodash'
 
+const log = createLogger('xo:backup')
+
 // ===================================================================
 
 export function list ({ remote }) {
@@ -62,7 +65,7 @@ function handleFetchFiles (
 
       const archive = archiver(archiveFormat)
       archive.on('error', error => {
-        console.error(error)
+        log.error(error)
         res.end(format.error(0, error))
       })
 
@@ -74,7 +77,7 @@ function handleFetchFiles (
       archive.pipe(res)
     })
     .catch(error => {
-      console.error(error)
+      log.error(error)
       res.writeHead(500)
       res.end(format.error(0, error))
     })

packages/xo-server/src/api/disk.js

@@ -1,23 +1,40 @@
+import createLogger from '@xen-orchestra/log'
 import pump from 'pump'
 import { format } from 'json-rpc-peer'
-import { unauthorized } from 'xo-common/api-errors'
+import { noSuchObject, unauthorized } from 'xo-common/api-errors'
 
 import { parseSize } from '../utils'
 
+const log = createLogger('xo:disk')
+
 // ===================================================================
 
 export async function create ({ name, size, sr, vm, bootable, position, mode }) {
   const attach = vm !== undefined
 
-  let resourceSet
-  if (attach && (resourceSet = vm.resourceSet) != null) {
-    await this.checkResourceSetConstraints(resourceSet, this.user.id, [sr.id])
-    await this.allocateLimitsInResourceSet({ disk: size }, resourceSet)
-  } else if (
-    !(await this.hasPermissions(this.user.id, [[sr.id, 'administrate']]))
-  ) {
-    throw unauthorized()
-  }
+  do {
+    let resourceSet
+    if (attach && (resourceSet = vm.resourceSet) != null) {
+      try {
+        await this.checkResourceSetConstraints(resourceSet, this.user.id, [
+          sr.id,
+        ])
+        await this.allocateLimitsInResourceSet({ disk: size }, resourceSet)
+
+        break
+      } catch (error) {
+        if (!noSuchObject.is(error, { data: { id: resourceSet } })) {
+          throw error
+        }
+      }
+
+      // the resource set does not exist, falls back to normal check
+    }
+
+    if (!(await this.hasPermissions(this.user.id, [[sr.id, 'administrate']]))) {
+      throw unauthorized()
+    }
+  } while (false)
 
   const xapi = this.getXapi(sr)
   const vdi = await xapi.createVdi({
@@ -72,7 +89,7 @@ async function handleExportContent (req, res, { xapi, id }) {
   )
   pump(stream, res, error => {
     if (error != null) {
-      console.warn('disk.exportContent', error)
+      log.warn('disk.exportContent', { error })
     }
   })
 }

packages/xo-server/src/api/vm.js

@@ -1,4 +1,5 @@
 import concat from 'lodash/concat'
+import defer from 'golike-defer'
 import { format } from 'json-rpc-peer'
 import { ignoreErrors } from 'promise-toolbox'
 import {
@@ -67,10 +68,12 @@ export async function create (params) {
   const xapi = this.getXapi(template)
 
   const objectIds = [template.id]
+  const cpus = extract(params, 'CPUs')
+  const memoryMax = extract(params, 'memoryMax')
   const limits = {
-    cpus: template.CPUs.number,
+    cpus: cpus !== undefined ? cpus : template.CPUs.number,
     disk: 0,
-    memory: template.memory.dynamic[1],
+    memory: memoryMax !== undefined ? memoryMax : template.memory.dynamic[1],
     vms: 1,
   }
   const vdiSizesByDevice = {}
@@ -647,17 +650,34 @@ restart.resolve = {
 
 // -------------------------------------------------------------------
 
-// TODO: implement resource sets
-export async function clone ({ vm, name, full_copy: fullCopy }) {
+export const clone = defer(async function (
+  $defer,
+  { vm, name, full_copy: fullCopy }
+) {
   await checkPermissionOnSrs.call(this, vm)
+  const xapi = this.getXapi(vm)
 
-  return this.getXapi(vm)
-    .cloneVm(vm._xapiRef, {
-      nameLabel: name,
-      fast: !fullCopy,
-    })
-    .then(vm => vm.$id)
-}
+  const { $id: cloneId } = await xapi.cloneVm(vm._xapiRef, {
+    nameLabel: name,
+    fast: !fullCopy,
+  })
+  $defer.onFailure(() => xapi.deleteVm(cloneId))
+
+  const isAdmin = this.user.permission === 'admin'
+  if (!isAdmin) {
+    await this.addAcl(this.user.id, cloneId, 'admin')
+  }
+
+  if (vm.resourceSet !== undefined) {
+    await this.allocateLimitsInResourceSet(
+      await this.computeVmResourcesUsage(vm),
+      vm.resourceSet,
+      isAdmin
+    )
+  }
+
+  return cloneId
+})
 
 clone.params = {
   id: { type: 'string' },
@@ -740,14 +760,22 @@ export { convertToTemplate as convert }
 // -------------------------------------------------------------------
 
 // TODO: implement resource sets
-export async function snapshot ({
-  vm,
-  name = `${vm.name_label}_${new Date().toISOString()}`,
-}) {
+export const snapshot = defer(async function (
+  $defer,
+  { vm, name = `${vm.name_label}_${new Date().toISOString()}` }
+) {
   await checkPermissionOnSrs.call(this, vm)
 
-  return (await this.getXapi(vm).snapshotVm(vm._xapiRef, name)).$id
-}
+  const xapi = this.getXapi(vm)
+  const { $id: snapshotId } = await xapi.snapshotVm(vm._xapiRef, name)
+  $defer.onFailure(() => xapi.deleteVm(snapshotId))
+
+  const { user } = this
+  if (user.permission !== 'admin') {
+    await this.addAcl(user.id, snapshotId, 'admin')
+  }
+  return snapshotId
+})
 
 snapshot.params = {
   id: { type: 'string' },
@@ -755,7 +783,7 @@ snapshot.params = {
 }
 
 snapshot.resolve = {
-  vm: ['id', 'VM', 'administrate'],
+  vm: ['id', 'VM', 'operate'],
 }
 
 // -------------------------------------------------------------------

packages/xo-server/src/api/xosan.js

@@ -1,5 +1,5 @@
 import asyncMap from '@xen-orchestra/async-map'
-import createLogger from 'debug'
+import createLogger from '@xen-orchestra/log'
 import defer from 'golike-defer'
 import execa from 'execa'
 import fs from 'fs-extra'
@@ -12,7 +12,7 @@ import { includes, remove, filter, find, range } from 'lodash'
 import { asInteger } from '../xapi/utils'
 import { parseXml, ensureArray } from '../utils'
 
-const debug = createLogger('xo:xosan')
+const log = createLogger('xo:xosan')
 
 const SSH_KEY_FILE = 'id_rsa_xosan'
 const DEFAULT_NETWORK_PREFIX = '172.31.100.'
@@ -73,7 +73,7 @@ async function rateLimitedRetry (action, shouldRetry, retryCount = 20) {
   let result
   while (retryCount > 0 && (result = await action()) && shouldRetry(result)) {
     retryDelay *= 1.1
-    debug('waiting ' + retryDelay + 'ms and retrying')
+    log.debug(`waiting ${retryDelay} ms and retrying`)
     await delay(retryDelay)
     retryCount--
   }
@@ -305,7 +305,7 @@ async function copyVm (xapi, originalVm, sr) {
 }
 
 async function callPlugin (xapi, host, command, params) {
-  debug('calling plugin', host.address, command)
+  log.debug(`calling plugin ${host.address} ${command}`)
   return JSON.parse(
     await xapi.call('host.call_plugin', host.$ref, 'xosan.py', command, params)
   )
@@ -346,15 +346,12 @@ async function remoteSsh (glusterEndpoint, cmd, ignoreError = false) {
         }
       }
     }
-    debug(
-      result.command.join(' '),
-      '\n  =>exit:',
-      result.exit,
-      '\n  =>err :',
-      result.stderr,
-      '\n  =>out (1000 chars) :',
-      result.stdout.substring(0, 1000)
-    )
+
+    log.debug(`result of ${result.command.join(' ')}`, {
+      exit: result.exit,
+      err: result.stderr,
+      out: result.stdout.substring(0, 1000),
+    })
     // 255 seems to be ssh's own error codes.
     if (result.exit !== 255) {
       if (!ignoreError && result.exit !== 0) {
@@ -552,7 +549,7 @@ async function configureGluster (
     creation +
     ' ' +
     brickVms.map(ipAndHost => ipAndHost.brickName).join(' ')
-  debug('creating volume: ', volumeCreation)
+  log.debug(`creating volume: ${volumeCreation}`)
   await glusterCmd(glusterEndpoint, volumeCreation)
   await glusterCmd(
     glusterEndpoint,
@@ -762,7 +759,7 @@ export const createSR = defer(async function (
       glusterType,
       arbiter
     )
-    debug('xosan gluster volume started')
+    log.debug('xosan gluster volume started')
     // We use 10 IPs of the gluster VM range as backup, in the hope that even if the first VM gets destroyed we find at least
     // one VM to give mount the volfile.
     // It is not possible to edit the device_config after the SR is created and this data is only used at mount time when rebooting
@@ -785,7 +782,7 @@ export const createSR = defer(async function (
       true,
       {}
     )
-    debug('sr created')
+    log.debug('sr created')
     // we just forget because the cleanup actions are stacked in the $onFailure system
     $defer.onFailure(() => xapi.forgetSr(xosanSrRef))
     if (arbiter) {
@@ -809,7 +806,7 @@ export const createSR = defer(async function (
       redundancy,
     })
     CURRENT_POOL_OPERATIONS[poolId] = { ...OPERATION_OBJECT, state: 6 }
-    debug('scanning new SR')
+    log.debug('scanning new SR')
     await xapi.call('SR.scan', xosanSrRef)
     await this.rebindLicense({
       licenseId: license.id,
@@ -1139,12 +1136,12 @@ async function _prepareGlusterVm (
     .find(vdi => vdi && vdi.name_label === 'xosan_root')
   const rootDiskSize = rootDisk.virtual_size
   await xapi.startVm(newVM)
-  debug('waiting for boot of ', ip)
+  log.debug(`waiting for boot of ${ip}`)
   // wait until we find the assigned IP in the networks, we are just checking the boot is complete
   const vmIsUp = vm =>
     Boolean(vm.$guest_metrics && includes(vm.$guest_metrics.networks, ip))
   const vm = await xapi._waitObjectState(newVM.$id, vmIsUp)
-  debug('booted ', ip)
+  log.debug(`booted ${ip}`)
   const localEndpoint = { xapi: xapi, hosts: [host], addresses: [ip] }
   const srFreeSpace = sr.physical_size - sr.physical_utilisation
   // we use a percentage because it looks like the VDI overhead is proportional

packages/xo-server/src/index.js

@@ -3,7 +3,7 @@ import assert from 'assert'
 import bind from 'lodash/bind'
 import blocked from 'blocked'
 import createExpress from 'express'
-import createLogger from 'debug'
+import createLogger from '@xen-orchestra/log'
 import has from 'lodash/has'
 import helmet from 'helmet'
 import includes from 'lodash/includes'
@@ -40,13 +40,20 @@ import passport from 'passport'
 import { parse as parseCookies } from 'cookie'
 import { Strategy as LocalStrategy } from 'passport-local'
 
+import transportConsole from '@xen-orchestra/log/transports/console'
+import { configure } from '@xen-orchestra/log/configure'
+
 // ===================================================================
 
-const debug = createLogger('xo:main')
+configure([
+  {
+    filter: process.env.DEBUG,
+    level: 'info',
+    transport: transportConsole(),
+  },
+])
 
-const warn = (...args) => {
-  console.warn('[Warn]', ...args)
-}
+const log = createLogger('xo:main')
 
 // ===================================================================
 
@@ -58,12 +65,12 @@ async function loadConfiguration () {
     ignoreUnknownFormats: true,
   })
 
-  debug('Configuration loaded.')
+  log.info('Configuration loaded.')
 
   // Print a message if deprecated entries are specified.
   forEach(DEPRECATED_ENTRIES, entry => {
     if (has(config, entry)) {
-      warn(`${entry} configuration is deprecated.`)
+      log.warn(`${entry} configuration is deprecated.`)
     }
   })
 
@@ -248,18 +255,18 @@ async function registerPlugin (pluginPath, pluginName) {
   )
 }
 
-const debugPlugin = createLogger('xo:plugin')
+const logPlugin = createLogger('xo:plugin')
 
 function registerPluginWrapper (pluginPath, pluginName) {
-  debugPlugin('register %s', pluginName)
+  logPlugin.info(`register ${pluginName}`)
 
   return registerPlugin.call(this, pluginPath, pluginName).then(
     () => {
-      debugPlugin(`successfully register ${pluginName}`)
+      logPlugin.info(`successfully register ${pluginName}`)
     },
     error => {
-      debugPlugin(`failed register ${pluginName}`)
-      debugPlugin(error)
+      logPlugin.info(`failed register ${pluginName}`)
+      logPlugin.info(error)
     }
   )
 }
@@ -323,20 +330,20 @@ async function makeWebServerListen (
   }
   try {
     const niceAddress = await webServer.listen(opts)
-    debug(`Web server listening on ${niceAddress}`)
+    log.info(`Web server listening on ${niceAddress}`)
   } catch (error) {
     if (error.niceAddress) {
-      warn(`Web server could not listen on ${error.niceAddress}`)
+      log.warn(`Web server could not listen on ${error.niceAddress}`, { error })
 
       const { code } = error
       if (code === 'EACCES') {
-        warn('  Access denied.')
-        warn('  Ports < 1024 are often reserved to privileges users.')
+        log.warn('  Access denied.')
+        log.warn('  Ports < 1024 are often reserved to privileges users.')
       } else if (code === 'EADDRINUSE') {
-        warn('  Address already in use.')
+        log.warn('  Address already in use.')
       }
     } else {
-      warn('Web server could not listen:', error.message)
+      log.warn('Web server could not listen:', { error })
     }
   }
 }
@@ -417,7 +424,7 @@ const setUpStaticFiles = (express, opts) => {
     }
 
     forEach(paths, path => {
-      debug('Setting up %s → %s', url, path)
+      log.info(`Setting up ${url} → ${path}`)
 
       express.use(url, serveStatic(path))
     })
@@ -435,7 +442,7 @@ const setUpApi = (webServer, xo, verboseLogsOnErrors) => {
   const onConnection = (socket, upgradeReq) => {
     const { remoteAddress } = upgradeReq.socket
 
-    debug('+ WebSocket connection (%s)', remoteAddress)
+    log.info(`+ WebSocket connection (${remoteAddress})`)
 
     // Create the abstract XO object for this connection.
     const connection = xo.createUserConnection()
@@ -453,7 +460,7 @@ const setUpApi = (webServer, xo, verboseLogsOnErrors) => {
 
     // Close the XO connection with this WebSocket.
     socket.once('close', () => {
-      debug('- WebSocket connection (%s)', remoteAddress)
+      log.info(`- WebSocket connection (${remoteAddress})`)
 
       connection.close()
     })
@@ -465,7 +472,7 @@ const setUpApi = (webServer, xo, verboseLogsOnErrors) => {
 
     const onSend = error => {
       if (error) {
-        warn('WebSocket send:', error.stack)
+        log.warn('WebSocket send:', { error })
       }
     }
     jsonRpc.on('data', data => {
@@ -513,9 +520,9 @@ const setUpConsoleProxy = (webServer, xo) => {
         }
 
         const { remoteAddress } = socket
-        debug('+ Console proxy (%s - %s)', user.name, remoteAddress)
+        log.info(`+ Console proxy (${user.name} - ${remoteAddress})`)
         socket.on('close', () => {
-          debug('- Console proxy (%s - %s)', user.name, remoteAddress)
+          log.info(`- Console proxy (${user.name} - ${remoteAddress})`)
         })
       }
 
@@ -549,10 +556,10 @@ export default async function main (args) {
   }
 
   {
-    const debug = createLogger('xo:perf')
+    const logPerf = createLogger('xo:perf')
     blocked(
       ms => {
-        debug('blocked for %sms', ms | 0)
+        logPerf.info(`blocked for ${ms | 0}ms`)
       },
       {
         threshold: 500,
@@ -569,14 +576,14 @@ export default async function main (args) {
     const { user, group } = config
     if (group) {
       process.setgid(group)
-      debug('Group changed to', group)
+      log.info(`Group changed to ${group}`)
     }
     if (user) {
       process.setuid(user)
-      debug('User changed to', user)
+      log.info(`User changed to ${user}`)
     }
   } catch (error) {
-    warn('Failed to change user/group:', error)
+    log.warn('Failed to change user/group:', { error })
   }
 
   // Creates main object.
@@ -604,7 +611,7 @@ export default async function main (args) {
     })
 
     if (port === undefined) {
-      warn('Could not setup HTTPs redirection: no HTTPs port found')
+      log.warn('Could not setup HTTPs redirection: no HTTPs port found')
     } else {
       express.use((req, res, next) => {
         if (req.secure) {
@@ -653,17 +660,17 @@ export default async function main (args) {
 
     process.on(signal, () => {
       if (alreadyCalled) {
-        warn('forced exit')
+        log.warn('forced exit')
         process.exit(1)
       }
       alreadyCalled = true
 
-      debug('%s caught, closing…', signal)
+      log.info(`${signal} caught, closing…`)
       xo.stop()
     })
   })
 
   await fromEvent(xo, 'stopped')
 
-  debug('bye :-)')
+  log.info('bye :-)')
 }

packages/xo-server/src/models/plugin-metadata.js

@@ -1,7 +1,10 @@
 import Collection from '../collection/redis'
+import createLogger from '@xen-orchestra/log'
 import Model from '../model'
 import { forEach } from '../utils'
 
+const log = createLogger('xo:plugin-metadata')
+
 // ===================================================================
 
 export default class PluginMetadata extends Model {}
@@ -44,10 +47,7 @@ export class PluginsMetadata extends Collection {
         pluginMetadata.configuration =
           configuration && JSON.parse(configuration)
       } catch (error) {
-        console.warn(
-          'cannot parse pluginMetadata.configuration:',
-          configuration
-        )
+        log.warn(`cannot parse pluginMetadata.configuration: ${configuration}`)
         pluginMetadata.configuration = []
       }
     })

packages/xo-server/src/proxy-console.js

@@ -1,22 +1,22 @@
-import createDebug from 'debug'
+import createLogger from '@xen-orchestra/log'
 import partialStream from 'partial-stream'
 import { connect } from 'tls'
 import { parse } from 'url'
 
-const debug = createDebug('xo:proxy-console')
+const log = createLogger('xo:proxy-console')
 
 export default function proxyConsole (ws, vmConsole, sessionId) {
   const url = parse(vmConsole.location)
   let { hostname } = url
   if (hostname === null || hostname === '') {
-    console.warn(
-      'host is missing in console (%s) URI (%s)',
-      vmConsole.uuid,
-      vmConsole.location
-    )
     const { address } = vmConsole.$VM.$resident_on
-    console.warn('  using host address (%s) as fallback', address)
     hostname = address
+
+    log.warn(
+      `host is missing in console (${vmConsole.uuid}) URI (${
+        vmConsole.location
+      }) using host address (${address}) as fallback`
+    )
   }
 
   let closed = false
@@ -41,10 +41,7 @@ export default function proxyConsole (ws, vmConsole, sessionId) {
 
       const onSend = error => {
         if (error) {
-          debug(
-            'error sending to the XO client: %s',
-            error.stack || error.message || error
-          )
+          log.debug('error sending to the XO client:', { error })
         }
       }
 
@@ -52,7 +49,7 @@ export default function proxyConsole (ws, vmConsole, sessionId) {
         .pipe(
           partialStream('\r\n\r\n', headers => {
             // TODO: check status code 200.
-            debug('connected')
+            log.debug('connected')
           })
         )
         .on('data', data => {
@@ -63,7 +60,7 @@ export default function proxyConsole (ws, vmConsole, sessionId) {
         .on('end', () => {
           if (!closed) {
             closed = true
-            debug('disconnected from the console')
+            log.debug('disconnected from the console')
           }
 
           ws.close()
@@ -71,10 +68,7 @@ export default function proxyConsole (ws, vmConsole, sessionId) {
 
       ws.on('error', error => {
         closed = true
-        debug(
-          'error from the XO client: %s',
-          error.stack || error.message || error
-        )
+        log.debug('error from the XO client:', { error })
 
         socket.end()
       })
@@ -86,7 +80,7 @@ export default function proxyConsole (ws, vmConsole, sessionId) {
         .on('close', () => {
           if (!closed) {
             closed = true
-            debug('disconnected from the XO client')
+            log.debug('disconnected from the XO client')
           }
 
           socket.end()
@@ -94,7 +88,7 @@ export default function proxyConsole (ws, vmConsole, sessionId) {
     }
   ).on('error', error => {
     closed = true
-    debug('error from the console: %s', error.stack || error.message || error)
+    log.debug('error from the console:', { error })
 
     ws.close()
   })

packages/xo-server/src/schemas/log/taskWarning.js

@@ -0,0 +1,15 @@
+export default {
+  $schema: 'http://json-schema.org/draft-04/schema#',
+  type: 'object',
+  properties: {
+    event: {
+      enum: ['task.warning'],
+    },
+    taskId: {
+      type: 'string',
+      description: 'identifier of the parent task or job',
+    },
+    data: {},
+  },
+  required: ['event', 'taskId'],
+}

packages/xo-server/src/utils.js

@@ -6,7 +6,6 @@ import humanFormat from 'human-format'
 import isArray from 'lodash/isArray'
 import isString from 'lodash/isString'
 import keys from 'lodash/keys'
-import kindOf from 'kindof'
 import multiKeyHashInt from 'multikey-hash'
 import pick from 'lodash/pick'
 import tmp from 'tmp'
@@ -192,35 +191,6 @@ export const noop = () => {}
 
 // -------------------------------------------------------------------
 
-// Usage: pDebug(promise, name) or promise::pDebug(name)
-export function pDebug (promise, name) {
-  if (arguments.length === 1) {
-    name = promise
-    promise = this
-  }
-
-  Promise.resolve(promise).then(
-    value => {
-      console.log(
-        '%s',
-        `Promise ${name} resolved${
-          value !== undefined ? ` with ${kindOf(value)}` : ''
-        }`
-      )
-    },
-    reason => {
-      console.log(
-        '%s',
-        `Promise ${name} rejected${
-          reason !== undefined ? ` with ${kindOf(reason)}` : ''
-        }`
-      )
-    }
-  )
-
-  return promise
-}
-
 // Given a collection (array or object) which contains promises,
 // return a promise that is fulfilled when all the items in the
 // collection are either fulfilled or rejected.

packages/xo-server/src/xapi-stats.js

@@ -230,6 +230,16 @@ const STATS = {
         getPath: matches => ['xvds', 'w', matches[1]],
       },
     },
+    iops: {
+      r: {
+        test: /^vbd_xvd(.)_iops_read$/,
+        getPath: matches => ['iops', 'r', matches[1]],
+      },
+      w: {
+        test: /^vbd_xvd(.)_iops_write$/,
+        getPath: matches => ['iops', 'w', matches[1]],
+      },
+    },
   },
 }
 

packages/xo-server/src/xapi/index.js

@@ -1,6 +1,7 @@
 /* eslint-disable camelcase */
 import asyncMap from '@xen-orchestra/async-map'
 import concurrency from 'limit-concurrency-decorator'
+import createLogger from '@xen-orchestra/log'
 import deferrable from 'golike-defer'
 import fatfs from 'fatfs'
 import mixin from '@xen-orchestra/mixin'
@@ -16,7 +17,7 @@ import {
 } from 'promise-toolbox'
 import { PassThrough } from 'stream'
 import { forbiddenOperation } from 'xo-common/api-errors'
-import { Xapi as XapiBase } from 'xen-api'
+import { Xapi as XapiBase, NULL_REF } from 'xen-api'
 import {
   every,
   find,
@@ -55,7 +56,6 @@ import { type DeltaVmExport } from './'
 import {
   asBoolean,
   asInteger,
-  debug,
   extractOpaqueRef,
   filterUndefineds,
   getNamespaceForType,
@@ -63,12 +63,13 @@ import {
   canSrHaveNewVdiOfSize,
   isVmHvm,
   isVmRunning,
-  NULL_REF,
   optional,
   parseDateTime,
   prepareXapiParam,
 } from './utils'
 
+const log = createLogger('xo:xapi')
+
 // ===================================================================
 
 const TAG_BASE_DELTA = 'xo:base_delta'
@@ -365,7 +366,7 @@ export default class Xapi extends XapiBase {
   async emergencyShutdownHost (hostId) {
     const host = this.getObject(hostId)
     const vms = host.$resident_VMs
-    debug(`Emergency shutdown: ${host.name_label}`)
+    log.debug(`Emergency shutdown: ${host.name_label}`)
     await pSettle(
       mapToArray(vms, vm => {
         if (!vm.is_control_domain) {
@@ -448,7 +449,7 @@ export default class Xapi extends XapiBase {
   // Clone a VM: make a fast copy by fast copying each of its VDIs
   // (using snapshots where possible) on the same SRs.
   _cloneVm (vm, nameLabel = vm.name_label) {
-    debug(
+    log.debug(
       `Cloning VM ${vm.name_label}${
         nameLabel !== vm.name_label ? ` as ${nameLabel}` : ''
       }`
@@ -467,7 +468,7 @@ export default class Xapi extends XapiBase {
       snapshot = await this._snapshotVm(vm)
     }
 
-    debug(
+    log.debug(
       `Copying VM ${vm.name_label}${
         nameLabel !== vm.name_label ? ` as ${nameLabel}` : ''
       }${sr ? ` on ${sr.name_label}` : ''}`
@@ -588,7 +589,7 @@ export default class Xapi extends XapiBase {
     version,
     xenstore_data,
   }) {
-    debug(`Creating VM ${name_label}`)
+    log.debug(`Creating VM ${name_label}`)
 
     return this.call(
       'VM.create',
@@ -649,7 +650,7 @@ export default class Xapi extends XapiBase {
     force = false,
     forceDeleteDefaultTemplate = false
   ) {
-    debug(`Deleting VM ${vm.name_label}`)
+    log.debug(`Deleting VM ${vm.name_label}`)
 
     const { $ref } = vm
 
@@ -690,13 +691,13 @@ export default class Xapi extends XapiBase {
         asyncMap(disks, ({ $ref: vdiRef }) => {
           let onFailure = () => {
             onFailure = vdi => {
-              console.error(
+              log.error(
                 `cannot delete VDI ${vdi.name_label} (from VM ${vm.name_label})`
               )
               forEach(vdi.$VBDs, vbd => {
                 if (vbd.VM !== $ref) {
                   const vm = vbd.$VM
-                  console.error('- %s (%s)', vm.name_label, vm.uuid)
+                  log.error(`- ${vm.name_label} (${vm.uuid})`)
                 }
               })
             }
@@ -744,28 +745,28 @@ export default class Xapi extends XapiBase {
   @cancelable
   async exportVm ($cancelToken, vmId, { compress = true } = {}) {
     const vm = this.getObject(vmId)
-
-    let snapshotRef
-    if (isVmRunning(vm)) {
-      snapshotRef = (await this._snapshotVm(
-        $cancelToken,
-        vm,
-        `[XO Export] ${vm.name_label}`
-      )).$ref
-    }
+    const useSnapshot = isVmRunning(vm)
+    const exportedVm = useSnapshot
+      ? await this._snapshotVm($cancelToken, vm, `[XO Export] ${vm.name_label}`)
+      : vm
 
     const promise = this.getResource($cancelToken, '/export/', {
       query: {
-        ref: snapshotRef || vm.$ref,
+        ref: exportedVm.$ref,
         use_compression: compress ? 'true' : 'false',
       },
       task: this.createTask('VM export', vm.name_label),
+    }).catch(error => {
+      // augment the error with as much relevant info as possible
+      error.pool_master = this.pool.$master
+      error.VM = exportedVm
+
+      throw error
     })
 
-    if (snapshotRef !== undefined) {
-      promise.then(_ =>
-        _.task::pFinally(() => this.deleteVm(snapshotRef)::ignoreErrors())
-      )
+    if (useSnapshot) {
+      const destroySnapshot = () => this.deleteVm(exportedVm)::ignoreErrors()
+      promise.then(_ => _.task::pFinally(destroySnapshot), destroySnapshot)
     }
 
     return promise
@@ -1093,6 +1094,7 @@ export default class Xapi extends XapiBase {
               transferSize += sizeStream.size
             })
           sizeStream.task = stream.task
+          sizeStream.length = stream.length
           await this._importVdiContent(vdi, sizeStream, VDI_FORMAT_VHD)
         }
       }),
@@ -1218,7 +1220,7 @@ export default class Xapi extends XapiBase {
       { vdi }
     ).catch(error => {
       if (error.code !== 'XENAPI_PLUGIN_FAILURE') {
-        console.warn('_callInstallationPlugin', error)
+        log.warn('_callInstallationPlugin', { error })
         throw error
       }
     })
@@ -1326,7 +1328,13 @@ export default class Xapi extends XapiBase {
     const vmRef = await this.putResource($cancelToken, stream, '/import/', {
       query,
       task: taskRef,
-    }).then(extractOpaqueRef)
+    }).then(extractOpaqueRef, error => {
+      // augment the error with as much relevant info as possible
+      error.pool_master = this.pool.$master
+      error.SR = sr
+
+      throw error
+    })
 
     return vmRef
   }
@@ -1486,7 +1494,7 @@ export default class Xapi extends XapiBase {
   @concurrency(2)
   @cancelable
   async _snapshotVm ($cancelToken, vm, nameLabel = vm.name_label) {
-    debug(
+    log.debug(
       `Snapshotting VM ${vm.name_label}${
         nameLabel !== vm.name_label ? ` as ${nameLabel}` : ''
       }`
@@ -1548,7 +1556,7 @@ export default class Xapi extends XapiBase {
   }
 
   async _startVm (vm, host, force) {
-    debug(`Starting VM ${vm.name_label}`)
+    log.debug(`Starting VM ${vm.name_label}`)
 
     if (force) {
       await this._updateObjectMapProperty(vm, 'blocked_operations', {
@@ -1696,7 +1704,7 @@ export default class Xapi extends XapiBase {
     vdi = this.getObject(vdi)
     vm = this.getObject(vm)
 
-    debug(`Creating VBD for VDI ${vdi.name_label} on VM ${vm.name_label}`)
+    log.debug(`Creating VBD for VDI ${vdi.name_label} on VM ${vm.name_label}`)
 
     if (userdevice == null) {
       const allowed = await this.call('VM.get_allowed_VBD_devices', vm.$ref)
@@ -1739,7 +1747,7 @@ export default class Xapi extends XapiBase {
   }
 
   _cloneVdi (vdi) {
-    debug(`Cloning VDI ${vdi.name_label}`)
+    log.debug(`Cloning VDI ${vdi.name_label}`)
 
     return this.call('VDI.clone', vdi.$ref)
   }
@@ -1761,7 +1769,7 @@ export default class Xapi extends XapiBase {
     sr = SR !== undefined && SR !== NULL_REF ? SR : this.pool.default_SR,
   }) {
     sr = this.getObject(sr)
-    debug(`Creating VDI ${name_label} on ${sr.name_label}`)
+    log.debug(`Creating VDI ${name_label} on ${sr.name_label}`)
 
     return this._getOrWaitObject(
       await this.call('VDI.create', {
@@ -1788,7 +1796,7 @@ export default class Xapi extends XapiBase {
       return // nothing to do
     }
 
-    debug(
+    log.debug(
       `Moving VDI ${vdi.name_label} from ${vdi.$SR.name_label} to ${
         sr.name_label
       }`
@@ -1821,13 +1829,15 @@ export default class Xapi extends XapiBase {
 
   // TODO: check whether the VDI is attached.
   async _deleteVdi (vdi) {
-    debug(`Deleting VDI ${vdi.name_label}`)
+    log.debug(`Deleting VDI ${vdi.name_label}`)
 
     await this.call('VDI.destroy', vdi.$ref)
   }
 
   _resizeVdi (vdi, size) {
-    debug(`Resizing VDI ${vdi.name_label} from ${vdi.virtual_size} to ${size}`)
+    log.debug(
+      `Resizing VDI ${vdi.name_label} from ${vdi.virtual_size} to ${size}`
+    )
 
     return this.call('VDI.resize', vdi.$ref, size)
   }
@@ -1958,7 +1968,7 @@ export default class Xapi extends XapiBase {
       query.base = base.$ref
     }
 
-    debug(
+    log.debug(
       `exporting VDI ${vdi.name_label}${
         base ? ` (from base ${vdi.name_label})` : ''
       }`
@@ -2029,7 +2039,7 @@ export default class Xapi extends XapiBase {
       qos_algorithm_type = '',
     } = {}
   ) {
-    debug(
+    log.debug(
       `Creating VIF for VM ${vm.name_label} on network ${network.name_label}`
     )
 
@@ -2292,9 +2302,9 @@ export default class Xapi extends XapiBase {
 
     // ignore errors, I (JFT) don't understand why they are emitted
     // because it works
-    await this._importVdiContent(vdi, buffer, VDI_FORMAT_RAW).catch(
-      console.warn
-    )
+    await this._importVdiContent(vdi, buffer, VDI_FORMAT_RAW).catch(error => {
+      log.warn('importVdiContent: ', { error })
+    })
 
     await this.createVbd({ vdi, vm })
   }

packages/xo-server/src/xapi/mixins/patching.js

@@ -1,4 +1,5 @@
 import asyncMap from '@xen-orchestra/async-map'
+import createLogger from '@xen-orchestra/log'
 import deferrable from 'golike-defer'
 import every from 'lodash/every'
 import filter from 'lodash/filter'
@@ -20,7 +21,9 @@ import {
   parseXml,
 } from '../../utils'
 
-import { debug, extractOpaqueRef, useUpdateSystem } from '../utils'
+import { extractOpaqueRef, useUpdateSystem } from '../utils'
+
+const log = createLogger('xo:xapi')
 
 export default {
   // FIXME: should be static
@@ -227,7 +230,7 @@ export default {
       return this.getObjectByUuid(uuid)
     } catch (error) {}
 
-    debug('downloading patch %s', uuid)
+    log.debug(`downloading patch ${uuid}`)
 
     const patchInfo = (await this._getXenUpdates()).patches[uuid]
     if (!patchInfo) {
@@ -255,7 +258,7 @@ export default {
 
   // patform_version >= 2.1.1 ----------------------------------------
   async _getUpdateVdi ($defer, patchUuid, hostId) {
-    debug('downloading patch %s', patchUuid)
+    log.debug(`downloading patch ${patchUuid}`)
 
     const patchInfo = (await this._getXenUpdates()).patches[patchUuid]
     if (!patchInfo) {
@@ -339,7 +342,7 @@ export default {
   // -----------------------------------------------------------------
 
   async installPoolPatchOnHost (patchUuid, host) {
-    debug('installing patch %s', patchUuid)
+    log.debug(`installing patch ${patchUuid}`)
     if (!isObject(host)) {
       host = this.getObject(host)
     }
@@ -380,7 +383,7 @@ export default {
   }),
 
   async installPoolPatchOnAllHosts (patchUuid) {
-    debug('installing patch %s on all hosts', patchUuid)
+    log.debug(`installing patch ${patchUuid} on all hosts`)
 
     return useUpdateSystem(this.pool.$master)
       ? this._installPatchUpdateOnAllHosts(patchUuid)

packages/xo-server/src/xapi/mixins/storage.js

@@ -1,7 +1,10 @@
+import createLogger from '@xen-orchestra/log'
 import { forEach, groupBy } from 'lodash'
 
 import { mapToArray } from '../../utils'
 
+const log = createLogger('xo:storage')
+
 export default {
   _connectAllSrPbds (sr) {
     return Promise.all(mapToArray(sr.$PBDs, pbd => this._plugPbd(pbd)))
@@ -58,7 +61,7 @@ export default {
           length += this._getUnhealthyVdiChainLength(parent, childrenMap, cache)
         }
       } catch (error) {
-        console.warn('Xapi#_getUnhealthyVdiChainLength(%s)', uuid, error)
+        log.warn(`Xapi#_getUnhealthyVdiChainLength(${uuid})`, { error })
       }
       cache[uuid] = length
     }

packages/xo-server/src/xapi/mixins/vm.js

@@ -1,10 +1,11 @@
 import deferrable from 'golike-defer'
-import { ignoreErrors, pCatch } from 'promise-toolbox'
 import { find, gte, includes, isEmpty, lte, noop } from 'lodash'
+import { ignoreErrors, pCatch } from 'promise-toolbox'
+import { NULL_REF } from 'xen-api'
 
 import { forEach, mapToArray, parseSize } from '../../utils'
 
-import { isVmHvm, isVmRunning, makeEditObject, NULL_REF } from '../utils'
+import { isVmHvm, isVmRunning, makeEditObject } from '../utils'
 
 // According to: https://xenserver.org/blog/entry/vga-over-cirrus-in-xenserver-6-2.html.
 const XEN_VGA_VALUES = ['std', 'cirrus']

packages/xo-server/src/xapi/other-config-template.js

@@ -1,4 +1,4 @@
-import { NULL_REF } from './utils'
+import { NULL_REF } from 'xen-api'
 
 const OTHER_CONFIG_TEMPLATE = {
   actions_after_crash: 'restart',

packages/xo-server/src/xapi/utils.js

@@ -1,6 +1,5 @@
 // import isFinite from 'lodash/isFinite'
 import camelCase from 'lodash/camelCase'
-import createDebug from 'debug'
 import isEqual from 'lodash/isEqual'
 import isPlainObject from 'lodash/isPlainObject'
 import pickBy from 'lodash/pickBy'
@@ -61,10 +60,6 @@ export const prepareXapiParam = param => {
 
 // -------------------------------------------------------------------
 
-export const debug = createDebug('xo:xapi')
-
-// -------------------------------------------------------------------
-
 const OPAQUE_REF_RE = /OpaqueRef:[0-9a-z-]+/
 export const extractOpaqueRef = str => {
   const matches = OPAQUE_REF_RE.exec(str)
@@ -366,10 +361,6 @@ export const makeEditObject = specs => {
 
 // ===================================================================
 
-export const NULL_REF = 'OpaqueRef:NULL'
-
-// ===================================================================
-
 export const useUpdateSystem = host => {
   // Match Xen Center's condition: https://github.com/xenserver/xenadmin/blob/f3a64fc54bbff239ca6f285406d9034f57537d64/XenModel/Utils/Helpers.cs#L420
   return versionSatisfies(host.software_version.platform_version, '^2.1.1')

packages/xo-server/src/xo-mixins/api.js

@@ -1,4 +1,4 @@
-import createDebug from 'debug'
+import createLogger from '@xen-orchestra/log'
 import kindOf from 'kindof'
 import ms from 'ms'
 import schemaInspector from 'schema-inspector'
@@ -12,7 +12,7 @@ import * as errors from 'xo-common/api-errors'
 
 // ===================================================================
 
-const debug = createDebug('xo:api')
+const log = createLogger('xo:api')
 
 const PERMISSIONS = {
   none: 0,
@@ -284,12 +284,10 @@ export default class Api {
         result = true
       }
 
-      debug(
-        '%s | %s(...) [%s] ==> %s',
-        userName,
-        name,
-        ms(Date.now() - startTime),
-        kindOf(result)
+      log.debug(
+        `${userName} | ${name}(...) [${ms(
+          Date.now() - startTime
+        )}] ==> ${kindOf(result)}`
       )
 
       return result
@@ -308,19 +306,12 @@ export default class Api {
       this._logger.error(message, data)
 
       if (this._xo._config.verboseLogsOnErrors) {
-        debug(message)
-
-        const stack = error && error.stack
-        if (stack) {
-          console.error(stack)
-        }
+        log.warn(message, { error })
       } else {
-        debug(
-          '%s | %s(...) [%s] =!> %s',
-          userName,
-          name,
-          ms(Date.now() - startTime),
-          error
+        log.warn(
+          `${userName} | ${name}(...) [${ms(
+            Date.now() - startTime
+          )}] =!> ${error}`
         )
       }
 

packages/xo-server/src/xo-mixins/authentication.js

@@ -1,3 +1,4 @@
+import createLogger from '@xen-orchestra/log'
 import ms from 'ms'
 import { noSuchObject } from 'xo-common/api-errors'
 import { ignoreErrors } from 'promise-toolbox'
@@ -6,6 +7,7 @@ import Token, { Tokens } from '../models/token'
 import { forEach, generateToken } from '../utils'
 
 // ===================================================================
+const log = createLogger('xo:authentification')
 
 const noSuchAuthenticationToken = id => noSuchObject(id, 'authenticationToken')
 
@@ -103,7 +105,7 @@ export default class {
         // DEPRECATED: Authentication providers may just throw `null`
         // to indicate they could not authenticate the user without
         // any special errors.
-        if (error) console.error(error.stack || error)
+        if (error) log.error(error)
       }
     }
 

packages/xo-server/src/xo-mixins/backups-ng-logs.js

@@ -1,8 +1,9 @@
 import { forEach } from 'lodash'
+import { noSuchObject } from 'xo-common/api-errors'
 
 const isSkippedError = error =>
   error.message === 'no disks found' ||
-  error.message === 'no such object' ||
+  noSuchObject.is(error) ||
   error.message === 'no VMs match this pattern' ||
   error.message === 'unhealthy VDI chain'
 
@@ -49,10 +50,16 @@ const taskTimeComparator = ({ start: s1, end: e1 }, { start: s2, end: e2 }) => {
 
 export default {
   async getBackupNgLogs (runId?: string) {
-    const { runningJobs } = this
+    const [jobLogs, restoreLogs] = await Promise.all([
+      this.getLogs('jobs'),
+      this.getLogs('restore'),
+    ])
+
+    const { runningJobs, runningRestores } = this
     const consolidated = {}
     const started = {}
-    forEach(await this.getLogs('jobs'), ({ data, time, message }, id) => {
+
+    const handleLog = ({ data, time, message }, id) => {
       const { event } = data
       if (event === 'job.start') {
         if (
@@ -82,17 +89,26 @@ export default {
           )
         }
       } else if (event === 'task.start') {
-        const parent = started[data.parentId]
-        if (parent !== undefined) {
-          ;(parent.tasks || (parent.tasks = [])).push(
-            (started[id] = {
-              data: data.data,
-              id,
-              message,
-              start: time,
-              status: parent.status,
-            })
-          )
+        const task = {
+          data: data.data,
+          id,
+          message,
+          start: time,
+        }
+        const { parentId } = data
+        let parent
+        if (parentId === undefined && (runId === undefined || runId === id)) {
+          // top level task
+          task.status =
+            message === 'restore' && !runningRestores.has(id)
+              ? 'interrupted'
+              : 'pending'
+          consolidated[id] = started[id] = task
+        } else if ((parent = started[parentId]) !== undefined) {
+          // sub-task for which the parent exists
+          task.status = parent.status
+          started[id] = task
+          ;(parent.tasks || (parent.tasks = [])).push(task)
         }
       } else if (event === 'task.end') {
         const { taskId } = data
@@ -106,6 +122,13 @@ export default {
             log.tasks
           )
         }
+      } else if (event === 'task.warning') {
+        const parent = started[data.taskId]
+        parent !== undefined &&
+          (parent.warnings || (parent.warnings = [])).push({
+            data: data.data,
+            message,
+          })
       } else if (event === 'jobCall.start') {
         const parent = started[data.runJobId]
         if (parent !== undefined) {
@@ -133,7 +156,11 @@ export default {
           )
         }
       }
-    })
+    }
+
+    forEach(jobLogs, handleLog)
+    forEach(restoreLogs, handleLog)
+
     return runId === undefined ? consolidated : consolidated[runId]
   },
 }

packages/xo-server/src/xo-mixins/backups-ng/index.js

@@ -3,6 +3,7 @@
 // $FlowFixMe
 import type RemoteHandler from '@xen-orchestra/fs'
 import asyncMap from '@xen-orchestra/async-map'
+import createLogger from '@xen-orchestra/log'
 import defer from 'golike-defer'
 import limitConcurrency from 'limit-concurrency-decorator'
 import { type Pattern, createPredicate } from 'value-matcher'
@@ -11,8 +12,9 @@ import { AssertionError } from 'assert'
 import { basename, dirname } from 'path'
 import {
   countBy,
+  findLast,
   flatMap,
-  forEach,
+  forOwn,
   groupBy,
   isEmpty,
   last,
@@ -22,12 +24,18 @@ import {
   sum,
   values,
 } from 'lodash'
-import { CancelToken, pFromEvent, ignoreErrors } from 'promise-toolbox'
+import {
+  CancelToken,
+  ignoreErrors,
+  pFinally,
+  pFromEvent,
+} from 'promise-toolbox'
 import Vhd, {
   chainVhd,
   createSyntheticStream as createVhdReadStream,
 } from 'vhd-lib'
 
+import type Logger from '../logs/loggers/abstract'
 import { type CallJob, type Executor, type Job } from '../jobs'
 import { type Schedule } from '../scheduling'
 
@@ -49,6 +57,8 @@ import {
 
 import { translateLegacyJob } from './migration'
 
+const log = createLogger('xo:xo-mixins:backups-ng')
+
 export type Mode = 'full' | 'delta'
 export type ReportWhen = 'always' | 'failure' | 'never'
 
@@ -201,11 +211,13 @@ const importers: $Dict<
     metadataFilename: string,
     metadata: Metadata,
     xapi: Xapi,
-    sr: { $id: string }
+    sr: { $id: string },
+    taskId: string,
+    logger: Logger
   ) => Promise<string>,
   Mode
 > = {
-  async delta (handler, metadataFilename, metadata, xapi, sr) {
+  async delta (handler, metadataFilename, metadata, xapi, sr, taskId, logger) {
     metadata = ((metadata: any): MetadataDelta)
     const { vdis, vhds, vm } = metadata
 
@@ -230,15 +242,26 @@ const importers: $Dict<
       },
     }
 
-    const { vm: newVm } = await xapi.importDeltaVm(delta, {
-      detectBase: false,
-      disableStartAfterImport: false,
-      srId: sr,
-      // TODO: support mapVdisSrs
-    })
+    const { vm: newVm } = await wrapTask(
+      {
+        logger,
+        message: 'transfer',
+        parentId: taskId,
+        result: ({ transferSize, vm: { $id: id } }) => ({
+          size: transferSize,
+          id,
+        }),
+      },
+      xapi.importDeltaVm(delta, {
+        detectBase: false,
+        disableStartAfterImport: false,
+        srId: sr,
+        // TODO: support mapVdisSrs
+      })
+    )
     return newVm.$id
   },
-  async full (handler, metadataFilename, metadata, xapi, sr) {
+  async full (handler, metadataFilename, metadata, xapi, sr, taskId, logger) {
     metadata = ((metadata: any): MetadataFull)
 
     const xva = await handler.createReadStream(
@@ -248,7 +271,16 @@ const importers: $Dict<
         ignoreMissingChecksum: true, // provide an easy way to opt-out
       }
     )
-    const vm = await xapi.importVm(xva, { srId: sr.$id })
+
+    const vm = await wrapTask(
+      {
+        logger,
+        message: 'transfer',
+        parentId: taskId,
+        result: ({ $id: id }) => ({ size: xva.length, id }),
+      },
+      xapi.importVm(xva, { srId: sr.$id })
+    )
     await Promise.all([
       xapi.addTag(vm.$id, 'restored from backup'),
       xapi.editVm(vm.$id, {
@@ -386,6 +418,35 @@ const wrapTaskFn = <T>(
     }
   }
 
+const extractIdsFromSimplePattern = (pattern: mixed) => {
+  if (pattern === null || typeof pattern !== 'object') {
+    return
+  }
+
+  let keys = Object.keys(pattern)
+  if (keys.length !== 1 || keys[0] !== 'id') {
+    return
+  }
+
+  pattern = pattern.id
+  if (typeof pattern === 'string') {
+    return [pattern]
+  }
+  if (pattern === null || typeof pattern !== 'object') {
+    return
+  }
+
+  keys = Object.keys(pattern)
+  if (
+    keys.length === 1 &&
+    keys[0] === '__or' &&
+    Array.isArray((pattern = pattern.__or)) &&
+    pattern.every(_ => typeof _ === 'string')
+  ) {
+    return pattern
+  }
+}
+
 // File structure on remotes:
 //
 // <remote>
@@ -425,11 +486,21 @@ export default class BackupNg {
     removeJob: (id: string) => Promise<void>,
     worker: $Dict<any>,
   }
+  _logger: Logger
+  _runningRestores: Set<string>
+
+  get runningRestores () {
+    return this._runningRestores
+  }
 
   constructor (app: any) {
     this._app = app
+    this._logger = undefined
+    this._runningRestores = new Set()
+
+    app.on('start', async () => {
+      this._logger = await app.getLogger('restore')
 
-    app.on('start', () => {
       const executor: Executor = async ({
         cancelToken,
         data: vmsId,
@@ -443,21 +514,48 @@ export default class BackupNg {
         }
 
         const job: BackupJob = (job_: any)
+        const vmsPattern = job.vms
 
-        const vms: $Dict<Vm> = app.getObjects({
-          filter: createPredicate({
-            type: 'VM',
-            ...(vmsId !== undefined
-              ? {
-                  id: {
-                    __or: vmsId,
-                  },
-                }
-              : job.vms),
-          }),
-        })
-        if (isEmpty(vms)) {
-          throw new Error('no VMs match this pattern')
+        let vms: $Dict<Vm>
+        if (
+          vmsId !== undefined ||
+          (vmsId = extractIdsFromSimplePattern(vmsPattern)) !== undefined
+        ) {
+          vms = vmsId
+            .map(id => {
+              try {
+                return app.getObject(id, 'VM')
+              } catch (error) {
+                const taskId: string = logger.notice(
+                  `Starting backup of ${id}. (${job.id})`,
+                  {
+                    event: 'task.start',
+                    parentId: runJobId,
+                    data: {
+                      type: 'VM',
+                      id,
+                    },
+                  }
+                )
+                logger.error(`Backuping ${id} has failed. (${job.id})`, {
+                  event: 'task.end',
+                  taskId,
+                  status: 'failure',
+                  result: serializeError(error),
+                })
+              }
+            })
+            .filter(vm => vm !== undefined)
+        } else {
+          vms = app.getObjects({
+            filter: createPredicate({
+              type: 'VM',
+              ...vmsPattern,
+            }),
+          })
+          if (isEmpty(vms)) {
+            throw new Error('no VMs match this pattern')
+          }
         }
         const jobId = job.id
         const srs = unboxIds(job.srs).map(id => {
@@ -627,14 +725,33 @@ export default class BackupNg {
     }
 
     const xapi = app.getXapi(srId)
-
-    return importer(
-      handler,
-      metadataFilename,
-      metadata,
-      xapi,
-      xapi.getObject(srId)
-    )
+    const { jobId, timestamp: time } = metadata
+    const logger = this._logger
+    return wrapTaskFn(
+      {
+        data: {
+          jobId,
+          srId,
+          time,
+        },
+        logger,
+        message: 'restore',
+      },
+      taskId => {
+        this._runningRestores.add(taskId)
+        return importer(
+          handler,
+          metadataFilename,
+          metadata,
+          xapi,
+          xapi.getObject(srId),
+          taskId,
+          logger
+        )::pFinally(() => {
+          this._runningRestores.delete(taskId)
+        })
+      }
+    )()
   }
 
   async listVmBackupsNg (remotes: string[]) {
@@ -685,7 +802,7 @@ export default class BackupNg {
             })
           )
         } catch (error) {
-          console.warn('[Warn] listVmBackups for remote %s:', remoteId, error)
+          log.warn(`listVmBackups for remote ${remoteId}:`, { error })
         }
       })
     )
@@ -769,6 +886,7 @@ export default class BackupNg {
         },
         xapi._updateObjectMapProperty(vm, 'other_config', {
           'xo:backup:datetime': null,
+          'xo:backup:exported': null,
           'xo:backup:job': null,
           'xo:backup:schedule': null,
           'xo:backup:vm': null,
@@ -889,7 +1007,19 @@ export default class BackupNg {
 
     snapshot = await xapi.barrier(snapshot.$ref)
 
-    let baseSnapshot = mode === 'delta' ? last(snapshots) : undefined
+    let baseSnapshot
+    if (mode === 'delta') {
+      baseSnapshot = findLast(
+        snapshots,
+        _ => 'xo:backup:exported' in _.other_config
+      )
+
+      // JFT 2018-10-02: support previous snapshots which did not have this
+      // entry, can be removed after 2018-12.
+      if (baseSnapshot === undefined) {
+        baseSnapshot = last(snapshots)
+      }
+    }
     snapshots.push(snapshot)
 
     // snapshots to delete due to the snapshot retention settings
@@ -1135,6 +1265,15 @@ export default class BackupNg {
         const fullRequired = { __proto__: null }
         const vdis: $Dict<Vdi> = getVmDisks(baseSnapshot)
 
+        // ignore VDI snapshots which no longer have a parent
+        forOwn(vdis, (vdi, key, vdis) => {
+          // `vdi.snapshot_of` is not always set to the null ref, it can contain
+          // an invalid ref, that's why the test is on `vdi.$snapshot_of`
+          if (vdi.$snapshot_of === undefined) {
+            delete vdis[key]
+          }
+        })
+
         for (const { $id: srId, xapi } of srs) {
           const replicatedVm = listReplicatedVms(
             xapi,
@@ -1151,7 +1290,7 @@ export default class BackupNg {
             getVmDisks(replicatedVm),
             vdi => vdi.other_config[TAG_COPY_SRC]
           )
-          forEach(vdis, vdi => {
+          forOwn(vdis, vdi => {
             if (!(vdi.uuid in replicatedVdis)) {
               fullRequired[vdi.$snapshot_of.$id] = true
             }
@@ -1439,6 +1578,17 @@ export default class BackupNg {
     } else {
       throw new Error(`no exporter for backup mode ${mode}`)
     }
+
+    await wrapTask(
+      {
+        logger,
+        message: 'set snapshot.other_config[xo:backup:exported]',
+        parentId: taskId,
+      },
+      xapi._updateObjectMapProperty(snapshot, 'other_config', {
+        'xo:backup:exported': 'true',
+      })
+    )
   }
 
   async _deleteDeltaVmBackups (
@@ -1492,7 +1642,7 @@ export default class BackupNg {
           // Do not fail on corrupted VHDs (usually uncleaned temporary files),
           // they are probably inconsequent to the backup process and should not
           // fail it.
-          console.warn('BackupNg#_deleteVhd', path, error)
+          log.warn(`BackupNg#_deleteVhd ${path}`, { error })
         }
       }
     )
@@ -1544,7 +1694,7 @@ export default class BackupNg {
               backups.push(metadata)
             }
           } catch (error) {
-            console.warn('_listVmBackups', path, error)
+            log.warn(`_listVmBackups ${path}`, { error })
           }
         })
       )

packages/xo-server/src/xo-mixins/backups.js

@@ -1,4 +1,5 @@
 import asyncMap from '@xen-orchestra/async-map'
+import createLogger from '@xen-orchestra/log'
 import deferrable from 'golike-defer'
 import escapeStringRegexp from 'escape-string-regexp'
 import execa from 'execa'
@@ -51,6 +52,7 @@ const TAG_SOURCE_VM = 'xo:source_vm'
 const TAG_EXPORT_TIME = 'xo:export_time'
 
 const shortDate = utcFormat('%Y-%m-%d')
+const log = createLogger('xo:xo-mixins:backups')
 
 // Test if a file is a vdi backup. (full or delta)
 const isVdiBackup = name => /^\d+T\d+Z_(?:full|delta)\.vhd$/.test(name)
@@ -225,7 +227,7 @@ const mountPartition = (device, partitionId) =>
             unmount: once(() => execa('umount', ['--lazy', path])),
           }),
           error => {
-            console.log(error)
+            log.error(error)
 
             throw error
           }
@@ -300,14 +302,6 @@ const mountLvmPv = (device, partition) => {
 export default class {
   constructor (xo) {
     this._xo = xo
-
-    // clean any LVM volumes that might have not been properly
-    // unmounted
-    xo.on('start', () =>
-      Promise.all([execa('losetup', ['-D']), execa('vgchange', ['-an'])]).then(
-        () => execa('pvscan', ['--cache'])
-      )
-    )
   }
 
   async listRemoteBackups (remoteId) {
@@ -557,9 +551,9 @@ export default class {
 
       try {
         mergedDataSize += await mergeVhd(handler, parent, handler, backup)
-      } catch (e) {
-        console.error('Unable to use vhd-util.', e)
-        throw e
+      } catch (error) {
+        log.error('unable to use vhd-util', { error })
+        throw error
       }
 
       await handler.unlink(backup)
@@ -731,7 +725,7 @@ export default class {
         fulFilledVdiBackups.push(vdiBackup)
       } else {
         error = vdiBackup.reason()
-        console.error('Rejected backup:', error)
+        log.error('Rejected backup:', { error })
       }
     }
 

packages/xo-server/src/xo-mixins/config-management.js

@@ -1,9 +1,9 @@
-import createDebug from 'debug'
+import createLogger from '@xen-orchestra/log'
 import DepTree from 'deptree'
 import { mapValues } from 'lodash'
 import { pAll } from 'promise-toolbox'
 
-const debug = createDebug('xo:config-management')
+const log = createLogger('xo:config-management')
 
 export default class ConfigManagement {
   constructor (app) {
@@ -33,7 +33,7 @@ export default class ConfigManagement {
 
       const data = config[key]
       if (data !== undefined) {
-        debug('importing', key)
+        log.debug(`importing ${key}`)
         await manager.importer(data)
       }
     }

packages/xo-server/src/xo-mixins/file-restore-ng.js

@@ -163,6 +163,13 @@ export default class BackupNgFileRestore {
   constructor (app) {
     this._app = app
     this._mounts = { __proto__: null }
+
+    // clean any LVM volumes that might have not been properly
+    // unmounted
+    app.on('start', async () => {
+      await Promise.all([execa('losetup', ['-D']), execa('vgchange', ['-an'])])
+      await execa('pvscan', ['--cache'])
+    })
   }
 
   @defer

packages/xo-server/src/xo-mixins/hooks.js

@@ -1,7 +1,7 @@
-import createLogger from 'debug'
+import createLogger from '@xen-orchestra/log'
 import emitAsync from '@xen-orchestra/emit-async'
 
-const debug = createLogger('xo:hooks')
+const log = createLogger('xo:xo-mixins:hooks')
 
 const makeSingletonHook = (hook, postEvent) => {
   let promise
@@ -19,20 +19,16 @@ const makeSingletonHook = (hook, postEvent) => {
 }
 
 const runHook = (app, hook) => {
-  debug(`${hook} start…`)
+  log.debug(`${hook} start…`)
   const promise = emitAsync.call(
     app,
     {
-      onError: error =>
-        console.error(
-          `[WARN] hook ${hook} failure:`,
-          (error != null && error.stack) || error
-        ),
+      onError: error => log.warn(`hook ${hook} failure:`, { error }),
     },
     hook
   )
   promise.then(() => {
-    debug(`${hook} finished`)
+    log.debug(`${hook} finished`)
   })
   return promise
 }

packages/xo-server/src/xo-mixins/jobs/index.js

@@ -144,38 +144,35 @@ export default class Jobs {
     executors.call = executeCall
 
     xo.on('clean', () => jobsDb.rebuildIndexes())
-    xo.on('start', () => {
+    xo.on('start', async () => {
+      this._logger = await xo.getLogger('jobs')
+
       xo.addConfigManager(
         'jobs',
         () => jobsDb.get(),
         jobs => Promise.all(mapToArray(jobs, job => jobsDb.save(job))),
         ['users']
       )
-
-      xo.getLogger('jobs').then(logger => {
-        this._logger = logger
-      })
-
-      // it sends a report for the interrupted backup jobs
-      this._app.on('plugins:registered', () =>
-        asyncMap(this._jobs.get(), job => {
-          // only the interrupted backup jobs have the runId property
-          if (job.runId === undefined) {
-            return
-          }
-
-          this._app.emit(
-            'job:terminated',
-            undefined,
-            job,
-            undefined,
-            // This cast can be removed after merging the PR: https://github.com/vatesfr/xen-orchestra/pull/3209
-            String(job.runId)
-          )
-          return this.updateJob({ id: job.id, runId: null })
-        })
-      )
     })
+    // it sends a report for the interrupted backup jobs
+    xo.on('plugins:registered', () =>
+      asyncMap(this._jobs.get(), job => {
+        // only the interrupted backup jobs have the runId property
+        if (job.runId === undefined) {
+          return
+        }
+
+        xo.emit(
+          'job:terminated',
+          undefined,
+          job,
+          undefined,
+          // This cast can be removed after merging the PR: https://github.com/vatesfr/xen-orchestra/pull/3209
+          String(job.runId)
+        )
+        return this.updateJob({ id: job.id, runId: null })
+      })
+    )
   }
 
   cancelJobRun (id: string) {
@@ -293,8 +290,8 @@ export default class Jobs {
     runs[runJobId] = { cancel }
 
     let session
+    const app = this._app
     try {
-      const app = this._app
       session = app.createUserConnection()
       session.set('user_id', job.userId)
 
@@ -319,11 +316,16 @@ export default class Jobs {
 
       app.emit('job:terminated', status, job, schedule, runJobId)
     } catch (error) {
-      logger.error(`The execution of ${id} has failed.`, {
-        event: 'job.end',
-        runJobId,
-        error: serializeError(error),
-      })
+      await logger.error(
+        `The execution of ${id} has failed.`,
+        {
+          event: 'job.end',
+          runJobId,
+          error: serializeError(error),
+        },
+        true
+      )
+      app.emit('job:terminated', undefined, job, schedule, runJobId)
       throw error
     } finally {
       ;this.updateJob({ id, runId: null })::ignoreErrors()

packages/xo-server/src/xo-mixins/plugins.js

@@ -1,4 +1,5 @@
 import Ajv from 'ajv'
+import createLogger from '@xen-orchestra/log'
 
 import { PluginsMetadata } from '../models/plugin-metadata'
 import { invalidParameters, noSuchObject } from 'xo-common/api-errors'
@@ -6,6 +7,8 @@ import { isFunction, mapToArray } from '../utils'
 
 // ===================================================================
 
+const log = createLogger('xo:xo-mixins:plugins')
+
 export default class {
   constructor (xo) {
     this._ajv = new Ajv({
@@ -73,7 +76,7 @@ export default class {
     if (metadata !== undefined) {
       ;({ autoload, configuration } = metadata)
     } else {
-      console.log(`[NOTICE] register plugin ${name} for the first time`)
+      log.info(`[NOTICE] register plugin ${name} for the first time`)
       await this._pluginsMetadata.save({
         id,
         autoload,

packages/xo-server/src/xo-mixins/remotes.js

@@ -44,12 +44,12 @@ export default class {
     })
   }
 
-  async getRemoteHandler (remote, ignoreDisabled) {
+  async getRemoteHandler (remote) {
     if (typeof remote === 'string') {
       remote = await this.getRemote(remote)
     }
 
-    if (!(ignoreDisabled || remote.enabled)) {
+    if (!remote.enabled) {
       throw new Error('remote is disabled')
     }
 
@@ -72,7 +72,7 @@ export default class {
   }
 
   async testRemote (remote) {
-    const handler = await this.getRemoteHandler(remote, true)
+    const handler = await this.getRemoteHandler(remote)
     return handler.test()
   }
 
@@ -126,8 +126,11 @@ export default class {
   }
 
   async removeRemote (id) {
-    const handler = await this.getRemoteHandler(id, true)
-    await handler.forget()
+    const handler = this._handlers[id]
+    if (handler !== undefined) {
+      ignoreErrors.call(handler.forget())
+    }
+
     await this._remotes.remove(id)
   }
 }

packages/xo-server/src/xo-mixins/resource-sets.js

@@ -284,7 +284,7 @@ export default class {
   }
 
   @synchronizedResourceSets
-  async allocateLimitsInResourceSet (limits, setId) {
+  async allocateLimitsInResourceSet (limits, setId, force = false) {
     const set = await this.getResourceSet(setId)
     forEach(limits, (quantity, id) => {
       const limit = set.limits[id]
@@ -292,7 +292,7 @@ export default class {
         return
       }
 
-      if ((limit.available -= quantity) < 0) {
+      if ((limit.available -= quantity) < 0 && !force) {
         throw new Error(`not enough ${id} available in the set ${setId}`)
       }
     })
@@ -333,6 +333,9 @@ export default class {
         if (
           object.$type !== 'vm' ||
           object.is_a_snapshot ||
+          ('start' in object.blocked_operations &&
+            (object.tags.includes('Disaster Recovery') ||
+              object.tags.includes('Continuous Replication'))) ||
           // No set for this VM.
           !(id = xapi.xo.getData(object, 'resourceSet')) ||
           // Not our set.

packages/xo-server/src/xo-mixins/subjects.js

@@ -1,3 +1,4 @@
+import createLogger from '@xen-orchestra/log'
 import { filter, includes } from 'lodash'
 import { ignoreErrors } from 'promise-toolbox'
 import { hash, needsRehash, verify } from 'hashy'
@@ -9,6 +10,8 @@ import { forEach, isEmpty, lightSet, mapToArray } from '../utils'
 
 // ===================================================================
 
+const log = createLogger('xo:xo-mixins:subjects')
+
 const addToArraySet = (set, value) =>
   set && !includes(set, value) ? set.concat(value) : [value]
 const removeFromArraySet = (set, value) =>
@@ -69,12 +72,7 @@ export default class {
         const password = 'admin'
 
         await this.createUser({ email, password, permission: 'admin' })
-        console.log(
-          '[INFO] Default user created:',
-          email,
-          ' with password',
-          password
-        )
+        log.info(`Default user created: ${email} with password ${password}`)
       }
     })
   }

packages/xo-server/src/xo-mixins/workers/worker.js

@@ -3,6 +3,10 @@
 import { type Remote, getHandler } from '@xen-orchestra/fs'
 import { mergeVhd as mergeVhd_ } from 'vhd-lib'
 
+// Use Bluebird for all promises as it provides better performance and
+// less memory usage.
+global.Promise = require('bluebird')
+
 export function mergeVhd (
   parentRemote: Remote,
   parentPath: string,

packages/xo-server/src/xo-mixins/xen-servers.js

@@ -1,3 +1,4 @@
+import createLogger from '@xen-orchestra/log'
 import { ignoreErrors } from 'promise-toolbox'
 import { noSuchObject } from 'xo-common/api-errors'
 
@@ -16,6 +17,8 @@ import { Servers } from '../models/server'
 
 // ===================================================================
 
+const log = createLogger('xo:xo-mixins:xen-servers')
+
 export default class {
   constructor (xo) {
     this._objectConflicts = { __proto__: null } // TODO: clean when a server is disconnected.
@@ -42,10 +45,10 @@ export default class {
       for (const server of servers) {
         if (server.enabled) {
           this.connectXenServer(server.id).catch(error => {
-            console.error(
-              `[WARN] ${server.host}:`,
-              error[0] || error.stack || error.code || error
-            )
+            log.warn('failed to connect to XenServer', {
+              host: server.host,
+              error,
+            })
           })
         }
       }
@@ -178,7 +181,7 @@ export default class {
           objects.set(xoId, xoObject)
         }
       } catch (error) {
-        console.error('ERROR: xapiObjectToXo', error)
+        log.error('xapiObjectToXo', { error })
 
         toRetry[xapiId] = xapiObject
       }

packages/xo-server/src/xo.js

@@ -1,3 +1,4 @@
+import createLogger from '@xen-orchestra/log'
 import XoCollection from 'xo-collection'
 import XoUniqueIndex from 'xo-collection/unique-index'
 import mixin from '@xen-orchestra/mixin'
@@ -21,6 +22,8 @@ import { generateToken, noop } from './utils'
 
 // ===================================================================
 
+const log = createLogger('xo:xo')
+
 @mixin(mapToArray(mixins))
 export default class Xo extends EventEmitter {
   constructor (config) {
@@ -151,7 +154,7 @@ export default class Xo extends EventEmitter {
         }
       },
       error => {
-        console.error('HTTP request error', error.stack || error)
+        log.error('HTTP request error', { error })
 
         if (!res.headersSent) {
           res.writeHead(500)

packages/xo-vmdk-to-vhd/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-vmdk-to-vhd",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "license": "AGPL-3.0",
   "description": "JS lib streaming a vmdk file to a vhd",
   "keywords": [
@@ -24,10 +24,11 @@
   },
   "dependencies": {
     "child-process-promise": "^2.0.3",
+    "core-js": "3.0.0-beta.3",
     "pipette": "^0.9.3",
     "promise-toolbox": "^0.10.1",
     "tmp": "^0.0.33",
-    "vhd-lib": "^0.3.0"
+    "vhd-lib": "^0.4.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/xo-vmdk-to-vhd/src/vmdk-read.js

@@ -1,4 +1,5 @@
-'use strict'
+// see https://github.com/babel/babel/issues/8450
+import 'core-js/features/symbol/async-iterator'
 
 import zlib from 'zlib'
 

packages/xo-web/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "xo-web",
-  "version": "5.27.0",
+  "version": "5.29.1",
   "license": "AGPL-3.0",
   "description": "Web interface client for Xen-Orchestra",
   "keywords": [
@@ -30,7 +30,6 @@
     "node": ">=6"
   },
   "devDependencies": {
-    "@julien-f/freactal": "0.4.0",
     "@nraynaud/novnc": "0.6.1",
     "@xen-orchestra/cron": "^1.0.3",
     "@xen-orchestra/defined": "^0.0.0",
@@ -58,7 +57,7 @@
     "chartist-plugin-legend": "^0.6.1",
     "chartist-plugin-tooltip": "0.0.11",
     "classnames": "^2.2.3",
-    "complex-matcher": "^0.4.0",
+    "complex-matcher": "^0.5.0",
     "cookies-js": "^1.2.2",
     "copy-to-clipboard": "^3.0.8",
     "d3": "^5.0.0",
@@ -98,6 +97,7 @@
     "promise-toolbox": "^0.10.1",
     "prop-types": "^15.6.0",
     "random-password": "^0.1.2",
+    "reaclette": "^0.7.0",
     "react": "^15.4.1",
     "react-addons-shallow-compare": "^15.6.2",
     "react-addons-test-utils": "^15.6.2",
@@ -135,11 +135,11 @@
     "watchify": "^3.7.0",
     "whatwg-fetch": "^2.0.3",
     "xml2js": "^0.4.19",
-    "xo-acl-resolver": "^0.2.4",
-    "xo-common": "^0.1.1",
+    "xo-acl-resolver": "^0.3.0",
+    "xo-common": "^0.1.2",
     "xo-lib": "^0.8.0",
     "xo-remote-parser": "^0.5.0",
-    "xo-vmdk-to-vhd": "^0.1.3"
+    "xo-vmdk-to-vhd": "^0.1.5"
   },
   "scripts": {
     "build": "NODE_ENV=production gulp build",

packages/xo-web/src/common/action-bar.js

@@ -1,5 +1,5 @@
 import ActionButton from 'action-button'
-import propTypes from 'prop-types-decorator'
+import PropTypes from 'prop-types'
 import React, { cloneElement } from 'react'
 import { noop } from 'lodash'
 
@@ -28,12 +28,12 @@ export const Action = ({
 )
 
 Action.propTypes = {
-  display: propTypes.oneOf(['icon', 'both']),
-  handler: propTypes.func.isRequired,
-  icon: propTypes.string.isRequired,
-  label: propTypes.node,
-  pending: propTypes.bool,
-  redirectOnSuccess: propTypes.string,
+  display: PropTypes.oneOf(['icon', 'both']),
+  handler: PropTypes.func.isRequired,
+  icon: PropTypes.string.isRequired,
+  label: PropTypes.node,
+  pending: PropTypes.bool,
+  redirectOnSuccess: PropTypes.string,
 }
 
 const ActionBar = ({ children, handlerParam = noop, display = 'both' }) => (
@@ -54,7 +54,7 @@ const ActionBar = ({ children, handlerParam = noop, display = 'both' }) => (
 )
 
 ActionBar.propTypes = {
-  display: propTypes.oneOf(['icon', 'both']),
-  handlerParam: propTypes.any,
+  display: PropTypes.oneOf(['icon', 'both']),
+  handlerParam: PropTypes.any,
 }
 export { ActionBar as default }

packages/xo-web/src/common/action-button.js

@@ -1,3 +1,4 @@
+import PropTypes from 'prop-types'
 import React from 'react'
 import { isFunction, startsWith } from 'lodash'
 
@@ -5,54 +6,55 @@ import Button from './button'
 import Component from './base-component'
 import Icon from './icon'
 import logError from './log-error'
-import propTypes from './prop-types-decorator'
 import Tooltip from './tooltip'
+import UserError from './user-error'
 import { error as _error } from './notification'
 
-@propTypes({
-  // React element to use as button content
-  children: propTypes.node,
-
-  // whether this button is disabled (default to false)
-  disabled: propTypes.bool,
-
-  // form identifier
-  //
-  // if provided, this button and its action are associated to this
-  // form for the submit event
-  form: propTypes.string,
-
-  // function to call when the action is triggered (via a clik on the
-  // button or submit on the form)
-  handler: propTypes.func.isRequired,
-
-  // optional value which will be passed as first param to the handler
-  //
-  // if you need multiple values, you can provide `data-*` props instead of
-  // `handlerParam`
-  handlerParam: propTypes.any,
-
-  // XO icon to use for this button
-  icon: propTypes.string.isRequired,
-
-  // the color of the xo icon
-  iconColor: propTypes.string,
-
-  // whether the action of this action is already underway
-  pending: propTypes.bool,
-
-  // path to redirect to when the triggered action finish successfully
-  //
-  // if a function, it will be called with the result of the action to
-  // compute the path
-  redirectOnSuccess: propTypes.oneOfType([propTypes.func, propTypes.string]),
-
-  // React element to use tooltip for the component
-  tooltip: propTypes.node,
-})
 export default class ActionButton extends Component {
   static contextTypes = {
-    router: propTypes.object,
+    router: PropTypes.object,
+  }
+
+  static propTypes = {
+    // React element to use as button content
+    children: PropTypes.node,
+
+    // whether this button is disabled (default to false)
+    disabled: PropTypes.bool,
+
+    // form identifier
+    //
+    // if provided, this button and its action are associated to this
+    // form for the submit event
+    form: PropTypes.string,
+
+    // function to call when the action is triggered (via a clik on the
+    // button or submit on the form)
+    handler: PropTypes.func.isRequired,
+
+    // optional value which will be passed as first param to the handler
+    //
+    // if you need multiple values, you can provide `data-*` props instead of
+    // `handlerParam`
+    handlerParam: PropTypes.any,
+
+    // XO icon to use for this button
+    icon: PropTypes.string.isRequired,
+
+    // the color of the xo icon
+    iconColor: PropTypes.string,
+
+    // whether the action of this action is already underway
+    pending: PropTypes.bool,
+
+    // path to redirect to when the triggered action finish successfully
+    //
+    // if a function, it will be called with the result of the action to
+    // compute the path
+    redirectOnSuccess: PropTypes.oneOfType([PropTypes.func, PropTypes.string]),
+
+    // React element to use tooltip for the component
+    tooltip: PropTypes.node,
   }
 
   async _execute () {
@@ -110,11 +112,15 @@ export default class ActionButton extends Component {
 
       // ignore when undefined because it usually means that the action has been canceled
       if (error !== undefined) {
-        logError(error)
-        _error(
-          children || tooltip || error.name,
-          error.message || String(error)
-        )
+        if (error instanceof UserError) {
+          _error(error.title, error.body)
+        } else {
+          logError(error)
+          _error(
+            children || tooltip || error.name,
+            error.message || String(error)
+          )
+        }
       }
     }
   }

packages/xo-web/src/common/action-toggle.js

@@ -1,7 +1,7 @@
 import React from 'react'
+import PropTypes from 'prop-types'
 
 import ActionButton from './action-button'
-import propTypes from './prop-types-decorator'
 
 const ActionToggle = ({ className, value, ...props }) => (
   <ActionButton
@@ -10,7 +10,8 @@ const ActionToggle = ({ className, value, ...props }) => (
     icon={value ? 'toggle-on' : 'toggle-off'}
   />
 )
+ActionToggle.propTypes = {
+  value: PropTypes.bool,
+}
 
-export default propTypes({
-  value: propTypes.bool,
-})(ActionToggle)
+export { ActionToggle as default }

packages/xo-web/src/common/apply-decorators.js

@@ -0,0 +1,2 @@
+const apply = (value, fn) => fn(value)
+export default fns => fns.reduceRight(apply)

packages/xo-web/src/common/button-link.js

@@ -1,8 +1,8 @@
 import React from 'react'
 import { routerShape } from 'react-router/lib/PropTypes'
+import PropTypes from 'prop-types'
 
 import Button from './button'
-import propTypes from './prop-types-decorator'
 
 const ButtonLink = ({ to, ...props }, { router }) => {
   props.onClick = () => {
@@ -12,17 +12,12 @@ const ButtonLink = ({ to, ...props }, { router }) => {
   return <Button {...props} />
 }
 
-propTypes(
-  {
-    to: propTypes.oneOfType([
-      propTypes.func,
-      propTypes.object,
-      propTypes.string,
-    ]),
-  },
-  {
-    router: routerShape,
-  }
-)(ButtonLink)
+ButtonLink.contextTypes = {
+  router: routerShape,
+}
+
+ButtonLink.propTypes = {
+  to: PropTypes.oneOfType([PropTypes.func, PropTypes.object, PropTypes.string]),
+}
 
 export { ButtonLink as default }

packages/xo-web/src/common/button.js

@@ -1,7 +1,6 @@
 import classNames from 'classnames'
 import React from 'react'
-
-import propTypes from './prop-types-decorator'
+import PropTypes from 'prop-types'
 
 const Button = ({
   active,
@@ -27,9 +26,9 @@ const Button = ({
   return <button {...props}>{children}</button>
 }
 
-propTypes({
-  active: propTypes.bool,
-  block: propTypes.bool,
+Button.propTypes = {
+  active: PropTypes.bool,
+  block: PropTypes.bool,
 
   // Bootstrap button style
   //
@@ -37,7 +36,7 @@ propTypes({
   //
   // The default value (secondary) is not listed here because it does
   // not make sense to explicit it.
-  btnStyle: propTypes.oneOf([
+  btnStyle: PropTypes.oneOf([
     'danger',
     'info',
     'link',
@@ -46,8 +45,8 @@ propTypes({
     'warning',
   ]),
 
-  outline: propTypes.bool,
-  size: propTypes.oneOf(['large', 'small']),
-})(Button)
+  outline: PropTypes.bool,
+  size: PropTypes.oneOf(['large', 'small']),
+}
 
 export { Button as default }

packages/xo-web/src/common/card.js

@@ -1,6 +1,5 @@
 import React from 'react'
-
-import propTypes from './prop-types-decorator'
+import PropTypes from 'prop-types'
 
 const CARD_STYLE = {
   minHeight: '100%',
@@ -16,9 +15,7 @@ const CARD_HEADER_STYLE = {
   textAlign: 'center',
 }
 
-export const Card = propTypes({
-  shadow: propTypes.bool,
-})(({ shadow, ...props }) => {
+export const Card = ({ shadow, ...props }) => {
   props.className = 'card'
   props.style = {
     ...props.style,
@@ -26,18 +23,26 @@ export const Card = propTypes({
   }
 
   return <div {...props} />
-})
+}
 
-export const CardHeader = propTypes({
-  className: propTypes.string,
-})(({ children, className }) => (
+Card.propTypes = {
+  shadow: PropTypes.bool,
+}
+
+export const CardHeader = ({ children, className }) => (
   <h4 className={`card-header ${className || ''}`} style={CARD_HEADER_STYLE}>
     {children}
   </h4>
-))
+)
 
-export const CardBlock = propTypes({
-  className: propTypes.string,
-})(({ children, className }) => (
+CardHeader.propTypes = {
+  className: PropTypes.string,
+}
+
+export const CardBlock = ({ children, className }) => (
   <div className={`card-block ${className || ''}`}>{children}</div>
-))
+)
+
+CardBlock.propTypes = {
+  className: PropTypes.string,
+}

packages/xo-web/src/common/collapse.js

@@ -1,18 +1,19 @@
 import React from 'react'
+import PropTypes from 'prop-types'
 
 import Button from './button'
 import Component from './base-component'
 import Icon from './icon'
-import propTypes from './prop-types-decorator'
 
-@propTypes({
-  buttonText: propTypes.any.isRequired,
-  children: propTypes.any.isRequired,
-  className: propTypes.string,
-  defaultOpen: propTypes.bool,
-  size: propTypes.string,
-})
 export default class Collapse extends Component {
+  static propTypes = {
+    buttonText: PropTypes.any.isRequired,
+    children: PropTypes.any.isRequired,
+    className: PropTypes.string,
+    defaultOpen: PropTypes.bool,
+    size: PropTypes.string,
+  }
+
   state = {
     isOpened: this.props.defaultOpen,
   }

packages/xo-web/src/common/combobox.js

@@ -1,24 +1,25 @@
 import React from 'react'
+import PropTypes from 'prop-types'
 import uncontrollableInput from 'uncontrollable-input'
 import { isEmpty, map } from 'lodash'
 import { DropdownButton, MenuItem } from 'react-bootstrap-4/lib'
 
 import Component from './base-component'
-import propTypes from './prop-types-decorator'
 
 @uncontrollableInput({
   defaultValue: '',
 })
-@propTypes({
-  disabled: propTypes.bool,
-  options: propTypes.oneOfType([
-    propTypes.arrayOf(propTypes.string),
-    propTypes.objectOf(propTypes.string),
-  ]),
-  onChange: propTypes.func.isRequired,
-  value: propTypes.string.isRequired,
-})
 export default class Combobox extends Component {
+  static propTypes = {
+    disabled: PropTypes.bool,
+    options: PropTypes.oneOfType([
+      PropTypes.arrayOf(PropTypes.string),
+      PropTypes.objectOf(PropTypes.string),
+    ]),
+    onChange: PropTypes.func.isRequired,
+    value: PropTypes.string.isRequired,
+  }
+
   _handleChange = event => {
     this.props.onChange(event.target.value)
   }

packages/xo-web/src/common/copiable/index.js

@@ -1,19 +1,16 @@
 import CopyToClipboard from 'react-copy-to-clipboard'
 import classNames from 'classnames'
 import React, { createElement } from 'react'
+import PropTypes from 'prop-types'
 
 import _ from '../intl'
 import Button from '../button'
 import Icon from '../icon'
-import propTypes from '../prop-types-decorator'
 import Tooltip from '../tooltip'
 
 import styles from './index.css'
 
-const Copiable = propTypes({
-  data: propTypes.string,
-  tagName: propTypes.string,
-})(({ className, tagName = 'span', ...props }) =>
+const Copiable = ({ className, tagName = 'span', ...props }) =>
   createElement(
     tagName,
     {
@@ -30,5 +27,10 @@ const Copiable = propTypes({
       </CopyToClipboard>
     </Tooltip>
   )
-)
+
+Copiable.propTypes = {
+  data: PropTypes.string,
+  tagName: PropTypes.string,
+}
+
 export { Copiable as default }

packages/xo-web/src/common/dropzone/index.js

@@ -1,16 +1,17 @@
 import Component from 'base-component'
-import propTypes from 'prop-types-decorator'
+import PropTypes from 'prop-types'
 import React from 'react'
 import ReactDropzone from 'react-dropzone'
 
 import styles from './index.css'
 
-@propTypes({
-  onDrop: propTypes.func,
-  message: propTypes.node,
-  multiple: propTypes.bool,
-})
 export default class Dropzone extends Component {
+  static propTypes = {
+    onDrop: PropTypes.func,
+    message: PropTypes.node,
+    multiple: PropTypes.bool,
+  }
+
   render () {
     const { onDrop, message, multiple } = this.props
 

packages/xo-web/src/common/editable/index.js

@@ -1,5 +1,6 @@
 import classNames from 'classnames'
 import React from 'react'
+import PropTypes from 'prop-types'
 import {
   findKey,
   isEmpty,
@@ -15,7 +16,6 @@ import Component from '../base-component'
 import getEventValue from '../get-event-value'
 import Icon from '../icon'
 import logError from '../log-error'
-import propTypes from '../prop-types-decorator'
 import Tooltip from '../tooltip'
 import { formatSize } from '../utils'
 import { SizeInput } from '../form'
@@ -38,10 +38,11 @@ import styles from './index.css'
 
 const LONG_CLICK = 400
 
-@propTypes({
-  alt: propTypes.node.isRequired,
-})
 class Hover extends Component {
+  static propTypes = {
+    alt: PropTypes.node.isRequired,
+  }
+
   constructor () {
     super()
 
@@ -64,13 +65,14 @@ class Hover extends Component {
 
 // it supports 'data-*': optional params,
 // wich will be passed as an object to the 'onChange' and the 'onUndo' functions
-@propTypes({
-  onChange: propTypes.func.isRequired,
-  onUndo: propTypes.oneOfType([propTypes.bool, propTypes.func]),
-  useLongClick: propTypes.bool,
-  value: propTypes.any.isRequired,
-})
 class Editable extends Component {
+  static propTypes = {
+    onChange: PropTypes.func.isRequired,
+    onUndo: PropTypes.oneOfType([PropTypes.bool, PropTypes.func]),
+    useLongClick: PropTypes.bool,
+    value: PropTypes.any.isRequired,
+  }
+
   get value () {
     throw new Error('not implemented')
   }
@@ -221,14 +223,15 @@ class Editable extends Component {
   }
 }
 
-@propTypes({
-  autoComplete: propTypes.string,
-  maxLength: propTypes.number,
-  minLength: propTypes.number,
-  pattern: propTypes.string,
-  value: propTypes.string.isRequired,
-})
 export class Text extends Editable {
+  static propTypes = {
+    autoComplete: PropTypes.string,
+    maxLength: PropTypes.number,
+    minLength: PropTypes.number,
+    pattern: PropTypes.string,
+    value: PropTypes.string.isRequired,
+  }
+
   get value () {
     const { input } = this.refs
 
@@ -302,11 +305,12 @@ export class Password extends Text {
   _isPassword = true
 }
 
-@propTypes({
-  nullable: propTypes.bool,
-  value: propTypes.number,
-})
 export class Number extends Component {
+  static propTypes = {
+    nullable: PropTypes.bool,
+    value: PropTypes.number,
+  }
+
   get value () {
     return +this.refs.input.value
   }
@@ -337,11 +341,13 @@ export class Number extends Component {
   }
 }
 
-@propTypes({
-  options: propTypes.oneOfType([propTypes.array, propTypes.object]).isRequired,
-  renderer: propTypes.func,
-})
 export class Select extends Editable {
+  static propTypes = {
+    options: PropTypes.oneOfType([PropTypes.array, PropTypes.object])
+      .isRequired,
+    renderer: PropTypes.func,
+  }
+
   componentWillReceiveProps (props) {
     if (
       props.value !== this.props.value ||
@@ -418,10 +424,11 @@ const MAP_TYPE_SELECT = {
   'VM-template': SelectVmTemplate,
 }
 
-@propTypes({
-  value: propTypes.oneOfType([propTypes.string, propTypes.object]),
-})
 export class XoSelect extends Editable {
+  static propTypes = {
+    value: PropTypes.oneOfType([PropTypes.string, PropTypes.object]),
+  }
+
   get value () {
     return this.state.value
   }
@@ -461,10 +468,11 @@ export class XoSelect extends Editable {
   }
 }
 
-@propTypes({
-  value: propTypes.number.isRequired,
-})
 export class Size extends Editable {
+  static propTypes = {
+    value: PropTypes.number.isRequired,
+  }
+
   get value () {
     return this.refs.input.value
   }

packages/xo-web/src/common/form-grid.js

@@ -1,18 +1,28 @@
 import React from 'react'
+import PropTypes from 'prop-types'
 
 import * as Grid from './grid'
-import propTypes from './prop-types-decorator'
 
-export const LabelCol = propTypes({
-  children: propTypes.any.isRequired,
-})(({ children }) => (
+export const LabelCol = ({ children }) => (
   <label className='col-md-2 form-control-label'>{children}</label>
-))
+)
 
-export const InputCol = propTypes({
-  children: propTypes.any.isRequired,
-})(({ children }) => <Grid.Col mediumSize={10}>{children}</Grid.Col>)
+LabelCol.propTypes = {
+  children: PropTypes.any.isRequired,
+}
 
-export const Row = propTypes({
-  children: propTypes.arrayOf(propTypes.element).isRequired,
-})(({ children }) => <Grid.Row className='form-group'>{children}</Grid.Row>)
+export const InputCol = ({ children }) => (
+  <Grid.Col mediumSize={10}>{children}</Grid.Col>
+)
+
+InputCol.propTypes = {
+  children: PropTypes.any.isRequired,
+}
+
+export const Row = ({ children }) => (
+  <Grid.Row className='form-group'>{children}</Grid.Row>
+)
+
+Row.propTypes = {
+  children: PropTypes.arrayOf(PropTypes.element).isRequired,
+}

packages/xo-web/src/common/form/index.js

@@ -3,6 +3,7 @@ import classNames from 'classnames'
 import defined from '@xen-orchestra/defined'
 import Icon from 'icon'
 import map from 'lodash/map'
+import PropTypes from 'prop-types'
 import randomPassword from 'random-password'
 import React from 'react'
 import round from 'lodash/round'
@@ -13,7 +14,6 @@ import { DropdownButton, MenuItem } from 'react-bootstrap-4/lib'
 import Button from '../button'
 import Component from '../base-component'
 import getEventValue from '../get-event-value'
-import propTypes from '../prop-types-decorator'
 import { formatSizeRaw, parseSize } from '../utils'
 
 export Number from './number'
@@ -21,10 +21,11 @@ export Select from './select'
 
 // ===================================================================
 
-@propTypes({
-  enableGenerator: propTypes.bool,
-})
 export class Password extends Component {
+  static propTypes = {
+    enableGenerator: PropTypes.bool,
+  }
+
   get value () {
     return this.refs.field.value
   }
@@ -86,19 +87,21 @@ export class Password extends Component {
 
 // ===================================================================
 
-@propTypes({
-  max: propTypes.number.isRequired,
-  min: propTypes.number.isRequired,
-  onChange: propTypes.func,
-  step: propTypes.number,
-  value: propTypes.number,
-})
 export class Range extends Component {
-  componentDidMount () {
-    const { min, onChange, value } = this.props
+  static propTypes = {
+    max: PropTypes.number.isRequired,
+    min: PropTypes.number.isRequired,
+    onChange: PropTypes.func,
+    required: PropTypes.bool,
+    step: PropTypes.number,
+    value: PropTypes.number,
+  }
 
-    if (!value) {
-      onChange && onChange(min)
+  componentDidMount () {
+    const { min, onChange, required, value } = this.props
+
+    if (value === undefined && required) {
+      onChange !== undefined && onChange(min)
     }
   }
 
@@ -111,7 +114,7 @@ export class Range extends Component {
       <Container>
         <SingleLineRow>
           <Col size={2}>
-            <span className='pull-right'>{value}</span>
+            {value !== undefined && <span className='pull-right'>{value}</span>}
           </Col>
           <Col size={10}>
             <input
@@ -121,7 +124,7 @@ export class Range extends Component {
               onChange={this._onChange}
               step={step}
               type='range'
-              value={value}
+              value={value !== undefined ? value : min}
             />
           </Col>
         </SingleLineRow>
@@ -134,19 +137,21 @@ export Toggle from './toggle'
 
 const UNITS = ['kiB', 'MiB', 'GiB']
 const DEFAULT_UNIT = 'GiB'
-@propTypes({
-  autoFocus: propTypes.bool,
-  className: propTypes.string,
-  defaultUnit: propTypes.oneOf(UNITS),
-  defaultValue: propTypes.number,
-  onChange: propTypes.func,
-  placeholder: propTypes.string,
-  readOnly: propTypes.bool,
-  required: propTypes.bool,
-  style: propTypes.object,
-  value: propTypes.oneOfType([propTypes.number, propTypes.oneOf([null])]),
-})
+
 export class SizeInput extends BaseComponent {
+  static propTypes = {
+    autoFocus: PropTypes.bool,
+    className: PropTypes.string,
+    defaultUnit: PropTypes.oneOf(UNITS),
+    defaultValue: PropTypes.number,
+    onChange: PropTypes.func,
+    placeholder: PropTypes.string,
+    readOnly: PropTypes.bool,
+    required: PropTypes.bool,
+    style: PropTypes.object,
+    value: PropTypes.oneOfType([PropTypes.number, PropTypes.oneOf([null])]),
+  }
+
   constructor (props) {
     super(props)
 

packages/xo-web/src/common/form/number.js

@@ -1,8 +1,10 @@
 import PropTypes from 'prop-types'
 import React from 'react'
-import { injectState, provideState } from '@julien-f/freactal'
+import { injectState, provideState } from 'reaclette'
 
-const Number_ = [
+import decorate from '../apply-decorators'
+
+const Number_ = decorate([
   provideState({
     effects: {
       onChange: (_, { target: { value } }) => (state, props) => {
@@ -30,7 +32,7 @@ const Number_ = [
       value={value === undefined ? '' : String(value)}
     />
   ),
-].reduceRight((value, decorator) => decorator(value))
+])
 
 Number_.propTypes = {
   onChange: PropTypes.func.isRequired,