feat(smart-selector): initial commit

Fixes xoa-support#1378
Fixes xoa-support#1384
Fixes xoa-support#1399

.eslintrc.js

@@ -1,5 +1,11 @@
 module.exports = {
-  extends: ['standard', 'standard-jsx', 'prettier'],
+  extends: [
+    'standard',
+    'standard-jsx',
+    'prettier',
+    'prettier/standard',
+    'prettier/react',
+  ],
   globals: {
     __DEV__: true,
     $Dict: true,
@@ -10,6 +16,16 @@ module.exports = {
     $PropertyType: true,
     $Shape: true,
   },
+
+  overrides: [
+    {
+      files: ['packages/*cli*/**/*.js', '*-cli.js'],
+      rules: {
+        'no-console': 'off',
+      },
+    },
+  ],
+
   parser: 'babel-eslint',
   parserOptions: {
     ecmaFeatures: {
@@ -17,12 +33,10 @@ module.exports = {
     },
   },
   rules: {
+    'no-console': ['error', { allow: ['warn', 'error'] }],
     'no-var': 'error',
     'node/no-extraneous-import': 'error',
     'node/no-extraneous-require': 'error',
     'prefer-const': 'error',
-
-    // See https://github.com/prettier/eslint-config-prettier/issues/65
-    'react/jsx-indent': 'off',
   },
 }

@xen-orchestra/async-map/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/async-map",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/async-map",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

@xen-orchestra/babel-config/package.json

@@ -5,6 +5,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/babel-config",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/babel-config",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   }

@xen-orchestra/cr-seed-cli/index.js

@@ -82,35 +82,26 @@ ${cliName} v${pkg.version}
   )
 
   await Promise.all([
-    srcXapi.setFieldEntries(srcSnapshot, 'other_config', metadata),
-    srcXapi.setFieldEntries(srcSnapshot, 'other_config', {
-      'xo:backup:exported': 'true',
-    }),
-    tgtXapi.setField(
-      tgtVm,
-      'name_label',
-      `${srcVm.name_label} (${srcSnapshot.snapshot_time})`
-    ),
-    tgtXapi.setFieldEntries(tgtVm, 'other_config', metadata),
-    tgtXapi.setFieldEntries(tgtVm, 'other_config', {
+    srcSnapshot.update_other_config(metadata),
+    srcSnapshot.update_other_config('xo:backup:exported', 'true'),
+    tgtVm.set_name_label(`${srcVm.name_label} (${srcSnapshot.snapshot_time})`),
+    tgtVm.update_other_config(metadata),
+    tgtVm.update_other_config({
       'xo:backup:sr': tgtSr.uuid,
       'xo:copy_of': srcSnapshotUuid,
     }),
-    tgtXapi.setFieldEntries(tgtVm, 'blocked_operations', {
-      start:
-        'Start operation for this vm is blocked, clone it if you want to use it.',
-    }),
+    tgtVm.update_blocked_operations(
+      'start',
+      'Start operation for this vm is blocked, clone it if you want to use it.'
+    ),
     Promise.all(
       userDevices.map(userDevice => {
         const srcDisk = srcDisks[userDevice]
         const tgtDisk = tgtDisks[userDevice]
 
-        return tgtXapi.setFieldEntry(
-          tgtDisk,
-          'other_config',
-          'xo:copy_of',
-          srcDisk.uuid
-        )
+        return tgtDisk.update_other_config({
+          'xo:copy_of': srcDisk.uuid,
+        })
       })
     ),
   ])

@xen-orchestra/cr-seed-cli/package.json

@@ -4,6 +4,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/cr-seed-cli",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/cr-seed-cli",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -15,6 +16,6 @@
   },
   "dependencies": {
     "golike-defer": "^0.4.1",
-    "xen-api": "^0.24.2"
+    "xen-api": "^0.24.5"
   }
 }

@xen-orchestra/cron/package.json

@@ -17,6 +17,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/cron",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/cron",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

@xen-orchestra/defined/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/defined",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/defined",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

@xen-orchestra/emit-async/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/emit-async",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/emit-async",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

@xen-orchestra/fs/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@xen-orchestra/fs",
-  "version": "0.6.0",
+  "version": "0.7.1",
   "license": "AGPL-3.0",
   "description": "The File System for Xen Orchestra backups.",
   "keywords": [],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/fs",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -23,11 +24,12 @@
     "@marsaud/smb2": "^0.13.0",
     "@sindresorhus/df": "^2.1.0",
     "@xen-orchestra/async-map": "^0.0.0",
+    "decorator-synchronized": "^0.5.0",
     "execa": "^1.0.0",
     "fs-extra": "^7.0.0",
     "get-stream": "^4.0.0",
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.11.0",
+    "promise-toolbox": "^0.12.1",
     "readable-stream": "^3.0.6",
     "through2": "^3.0.0",
     "tmp": "^0.0.33",
@@ -43,7 +45,7 @@
     "async-iterator-to-stream": "^1.1.0",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^5.1.3",
-    "dotenv": "^6.1.0",
+    "dotenv": "^7.0.0",
     "index-modules": "^0.3.0",
     "rimraf": "^2.6.2"
   },

@xen-orchestra/fs/src/_mount.js

@@ -1,5 +1,6 @@
 import execa from 'execa'
 import fs from 'fs-extra'
+import { ignoreErrors } from 'promise-toolbox'
 import { join } from 'path'
 import { tmpdir } from 'os'
 
@@ -21,11 +22,12 @@ export default class MountHandler extends LocalHandler {
     super(remote, opts)
 
     this._execa = useSudo ? sudoExeca : execa
+    this._keeper = undefined
     this._params = {
       ...params,
-      options: [params.options, remote.options].filter(
-        _ => _ !== undefined
-      ).join(','),
+      options: [params.options, remote.options]
+        .filter(_ => _ !== undefined)
+        .join(','),
     }
     this._realPath = join(
       mountsDir,
@@ -37,19 +39,20 @@ export default class MountHandler extends LocalHandler {
   }
 
   async _forget() {
-    await this._execa('umount', ['--force', this._getRealPath()], {
-      env: {
-        LANG: 'C',
-      },
-    }).catch(error => {
-      if (
-        error == null ||
-        typeof error.stderr !== 'string' ||
-        !error.stderr.includes('not mounted')
-      ) {
-        throw error
-      }
-    })
+    const keeper = this._keeper
+    if (keeper === undefined) {
+      return
+    }
+    this._keeper = undefined
+    await fs.close(keeper)
+
+    await ignoreErrors.call(
+      this._execa('umount', [this._getRealPath()], {
+        env: {
+          LANG: 'C',
+        },
+      })
+    )
   }
 
   _getRealPath() {
@@ -57,26 +60,49 @@ export default class MountHandler extends LocalHandler {
   }
 
   async _sync() {
-    await fs.ensureDir(this._getRealPath())
-    const { type, device, options, env } = this._params
-    return this._execa(
-      'mount',
-      ['-t', type, device, this._getRealPath(), '-o', options],
-      {
-        env: {
-          LANG: 'C',
-          ...env,
-        },
+    // in case of multiple `sync`s, ensure we properly close previous keeper
+    {
+      const keeper = this._keeper
+      if (keeper !== undefined) {
+        this._keeper = undefined
+        ignoreErrors.call(fs.close(keeper))
       }
-    ).catch(error => {
-      let stderr
-      if (
-        error == null ||
-        typeof (stderr = error.stderr) !== 'string' ||
-        !(stderr.includes('already mounted') || stderr.includes('busy'))
-      ) {
+    }
+
+    const realPath = this._getRealPath()
+
+    await fs.ensureDir(realPath)
+
+    try {
+      const { type, device, options, env } = this._params
+      await this._execa(
+        'mount',
+        ['-t', type, device, realPath, '-o', options],
+        {
+          env: {
+            LANG: 'C',
+            ...env,
+          },
+        }
+      )
+    } catch (error) {
+      try {
+        // the failure may mean it's already mounted, use `findmnt` to check
+        // that's the case
+        await this._execa('findmnt', [realPath], {
+          stdio: 'ignore',
+        })
+      } catch (_) {
         throw error
       }
-    })
+    }
+
+    // keep an open file on the mount to prevent it from being unmounted if used
+    // by another handler/process
+    const keeperPath = `${realPath}/.keeper_${Math.random()
+      .toString(36)
+      .slice(2)}`
+    this._keeper = await fs.open(keeperPath, 'w')
+    ignoreErrors.call(fs.unlink(keeperPath))
   }
 }

@xen-orchestra/fs/src/abstract.js

@@ -5,6 +5,7 @@ import getStream from 'get-stream'
 
 import asyncMap from '@xen-orchestra/async-map'
 import path from 'path'
+import synchronized from 'decorator-synchronized'
 import { fromCallback, fromEvent, ignoreErrors, timeout } from 'promise-toolbox'
 import { parse } from 'xo-remote-parser'
 import { randomBytes } from 'crypto'
@@ -24,6 +25,10 @@ type RemoteInfo = { used?: number, size?: number }
 type File = FileDescriptor | string
 
 const checksumFile = file => file + '.checksum'
+const computeRate = (hrtime: number[], size: number) => {
+  const seconds = hrtime[0] + hrtime[1] / 1e9
+  return size / seconds
+}
 
 const DEFAULT_TIMEOUT = 6e5 // 10 min
 
@@ -34,18 +39,18 @@ const ignoreEnoent = error => {
 }
 
 class PrefixWrapper {
-  constructor(remote, prefix) {
+  constructor(handler, prefix) {
     this._prefix = prefix
-    this._remote = remote
+    this._handler = handler
   }
 
   get type() {
-    return this._remote.type
+    return this._handler.type
   }
 
   // necessary to remove the prefix from the path with `prependDir` option
   async list(dir, opts) {
-    const entries = await this._remote.list(this._resolve(dir), opts)
+    const entries = await this._handler.list(this._resolve(dir), opts)
     if (opts != null && opts.prependDir) {
       const n = this._prefix.length
       entries.forEach((entry, i, entries) => {
@@ -56,7 +61,7 @@ class PrefixWrapper {
   }
 
   rename(oldPath, newPath) {
-    return this._remote.rename(this._resolve(oldPath), this._resolve(newPath))
+    return this._handler.rename(this._resolve(oldPath), this._resolve(newPath))
   }
 
   _resolve(path) {
@@ -216,6 +221,7 @@ export default class RemoteHandlerAbstract {
   // FIXME: Some handlers are implemented based on system-wide mecanisms (such
   // as mount), forgetting them might breaking other processes using the same
   // remote.
+  @synchronized()
   async forget(): Promise<void> {
     await this._forget()
   }
@@ -354,23 +360,33 @@ export default class RemoteHandlerAbstract {
   // metadata
   //
   // This method MUST ALWAYS be called before using the handler.
+  @synchronized()
   async sync(): Promise<void> {
     await this._sync()
   }
 
   async test(): Promise<Object> {
+    const SIZE = 1024 * 1024 * 10
     const testFileName = normalizePath(`${Date.now()}.test`)
-    const data = await fromCallback(cb => randomBytes(1024 * 1024, cb))
+    const data = await fromCallback(cb => randomBytes(SIZE, cb))
     let step = 'write'
     try {
+      const writeStart = process.hrtime()
       await this._outputFile(testFileName, data, { flags: 'wx' })
+      const writeDuration = process.hrtime(writeStart)
+
       step = 'read'
+      const readStart = process.hrtime()
       const read = await this._readFile(testFileName, { flags: 'r' })
+      const readDuration = process.hrtime(readStart)
+
       if (!data.equals(read)) {
         throw new Error('output and input did not match')
       }
       return {
         success: true,
+        writeRate: computeRate(writeDuration, SIZE),
+        readRate: computeRate(readDuration, SIZE),
       }
     } catch (error) {
       return {
@@ -565,7 +581,7 @@ function createPrefixWrapperMethods() {
       if (arguments.length !== 0 && typeof (path = arguments[0]) === 'string') {
         arguments[0] = this._resolve(path)
       }
-      return value.apply(this._remote, arguments)
+      return value.apply(this._handler, arguments)
     }
 
     defineProperty(pPw, name, descriptor)

@xen-orchestra/fs/src/fs.spec.js

@@ -290,9 +290,11 @@ handlers.forEach(url => {
 
     describe('#test()', () => {
       it('tests the remote appears to be working', async () => {
-        expect(await handler.test()).toEqual({
-          success: true,
-        })
+        const answer = await handler.test()
+
+        expect(answer.success).toBe(true)
+        expect(typeof answer.writeRate).toBe('number')
+        expect(typeof answer.readRate).toBe('number')
       })
     })
 

@xen-orchestra/log/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/log",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/log",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -30,7 +31,7 @@
   },
   "dependencies": {
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.11.0"
+    "promise-toolbox": "^0.12.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

@xen-orchestra/mixin/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/mixin",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/mixin",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

CHANGELOG.md

@@ -1,5 +1,99 @@
 # ChangeLog
 
+## Next (2019-03-19)
+
+### Enhancements
+
+- [SR/Disk] Disable actions on unmanaged VDIs [#3988](https://github.com/vatesfr/xen-orchestra/issues/3988) (PR [#4000](https://github.com/vatesfr/xen-orchestra/pull/4000))
+- [Pool] Specify automatic networks on a Pool [#3916](https://github.com/vatesfr/xen-orchestra/issues/3916) (PR [#3958](https://github.com/vatesfr/xen-orchestra/pull/3958))
+- [VM/advanced] Manage start delay for VM [#3909](https://github.com/vatesfr/xen-orchestra/issues/3909) (PR [#4002](https://github.com/vatesfr/xen-orchestra/pull/4002))
+- [New/Vm] SR section: Display warning message when the selected SRs aren't in the same host [#3911](https://github.com/vatesfr/xen-orchestra/issues/3911) (PR [#3967](https://github.com/vatesfr/xen-orchestra/pull/3967))
+- Enable compression for HTTP requests (and initial objects fetch)
+- [VDI migration] Display same-pool SRs first in the selector [#3945](https://github.com/vatesfr/xen-orchestra/issues/3945) (PR [#3996](https://github.com/vatesfr/xen-orchestra/pull/3996))
+- [Home] Save the current page in url [#3993](https://github.com/vatesfr/xen-orchestra/issues/3993) (PR [#3999](https://github.com/vatesfr/xen-orchestra/pull/3999))
+- [VDI] Ensure suspend VDI is destroyed when destroying a VM  [#4027](https://github.com/vatesfr/xen-orchestra/issues/4027) (PR [#4038](https://github.com/vatesfr/xen-orchestra/pull/4038))
+- [VM/disk]: Warning when 2 VDIs are on 2 different hosts' local SRs [#3911](https://github.com/vatesfr/xen-orchestra/issues/3911) (PR [#3969](https://github.com/vatesfr/xen-orchestra/pull/3969))
+
+### Bug fixes
+
+- [New network] PIF was wrongly required which prevented from creating a private network (PR [#4010](https://github.com/vatesfr/xen-orchestra/pull/4010))
+- [Google authentication] Migrate to new endpoint
+- [Backup NG] Better handling of huge logs [#4025](https://github.com/vatesfr/xen-orchestra/issues/4025) (PR [#4026](https://github.com/vatesfr/xen-orchestra/pull/4026))
+- [Home/VM] Bulk migration: fixed VM VDIs not migrated to the selected SR [#3986](https://github.com/vatesfr/xen-orchestra/issues/3986) (PR [#3987](https://github.com/vatesfr/xen-orchestra/pull/3987))
+- [Stats] Fix cache usage with simultaneous requests [#4017](https://github.com/vatesfr/xen-orchestra/issues/4017) (PR [#4028](https://github.com/vatesfr/xen-orchestra/pull/4028))
+- [Backup NG] Fix compression displayed for the wrong backup mode (PR [#4021](https://github.com/vatesfr/xen-orchestra/pull/4021))
+
+## **5.32.2** (2019-02-28)
+
+### Bug fixes
+
+- Fix XAPI events monitoring on old version (XenServer 7.2)
+
+## **5.32.1** (2019-02-28)
+
+### Bug fixes
+
+- Fix a very short timeout in the monitoring of XAPI events which may lead to unresponsive XenServer hosts
+
+## **5.32.0** (2019-02-28)
+
+### Enhancements
+
+- [VM migration] Display same-pool hosts first in the selector [#3262](https://github.com/vatesfr/xen-orchestra/issues/3262) (PR [#3890](https://github.com/vatesfr/xen-orchestra/pull/3890))
+- [Home/VM] Sort VM by start time [#3955](https://github.com/vatesfr/xen-orchestra/issues/3955) (PR [#3970](https://github.com/vatesfr/xen-orchestra/pull/3970))
+- [Editable fields] Unfocusing (clicking outside) submits the change instead of canceling (PR [#3980](https://github.com/vatesfr/xen-orchestra/pull/3980))
+- [Network] Dedicated page for network creation [#3895](https://github.com/vatesfr/xen-orchestra/issues/3895) (PR [#3906](https://github.com/vatesfr/xen-orchestra/pull/3906))
+- [Logs] Add button to download the log [#3957](https://github.com/vatesfr/xen-orchestra/issues/3957) (PR [#3985](https://github.com/vatesfr/xen-orchestra/pull/3985))
+- [Continuous Replication] Share full copy between schedules [#3973](https://github.com/vatesfr/xen-orchestra/issues/3973) (PR [#3995](https://github.com/vatesfr/xen-orchestra/pull/3995))
+- [Backup] Ability to backup XO configuration and pool metadata [#808](https://github.com/vatesfr/xen-orchestra/issues/808) [#3501](https://github.com/vatesfr/xen-orchestra/issues/3501) (PR [#3912](https://github.com/vatesfr/xen-orchestra/pull/3912))
+
+### Bug fixes
+
+- [Host] Fix multipathing status for XenServer < 7.5 [#3956](https://github.com/vatesfr/xen-orchestra/issues/3956) (PR [#3961](https://github.com/vatesfr/xen-orchestra/pull/3961))
+- [Home/VM] Show creation date of the VM on if it available [#3953](https://github.com/vatesfr/xen-orchestra/issues/3953) (PR [#3959](https://github.com/vatesfr/xen-orchestra/pull/3959))
+- [Notifications] Fix invalid notifications when not registered (PR [#3966](https://github.com/vatesfr/xen-orchestra/pull/3966))
+- [Import] Fix import of some OVA files [#3962](https://github.com/vatesfr/xen-orchestra/issues/3962) (PR [#3974](https://github.com/vatesfr/xen-orchestra/pull/3974))
+- [Servers] Fix *already connected error* after a server has been removed during connection [#3976](https://github.com/vatesfr/xen-orchestra/issues/3976) (PR [#3977](https://github.com/vatesfr/xen-orchestra/pull/3977))
+- [Backup] Fix random _mount_ issues with NFS/SMB remotes [#3973](https://github.com/vatesfr/xen-orchestra/issues/3973) (PR [#4003](https://github.com/vatesfr/xen-orchestra/pull/4003))
+
+### Released packages
+
+- @xen-orchestra/fs v0.7.0
+- xen-api v0.24.3
+- xoa-updater v0.15.2
+- xo-server v5.36.0
+- xo-web v5.36.0
+
+## **5.31.2** (2019-02-08)
+
+### Enhancements
+
+- [Home] Set description on bulk snapshot [#3925](https://github.com/vatesfr/xen-orchestra/issues/3925) (PR [#3933](https://github.com/vatesfr/xen-orchestra/pull/3933))
+- Work-around the XenServer issue when `VBD#VDI` is an empty string instead of an opaque reference (PR [#3950](https://github.com/vatesfr/xen-orchestra/pull/3950))
+- [VDI migration] Retry when XenServer fails with `TOO_MANY_STORAGE_MIGRATES` (PR [#3940](https://github.com/vatesfr/xen-orchestra/pull/3940))
+- [VM]
+  - [General] The creation date of the VM is now visible [#3932](https://github.com/vatesfr/xen-orchestra/issues/3932) (PR [#3947](https://github.com/vatesfr/xen-orchestra/pull/3947))
+  - [Disks] Display device name [#3902](https://github.com/vatesfr/xen-orchestra/issues/3902) (PR [#3946](https://github.com/vatesfr/xen-orchestra/pull/3946))
+- [VM Snapshotting]
+  - Detect and destroy broken quiesced snapshot left by XenServer [#3936](https://github.com/vatesfr/xen-orchestra/issues/3936) (PR [#3937](https://github.com/vatesfr/xen-orchestra/pull/3937))
+  - Retry twice after a 1 minute delay if quiesce failed [#3938](https://github.com/vatesfr/xen-orchestra/issues/3938) (PR [#3952](https://github.com/vatesfr/xen-orchestra/pull/3952))
+
+### Bug fixes
+
+- [Import] Fix import of big OVA files
+- [Host] Show the host's memory usage instead of the sum of the VMs' memory usage (PR [#3924](https://github.com/vatesfr/xen-orchestra/pull/3924))
+- [SAML] Make `AssertionConsumerServiceURL` matches the callback URL
+- [Backup NG] Correctly delete broken VHD chains [#3875](https://github.com/vatesfr/xen-orchestra/issues/3875) (PR [#3939](https://github.com/vatesfr/xen-orchestra/pull/3939))
+- [Remotes] Don't ignore `mount` options [#3935](https://github.com/vatesfr/xen-orchestra/issues/3935) (PR [#3931](https://github.com/vatesfr/xen-orchestra/pull/3931))
+
+### Released packages
+
+- xen-api v0.24.2
+- @xen-orchestra/fs v0.6.1
+- xo-server-auth-saml v0.5.3
+- xo-server v5.35.0
+- xo-web v5.35.0
+
 ## **5.31.0** (2019-01-31)
 
 ### Enhancements

CHANGELOG.unreleased.md

@@ -2,28 +2,19 @@
 
 ### Enhancements
 
-- [Home] Set description on bulk snapshot [#3925](https://github.com/vatesfr/xen-orchestra/issues/3925) (PR [#3933](https://github.com/vatesfr/xen-orchestra/pull/3933))
-- Work-around the XenServer issue when `VBD#VDI` is an empty string instead of an opaque reference (PR [#3950](https://github.com/vatesfr/xen-orchestra/pull/3950))
-- [VDI migration] Retry when XenServer fails with `TOO_MANY_STORAGE_MIGRATES` (PR [#3940](https://github.com/vatesfr/xen-orchestra/pull/3940))
-- [VM]
-  - [General] The creation date of the VM is now visible [#3932](https://github.com/vatesfr/xen-orchestra/issues/3932) (PR [#3947](https://github.com/vatesfr/xen-orchestra/pull/3947))
-  - [Disks] Display device name [#3902](https://github.com/vatesfr/xen-orchestra/issues/3902) (PR [#3946](https://github.com/vatesfr/xen-orchestra/pull/3946))
-- [VM Snapshotting]
-  - Detect and destroy broken quiesced snapshot left by XenServer [#3936](https://github.com/vatesfr/xen-orchestra/issues/3936) (PR [#3937](https://github.com/vatesfr/xen-orchestra/pull/3937))
-  - Retry twice after a 1 minute delay if quiesce failed [#3938](https://github.com/vatesfr/xen-orchestra/issues/3938) (PR [#3952](https://github.com/vatesfr/xen-orchestra/pull/3952))
+- [Remotes] Benchmarks (read and write rate speed) added when remote is tested [#3991](https://github.com/vatesfr/xen-orchestra/issues/3991) (PR [#4015](https://github.com/vatesfr/xen-orchestra/pull/4015))
+- [Cloud Config] Support both NoCloud and Config Drive 2 datasources for maximum compatibility (PR [#4053](https://github.com/vatesfr/xen-orchestra/pull/4053))
+- [Advanced] Configurable cookie validity (PR [#4059](https://github.com/vatesfr/xen-orchestra/pull/4059))
+- [Plugins] Display number of installed plugins [#4008](https://github.com/vatesfr/xen-orchestra/issues/4008) (PR [#4050](https://github.com/vatesfr/xen-orchestra/pull/4050))
 
 ### Bug fixes
 
-- [Import] Fix import of big OVA files
-- [Host] Show the host's memory usage instead of the sum of the VMs' memory usage (PR [#3924](https://github.com/vatesfr/xen-orchestra/pull/3924))
-- [SAML] Make `AssertionConsumerServiceURL` matches the callback URL
-- [Backup NG] Correctly delete broken VHD chains [#3875](https://github.com/vatesfr/xen-orchestra/issues/3875) (PR [#3939](https://github.com/vatesfr/xen-orchestra/pull/3939))
-- [Remotes] Don't ignore `mount` options [#3935](https://github.com/vatesfr/xen-orchestra/issues/3935) (PR [#3931](https://github.com/vatesfr/xen-orchestra/pull/3931))
+- [Home] Always sort the items by their names as a secondary sort criteria [#3983](https://github.com/vatesfr/xen-orchestra/issues/3983) (PR [#4047](https://github.com/vatesfr/xen-orchestra/pull/4047))
+- [Remotes] Fixes `spawn mount EMFILE` error during backup
+- Properly redirect to sign in page instead of being stuck in a refresh loop
 
 ### Released packages
 
-- xen-api v0.24.2
-- @xen-orchestra/fs v0.6.1
-- xo-server-auth-saml v0.5.3
-- xo-server v5.35.0
-- xo-web v5.35.0
+- @xen-orchestra/fs v0.8.0
+- xo-server v5.38.0
+- xo-web v5.38.0

docs/SUMMARY.md

@@ -33,6 +33,7 @@
        * [Disaster recovery](disaster_recovery.md)
        * [Smart Backup](smart_backup.md)
        * [File level Restore](file_level_restore.md)
+       * [Metadata Backup](metadata_backup.md)
        * [Backup Concurrency](concurrency.md)
        * [Configure backup reports](backup_reports.md)
        * [Backup troubleshooting](backup_troubleshooting.md)
@@ -51,6 +52,7 @@
    * [Job manager](scheduler.md)
    * [Alerts](alerts.md)
    * [Load balancing](load_balancing.md)
+   * [Emergency Shutdown](emergency_shutdown.md)
    * [Auto scalability](auto_scalability.md)
    * [Forecaster](forecaster.md)
 * [Recipes](recipes.md)

docs/backup_troubleshooting.md

@@ -12,7 +12,9 @@ Another good way to check if there is activity is the XOA VM stats view (on the
 
 ### VDI chain protection
 
-This means your previous VM disks and snapshots should be "merged" (*coalesced* in the XenServer world) before we can take a new snapshot. This mechanism is handled by XenServer itself, not Xen Orchestra. But we can check your existing VDI chain and avoid creating more snapshots than your storage can merge. Otherwise, this will lead to catastrophic consequences. Xen Orchestra is the **only** XenServer/XCP backup product dealing with this.
+Backup jobs regularly delete snapshots. When a snapshot is deleted, either manually or via a backup job, it triggers the need for Xenserver to coalesce the VDI chain - to merge the remaining VDIs and base copies in the chain. This means generally we cannot take too many new snapshots on said VM until Xenserver has finished running a coalesce job on the VDI chain.  
+
+This mechanism and scheduling is handled by XenServer itself, not Xen Orchestra. But we can check your existing VDI chain and avoid creating more snapshots than your storage can merge. If we don't, this will lead to catastrophic consequences. Xen Orchestra is the **only** XenServer/XCP backup product that takes this into account and offers protection.
 
 Without this detection, you could have 2 potential issues:
 
@@ -21,9 +23,9 @@ Without this detection, you could have 2 potential issues:
 
 The first issue is a chain that contains more than 30 elements (fixed XenServer limit), and the other one means it's full because the "coalesce" process couldn't keep up the pace and the storage filled up.
 
-In the end, this message is a **protection mechanism against damaging your SR**. The backup job will fail, but XenServer itself should eventually automatically coalesce the snapshot chain, and the the next time the backup job should complete.
+In the end, this message is a **protection mechanism preventing damage to your SR**. The backup job will fail, but XenServer itself should eventually automatically coalesce the snapshot chain, and the the next time the backup job should complete.
 
-Just remember this: **coalesce will happen every time a snapshot is removed**.
+Just remember this: **a coalesce should happen every time a snapshot is removed**.
 
 > You can read more on this on our dedicated blog post regarding [XenServer coalesce detection](https://xen-orchestra.com/blog/xenserver-coalesce-detection-in-xen-orchestra/).
 
@@ -37,7 +39,9 @@ Coalesce jobs can also fail to run if the SR does not have enough free space. Ch
 
 You can check if a coalesce job is currently active by running `ps axf | grep vhd` on the XenServer host and looking for a VHD process in the results (one of the resulting processes will be the grep command you just ran, ignore that one).  
 
-If you don't see any running coalesce jobs, and can't find any other reason that XenServer has not started one, you can attempt to make it start a coalesce job by rescanning the SR. This is harmless to try, but will not always result in a coalesce. Visit the problematic SR in the XOA UI, then click the "Rescan All Disks" button towards the top right: it looks like a refresh circle icon. This should begin the coalesce process - if you click the Advanced tab in the SR view, the "disks needing to be coalesced" list should become smaller and smaller.
+If you don't see any running coalesce jobs, and can't find any other reason that XenServer has not started one, you can attempt to make it start a coalesce job by rescanning the SR. This is harmless to try, but will not always result in a coalesce. Visit the problematic SR in the XOA UI, then click the "Rescan All Disks" button towards the top right: it looks like a refresh circle icon. This should begin the coalesce process - if you click the Advanced tab in the SR view, the "disks needing to be coalesced" list should become smaller and smaller.  
+
+As a last resort, migrating the VM (more specifically, its disks) to a new storage repository will also force a coalesce and solve this issue. That means migrating a VM to another host (with its own storage) and back will force the VDI chain for that VM to be coalesced, and get rid of the `VDI Chain Protection` message.
 
 ### Parse Error
 

docs/backups.md

@@ -10,6 +10,7 @@ There are several ways to protect your VMs:
 * [Rolling Snapshots](rolling_snapshots.md) [*Starter Edition*]
 * [Delta Backups](delta_backups.md) (best of both previous ones) [*Enterprise Edition*]
 * [Disaster Recovery](disaster_recovery.md) [*Enterprise Edition*]
+* [Metadata Backups](metadata_backup.md) [*Enterprise Edition*]
 * [Continuous Replication](continuous_replication.md) [*Premium Edition*]
 * [File Level Restore](file_level_restore.md) [*Premium Edition*]
 
@@ -41,7 +42,7 @@ Each backups' job execution is identified by a `runId`. You can find this `runId
 
 All backup types rely on snapshots. But what about data consistency? By default, Xen Orchestra will try to take a **quiesced snapshot** every time a snapshot is done (and fall back to normal snapshots if it's not possible).
 
-Snapshots of Windows VMs can be quiesced (especially MS SQL or Exchange services) after you have installed Xen Tools in your VMs. However, [there is an extra step to install the VSS provider on windows](quiesce). A quiesced snapshot means the operating system will be notified and the cache will be flushed to disks. This way, your backups will always be consistent.
+Snapshots of Windows VMs can be quiesced (especially MS SQL or Exchange services) after you have installed Xen Tools in your VMs. However, [there is an extra step to install the VSS provider on windows](https://xen-orchestra.com/blog/xenserver-quiesce-snapshots/). A quiesced snapshot means the operating system will be notified and the cache will be flushed to disks. This way, your backups will always be consistent.
 
 To see if you have quiesced snapshots for a VM, just go into its snapshot tab, then the "info" icon means it is a quiesced snapshot:
 

docs/continuous_replication.md

@@ -43,11 +43,19 @@ To protect the replication, we removed the possibility to boot your copied VM di
 
 ### Job creation
 
-Create the Continuous Replication backup job, and leave it disabled for now. On the main Backup-NG page, note its identifiers, the main `backupJobId` and the ID of one on the schedules for the job, `backupScheduleId`.
+Create the Continuous Replication backup job, and leave it disabled for now. On the main Backup-NG page, copy the job's `backupJobId` by hovering to the left of the shortened ID and clicking the  copy to clipboard button:
+
+![](./assets/cr-seed-1.png)
+
+Copy it somewhere temporarily. Now we need to also copy the ID of the job schedule, `backupScheduleId`. Do this by hovering over the schedule name in the same panel as before, and clicking the copy to clipboard button. Keep it with the `backupJobId` you copied previously as we will need them all later:
+
+![](./assets/cr-seed-2.png)
 
 ### Seed creation
 
-Manually create a snapshot on the VM to backup, and note its UUID as `snapshotUuid` from the snapshot panel for the VM.
+Manually create a snapshot on the VM being backed up, then copy this snapshot UUID, `snapshotUuid` from the snapshot panel of the VM:
+
+![](./assets/cr-seed-3.png)
 
 > DO NOT ever delete or alter this snapshot, feel free to rename it to make that clear.
 
@@ -55,7 +63,9 @@ Manually create a snapshot on the VM to backup, and note its UUID as `snapshotUu
 
 Export this snapshot to a file, then import it on the target SR.
 
-Note the UUID of this newly created VM as `targetVmUuid`.
+We need to copy the UUID of this newly created VM as well, `targetVmUuid`:
+
+![](./assets/cr-seed-4.png)
 
 > DO not start this VM or it will break the Continuous Replication job! You can rename this VM to more easily remember this.
 
@@ -66,7 +76,7 @@ The XOA backup system requires metadata to correctly associate the source snapsh
 First install the tool (all the following is done from the XOA VM CLI):
 
 ```
-npm i -g xo-cr-seed
+sudo npm i -g --unsafe-perm @xen-orchestra/cr-seed-cli
 ```
 
 Here is an example of how the utility expects the UUIDs and info passed to it:

docs/emergency_shutdown.md

@@ -0,0 +1,27 @@
+# Emergency Shutdown
+
+If you have a UPS for your hosts, and lose power, you may have a limited amount of time to shut down all of your VM infrastructure before the batteries run out. If you find yourself in this situation, or any other situation requiring the fast shutdown of everything, you can use the **Emergency Shutdown** feature.
+
+## How to activate
+On the host view, clicking on this button will trigger the  _Emergency Shutdown_ procedure:
+
+![](./assets/e-shutdown-1.png)
+
+1.  **All running VMs will be suspended**  (think of it like "hibernate" on your laptop: the RAM will be stored in the storage repository).
+2.  Only after this is complete, the host will be halted.
+
+Here, you can see the running VMs are being suspended:
+
+![](./assets/e-shutdown-2.png)
+
+And finally, that's it. They are cleanly shut down with the RAM saved to disk to be resumed later:
+
+![](./assets/e-shutdown-3.png)
+
+Now the host is halted automatically.
+
+## Powering back on
+When the power outage is over, all you need to do is:
+
+1.  Start your host.
+2.  All your VMs can be resumed, your RAM is preserved and therefore your VMs will be in the exact same state as they were before the power outage.

docs/from_the_sources.md

@@ -6,9 +6,9 @@
 
 > Please take time to read this guide carefully.
 
-This installation has been validated against a fresh Debian 8 (Jessie) x64 install. It should be nearly the same on other dpkg systems. For RPM based OS's, it should be close, as most of our dependencies come from NPM and not the OS itself.
+This installation has been validated against a fresh Debian 9 (Stretch) x64 install. It should be nearly the same on other dpkg systems. For RPM based OS's, it should be close, as most of our dependencies come from NPM and not the OS itself.
 
-As you may have seen,in other parts of the documentation, XO is composed of two parts: [xo-server](https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server/) and [xo-web](https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-web/). They can be installed separately, even on different machines, but for the sake of simplicity we will set them up together.
+As you may have seen in other parts of the documentation, XO is composed of two parts: [xo-server](https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server/) and [xo-web](https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-web/). They can be installed separately, even on different machines, but for the sake of simplicity we will set them up together.
 
 ## Packages and Pre-requisites
 
@@ -49,13 +49,14 @@ You need to use the `git` source code manager to fetch the code. Ideally, you sh
 git clone -b master http://github.com/vatesfr/xen-orchestra
 ```
 
-> Note: xo-server and xo-web have been migrated to the [xen-orchestra](https://github.com/vatesfr/xen-orchestra) mono-repository.
+> Note: xo-server and xo-web have been migrated to the [xen-orchestra](https://github.com/vatesfr/xen-orchestra) mono-repository - so you only need the single clone command above
 
 ## Installing dependencies
 
-Once you have it, use `yarn`, as the non-root (or root) user owning the fetched code, to install the other dependencies. Enter the `xen-orchestra` directory and run the following commands:
+Now that you have the code, you can enter the `xen-orchestra` directory and use `yarn` to install other dependencies. Then finally build it using `yarn build`. Be sure to run `yarn` commands as the same user you will be using to run Xen Orchestra:
 
 ```
+$ cd xen-orchestra
 $ yarn
 $ yarn build
 ```
@@ -86,7 +87,7 @@ WebServer listening on localhost:80
 
 ## Running XO
 
-The only part you need to launch is xo-server which is quite easy to do. From the `xen-orchestra/packages/xo-server` directory, run the following:
+The only part you need to launch is xo-server, which is quite easy to do. From the `xen-orchestra/packages/xo-server` directory, run the following:
 
 ```
 $ yarn start

docs/metadata_backup.md

@@ -0,0 +1,31 @@
+# Metadata backup
+
+> WARNING: Metadata backup is an experimental feature. Restore is not yet available and some unexpected issues may occur.
+
+## Introduction
+
+XCP-ng and Citrix Hypervisor (Xenserver) hosts use a database to store metadata about VMs and their associated resources such as storage and networking. Metadata forms this complete view of all VMs available on your pool. Backing up the metadata of your pool allows you to recover from a physical hardware failure scenario in which you lose your hosts without losing your storage (SAN, NAS...).
+
+In Xen Orchestra, Metadata backup is divided into two different options:
+
+* Pool metadata backup
+* XO configuration backup
+
+### How to use metadata backup
+
+In the backup job section, when creating a new backup job, you will now have a choice between backing up VMs and backing up Metadata.
+![](https://user-images.githubusercontent.com/21563339/53413921-bd636f00-39cd-11e9-8a3c-d4f893135fa4.png)
+
+When you select Metadata backup, you will have a new backup job screen, letting you choose between a pool metadata backup and an XO configuration backup (or both at the same time):
+
+![](https://user-images.githubusercontent.com/21563339/52416838-d2de2b00-2aea-11e9-8da0-340fcb2767db.png)
+
+Define the name and retention for the job.
+
+![](https://user-images.githubusercontent.com/21563339/52471527-65390a00-2b91-11e9-8019-600a4d9eeafb.png)
+
+Once created, the job is displayed with the other classic jobs.
+
+![](https://user-images.githubusercontent.com/21563339/52416802-c0fc8800-2aea-11e9-8ef0-b0c1bd0e48b8.png)
+
+> Restore for metadata backup jobs should be available in XO 5.33

package.json

@@ -4,10 +4,10 @@
     "@babel/register": "^7.0.0",
     "babel-core": "^7.0.0-0",
     "babel-eslint": "^10.0.1",
-    "babel-jest": "^23.0.1",
+    "babel-jest": "^24.1.0",
     "benchmark": "^2.1.4",
     "eslint": "^5.1.0",
-    "eslint-config-prettier": "^3.3.0",
+    "eslint-config-prettier": "^4.1.0",
     "eslint-config-standard": "12.0.0",
     "eslint-config-standard-jsx": "^6.0.2",
     "eslint-plugin-import": "^2.8.0",
@@ -16,13 +16,13 @@
     "eslint-plugin-react": "^7.6.1",
     "eslint-plugin-standard": "^4.0.0",
     "exec-promise": "^0.7.0",
-    "flow-bin": "^0.90.0",
+    "flow-bin": "^0.95.1",
     "globby": "^9.0.0",
     "husky": "^1.2.1",
-    "jest": "^23.0.1",
+    "jest": "^24.1.0",
     "lodash": "^4.17.4",
     "prettier": "^1.10.2",
-    "promise-toolbox": "^0.11.0",
+    "promise-toolbox": "^0.12.1",
     "sorted-object": "^2.0.1"
   },
   "engines": {

packages/complex-matcher/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/complex-matcher",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/complex-matcher",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/smart-selector/.babelrc.js

@@ -0,0 +1,3 @@
+module.exports = require('../../@xen-orchestra/babel-config')(
+  require('./package.json')
+)

packages/smart-selector/.npmignore

@@ -0,0 +1,24 @@
+/benchmark/
+/benchmarks/
+*.bench.js
+*.bench.js.map
+
+/examples/
+example.js
+example.js.map
+*.example.js
+*.example.js.map
+
+/fixture/
+/fixtures/
+*.fixture.js
+*.fixture.js.map
+*.fixtures.js
+*.fixtures.js.map
+
+/test/
+/tests/
+*.spec.js
+*.spec.js.map
+
+__snapshots__/

packages/smart-selector/README.md

@@ -0,0 +1,76 @@
+# ${pkg.name} [![Build Status](https://travis-ci.org/${pkg.shortGitHubPath}.png?branch=master)](https://travis-ci.org/${pkg.shortGitHubPath})
+
+> ${pkg.description}
+
+Differences with [reselect](https://github.com/reactjs/reselect):
+
+- simpler: no custom memoization
+- inputs (and their selectors): are stored in objects, not arrays
+- lazy:
+   - inputs are not computed before accessed
+   - unused inputs do not trigger a call to the transform function
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/${pkg.name}):
+
+```
+> npm install --save ${pkg.name}
+```
+
+## Usage
+
+```js
+import createSelector from 'smart-selector'
+
+const getVisibleTodos = createSelector(
+  {
+    filter: state => state.filter,
+    todos: state => state.todos,
+  },
+  inputs => {
+    switch (inputs.filter) {
+      case 'ALL':
+        return inputs.todos
+      case 'COMPLETED':
+        return inputs.todos.filter(todo => todo.completed)
+      case 'ACTIVE':
+        return inputs.todos.filter(todo => !todo.completed)
+    }
+  }
+)
+```
+
+## Development
+
+```
+# Install dependencies
+> yarn
+
+# Run the tests
+> yarn test
+
+# Continuously compile
+> yarn dev
+
+# Continuously run the tests
+> yarn dev-test
+
+# Build for production (automatically called by npm install)
+> yarn build
+```
+
+## Contributions
+
+Contributions are *very* welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xo-web/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+ISC © [Vates SAS](https://vates.fr)

packages/smart-selector/package.json

@@ -0,0 +1,43 @@
+{
+  "private": true,
+  "name": "smart-selector",
+  "version": "0.0.0",
+  "license": "ISC",
+  "description": "",
+  "keywords": [],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/smart-selector",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Julien Fontanet",
+    "email": "julien.fontanet@isonoe.net"
+  },
+  "preferGlobal": false,
+  "main": "dist/",
+  "bin": {},
+  "files": [
+    "dist/"
+  ],
+  "engines": {
+    "node": ">=8"
+  },
+  "devDependencies": {
+    "@babel/cli": "7.1.5",
+    "@babel/core": "7.1.5",
+    "@babel/preset-env": "7.1.5",
+    "babel-plugin-lodash": "^3.3.2",
+    "cross-env": "^5.1.1",
+    "rimraf": "^2.6.2"
+  },
+  "scripts": {
+    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
+    "clean": "rimraf dist/",
+    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
+    "prebuild": "yarn run clean",
+    "predev": "yarn run clean",
+    "prepublishOnly": "yarn run build"
+  }
+}

packages/smart-selector/src/index.js

@@ -0,0 +1,82 @@
+const { create, keys } = Object
+
+const createSelector = (inputSelectors, transform) => {
+  const previousArgs = [{}] // initialize with non-repeatable args
+  let cache, previousResult, previousThisArg
+  let previousInputs = {}
+
+  const spyDescriptors = {}
+  const inputs = keys(inputSelectors)
+  for (let i = 0, n = inputs.length; i < n; ++i) {
+    const input = inputs[i]
+    spyDescriptors[input] = {
+      enumerable: true,
+      get: () =>
+        input in previousInputs
+          ? previousInputs[input]
+          : (previousInputs[input] =
+              input in cache
+                ? cache[input]
+                : inputSelectors[input].apply(previousThisArg, previousArgs)),
+    }
+  }
+  const spy = create(null, spyDescriptors)
+
+  function selector () {
+    // handle arguments
+    {
+      const { length } = arguments
+      let i = 0
+      if (this === previousThisArg && length === previousArgs.length) {
+        while (i < length && arguments[i] === previousArgs[i]) {
+          ++i
+        }
+        if (i === length) {
+          return previousResult
+        }
+      } else {
+        previousArgs.length = length
+        previousThisArg = this
+      }
+      while (i < length) {
+        previousArgs[i] = arguments[i]
+        ++i
+      }
+    }
+
+    // handle inputs
+    cache = previousInputs
+    previousInputs = {}
+    {
+      const inputs = keys(cache)
+      const { length } = inputs
+      if (length !== 0) {
+        let i = 0
+        while (true) {
+          if (i === length) {
+            // inputs are unchanged
+            return previousResult
+          }
+
+          const input = inputs[i++]
+          const value = inputSelectors[input].apply(this, arguments)
+          if (value !== cache[input]) {
+            // update the value
+            cache[input] = value
+
+            // remove non-computed values
+            while (i < length) {
+              delete cache[inputs[i++]]
+            }
+
+            break
+          }
+        }
+      }
+    }
+
+    return (previousResult = transform(spy))
+  }
+  return selector
+}
+export { createSelector as default }

packages/smart-selector/src/index.spec.js

@@ -0,0 +1,99 @@
+/* eslint-env jest */
+
+import createSelector from './'
+
+const noop = () => {}
+
+describe('createSelector', () => {
+  it('calls input selectors with this and arguments', () => {
+    const thisArg = {}
+    const args = ['arg1', 'arg2']
+    const foo = jest.fn()
+
+    createSelector({ foo }, ({ foo }) => {}).apply(thisArg, args)
+
+    expect(foo.mock.instances).toEqual([thisArg])
+    expect(foo.mock.calls).toEqual([args])
+  })
+
+  it('calls input selectors only when accessed', () => {
+    const foo = jest.fn()
+    createSelector({ foo }, inputs => {
+      expect(foo.mock.calls.length).toBe(0)
+      noop(inputs.foo)
+      expect(foo.mock.calls.length).toBe(1)
+    })()
+  })
+
+  it('does not call the input selectors if this arguments did not change', () => {
+    const foo = jest.fn()
+    const selector = createSelector({ foo }, ({ foo }) => {})
+
+    selector('arg1')
+    expect(foo.mock.calls.length).toBe(1)
+
+    selector('arg1')
+    expect(foo.mock.calls.length).toBe(1)
+
+    selector('arg1', 'arg2')
+    expect(foo.mock.calls.length).toBe(2)
+
+    selector.call({}, 'arg1', 'arg2')
+    expect(foo.mock.calls.length).toBe(3)
+  })
+
+  it('does not call the transform if inputs did not change', () => {
+    const transform = jest.fn(({ foo }) => {})
+    const selector = createSelector(
+      {
+        foo: () => 'foo',
+      },
+      transform
+    )
+
+    selector({})
+    expect(transform.mock.calls.length).toBe(1)
+
+    selector({})
+    expect(transform.mock.calls.length).toBe(1)
+  })
+
+  it('computes only the necessary inputs to determine if transform should be called', () => {
+    let foo = 'foo 1'
+    const bar = 'bar 1'
+    const inputs = {
+      foo: jest.fn(() => foo),
+      bar: jest.fn(() => bar),
+    }
+    const transform = jest.fn(inputs => {
+      if (inputs.foo !== 'foo 1') {
+        return inputs.bar
+      }
+    })
+    const selector = createSelector(inputs, transform)
+
+    selector({})
+    expect(inputs.foo.mock.calls.length).toBe(1)
+    expect(inputs.bar.mock.calls.length).toBe(0)
+
+    selector({})
+    expect(inputs.foo.mock.calls.length).toBe(2)
+    expect(inputs.bar.mock.calls.length).toBe(0)
+
+    foo = 'foo 2'
+
+    selector({})
+    expect(inputs.foo.mock.calls.length).toBe(3)
+    expect(inputs.bar.mock.calls.length).toBe(1)
+
+    foo = 'foo 1'
+
+    selector({})
+    expect(inputs.foo.mock.calls.length).toBe(4)
+    expect(inputs.bar.mock.calls.length).toBe(1)
+
+    selector({})
+    expect(inputs.foo.mock.calls.length).toBe(5)
+    expect(inputs.bar.mock.calls.length).toBe(1)
+  })
+})

packages/value-matcher/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/value-matcher",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/value-matcher",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/vhd-cli/package.json

@@ -1,12 +1,13 @@
 {
   "name": "vhd-cli",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "license": "ISC",
   "description": "",
   "keywords": [],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-cli",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/vhd-cli",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -26,7 +27,7 @@
     "node": ">=6"
   },
   "dependencies": {
-    "@xen-orchestra/fs": "^0.6.0",
+    "@xen-orchestra/fs": "^0.7.1",
     "cli-progress": "^2.0.0",
     "exec-promise": "^0.7.0",
     "getopts": "^2.2.3",
@@ -41,7 +42,7 @@
     "cross-env": "^5.1.3",
     "execa": "^1.0.0",
     "index-modules": "^0.3.0",
-    "promise-toolbox": "^0.11.0",
+    "promise-toolbox": "^0.12.1",
     "rimraf": "^2.6.1",
     "tmp": "^0.0.33"
   },

packages/vhd-cli/src/commands/repl.js

@@ -0,0 +1,33 @@
+import { asCallback, fromCallback, fromEvent } from 'promise-toolbox'
+import { getHandler } from '@xen-orchestra/fs'
+import { relative } from 'path'
+import { start as createRepl } from 'repl'
+import Vhd, * as vhdLib from 'vhd-lib'
+
+export default async args => {
+  const cwd = process.cwd()
+  const handler = getHandler({ url: 'file://' + cwd })
+  await handler.sync()
+  try {
+    const repl = createRepl({
+      prompt: 'vhd> ',
+    })
+    Object.assign(repl.context, vhdLib)
+    repl.context.handler = handler
+    repl.context.open = path => new Vhd(handler, relative(cwd, path))
+
+    // Make the REPL waits for promise completion.
+    repl.eval = (evaluate => (cmd, context, filename, cb) => {
+      asCallback.call(
+        fromCallback(cb => {
+          evaluate.call(repl, cmd, context, filename, cb)
+        }).then(value => (Array.isArray(value) ? Promise.all(value) : value)),
+        cb
+      )
+    })(repl.eval)
+
+    await fromEvent(repl, 'exit')
+  } finally {
+    await handler.forget()
+  }
+}

packages/vhd-lib/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-lib",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/vhd-lib",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -21,11 +22,11 @@
   },
   "dependencies": {
     "async-iterator-to-stream": "^1.0.2",
-    "core-js": "3.0.0-beta.3",
+    "core-js": "3.0.0",
     "from2": "^2.3.0",
     "fs-extra": "^7.0.0",
     "limit-concurrency-decorator": "^0.4.0",
-    "promise-toolbox": "^0.11.0",
+    "promise-toolbox": "^0.12.1",
     "struct-fu": "^1.2.0",
     "uuid": "^3.0.1"
   },
@@ -34,7 +35,7 @@
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
     "@babel/preset-flow": "^7.0.0",
-    "@xen-orchestra/fs": "^0.6.0",
+    "@xen-orchestra/fs": "^0.7.1",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^5.1.3",
     "execa": "^1.0.0",

packages/xapi-explore-sr/package.json

@@ -15,6 +15,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xapi-explore-sr",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xapi-explore-sr",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -40,7 +41,7 @@
     "human-format": "^0.10.0",
     "lodash": "^4.17.4",
     "pw": "^0.0.4",
-    "xen-api": "^0.24.2"
+    "xen-api": "^0.24.5"
   },
   "devDependencies": {
     "@babel/cli": "^7.1.5",

packages/xen-api/README.md

@@ -95,7 +95,7 @@ root@xen1.company.net> xapi.pool.$master.name_label
 To ease searches, `find()` and `findAll()` functions are available:
 
 ```
-root@xen1.company.net> findAll({ $type: 'vm' }).length
+root@xen1.company.net> findAll({ $type: 'VM' }).length
 183
 ```
 

packages/xen-api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xen-api",
-  "version": "0.24.2",
+  "version": "0.24.5",
   "license": "ISC",
   "description": "Connector to the Xen API",
   "keywords": [
@@ -13,6 +13,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xen-api",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xen-api",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -36,16 +37,16 @@
     "debug": "^4.0.1",
     "event-to-promise": "^0.8.0",
     "exec-promise": "^0.7.0",
-    "http-request-plus": "^0.7.1",
-    "iterable-backoff": "^0.0.0",
-    "jest-diff": "^23.5.0",
+    "http-request-plus": "^0.8.0",
+    "iterable-backoff": "^0.1.0",
+    "jest-diff": "^24.0.0",
     "json-rpc-protocol": "^0.13.1",
     "kindof": "^2.0.0",
     "lodash": "^4.17.4",
     "make-error": "^1.3.0",
     "minimist": "^1.2.0",
     "ms": "^2.1.1",
-    "promise-toolbox": "^0.11.0",
+    "promise-toolbox": "^0.12.1",
     "pw": "0.0.4",
     "xmlrpc": "^1.3.2",
     "xo-collection": "^0.4.1"

packages/xen-api/src/index.js

@@ -7,8 +7,8 @@ import { BaseError } from 'make-error'
 import { EventEmitter } from 'events'
 import { fibonacci } from 'iterable-backoff'
 import {
-  filter,
   forEach,
+  forOwn,
   isArray,
   isInteger,
   map,
@@ -37,7 +37,7 @@ const debug = createDebug('xen-api')
 
 // ===================================================================
 
-// in seconds
+// in seconds!
 const EVENT_TIMEOUT = 60
 
 // http://www.gnu.org/software/libc/manual/html_node/Error-Codes.html
@@ -248,6 +248,11 @@ const RESERVED_FIELDS = {
   pool: true,
   ref: true,
   type: true,
+  xapi: true,
+}
+
+function getPool() {
+  return this.$xapi.pool
 }
 
 // -------------------------------------------------------------------
@@ -266,17 +271,14 @@ export class Xapi extends EventEmitter {
     super()
 
     this._allowUnauthorized = opts.allowUnauthorized
-    this._auth = opts.auth
     this._callTimeout = makeCallSetting(opts.callTimeout, 0)
-    this._debounce = opts.debounce == null ? 200 : opts.debounce
     this._pool = null
     this._readOnly = Boolean(opts.readOnly)
     this._RecordsByType = createObject(null)
     this._sessionId = null
-    ;(this._objects = new Collection()).getKey = getKey
-    ;(this._objectsByRef = createObject(null))[NULL_REF] = undefined
-    const url = (this._url = parseUrl(opts.url))
 
+    this._auth = opts.auth
+    const url = (this._url = parseUrl(opts.url))
     if (this._auth === undefined) {
       const user = url.username
       if (user !== undefined) {
@@ -289,10 +291,19 @@ export class Xapi extends EventEmitter {
       }
     }
 
-    // Memoize this function _addObject().
-    this._getPool = () => this._pool
+    ;(this._objects = new Collection()).getKey = getKey
+    this._debounce = opts.debounce == null ? 200 : opts.debounce
+    this._watchedTypes = undefined
+    this._watching = false
 
-    if (opts.watchEvents !== false) {
+    this.on(DISCONNECTED, this._clearObjects)
+    this._clearObjects()
+
+    const { watchEvents } = opts
+    if (watchEvents !== false) {
+      if (Array.isArray(watchEvents)) {
+        this._watchedTypes = watchEvents
+      }
       this.watchEvents()
     }
   }
@@ -300,19 +311,14 @@ export class Xapi extends EventEmitter {
   watchEvents() {
     this._eventWatchers = createObject(null)
 
-    this._fromToken = ''
-
-    this._nTasks = 0
-
     this._taskWatchers = Object.create(null)
 
     if (this.status === CONNECTED) {
-      this._watchEvents()
+      this._watchEventsWrapper()
     }
 
-    this.on('connected', this._watchEvents)
+    this.on('connected', this._watchEventsWrapper)
     this.on('disconnected', () => {
-      this._fromToken = ''
       this._objects.clear()
     })
   }
@@ -401,42 +407,55 @@ export class Xapi extends EventEmitter {
     )
   }
 
-  connect() {
+  async connect() {
     const { status } = this
 
     if (status === CONNECTED) {
-      return Promise.reject(new Error('already connected'))
+      throw new Error('already connected')
     }
 
     if (status === CONNECTING) {
-      return Promise.reject(new Error('already connecting'))
+      throw new Error('already connecting')
     }
 
     const auth = this._auth
     if (auth === undefined) {
-      return Promise.reject(new Error('missing credentials'))
+      throw new Error('missing credentials')
     }
 
     this._sessionId = CONNECTING
 
-    return this._transportCall('session.login_with_password', [
-      auth.user,
-      auth.password,
-    ]).then(
-      async sessionId => {
-        this._sessionId = sessionId
-        this._pool = (await this.getAllRecords('pool'))[0]
+    try {
+      const [methods, sessionId] = await Promise.all([
+        this._transportCall('system.listMethods', []),
+        this._transportCall('session.login_with_password', [
+          auth.user,
+          auth.password,
+        ]),
+      ])
 
-        debug('%s: connected', this._humanId)
+      // Uses introspection to list available types.
+      const types = (this._types = methods
+        .filter(isGetAllRecordsMethod)
+        .map(method => method.slice(0, method.indexOf('.'))))
+      this._lcToTypes = { __proto__: null }
+      types.forEach(type => {
+        const lcType = type.toLowerCase()
+        if (lcType !== type) {
+          this._lcToTypes[lcType] = type
+        }
+      })
 
-        this.emit(CONNECTED)
-      },
-      error => {
-        this._sessionId = null
+      this._sessionId = sessionId
+      this._pool = (await this.getAllRecords('pool'))[0]
 
-        throw error
-      }
-    )
+      debug('%s: connected', this._humanId)
+      this.emit(CONNECTED)
+    } catch (error) {
+      this._sessionId = null
+
+      throw error
+    }
   }
 
   disconnect() {
@@ -499,6 +518,10 @@ export class Xapi extends EventEmitter {
     return promise
   }
 
+  getField(type, ref, field) {
+    return this._sessionCall(`${type}.get_${field}`, [ref])
+  }
+
   // Nice getter which returns the object for a given $id (internal to
   // this lib), UUID (unique identifier that some objects have) or
   // opaque reference (internal to XAPI).
@@ -550,6 +573,10 @@ export class Xapi extends EventEmitter {
     )
   }
 
+  getRecords(type, refs) {
+    return Promise.all(refs.map(ref => this.getRecord(type, ref)))
+  }
+
   async getAllRecords(type) {
     return map(
       await this._sessionCall(`${type}.get_all_records`),
@@ -565,7 +592,7 @@ export class Xapi extends EventEmitter {
   }
 
   @cancelable
-  getResource($cancelToken, pathname, { host, query, task }) {
+  getResource($cancelToken, pathname, { host, query, task } = {}) {
     return this._autoTask(task, `Xapi#getResource ${pathname}`).then(
       taskRef => {
         query = { ...query, session_id: this.sessionId }
@@ -718,41 +745,38 @@ export class Xapi extends EventEmitter {
     )
   }
 
-  setField({ $type, $ref }, field, value) {
-    return this.call(`${$type}.set_${field}`, $ref, value).then(noop)
+  setField(type, ref, field, value) {
+    return this.call(`${type}.set_${field}`, ref, value).then(noop)
   }
 
-  setFieldEntries(record, field, entries) {
+  setFieldEntries(type, ref, field, entries) {
     return Promise.all(
       getKeys(entries).map(entry => {
         const value = entries[entry]
         if (value !== undefined) {
-          return value === null
-            ? this.unsetFieldEntry(record, field, entry)
-            : this.setFieldEntry(record, field, entry, value)
+          return this.setFieldEntry(type, ref, field, entry, value)
         }
       })
     ).then(noop)
   }
 
-  async setFieldEntry({ $type, $ref }, field, entry, value) {
+  async setFieldEntry(type, ref, field, entry, value) {
+    if (value === null) {
+      return this.call(`${type}.remove_from_${field}`, ref, entry).then(noop)
+    }
     while (true) {
       try {
-        await this.call(`${$type}.add_to_${field}`, $ref, entry, value)
+        await this.call(`${type}.add_to_${field}`, ref, entry, value)
         return
       } catch (error) {
         if (error == null || error.code !== 'MAP_DUPLICATE_KEY') {
           throw error
         }
       }
-      await this.unsetFieldEntry({ $type, $ref }, field, entry)
+      await this.call(`${type}.remove_from_${field}`, ref, entry)
     }
   }
 
-  unsetFieldEntry({ $type, $ref }, field, entry) {
-    return this.call(`${$type}.remove_from_${field}`, $ref, entry)
-  }
-
   watchTask(ref) {
     const watchers = this._taskWatchers
     if (watchers === undefined) {
@@ -786,6 +810,15 @@ export class Xapi extends EventEmitter {
     return this._objects
   }
 
+  _clearObjects() {
+    ;(this._objectsByRef = createObject(null))[NULL_REF] = undefined
+    this._nTasks = 0
+    this._objects.clear()
+    this.objectsFetched = new Promise(resolve => {
+      this._resolveObjectsFetched = resolve
+    })
+  }
+
   // return a promise which resolves to a task ref or undefined
   _autoTask(task = this._taskWatchers !== undefined, name) {
     if (task === false) {
@@ -801,7 +834,7 @@ export class Xapi extends EventEmitter {
   }
 
   // Medium level call: handle session errors.
-  _sessionCall(method, args) {
+  _sessionCall(method, args, timeout = this._callTimeout(method, args)) {
     try {
       if (startsWith(method, 'session.')) {
         throw new Error('session.*() methods are disabled from this interface')
@@ -825,7 +858,7 @@ export class Xapi extends EventEmitter {
             return this.connect().then(() => this._sessionCall(method, args))
           }
         ),
-        this._callTimeout(method, args)
+        timeout
       )
     } catch (error) {
       return Promise.reject(error)
@@ -904,7 +937,12 @@ export class Xapi extends EventEmitter {
 
   _processEvents(events) {
     forEach(events, event => {
-      const { class: type, ref } = event
+      let type = event.class
+      const lcToTypes = this._lcToTypes
+      if (type in lcToTypes) {
+        type = lcToTypes[type]
+      }
+      const { ref } = event
       if (event.operation === 'del') {
         this._removeObject(type, ref)
       } else {
@@ -913,34 +951,112 @@ export class Xapi extends EventEmitter {
     })
   }
 
-  _watchEvents() {
-    const loop = () =>
-      this.status === CONNECTED &&
-      pTimeout
-        .call(
-          this._sessionCall('event.from', [
-            ['*'],
-            this._fromToken,
-            EVENT_TIMEOUT + 0.1, // Force float.
-          ]),
-          EVENT_TIMEOUT * 1.1e3 // 10% longer than the XenAPI timeout
+  // - prevent multiple watches
+  // - swallow errors
+  async _watchEventsWrapper() {
+    if (!this._watching) {
+      this._watching = true
+      try {
+        await this._watchEvents()
+      } catch (error) {
+        console.error('_watchEventsWrapper', error)
+      }
+      this._watching = false
+    }
+  }
+
+  // TODO: cancelation
+  async _watchEvents() {
+    this._clearObjects()
+
+    // compute the initial token for the event loop
+    //
+    // we need to do this before the initial fetch to avoid losing events
+    let fromToken
+    try {
+      fromToken = await this._sessionCall('event.inject', [
+        'pool',
+        this._pool.$ref,
+      ])
+    } catch (error) {
+      if (isMethodUnknown(error)) {
+        return this._watchEventsLegacy()
+      }
+    }
+
+    const types = this._watchedTypes || this._types
+
+    // initial fetch
+    const flush = this.objects.bufferEvents()
+    try {
+      await Promise.all(
+        types.map(async type => {
+          try {
+            // FIXME: use _transportCall to avoid auto-reconnection
+            forOwn(
+              await this._sessionCall(`${type}.get_all_records`),
+              (record, ref) => {
+                // we can bypass _processEvents here because they are all *add*
+                // event and all objects are of the same type
+                this._addObject(type, ref, record)
+              }
+            )
+          } catch (error) {
+            // there is nothing ideal to do here, do not interrupt event
+            // handling
+            if (error != null && error.code !== 'MESSAGE_REMOVED') {
+              console.warn('_watchEvents', 'initial fetch', type, error)
+            }
+          }
+        })
+      )
+    } finally {
+      flush()
+    }
+    this._resolveObjectsFetched()
+
+    // event loop
+    const debounce = this._debounce
+    while (true) {
+      if (debounce != null) {
+        await pDelay(debounce)
+      }
+
+      let result
+      try {
+        result = await this._sessionCall(
+          'event.from',
+          [
+            types,
+            fromToken,
+            EVENT_TIMEOUT + 0.1, // must be float for XML-RPC transport
+          ],
+          EVENT_TIMEOUT * 1e3 * 1.1
         )
-        .then(onSuccess, onFailure)
+      } catch (error) {
+        if (error instanceof TimeoutError) {
+          continue
+        }
+        if (areEventsLost(error)) {
+          return this._watchEvents()
+        }
+        throw error
+      }
 
-    const onSuccess = ({ events, token, valid_ref_counts: { task } }) => {
-      this._fromToken = token
-      this._processEvents(events)
+      fromToken = result.token
+      this._processEvents(result.events)
 
-      if (task !== this._nTasks) {
-        this._sessionCall('task.get_all_records')
-          .then(tasks => {
+      // detect and fix disappearing tasks (e.g. when toolstack restarts)
+      if (result.valid_ref_counts.task !== this._nTasks) {
+        await ignoreErrors.call(
+          this._sessionCall('task.get_all_records').then(tasks => {
             const toRemove = new Set()
-            forEach(this.objects.all, object => {
+            forOwn(this.objects.all, object => {
               if (object.$type === 'task') {
                 toRemove.add(object.$ref)
               }
             })
-            forEach(tasks, (task, ref) => {
+            forOwn(tasks, (task, ref) => {
               toRemove.delete(ref)
               this._addObject('task', ref, task)
             })
@@ -948,40 +1064,9 @@ export class Xapi extends EventEmitter {
               this._removeObject('task', ref)
             })
           })
-          .catch(noop)
+        )
       }
-
-      const debounce = this._debounce
-      return debounce != null ? pDelay(debounce).then(loop) : loop()
     }
-    const onFailure = error => {
-      if (error instanceof TimeoutError) {
-        return loop()
-      }
-
-      if (areEventsLost(error)) {
-        this._fromToken = ''
-        this._objects.clear()
-
-        return loop()
-      }
-
-      throw error
-    }
-
-    ignoreErrors.call(
-      pCatch.call(
-        loop(),
-        isMethodUnknown,
-
-        // If the server failed, it is probably due to an excessively
-        // large response.
-        // Falling back to legacy events watch should be enough.
-        error => error && error.res && error.res.statusCode === 500,
-
-        () => this._watchEventsLegacy()
-      )
-    )
   }
 
   // This method watches events using the legacy `event.next` XAPI
@@ -989,17 +1074,13 @@ export class Xapi extends EventEmitter {
   //
   // It also has to manually get all objects first.
   _watchEventsLegacy() {
-    const getAllObjects = () => {
-      return this._sessionCall('system.listMethods').then(methods => {
-        // Uses introspection to determine the methods to use to get
-        // all objects.
-        const getAllRecordsMethods = filter(methods, isGetAllRecordsMethod)
-
-        return Promise.all(
-          map(getAllRecordsMethods, method =>
-            this._sessionCall(method).then(
+    const getAllObjects = async () => {
+      const flush = this.objects.bufferEvents()
+      try {
+        await Promise.all(
+          this._types.map(type =>
+            this._sessionCall(`${type}.get_all_records`).then(
               objects => {
-                const type = method.slice(0, method.indexOf('.')).toLowerCase()
                 forEach(objects, (object, ref) => {
                   this._addObject(type, ref, object)
                 })
@@ -1012,7 +1093,10 @@ export class Xapi extends EventEmitter {
             )
           )
         )
-      })
+      } finally {
+        flush()
+      }
+      this._resolveObjectsFetched()
     }
 
     const watchEvents = () =>
@@ -1048,13 +1132,13 @@ export class Xapi extends EventEmitter {
       const nFields = fields.length
       const xapi = this
 
-      const objectsByRef = this._objectsByRef
-      const getObjectByRef = ref => objectsByRef[ref]
+      const getObjectByRef = ref => this._objectsByRef[ref]
 
       Record = function(ref, data) {
         defineProperties(this, {
           $id: { value: data.uuid || ref },
           $ref: { value: ref },
+          $xapi: { value: xapi },
         })
         for (let i = 0; i < nFields; ++i) {
           const field = fields[i]
@@ -1062,11 +1146,11 @@ export class Xapi extends EventEmitter {
         }
       }
 
-      const getters = { $pool: this._getPool }
+      const getters = { $pool: getPool }
       const props = { $type: type }
       fields.forEach(field => {
         props[`set_${field}`] = function(value) {
-          return xapi.setField(this, field, value)
+          return xapi.setField(this.$type, this.$ref, field, value)
         }
 
         const $field = (field in RESERVED_FIELDS ? '$$' : '$') + field
@@ -1090,19 +1174,21 @@ export class Xapi extends EventEmitter {
             const value = this[field]
             const result = {}
             getKeys(value).forEach(key => {
-              result[key] = objectsByRef[value[key]]
+              result[key] = xapi._objectsByRef[value[key]]
             })
             return result
           }
-          props[`update_${field}`] = function(entries) {
-            return xapi.setFieldEntries(this, field, entries)
+          props[`update_${field}`] = function(entries, value) {
+            return typeof entries === 'string'
+              ? xapi.setFieldEntry(this.$type, this.$ref, field, entries, value)
+              : xapi.setFieldEntries(this.$type, this.$ref, field, entries)
           }
         } else if (value === '' || isOpaqueRef(value)) {
           // 2019-02-07 - JFT: even if `value` should not be an empty string for
           // a ref property, an user had the case on XenServer 7.0 on the CD VBD
           // of a VM created by XenCenter
           getters[$field] = function() {
-            return objectsByRef[this[field]]
+            return xapi._objectsByRef[this[field]]
           }
         }
       })
@@ -1131,17 +1217,25 @@ export class Xapi extends EventEmitter {
 Xapi.prototype._transportCall = reduce(
   [
     function(method, args) {
-      return this._call(method, args).catch(error => {
-        if (!(error instanceof Error)) {
-          error = wrapError(error)
-        }
+      return pTimeout
+        .call(this._call(method, args), HTTP_TIMEOUT)
+        .catch(error => {
+          if (!(error instanceof Error)) {
+            error = wrapError(error)
+          }
 
-        error.call = {
-          method,
-          params: replaceSensitiveValues(args, '* obfuscated *'),
-        }
-        throw error
-      })
+          // do not log the session ID
+          //
+          // TODO: should log at the session level to avoid logging sensitive
+          // values?
+          const params = args[0] === this._sessionId ? args.slice(1) : args
+
+          error.call = {
+            method,
+            params: replaceSensitiveValues(params, '* obfuscated *'),
+          }
+          throw error
+        })
     },
     call =>
       function() {

packages/xen-api/src/transports/json-rpc.js

@@ -3,6 +3,7 @@ import { format, parse } from 'json-rpc-protocol'
 
 import { UnsupportedTransport } from './_utils'
 
+// https://github.com/xenserver/xenadmin/blob/0df39a9d83cd82713f32d24704852a0fd57b8a64/XenModel/XenAPI/Session.cs#L403-L433
 export default ({ allowUnauthorized, url }) => {
   return (method, args) =>
     httpRequestPlus

packages/xo-acl-resolver/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-acl-resolver",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-acl-resolver",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-cli/package.json

@@ -12,6 +12,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-cli",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-cli",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -33,7 +34,7 @@
     "chalk": "^2.2.0",
     "exec-promise": "^0.7.0",
     "fs-promise": "^2.0.3",
-    "http-request-plus": "^0.7.1",
+    "http-request-plus": "^0.8.0",
     "human-format": "^0.10.0",
     "l33teral": "^3.0.3",
     "lodash": "^4.17.4",
@@ -42,7 +43,7 @@
     "nice-pipe": "0.0.0",
     "pretty-ms": "^4.0.0",
     "progress-stream": "^2.0.0",
-    "promise-toolbox": "^0.11.0",
+    "promise-toolbox": "^0.12.1",
     "pump": "^3.0.0",
     "pw": "^0.0.4",
     "strip-indent": "^2.0.0",

packages/xo-collection/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-collection",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-collection",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-common/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-common",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-common",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-import-servers-csv/package.json

@@ -16,6 +16,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-import-servers-csv",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-import-servers-csv",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -42,7 +43,7 @@
     "xo-lib": "^0.9.0"
   },
   "devDependencies": {
-    "@types/node": "^10.12.2",
+    "@types/node": "^11.11.4",
     "@types/through2": "^2.0.31",
     "tslint": "^5.9.1",
     "tslint-config-standard": "^8.0.1",

packages/xo-import-servers-csv/src/index.d.ts

@@ -1,8 +1,3 @@
-declare module 'csv-parser' {
-  function csvParser(opts?: Object): any
-  export = csvParser
-}
-
 declare module 'exec-promise' {
   function execPromise(cb: (args: string[]) => any): void
   export = execPromise

packages/xo-lib/package.json

@@ -11,6 +11,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-lib",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-lib",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-remote-parser/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-remote-parser",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-remote-parser",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-auth-github/package.json

@@ -12,6 +12,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-auth-github",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-auth-github",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-auth-google/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-server-auth-google",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "license": "AGPL-3.0",
   "description": "Google authentication plugin for XO-Server",
   "keywords": [
@@ -15,6 +15,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-auth-google",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-auth-google",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -32,7 +33,7 @@
     "node": ">=6"
   },
   "dependencies": {
-    "passport-google-oauth20": "^1.0.0"
+    "passport-google-oauth20": "^2.0.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/xo-server-auth-ldap/package.json

@@ -14,6 +14,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-auth-ldap",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-auth-ldap",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -38,7 +39,7 @@
     "inquirer": "^6.0.0",
     "ldapjs": "^1.0.1",
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.11.0"
+    "promise-toolbox": "^0.12.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/xo-server-auth-saml/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-server-auth-saml",
-  "version": "0.5.2",
+  "version": "0.5.3",
   "license": "AGPL-3.0",
   "description": "SAML authentication plugin for XO-Server",
   "keywords": [
@@ -15,6 +15,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-auth-saml",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-auth-saml",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-backup-reports/package.json

@@ -18,6 +18,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-backup-reports",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-backup-reports",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-backup-reports/src/index.js

@@ -154,6 +154,10 @@ class BackupReportsXoPlugin {
   }
 
   _wrapper(status, job, schedule, runJobId) {
+    if (job.type === 'metadataBackup') {
+      return
+    }
+
     return new Promise(resolve =>
       resolve(
         job.type === 'backup'

packages/xo-server-cloud/package.json

@@ -14,6 +14,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-cloud",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-cloud",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -31,7 +32,7 @@
     "node": ">=6"
   },
   "dependencies": {
-    "http-request-plus": "^0.7.1",
+    "http-request-plus": "^0.8.0",
     "jsonrpc-websocket-client": "^0.4.1"
   },
   "devDependencies": {

packages/xo-server-load-balancer/package.json

@@ -13,6 +13,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-load-balancer",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-load-balancer",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-perf-alert/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-perf-alert",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-perf-alert",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-perf-alert/src/index.js

@@ -673,8 +673,9 @@ ${entry.listItem}
     }
   }
 
-  async getRrd(xoObject, secondsAgo) {
-    const host = xoObject.$type === 'host' ? xoObject : xoObject.$resident_on
+  async getRrd(xapiObject, secondsAgo) {
+    const host =
+      xapiObject.$type === 'host' ? xapiObject : xapiObject.$resident_on
     if (host == null) {
       return null
     }
@@ -685,13 +686,13 @@ ${entry.listItem}
       host,
       query: {
         cf: 'AVERAGE',
-        host: (xoObject.$type === 'host').toString(),
+        host: (xapiObject.$type === 'host').toString(),
         json: 'true',
         start: serverTimestamp - secondsAgo,
       },
     }
-    if (xoObject.$type === 'vm') {
-      payload['vm_uuid'] = xoObject.uuid
+    if (xapiObject.$type === 'VM') {
+      payload['vm_uuid'] = xapiObject.uuid
     }
     // JSON is not well formed, can't use the default node parser
     return JSON5.parse(

packages/xo-server-test-plugin/package.json

@@ -5,6 +5,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-test-plugin",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-test-plugin",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   }

packages/xo-server-transport-email/package.json

@@ -14,6 +14,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-transport-email",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-transport-email",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -33,7 +34,7 @@
   "dependencies": {
     "nodemailer": "^5.0.0",
     "nodemailer-markdown": "^1.0.1",
-    "promise-toolbox": "^0.11.0"
+    "promise-toolbox": "^0.12.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/xo-server-transport-nagios/package.json

@@ -14,6 +14,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-transport-nagios",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-transport-nagios",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-transport-slack/package.json

@@ -15,6 +15,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-transport-slack",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-transport-slack",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -32,7 +33,7 @@
     "node": ">=6"
   },
   "dependencies": {
-    "promise-toolbox": "^0.11.0",
+    "promise-toolbox": "^0.12.1",
     "slack-node": "^0.1.8"
   },
   "devDependencies": {

packages/xo-server-transport-xmpp/package.json

@@ -15,6 +15,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-transport-xmpp",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-transport-xmpp",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-usage-report/package.json

@@ -15,6 +15,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-usage-report",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-usage-report",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -41,7 +42,7 @@
     "html-minifier": "^3.5.8",
     "human-format": "^0.10.0",
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.11.0"
+    "promise-toolbox": "^0.12.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/xo-server/bin/xo-server

@@ -4,6 +4,11 @@
 
 // ===================================================================
 
+// https://expressjs.com/en/advanced/best-practice-performance.html#set-node_env-to-production
+if (process.env.NODE_ENV === undefined) {
+  process.env.NODE_ENV = 'production'
+}
+
 // Better stack traces if possible.
 require('../better-stacks')
 

packages/xo-server/config.toml

@@ -21,6 +21,18 @@ verboseApiLogsOnErrors = false
 [apiWebSocketOptions]
 perMessageDeflate = { threshold = 524288 } # 512kiB
 
+[authentication]
+defaultTokenValidity = '30 days'
+maxTokenValidity = '0.5 year'
+
+# Default to `maxTokenValidity`
+#permanentCookieValidity = '30 days'
+
+# Default to `undefined`, ie as long as the browser is not restarted
+#
+# https://developer.mozilla.org/fr/docs/Web/HTTP/Headers/Set-Cookie#Session_cookie
+#sessionCookieValidity = '10 hours'
+
 [[http.listen]]
 port = 80
 

packages/xo-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-server",
-  "version": "5.34.1",
+  "version": "5.37.0",
   "license": "AGPL-3.0",
   "description": "Server part of Xen-Orchestra",
   "keywords": [
@@ -12,6 +12,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -34,8 +35,9 @@
     "@iarna/toml": "^2.2.1",
     "@xen-orchestra/async-map": "^0.0.0",
     "@xen-orchestra/cron": "^1.0.3",
+    "@xen-orchestra/defined": "^0.0.0",
     "@xen-orchestra/emit-async": "^0.0.0",
-    "@xen-orchestra/fs": "^0.6.0",
+    "@xen-orchestra/fs": "^0.7.1",
     "@xen-orchestra/log": "^0.1.4",
     "@xen-orchestra/mixin": "^0.0.0",
     "ajv": "^6.1.1",
@@ -46,12 +48,13 @@
     "blocked": "^1.2.1",
     "bluebird": "^3.5.1",
     "body-parser": "^1.18.2",
+    "compression": "^1.7.3",
     "connect-flash": "^0.1.1",
     "cookie": "^0.3.1",
     "cookie-parser": "^1.4.3",
     "d3-time-format": "^2.1.1",
     "debug": "^4.0.1",
-    "decorator-synchronized": "^0.3.0",
+    "decorator-synchronized": "^0.5.0",
     "deptree": "^1.0.0",
     "escape-string-regexp": "^1.0.5",
     "event-to-promise": "^0.8.0",
@@ -68,11 +71,11 @@
     "helmet": "^3.9.0",
     "highland": "^2.11.1",
     "http-proxy": "^1.16.2",
-    "http-request-plus": "^0.7.1",
+    "http-request-plus": "^0.8.0",
     "http-server-plus": "^0.10.0",
     "human-format": "^0.10.0",
     "is-redirect": "^1.0.0",
-    "iterable-backoff": "^0.0.0",
+    "iterable-backoff": "^0.1.0",
     "jest-worker": "^24.0.0",
     "js-yaml": "^3.10.0",
     "json-rpc-peer": "^0.15.3",
@@ -92,17 +95,18 @@
     "ms": "^2.1.1",
     "multikey-hash": "^1.0.4",
     "ndjson": "^1.5.0",
-    "otplib": "^10.0.1",
+    "otplib": "^11.0.0",
     "parse-pairs": "^0.2.2",
     "partial-stream": "0.0.0",
     "passport": "^0.4.0",
     "passport-local": "^1.0.0",
-    "pretty-format": "^23.0.0",
-    "promise-toolbox": "^0.11.0",
+    "pretty-format": "^24.0.0",
+    "promise-toolbox": "^0.12.1",
     "proxy-agent": "^3.0.0",
     "pug": "^2.0.0-rc.4",
     "pump": "^3.0.0",
     "pw": "^0.0.4",
+    "readable-stream": "^3.2.0",
     "redis": "^2.8.0",
     "schema-inspector": "^1.6.8",
     "semver": "^5.4.1",
@@ -111,14 +115,14 @@
     "stack-chain": "^2.0.0",
     "stoppable": "^1.0.5",
     "struct-fu": "^1.2.0",
-    "tar-stream": "^1.5.5",
+    "tar-stream": "^2.0.1",
     "through2": "^3.0.0",
     "tmp": "^0.0.33",
     "uuid": "^3.0.1",
     "value-matcher": "^0.2.0",
     "vhd-lib": "^0.5.1",
     "ws": "^6.0.0",
-    "xen-api": "^0.24.2",
+    "xen-api": "^0.24.5",
     "xml2js": "^0.4.19",
     "xo-acl-resolver": "^0.4.1",
     "xo-collection": "^0.4.1",

packages/xo-server/sample.config.toml

@@ -44,6 +44,14 @@
 
 #=====================================================================
 
+# Directory containing the database of XO.
+# Currently used for logs.
+#
+# Default: '/var/lib/xo-server/data'
+#datadir = '/var/lib/xo-server/data'
+
+#=====================================================================
+
 # Configuration of the embedded HTTP server.
 [http]
 # If set to true, all HTTP traffic will be redirected to the first HTTPs
@@ -136,12 +144,6 @@ port = 80
 #  del = '3dda29ad-3015-44f9-b13b-fa570de92489'
 #  srem = '3fd758c9-5610-4e9d-a058-dbf4cb6d8bf0'
 
-# Directory containing the database of XO.
-# Currently used for logs.
-#
-# Default: '/var/lib/xo-server/data'
-#datadir = '/var/lib/xo-server/data'
-
 #=====================================================================
 
 # Configuration for remotes

packages/xo-server/src/_MultiKeyMap.js

@@ -0,0 +1,87 @@
+class Node {
+  constructor(value) {
+    this.children = new Map()
+    this.value = value
+  }
+}
+
+function del(node, i, keys) {
+  if (i === keys.length) {
+    if (node instanceof Node) {
+      node.value = undefined
+      return node
+    }
+    return
+  }
+  if (!(node instanceof Node)) {
+    return node
+  }
+  const key = keys[i]
+  const { children } = node
+  const child = children.get(key)
+  if (child === undefined) {
+    return node
+  }
+  const newChild = del(child, i + 1, keys)
+  if (newChild === undefined) {
+    if (children.size === 1) {
+      return node.value
+    }
+    children.delete(key)
+  } else if (newChild !== child) {
+    children.set(key, newChild)
+  }
+  return node
+}
+
+function get(node, i, keys) {
+  return i === keys.length
+    ? node instanceof Node
+      ? node.value
+      : node
+    : node instanceof Node
+    ? get(node.children.get(keys[i]), i + 1, keys)
+    : undefined
+}
+
+function set(node, i, keys, value) {
+  if (i === keys.length) {
+    if (node instanceof Node) {
+      node.value = value
+      return node
+    }
+    return value
+  }
+  const key = keys[i]
+  if (!(node instanceof Node)) {
+    node = new Node(node)
+    node.children.set(key, set(undefined, i + 1, keys, value))
+  } else {
+    const { children } = node
+    const child = children.get(key)
+    const newChild = set(child, i + 1, keys, value)
+    if (newChild !== child) {
+      children.set(key, newChild)
+    }
+  }
+  return node
+}
+
+export default class MultiKeyMap {
+  constructor() {
+    // each node is either a value or a Node if it contains children
+    this._root = undefined
+  }
+
+  delete(keys) {
+    this._root = del(this._root, 0, keys)
+  }
+
+  get(keys) {
+    return get(this._root, 0, keys)
+  }
+
+  set(keys, value) {
+    this._root = set(this._root, 0, keys, value)
+  }
+}

packages/xo-server/src/_createNdJsonStream.js

@@ -0,0 +1,22 @@
+import asyncIteratorToStream from 'async-iterator-to-stream'
+
+function* values(object) {
+  const keys = Object.keys(object)
+  for (let i = 0, n = keys.length; i < n; ++i) {
+    yield object[keys[i]]
+  }
+}
+
+/**
+ * Creates a NDJSON stream of all the values
+ *
+ * @param {(Array|Object)} collection
+ */
+module.exports = asyncIteratorToStream(function*(collection) {
+  for (const value of Array.isArray(collection)
+    ? collection
+    : values(collection)) {
+    yield JSON.stringify(value)
+    yield '\n'
+  }
+})

packages/xo-server/src/_ensureArray.js

@@ -0,0 +1,3 @@
+// Ensure the value is an array, wrap it if necessary.
+export default value =>
+  value === undefined ? [] : Array.isArray(value) ? value : [value]

packages/xo-server/src/_ensureArray.spec.js

@@ -0,0 +1,21 @@
+/* eslint-env jest */
+
+import ensureArray from './_ensureArray'
+
+describe('ensureArray()', function() {
+  it('wrap the value in an array', function() {
+    const value = 'foo'
+
+    expect(ensureArray(value)).toEqual([value])
+  })
+
+  it('returns an empty array for undefined', function() {
+    expect(ensureArray(undefined)).toEqual([])
+  })
+
+  it('returns the object itself if is already an array', function() {
+    const array = ['foo', 'bar', 'baz']
+
+    expect(ensureArray(array)).toBe(array)
+  })
+})

packages/xo-server/src/_pDebounceWithKey.js

@@ -0,0 +1,39 @@
+import ensureArray from './_ensureArray'
+import MultiKeyMap from './_MultiKeyMap'
+
+function removeCacheEntry(cache, keys) {
+  cache.delete(keys)
+}
+
+function scheduleRemoveCacheEntry(keys, expires) {
+  const delay = expires - Date.now()
+  if (delay <= 0) {
+    removeCacheEntry(this, keys)
+  } else {
+    setTimeout(removeCacheEntry, delay, this, keys)
+  }
+}
+
+const defaultKeyFn = () => []
+
+// debounce an async function so that all subsequent calls in a delay receive
+// the same result
+//
+// similar to `p-debounce` with `leading` set to `true` but with key support
+export default (fn, delay, keyFn = defaultKeyFn) => {
+  const cache = new MultiKeyMap()
+  return function() {
+    const keys = ensureArray(keyFn.apply(this, arguments))
+    let promise = cache.get(keys)
+    if (promise === undefined) {
+      cache.set(keys, (promise = fn.apply(this, arguments)))
+      const remove = scheduleRemoveCacheEntry.bind(
+        cache,
+        keys,
+        Date.now() + delay
+      )
+      promise.then(remove, remove)
+    }
+    return promise
+  }
+}

packages/xo-server/src/_pRetry.spec.js

@@ -1,5 +1,7 @@
 /* eslint-env jest */
 
+import { forOwn } from 'lodash'
+
 import pRetry from './_pRetry'
 
 describe('pRetry()', () => {
@@ -43,4 +45,51 @@ describe('pRetry()', () => {
       expect(i).toBe(1)
     })
   })
+
+  it('does not retry if `stop` callback is called', async () => {
+    const e = new Error()
+    let i = 0
+    await expect(
+      pRetry(stop => {
+        ++i
+        stop(e)
+      })
+    ).rejects.toBe(e)
+    expect(i).toBe(1)
+  })
+
+  describe('`when` option', () => {
+    forOwn(
+      {
+        'with function predicate': _ => _.message === 'foo',
+        'with object predicate': { message: 'foo' },
+      },
+      (when, title) =>
+        describe(title, () => {
+          it('retries when error matches', async () => {
+            let i = 0
+            await pRetry(
+              () => {
+                ++i
+                throw new Error('foo')
+              },
+              { when, tries: 2 }
+            ).catch(Function.prototype)
+            expect(i).toBe(2)
+          })
+
+          it('does not retry when error does not match', async () => {
+            let i = 0
+            await pRetry(
+              () => {
+                ++i
+                throw new Error('bar')
+              },
+              { when, tries: 2 }
+            ).catch(Function.prototype)
+            expect(i).toBe(1)
+          })
+        })
+    )
+  })
 })

packages/xo-server/src/_parseDuration.js

@@ -0,0 +1,12 @@
+import ms from 'ms'
+
+export default value => {
+  if (typeof value === 'number') {
+    return value
+  }
+  const duration = ms(value)
+  if (duration === undefined) {
+    throw new TypeError(`not a valid duration: ${duration}`)
+  }
+  return duration
+}

packages/xo-server/src/api/backup-ng.js

@@ -1,5 +1,8 @@
 import { basename } from 'path'
+import { fromCallback } from 'promise-toolbox'
+import { pipeline } from 'readable-stream'
 
+import createNdJsonStream from '../_createNdJsonStream'
 import { safeDateFormat } from '../utils'
 
 export function createJob({ schedules, ...job }) {
@@ -150,12 +153,26 @@ runJob.params = {
 
 // -----------------------------------------------------------------------------
 
-export function getAllLogs() {
-  return this.getBackupNgLogs()
+async function handleGetAllLogs(req, res) {
+  const logs = await this.getBackupNgLogs()
+  res.set('Content-Type', 'application/json')
+  return fromCallback(cb => pipeline(createNdJsonStream(logs), res, cb))
+}
+
+export function getAllLogs({ ndjson = false }) {
+  return ndjson
+    ? this.registerHttpRequest(handleGetAllLogs).then($getFrom => ({
+        $getFrom,
+      }))
+    : this.getBackupNgLogs()
 }
 
 getAllLogs.permission = 'admin'
 
+getAllLogs.params = {
+  ndjson: { type: 'boolean', optional: true },
+}
+
 export function getLogs({ after, before, limit, ...filter }) {
   return this.getBackupNgLogsSorted({ after, before, limit, filter })
 }

packages/xo-server/src/api/metadata-backup.js

@@ -0,0 +1,103 @@
+export function createJob({ schedules, ...job }) {
+  job.userId = this.user.id
+  return this.createMetadataBackupJob(job, schedules)
+}
+
+createJob.permission = 'admin'
+createJob.params = {
+  name: {
+    type: 'string',
+    optional: true,
+  },
+  pools: {
+    type: 'object',
+    optional: true,
+  },
+  remotes: {
+    type: 'object',
+  },
+  schedules: {
+    type: 'object',
+  },
+  settings: {
+    type: 'object',
+  },
+  xoMetadata: {
+    type: 'boolean',
+    optional: true,
+  },
+}
+
+export function getAllJobs() {
+  return this.getAllJobs('metadataBackup')
+}
+
+getAllJobs.permission = 'admin'
+
+export function getJob({ id }) {
+  return this.getJob(id, 'metadataBackup')
+}
+
+getJob.permission = 'admin'
+getJob.params = {
+  id: {
+    type: 'string',
+  },
+}
+
+export function deleteJob({ id }) {
+  return this.deleteMetadataBackupJob(id)
+}
+
+deleteJob.permission = 'admin'
+deleteJob.params = {
+  id: {
+    type: 'string',
+  },
+}
+
+export function editJob(props) {
+  return this.updateJob(props)
+}
+
+editJob.permission = 'admin'
+editJob.params = {
+  id: {
+    type: 'string',
+  },
+  name: {
+    type: 'string',
+    optional: true,
+  },
+  pools: {
+    type: ['object', 'null'],
+    optional: true,
+  },
+  settings: {
+    type: 'object',
+    optional: true,
+  },
+  remotes: {
+    type: 'object',
+    optional: true,
+  },
+  xoMetadata: {
+    type: 'boolean',
+    optional: true,
+  },
+}
+
+export async function runJob({ id, schedule }) {
+  return this.runJobSequence([id], await this.getSchedule(schedule))
+}
+
+runJob.permission = 'admin'
+
+runJob.params = {
+  id: {
+    type: 'string',
+  },
+  schedule: {
+    type: 'string',
+  },
+}

packages/xo-server/src/api/network.js

@@ -85,34 +85,35 @@ createBonded.description =
 // ===================================================================
 
 export async function set({
-  network,
-
+  automatic,
+  defaultIsLocked,
   name_description: nameDescription,
   name_label: nameLabel,
-  defaultIsLocked,
-  id,
+  network,
 }) {
   await this.getXapi(network).setNetworkProperties(network._xapiId, {
+    automatic,
+    defaultIsLocked,
     nameDescription,
     nameLabel,
-    defaultIsLocked,
   })
 }
 
 set.params = {
-  id: {
-    type: 'string',
+  automatic: {
+    type: 'boolean',
+    optional: true,
   },
-  name_label: {
-    type: 'string',
+  defaultIsLocked: {
+    type: 'boolean',
     optional: true,
   },
   name_description: {
     type: 'string',
     optional: true,
   },
-  defaultIsLocked: {
-    type: 'boolean',
+  name_label: {
+    type: 'string',
     optional: true,
   },
 }

packages/xo-server/src/api/session.js

@@ -1,16 +1,20 @@
 import { deprecate } from 'util'
 
 import { getUserPublicProperties } from '../utils'
-import { invalidCredentials } from 'xo-common/api-errors'
 
 // ===================================================================
 
 export async function signIn(credentials) {
-  const user = await this.authenticateUser(credentials)
-  if (!user) {
-    throw invalidCredentials()
+  const { session } = this
+
+  const { user, expiration } = await this.authenticateUser(credentials)
+  session.set('user_id', user.id)
+
+  if (expiration === undefined) {
+    session.unset('expiration')
+  } else {
+    session.set('expiration', expiration)
   }
-  this.session.set('user_id', user.id)
 
   return getUserPublicProperties(user)
 }

packages/xo-server/src/api/sr.js

@@ -1,8 +1,9 @@
 import asyncMap from '@xen-orchestra/async-map'
 import { some } from 'lodash'
 
+import ensureArray from '../_ensureArray'
 import { asInteger } from '../xapi/utils'
-import { ensureArray, forEach, parseXml } from '../utils'
+import { forEach, parseXml } from '../utils'
 
 // ===================================================================
 

packages/xo-server/src/api/vm.js

@@ -612,6 +612,8 @@ set.params = {
 
   share: { type: 'boolean', optional: true },
 
+  startDelay: { type: 'integer', optional: true },
+
   // set the VM network interface controller
   nicType: { type: ['string', 'null'], optional: true },
 }
@@ -1461,14 +1463,25 @@ getCloudInitConfig.resolve = {
 
 // -------------------------------------------------------------------
 
-export async function createCloudInitConfigDrive({ vm, sr, config, coreos }) {
+export async function createCloudInitConfigDrive({
+  config,
+  coreos,
+  networkConfig,
+  sr,
+  vm,
+}) {
   const xapi = this.getXapi(vm)
   if (coreos) {
     // CoreOS is a special CloudConfig drive created by XS plugin
     await xapi.createCoreOsCloudInitConfigDrive(vm._xapiId, sr._xapiId, config)
   } else {
     // use generic Cloud Init drive
-    await xapi.createCloudInitConfigDrive(vm._xapiId, sr._xapiId, config)
+    await xapi.createCloudInitConfigDrive(
+      vm._xapiId,
+      sr._xapiId,
+      config,
+      networkConfig
+    )
   }
 }
 
@@ -1476,6 +1489,7 @@ createCloudInitConfigDrive.params = {
   vm: { type: 'string' },
   sr: { type: 'string' },
   config: { type: 'string' },
+  networkConfig: { type: 'string', optional: true },
 }
 
 createCloudInitConfigDrive.resolve = {

packages/xo-server/src/api/xo.js

@@ -1,5 +1,8 @@
 import getStream from 'get-stream'
-import { forEach } from 'lodash'
+import { fromCallback } from 'promise-toolbox'
+import { pipeline } from 'readable-stream'
+
+import createNdJsonStream from '../_createNdJsonStream'
 
 // ===================================================================
 
@@ -17,6 +20,7 @@ export async function exportConfig() {
       (req, res) => {
         res.writeHead(200, 'OK', {
           'content-disposition': 'attachment',
+          'content-type': 'application/json',
         })
 
         return this.exportConfig()
@@ -32,11 +36,9 @@ exportConfig.permission = 'admin'
 // -------------------------------------------------------------------
 
 function handleGetAllObjects(req, res, { filter, limit }) {
-  forEach(this.getObjects({ filter, limit }), object => {
-    res.write(JSON.stringify(object))
-    res.write('\n')
-  })
-  res.end()
+  const objects = this.getObjects({ filter, limit })
+  res.set('Content-Type', 'application/json')
+  return fromCallback(cb => pipeline(createNdJsonStream(objects), res, cb))
 }
 
 export function getAllObjects({ filter, limit, ndjson = false }) {

packages/xo-server/src/api/xosan.js

@@ -10,8 +10,9 @@ import { invalidParameters } from 'xo-common/api-errors'
 import { v4 as generateUuid } from 'uuid'
 import { includes, remove, filter, find, range } from 'lodash'
 
+import ensureArray from '../_ensureArray'
 import { asInteger } from '../xapi/utils'
-import { parseXml, ensureArray } from '../utils'
+import { parseXml } from '../utils'
 
 const log = createLogger('xo:xosan')
 

packages/xo-server/src/index.js

@@ -3,6 +3,7 @@ import assert from 'assert'
 import authenticator from 'otplib/authenticator'
 import bind from 'lodash/bind'
 import blocked from 'blocked'
+import compression from 'compression'
 import createExpress from 'express'
 import createLogger from '@xen-orchestra/log'
 import crypto from 'crypto'
@@ -14,17 +15,20 @@ import pw from 'pw'
 import serveStatic from 'serve-static'
 import startsWith from 'lodash/startsWith'
 import stoppable from 'stoppable'
+import WebServer from 'http-server-plus'
 import WebSocket from 'ws'
+
 import { compile as compilePug } from 'pug'
 import { createServer as createProxyServer } from 'http-proxy'
 import { fromEvent } from 'promise-toolbox'
+import { ifDef } from '@xen-orchestra/defined'
 import { join as joinPath } from 'path'
 
 import JsonRpcPeer from 'json-rpc-peer'
 import { invalidCredentials } from 'xo-common/api-errors'
 import { ensureDir, readdir, readFile } from 'fs-extra'
 
-import WebServer from 'http-server-plus'
+import parseDuration from './_parseDuration'
 import Xo from './xo'
 import {
   forEach,
@@ -91,6 +95,8 @@ function createExpressApp(config) {
 
   app.use(helmet())
 
+  app.use(compression())
+
   // Registers the cookie-parser and express-session middlewares,
   // necessary for connect-flash.
   app.use(cookieParser(null, config.http.cookies))
@@ -118,7 +124,7 @@ function createExpressApp(config) {
   return app
 }
 
-async function setUpPassport(express, xo) {
+async function setUpPassport(express, xo, { authentication: authCfg }) {
   const strategies = { __proto__: null }
   xo.registerPassportStrategy = strategy => {
     passport.use(strategy)
@@ -176,16 +182,24 @@ async function setUpPassport(express, xo) {
     }
   })
 
+  const PERMANENT_VALIDITY = ifDef(
+    authCfg.permanentCookieValidity,
+    parseDuration
+  )
+  const SESSION_VALIDITY = ifDef(authCfg.sessionCookieValidity, parseDuration)
   const setToken = async (req, res, next) => {
     const { user, isPersistent } = req.session
-    const token = (await xo.createAuthenticationToken({ userId: user.id })).id
+    const token = await xo.createAuthenticationToken({
+      expiresIn: isPersistent ? PERMANENT_VALIDITY : SESSION_VALIDITY,
+      userId: user.id,
+    })
 
-    // Persistent cookie ? => 1 year
-    // Non-persistent : external provider as Github, Twitter...
     res.cookie(
       'token',
-      token,
-      isPersistent ? { maxAge: 1000 * 60 * 60 * 24 * 365 } : undefined
+      token.id,
+      // a session (non-permanent) cookie must not have an expiration date
+      // because it must not survive browser restart
+      isPersistent ? { expires: new Date(token.expiration) } : undefined
     )
 
     delete req.session.isPersistent
@@ -237,7 +251,7 @@ async function setUpPassport(express, xo) {
   xo.registerPassportStrategy(
     new LocalStrategy(async (username, password, done) => {
       try {
-        const user = await xo.authenticateUser({ username, password })
+        const { user } = await xo.authenticateUser({ username, password })
         done(null, user)
       } catch (error) {
         done(null, false, { message: error.message })
@@ -356,6 +370,7 @@ async function makeWebServerListen(
     ;[opts.cert, opts.key] = await Promise.all([readFile(cert), readFile(key)])
     if (opts.key.includes('ENCRYPTED')) {
       opts.passphrase = await new Promise(resolve => {
+        // eslint-disable-next-line no-console
         console.log('Encrypted key %s', key)
         process.stdout.write(`Enter pass phrase: `)
         pw(resolve)
@@ -503,6 +518,11 @@ const setUpApi = (webServer, xo, config) => {
 
     // Connect the WebSocket to the JSON-RPC server.
     socket.on('message', message => {
+      const expiration = connection.get('expiration', undefined)
+      if (expiration !== undefined && expiration < Date.now()) {
+        return void connection.close()
+      }
+
       jsonRpc.write(message)
     })
 
@@ -550,7 +570,7 @@ const setUpConsoleProxy = (webServer, xo) => {
       {
         const { token } = parseCookies(req.headers.cookie)
 
-        const user = await xo.authenticateUser({ token })
+        const { user } = await xo.authenticateUser({ token })
         if (!(await xo.hasPermissions(user.id, [[id, 'operate']]))) {
           throw invalidCredentials()
         }
@@ -570,6 +590,9 @@ const setUpConsoleProxy = (webServer, xo) => {
         proxyConsole(connection, vmConsole, xapi.sessionId)
       })
     } catch (error) {
+      try {
+        socket.end()
+      } catch (_) {}
       console.error((error && error.stack) || error)
     }
   })
@@ -667,7 +690,7 @@ export default async function main(args) {
 
   // Everything above is not protected by the sign in, allowing xo-cli
   // to work properly.
-  await setUpPassport(express, xo)
+  await setUpPassport(express, xo, config)
 
   // Attaches express to the web server.
   webServer.on('request', express)

packages/xo-server/src/models/remote.js

@@ -14,6 +14,10 @@ export class Remotes extends Collection {
   async get(properties) {
     const remotes = await super.get(properties)
     forEach(remotes, remote => {
+      remote.benchmarks =
+        remote.benchmarks !== undefined
+          ? JSON.parse(remote.benchmarks)
+          : undefined
       remote.enabled = remote.enabled === 'true'
     })
     return remotes

packages/xo-server/src/recover-account-cli.js

@@ -10,7 +10,7 @@ const recoverAccount = async ([name]) => {
 xo-server-recover-account <user name or email>
 
     If the user does not exist, it is created, if it exists, updates
-    its password and resets its permission to Admin.
+    its password, remove any configured OTP and resets its permission to Admin.
 `
   }
 
@@ -32,7 +32,11 @@ xo-server-recover-account <user name or email>
 
   const user = await xo.getUserByName(name, true)
   if (user !== null) {
-    await xo.updateUser(user.id, { password, permission: 'admin' })
+    await xo.updateUser(user.id, {
+      password,
+      permission: 'admin',
+      preferences: { otp: null },
+    })
     console.log(`user ${name} has been successfully updated`)
   } else {
     await xo.createUser({ name, password, permission: 'admin' })

packages/xo-server/src/utils.js

@@ -3,7 +3,6 @@ import forEach from 'lodash/forEach'
 import has from 'lodash/has'
 import highland from 'highland'
 import humanFormat from 'human-format'
-import isArray from 'lodash/isArray'
 import isString from 'lodash/isString'
 import keys from 'lodash/keys'
 import multiKeyHashInt from 'multikey-hash'
@@ -15,6 +14,8 @@ import { dirname, resolve } from 'path'
 import { utcFormat, utcParse } from 'd3-time-format'
 import { fromCallback, pAll, pReflect, promisify } from 'promise-toolbox'
 
+import { type SimpleIdPattern } from './utils'
+
 // ===================================================================
 
 export function camelToSnakeCase(string) {
@@ -47,17 +48,6 @@ export const diffItems = (coll1, coll2) => {
 
 // -------------------------------------------------------------------
 
-// Ensure the value is an array, wrap it if necessary.
-export function ensureArray(value) {
-  if (value === undefined) {
-    return []
-  }
-
-  return isArray(value) ? value : [value]
-}
-
-// -------------------------------------------------------------------
-
 // Returns the value of a property and removes it from the object.
 export function extractProperty(obj, prop) {
   const value = obj[prop]
@@ -417,3 +407,13 @@ export const getFirstPropertyName = object => {
     }
   }
 }
+
+// -------------------------------------------------------------------
+
+export const unboxIdsFromPattern = (pattern?: SimpleIdPattern): string[] => {
+  if (pattern === undefined) {
+    return []
+  }
+  const { id } = pattern
+  return typeof id === 'string' ? [id] : id.__or
+}

packages/xo-server/src/utils.js.flow

@@ -11,3 +11,5 @@ declare export function safeDateFormat(timestamp: number): string
 declare export function serializeError(error: Error): Object
 
 declare export function streamToBuffer(stream: Readable): Promise<Buffer>
+
+export type SimpleIdPattern = {| id: string | {| __or: string[] |}, |}

packages/xo-server/src/utils.spec.js

@@ -3,7 +3,6 @@
 import {
   camelToSnakeCase,
   diffItems,
-  ensureArray,
   extractProperty,
   formatXml,
   generateToken,
@@ -42,26 +41,6 @@ describe('diffItems', () => {
 
 // -------------------------------------------------------------------
 
-describe('ensureArray()', function() {
-  it('wrap the value in an array', function() {
-    const value = 'foo'
-
-    expect(ensureArray(value)).toEqual([value])
-  })
-
-  it('returns an empty array for undefined', function() {
-    expect(ensureArray(undefined)).toEqual([])
-  })
-
-  it('returns the object itself if is already an array', function() {
-    const array = ['foo', 'bar', 'baz']
-
-    expect(ensureArray(array)).toBe(array)
-  })
-})
-
-// -------------------------------------------------------------------
-
 describe('extractProperty()', function() {
   it('returns the value of the property', function() {
     const value = {}

packages/xo-server/src/xapi-object-to-xo.js

@@ -1,7 +1,7 @@
 import { startsWith } from 'lodash'
 
+import ensureArray from './_ensureArray'
 import {
-  ensureArray,
   extractProperty,
   forEach,
   isArray,
@@ -54,12 +54,9 @@ function toTimestamp(date) {
     return timestamp
   }
 
-  const ms = parseDateTime(date)
-  if (!ms) {
-    return null
-  }
+  const ms = parseDateTime(date)?.getTime()
 
-  return Math.round(ms.getTime() / 1000)
+  return ms === undefined || ms === 0 ? null : Math.round(ms / 1000)
 }
 
 // ===================================================================
@@ -173,7 +170,7 @@ const TRANSFORMS = {
           total: 0,
         }
       })(),
-      multipathing: obj.multipathing,
+      multipathing: otherConfig.multipathing === 'true',
       patches: patches || link(obj, 'patches'),
       powerOnMode: obj.power_on_mode,
       power_state: metrics ? (isRunning ? 'Running' : 'Halted') : 'Unknown',
@@ -350,6 +347,7 @@ const TRANSFORMS = {
       hasVendorDevice: obj.has_vendor_device,
       resourceSet,
       snapshots: link(obj, 'snapshots'),
+      startDelay: +obj.start_delay,
       startTime: metrics && toTimestamp(metrics.start_time),
       tags: obj.tags,
       VIFs: link(obj, 'VIFs'),
@@ -581,6 +579,7 @@ const TRANSFORMS = {
 
   network(obj) {
     return {
+      automatic: obj.other_config?.automatic === 'true',
       bridge: obj.bridge,
       defaultIsLocked: obj.default_locking_mode === 'disabled',
       MTU: +obj.MTU,

packages/xo-server/src/xapi-stats.js

@@ -1,11 +1,12 @@
 import JSON5 from 'json5'
 import limitConcurrency from 'limit-concurrency-decorator'
+import synchronized from 'decorator-synchronized'
 import { BaseError } from 'make-error'
 import {
+  defaults,
   endsWith,
   findKey,
   forEach,
-  get,
   identity,
   map,
   mapValues,
@@ -51,11 +52,6 @@ const RRD_POINTS_PER_STEP = {
 // Utils
 // -------------------------------------------------------------------
 
-// Return current local timestamp in seconds
-function getCurrentTimestamp() {
-  return Date.now() / 1000
-}
-
 function convertNanToNull(value) {
   return isNaN(value) ? null : value
 }
@@ -77,23 +73,8 @@ const computeValues = (dataRow, legendIndex, transformValue = identity) =>
 const combineStats = (stats, path, combineValues) =>
   zipWith(...map(stats, path), (...values) => combineValues(values))
 
-// It browse the object in depth and initialise it's properties
-// The targerPath can be a string or an array containing the depth
-// targetPath: [a, b, c] => a.b.c
-const getValuesFromDepth = (obj, targetPath) => {
-  if (typeof targetPath === 'string') {
-    return (obj[targetPath] = [])
-  }
-
-  forEach(targetPath, (path, key) => {
-    if (obj[path] === undefined) {
-      obj = obj[path] = targetPath.length - 1 === key ? [] : {}
-      return
-    }
-    obj = obj[path]
-  })
-  return obj
-}
+const createGetProperty = (obj, property, defaultValue) =>
+  defaults(obj, { [property]: defaultValue })[property]
 
 const testMetric = (test, type) =>
   typeof test === 'string'
@@ -245,6 +226,34 @@ const STATS = {
 
 // -------------------------------------------------------------------
 
+// RRD
+// json: {
+//   meta: {
+//     start: Number,
+//     step: Number,
+//     end: Number,
+//     rows: Number,
+//     columns: Number,
+//     legend: String[rows]
+//   },
+//   data: Item[columns] // Item = { t: Number, values: Number[rows] }
+// }
+
+// Local cache
+// _statsByObject : {
+//   [uuid]: {
+//     [step]: {
+//       endTimestamp: Number, // the timestamp of the last statistic point
+//       interval: Number, // step
+//       stats: {
+//         [metric1]: Number[],
+//         [metric2]: {
+//           [subMetric]: Number[],
+//         }
+//       }
+//     }
+//   }
+// }
 export default class XapiStats {
   constructor() {
     this._statsByObject = {}
@@ -268,44 +277,26 @@ export default class XapiStats {
       .then(response => response.readAll().then(JSON5.parse))
   }
 
-  async _getNextTimestamp(xapi, host, step) {
-    const currentTimeStamp = await getServerTimestamp(xapi, host.$ref)
-    const maxDuration = step * RRD_POINTS_PER_STEP[step]
-    const lastTimestamp = get(this._statsByObject, [
-      host.uuid,
-      step,
-      'endTimestamp',
-    ])
+  // To avoid multiple requests, we keep a cash for the stats and
+  // only return it if we not exceed a step
+  _getCachedStats(uuid, step, currentTimeStamp) {
+    const statsByObject = this._statsByObject
 
-    if (
-      lastTimestamp === undefined ||
-      currentTimeStamp - lastTimestamp + step > maxDuration
-    ) {
-      return currentTimeStamp - maxDuration + step
+    const stats = statsByObject[uuid]?.[step]
+    if (stats === undefined) {
+      return
     }
-    return lastTimestamp
+
+    if (stats.endTimestamp + step < currentTimeStamp) {
+      delete statsByObject[uuid][step]
+      return
+    }
+
+    return stats
   }
 
-  _getStats(hostUuid, step, vmUuid) {
-    const hostStats = this._statsByObject[hostUuid][step]
-
-    // Return host stats
-    if (vmUuid === undefined) {
-      return {
-        interval: step,
-        ...hostStats,
-      }
-    }
-
-    // Return vm stats
-    return {
-      interval: step,
-      endTimestamp: hostStats.endTimestamp,
-      ...this._statsByObject[vmUuid][step],
-    }
-  }
-
-  async _getAndUpdateStats(xapi, { host, vmUuid, granularity }) {
+  @synchronized.withKey((_, { host }) => host.uuid)
+  async _getAndUpdateStats(xapi, { host, uuid, granularity }) {
     const step =
       granularity === undefined
         ? RRD_STEP_SECONDS
@@ -317,103 +308,93 @@ export default class XapiStats {
       )
     }
 
-    // Limit the number of http requests
-    const hostUuid = host.uuid
+    const currentTimeStamp = await getServerTimestamp(xapi, host.$ref)
 
-    if (
-      !(
-        vmUuid !== undefined &&
-        get(this._statsByObject, [vmUuid, step]) === undefined
-      ) &&
-      get(this._statsByObject, [hostUuid, step, 'localTimestamp']) + step >
-        getCurrentTimestamp()
-    ) {
-      return this._getStats(hostUuid, step, vmUuid)
+    const stats = this._getCachedStats(uuid, step, currentTimeStamp)
+    if (stats !== undefined) {
+      return stats
     }
 
-    const timestamp = await this._getNextTimestamp(xapi, host, step)
-    const json = await this._getJson(xapi, host, timestamp, step)
-    if (json.meta.step !== step) {
+    const maxDuration = step * RRD_POINTS_PER_STEP[step]
+
+    // To avoid crossing over the boundary, we ask for one less step
+    const optimumTimestamp = currentTimeStamp - maxDuration + step
+    const json = await this._getJson(xapi, host, optimumTimestamp, step)
+
+    const actualStep = json.meta.step
+    if (json.data.length > 0) {
+      // fetched data is organized from the newest to the oldest
+      // but this implementation requires it in the other direction
+      json.data.reverse()
+      json.meta.legend.forEach((legend, index) => {
+        const [, type, uuid, metricType] = /^AVERAGE:([^:]+):(.+):(.+)$/.exec(
+          legend
+        )
+
+        const metrics = STATS[type]
+        if (metrics === undefined) {
+          return
+        }
+
+        const { metric, testResult } = findMetric(metrics, metricType)
+        if (metric === undefined) {
+          return
+        }
+
+        const xoObjectStats = createGetProperty(this._statsByObject, uuid, {})
+        let stepStats = xoObjectStats[actualStep]
+        if (
+          stepStats === undefined ||
+          stepStats.endTimestamp !== json.meta.end
+        ) {
+          stepStats = xoObjectStats[actualStep] = {
+            endTimestamp: json.meta.end,
+            interval: actualStep,
+          }
+        }
+
+        const path =
+          metric.getPath !== undefined
+            ? metric.getPath(testResult)
+            : [findKey(metrics, metric)]
+
+        const lastKey = path.length - 1
+        let metricStats = createGetProperty(stepStats, 'stats', {})
+        path.forEach((property, key) => {
+          if (key === lastKey) {
+            metricStats[property] = computeValues(
+              json.data,
+              index,
+              metric.transformValue
+            )
+            return
+          }
+
+          metricStats = createGetProperty(metricStats, property, {})
+        })
+      })
+    }
+
+    if (actualStep !== step) {
       throw new FaultyGranularity(
-        `Unable to get the true granularity: ${json.meta.step}`
+        `Unable to get the true granularity: ${actualStep}`
       )
     }
 
-    // It exists data
-    if (json.data.length !== 0) {
-      // Warning: Sometimes, the json.xport.meta.start value does not match with the
-      // timestamp of the oldest data value
-      // So, we use the timestamp of the oldest data value !
-      const startTimestamp = json.data[json.meta.rows - 1].t
-      const endTimestamp = get(this._statsByObject, [
-        hostUuid,
-        step,
-        'endTimestamp',
-      ])
-
-      const statsOffset = endTimestamp - startTimestamp + step
-      if (endTimestamp !== undefined && statsOffset > 0) {
-        const parseOffset = statsOffset / step
-        // Remove useless data
-        // Note: Older values are at end of json.data.row
-        json.data.splice(json.data.length - parseOffset)
+    return (
+      this._statsByObject[uuid]?.[step] ?? {
+        endTimestamp: currentTimeStamp,
+        interval: step,
+        stats: {},
       }
-
-      // It exists useful data
-      if (json.data.length > 0) {
-        // reorder data
-        json.data.reverse()
-        forEach(json.meta.legend, (legend, index) => {
-          const [, type, uuid, metricType] = /^AVERAGE:([^:]+):(.+):(.+)$/.exec(
-            legend
-          )
-
-          const metrics = STATS[type]
-          if (metrics === undefined) {
-            return
-          }
-
-          const { metric, testResult } = findMetric(metrics, metricType)
-
-          if (metric === undefined) {
-            return
-          }
-
-          const path =
-            metric.getPath !== undefined
-              ? metric.getPath(testResult)
-              : [findKey(metrics, metric)]
-
-          const metricValues = getValuesFromDepth(this._statsByObject, [
-            uuid,
-            step,
-            'stats',
-            ...path,
-          ])
-
-          metricValues.push(
-            ...computeValues(json.data, index, metric.transformValue)
-          )
-
-          // remove older Values
-          metricValues.splice(
-            0,
-            metricValues.length - RRD_POINTS_PER_STEP[step]
-          )
-        })
-      }
-    }
-
-    // Update timestamp
-    const hostStats = this._statsByObject[hostUuid][step]
-    hostStats.endTimestamp = json.meta.end
-    hostStats.localTimestamp = getCurrentTimestamp()
-    return this._getStats(hostUuid, step, vmUuid)
+    )
   }
 
   getHostStats(xapi, hostId, granularity) {
+    const host = xapi.getObject(hostId)
     return this._getAndUpdateStats(xapi, {
-      host: xapi.getObject(hostId),
+      host,
+      uuid: host.uuid,
       granularity,
     })
   }
@@ -427,7 +408,7 @@ export default class XapiStats {
 
     return this._getAndUpdateStats(xapi, {
       host,
-      vmUuid: vm.uuid,
+      uuid: vm.uuid,
       granularity,
     })
   }

packages/xo-server/src/xapi/index.js

@@ -35,11 +35,11 @@ import {
 import { satisfies as versionSatisfies } from 'semver'
 
 import createSizeStream from '../size-stream'
+import ensureArray from '../_ensureArray'
 import fatfsBuffer, { init as fatfsBufferInit } from '../fatfs-buffer'
 import pRetry from '../_pRetry'
 import {
   camelToSnakeCase,
-  ensureArray,
   forEach,
   isFunction,
   map,
@@ -60,7 +60,6 @@ import {
   asInteger,
   extractOpaqueRef,
   filterUndefineds,
-  getNamespaceForType,
   getVmDisks,
   canSrHaveNewVdiOfSize,
   isVmHvm,
@@ -227,7 +226,7 @@ export default class Xapi extends XapiBase {
 
   _setObjectProperty(object, name, value) {
     return this.call(
-      `${getNamespaceForType(object.$type)}.set_${camelToSnakeCase(name)}`,
+      `${object.$type}.set_${camelToSnakeCase(name)}`,
       object.$ref,
       prepareXapiParam(value)
     )
@@ -236,15 +235,13 @@ export default class Xapi extends XapiBase {
   _setObjectProperties(object, props) {
     const { $ref: ref, $type: type } = object
 
-    const namespace = getNamespaceForType(type)
-
     // TODO: the thrown error should contain the name of the
     // properties that failed to be set.
     return Promise.all(
       mapToArray(props, (value, name) => {
         if (value != null) {
           return this.call(
-            `${namespace}.set_${camelToSnakeCase(name)}`,
+            `${type}.set_${camelToSnakeCase(name)}`,
             ref,
             prepareXapiParam(value)
           )
@@ -258,9 +255,8 @@ export default class Xapi extends XapiBase {
 
     prop = camelToSnakeCase(prop)
 
-    const namespace = getNamespaceForType(type)
-    const add = `${namespace}.add_to_${prop}`
-    const remove = `${namespace}.remove_from_${prop}`
+    const add = `${type}.add_to_${prop}`
+    const remove = `${type}.remove_from_${prop}`
 
     await Promise.all(
       mapToArray(values, (value, name) => {
@@ -309,17 +305,24 @@ export default class Xapi extends XapiBase {
 
   async setNetworkProperties(
     id,
-    { nameLabel, nameDescription, defaultIsLocked }
+    { automatic, defaultIsLocked, nameDescription, nameLabel }
   ) {
     let defaultLockingMode
     if (defaultIsLocked != null) {
       defaultLockingMode = defaultIsLocked ? 'disabled' : 'unlocked'
     }
-    await this._setObjectProperties(this.getObject(id), {
-      nameLabel,
-      nameDescription,
-      defaultLockingMode,
-    })
+    const network = this.getObject(id)
+    await Promise.all([
+      this._setObjectProperties(network, {
+        defaultLockingMode,
+        nameDescription,
+        nameLabel,
+      }),
+      this._updateObjectMapProperty(network, 'other_config', {
+        automatic:
+          automatic === undefined ? undefined : automatic ? 'true' : null,
+      }),
+    ])
   }
 
   // =================================================================
@@ -327,15 +330,13 @@ export default class Xapi extends XapiBase {
   async addTag(id, tag) {
     const { $ref: ref, $type: type } = this.getObject(id)
 
-    const namespace = getNamespaceForType(type)
-    await this.call(`${namespace}.add_tags`, ref, tag)
+    await this.call(`${type}.add_tags`, ref, tag)
   }
 
   async removeTag(id, tag) {
     const { $ref: ref, $type: type } = this.getObject(id)
 
-    const namespace = getNamespaceForType(type)
-    await this.call(`${namespace}.remove_tags`, ref, tag)
+    await this.call(`${type}.remove_tags`, ref, tag)
   }
 
   // =================================================================
@@ -416,10 +417,23 @@ export default class Xapi extends XapiBase {
     await this.call('host.enable', this.getObject(hostId).$ref)
   }
 
+  // Resources:
+  // - Citrix XenServer ® 7.0 Administrator's Guide ch. 5.4
+  // - https://github.com/xcp-ng/xenadmin/blob/60dd70fc36faa0ec91654ec97e24b7af36acff9f/XenModel/Actions/Host/EditMultipathAction.cs
+  // - https://github.com/serencorbett1/xenadmin/blob/1c3fb0c1112e4e316423afc6a028066001d3dea1/XenModel/XenAPI-Extensions/SR.cs
   @deferrable.onError(log.warn)
   async setHostMultipathing($defer, hostId, multipathing) {
     const host = this.getObject(hostId)
 
+    if (host.enabled) {
+      await this.disableHost(hostId)
+      $defer(() => this.enableHost(hostId))
+    }
+
+    // Xen center evacuate running VMs before unplugging the PBDs.
+    // The evacuate method uses the live migration to migrate running VMs
+    // from host to another. It only works when a shared SR is present
+    // in the host. For this reason we chose to show a warning instead.
     const pluggedPbds = host.$PBDs.filter(pbd => pbd.currently_attached)
     await asyncMap(pluggedPbds, async pbd => {
       const ref = pbd.$ref
@@ -427,11 +441,6 @@ export default class Xapi extends XapiBase {
       $defer(() => this.plugPbd(ref))
     })
 
-    if (host.enabled) {
-      await this.disableHost(hostId)
-      $defer(() => this.enableHost(hostId))
-    }
-
     return this._updateObjectMapProperty(
       host,
       'other_config',
@@ -677,17 +686,17 @@ export default class Xapi extends XapiBase {
   }
 
   async _deleteVm(
-    vm,
+    vmOrRef,
     deleteDisks = true,
     force = false,
     forceDeleteDefaultTemplate = false
   ) {
-    log.debug(`Deleting VM ${vm.name_label}`)
-
-    const { $ref } = vm
+    const $ref = typeof vmOrRef === 'string' ? vmOrRef : vmOrRef.$ref
 
     // ensure the vm record is up-to-date
-    vm = await this.barrier($ref)
+    const vm = await this.barrier($ref)
+
+    log.debug(`Deleting VM ${vm.name_label}`)
 
     if (!force && 'destroy' in vm.blocked_operations) {
       throw forbiddenOperation('destroy', vm.blocked_operations.destroy.reason)
@@ -728,6 +737,10 @@ export default class Xapi extends XapiBase {
         this._deleteVm(snapshot)
       )::ignoreErrors(),
 
+      vm.power_state === 'Suspended' &&
+        vm.suspend_VDI !== NULL_REF &&
+        this._deleteVdi(vm.suspend_VDI)::ignoreErrors(),
+
       deleteDisks &&
         asyncMap(disks, ({ $ref: vdiRef }) => {
           let onFailure = () => {
@@ -752,7 +765,7 @@ export default class Xapi extends XapiBase {
             return (
               // Only remove VBDs not attached to other VMs.
               vdi.VBDs.length < 2 || every(vdi.$VBDs, vbd => vbd.VM === $ref)
-                ? this._deleteVdi(vdi)
+                ? this._deleteVdi(vdiRef)
                 : onFailure(vdi)
             )
           }
@@ -922,7 +935,7 @@ export default class Xapi extends XapiBase {
         //
         // The snapshot must not exist otherwise it could break the
         // next export.
-        this._deleteVdi(vdi)::ignoreErrors()
+        this._deleteVdi(vdi.$ref)::ignoreErrors()
         return
       }
 
@@ -1078,7 +1091,7 @@ export default class Xapi extends XapiBase {
         }
 
         newVdi = await this._getOrWaitObject(await this._cloneVdi(baseVdi))
-        $defer.onFailure(() => this._deleteVdi(newVdi))
+        $defer.onFailure(() => this._deleteVdi(newVdi.$ref))
 
         await this._updateObjectMapProperty(newVdi, 'other_config', {
           [TAG_COPY_SRC]: vdi.uuid,
@@ -1093,7 +1106,7 @@ export default class Xapi extends XapiBase {
           },
           sr: mapVdisSrs[vdi.uuid] || srId,
         })
-        $defer.onFailure(() => this._deleteVdi(newVdi))
+        $defer.onFailure(() => this._deleteVdi(newVdi.$ref))
       }
 
       await asyncMap(vbds[vdiId], vbd =>
@@ -1256,7 +1269,7 @@ export default class Xapi extends XapiBase {
     return loop()
   }
 
-  @synchronized
+  @synchronized()
   _callInstallationPlugin(hostRef, vdi) {
     return this.call(
       'host.call_plugin',
@@ -1284,7 +1297,7 @@ export default class Xapi extends XapiBase {
       '[XO] Supplemental pack ISO',
       'small temporary VDI to store a supplemental pack ISO'
     )
-    $defer(() => this._deleteVdi(vdi))
+    $defer(() => this._deleteVdi(vdi.$ref))
 
     await this._callInstallationPlugin(this.getObject(hostId).$ref, vdi.uuid)
   }
@@ -1312,7 +1325,7 @@ export default class Xapi extends XapiBase {
         '[XO] Supplemental pack ISO',
         'small temporary VDI to store a supplemental pack ISO'
       )
-      $defer(() => this._deleteVdi(vdi))
+      $defer(() => this._deleteVdi(vdi.$ref))
 
       // Install pack sequentially to prevent concurrent access to the unique VDI
       for (const host of hosts) {
@@ -1343,7 +1356,7 @@ export default class Xapi extends XapiBase {
             '[XO] Supplemental pack ISO',
             'small temporary VDI to store a supplemental pack ISO'
           )
-          $defer(() => this._deleteVdi(vdi))
+          $defer(() => this._deleteVdi(vdi.$ref))
 
           await this._callInstallationPlugin(host.$ref, vdi.uuid)
         })
@@ -1429,7 +1442,7 @@ export default class Xapi extends XapiBase {
           size: disk.capacity,
           sr: sr.$ref,
         }))
-        $defer.onFailure(() => this._deleteVdi(vdi))
+        $defer.onFailure(() => this._deleteVdi(vdi.$ref))
 
         return this.createVbd({
           userdevice: disk.position,
@@ -1539,19 +1552,22 @@ export default class Xapi extends XapiBase {
 
   @concurrency(2)
   @cancelable
-  async _snapshotVm($cancelToken, vm, nameLabel = vm.name_label) {
+  async _snapshotVm($cancelToken, { $ref: vmRef }, nameLabel) {
+    const vm = await this.getRecord('VM', vmRef)
+    if (nameLabel === undefined) {
+      nameLabel = vm.name_label
+    }
+
     log.debug(
       `Snapshotting VM ${vm.name_label}${
         nameLabel !== vm.name_label ? ` as ${nameLabel}` : ''
       }`
     )
 
-    const vmRef = vm.$ref
     let ref
     do {
       if (!vm.tags.includes('xo-disable-quiesce')) {
         try {
-          vm = await this.barrier(vmRef)
           ref = await pRetry(
             async bail => {
               try {
@@ -1571,12 +1587,11 @@ export default class Xapi extends XapiBase {
                 // see https://github.com/vatesfr/xen-orchestra/issues/3936
                 const prevSnapshotRefs = new Set(vm.snapshots)
                 const snapshotNameLabelPrefix = `Snapshot of ${vm.uuid} [`
-                vm = await this.barrier(vmRef)
-                const createdSnapshots = vm.$snapshots.filter(
-                  _ =>
-                    !prevSnapshotRefs.has(_.$ref) &&
-                    _.name_label.startsWith(snapshotNameLabelPrefix)
-                )
+                vm.snapshots = await this.getField('VM', vmRef, 'snapshots')
+                const createdSnapshots = (await this.getRecords(
+                  'VM',
+                  vm.snapshots.filter(_ => !prevSnapshotRefs.has(_))
+                )).filter(_ => _.name_label.startsWith(snapshotNameLabelPrefix))
 
                 // be safe: only delete if there was a single match
                 if (createdSnapshots.length === 1) {
@@ -1591,7 +1606,7 @@ export default class Xapi extends XapiBase {
               tries: 3,
             }
           ).then(extractOpaqueRef)
-          this.addTag(ref, 'quiesce')::ignoreErrors()
+          ignoreErrors.call(this.call('VM.add_tags', ref, 'quiesce'))
 
           break
         } catch (error) {
@@ -1616,14 +1631,9 @@ export default class Xapi extends XapiBase {
       ).then(extractOpaqueRef)
     } while (false)
 
-    // Convert the template to a VM and wait to have receive the up-
-    // to-date object.
-    const [, snapshot] = await Promise.all([
-      this.call('VM.set_is_a_template', ref, false),
-      this.barrier(ref),
-    ])
+    await this.setField('VM', ref, 'is_a_template', false)
 
-    return snapshot
+    return this.getRecord('VM', ref)
   }
 
   async snapshotVm(vmId, nameLabel = undefined) {
@@ -1700,7 +1710,7 @@ export default class Xapi extends XapiBase {
         find(
           this.objects.all,
           obj =>
-            obj.$type === 'vm' &&
+            obj.$type === 'VM' &&
             obj.is_a_template &&
             obj.name_label === templateNameLabel
         )
@@ -1907,15 +1917,21 @@ export default class Xapi extends XapiBase {
           vdi: newVdi,
         })
       })
-      await this._deleteVdi(vdi)
+      await this._deleteVdi(vdi.$ref)
     }
   }
 
   // TODO: check whether the VDI is attached.
-  async _deleteVdi(vdi) {
-    log.debug(`Deleting VDI ${vdi.name_label}`)
+  async _deleteVdi(vdiRef) {
+    log.debug(`Deleting VDI ${vdiRef}`)
 
-    await this.call('VDI.destroy', vdi.$ref)
+    try {
+      await this.call('VDI.destroy', vdiRef)
+    } catch (error) {
+      if (error?.code !== 'HANDLE_INVALID') {
+        throw error
+      }
+    }
   }
 
   _resizeVdi(vdi, size) {
@@ -2010,7 +2026,7 @@ export default class Xapi extends XapiBase {
   }
 
   async deleteVdi(vdiId) {
-    await this._deleteVdi(this.getObject(vdiId))
+    await this._deleteVdi(this.getObject(vdiId).$ref)
   }
 
   async resizeVdi(vdiId, size) {
@@ -2200,7 +2216,7 @@ export default class Xapi extends XapiBase {
     const physPif = find(
       this.objects.all,
       obj =>
-        obj.$type === 'pif' &&
+        obj.$type === 'PIF' &&
         (obj.physical || !isEmpty(obj.bond_master_of)) &&
         obj.$pool === pif.$pool &&
         obj.device === pif.device
@@ -2363,7 +2379,13 @@ export default class Xapi extends XapiBase {
 
   // Generic Config Drive
   @deferrable
-  async createCloudInitConfigDrive($defer, vmId, srId, config) {
+  async createCloudInitConfigDrive(
+    $defer,
+    vmId,
+    srId,
+    userConfig,
+    networkConfig
+  ) {
     const vm = this.getObject(vmId)
     const sr = this.getObject(srId)
 
@@ -2374,14 +2396,35 @@ export default class Xapi extends XapiBase {
       size: buffer.length,
       sr: sr.$ref,
     })
-    $defer.onFailure(() => this._deleteVdi(vdi))
+    $defer.onFailure(() => this._deleteVdi(vdi.$ref))
 
     // Then, generate a FAT fs
-    const fs = promisifyAll(fatfs.createFileSystem(fatfsBuffer(buffer)))
+    const { mkdir, writeFile } = promisifyAll(
+      fatfs.createFileSystem(fatfsBuffer(buffer))
+    )
 
     await Promise.all([
-      fs.writeFile('meta-data', 'instance-id: ' + vm.uuid + '\n'),
-      fs.writeFile('user-data', config),
+      // preferred datasource: NoCloud
+      //
+      // https://cloudinit.readthedocs.io/en/latest/topics/datasources/nocloud.html
+      writeFile('meta-data', 'instance-id: ' + vm.uuid + '\n'),
+      writeFile('user-data', userConfig),
+      networkConfig !== undefined && writeFile('network-config', networkConfig),
+
+      // fallback datasource: Config Drive 2
+      //
+      // https://cloudinit.readthedocs.io/en/latest/topics/datasources/configdrive.html#version-2
+      mkdir('openstack').then(() =>
+        mkdir('openstack/latest').then(() =>
+          Promise.all([
+            writeFile(
+              'openstack/latest/meta_data.json',
+              JSON.stringify({ uuid: vm.uuid })
+            ),
+            writeFile('openstack/latest/user_data', userConfig),
+          ])
+        )
+      ),
     ])
 
     // ignore errors, I (JFT) don't understand why they are emitted
@@ -2407,7 +2450,7 @@ export default class Xapi extends XapiBase {
       size: stream.length,
       sr: sr.$ref,
     })
-    $defer.onFailure(() => this._deleteVdi(vdi))
+    $defer.onFailure(() => this._deleteVdi(vdi.$ref))
 
     await this.importVdiContent(vdi.$id, stream, { format: VDI_FORMAT_RAW })
 
@@ -2436,7 +2479,7 @@ export default class Xapi extends XapiBase {
     return find(
       this.objects.all,
       obj =>
-        obj.$type === 'sr' && obj.shared && canSrHaveNewVdiOfSize(obj, minSize)
+        obj.$type === 'SR' && obj.shared && canSrHaveNewVdiOfSize(obj, minSize)
     )
   }
 

packages/xo-server/src/xapi/mixins/patching.js

@@ -12,14 +12,9 @@ import sortBy from 'lodash/sortBy'
 import assign from 'lodash/assign'
 import unzip from 'julien-f-unzip'
 
+import ensureArray from '../../_ensureArray'
 import { debounce } from '../../decorators'
-import {
-  ensureArray,
-  forEach,
-  mapFilter,
-  mapToArray,
-  parseXml,
-} from '../../utils'
+import { forEach, mapFilter, mapToArray, parseXml } from '../../utils'
 
 import { extractOpaqueRef, useUpdateSystem } from '../utils'
 
@@ -166,7 +161,7 @@ export default {
   async _ejectToolsIsos(hostRef) {
     return Promise.all(
       mapFilter(this.objects.all, vm => {
-        if (vm.$type !== 'vm' || (hostRef && vm.resident_on !== hostRef)) {
+        if (vm.$type !== 'VM' || (hostRef && vm.resident_on !== hostRef)) {
           return
         }
 
@@ -300,7 +295,7 @@ export default {
         'small temporary VDI to store a patch ISO'
       )
     }
-    $defer(() => this._deleteVdi(vdi))
+    $defer(() => this._deleteVdi(vdi.$ref))
 
     return vdi
   },