feat(xo-server): v5.37.0

Fixes xoa-support#1346

.editorconfig

@@ -3,63 +3,12 @@
 # Julien Fontanet's configuration
 # https://gist.github.com/julien-f/8096213
 
-# Top-most EditorConfig file.
 root = true
 
-# Common config.
 [*]
 charset = utf-8
 end_of_line = lf
+indent_size = 2
+indent_style = space
 insert_final_newline = true
 trim_trailing_whitespace = true
-
-# CoffeeScript
-#
-# https://github.com/polarmobile/coffeescript-style-guide/blob/master/README.md
-[*.{,lit}coffee]
-indent_size = 2
-indent_style = space
-
-# Markdown
-[*.{md,mdwn,mdown,markdown}]
-indent_size = 4
-indent_style = space
-
-# Package.json
-#
-# This indentation style is the one used by npm.
-[package.json]
-indent_size = 2
-indent_style = space
-
-# Pug (Jade)
-[*.{jade,pug}]
-indent_size = 2
-indent_style = space
-
-# JavaScript
-#
-# Two spaces seems to be the standard most common style, at least in
-# Node.js (http://nodeguide.com/style.html#tabs-vs-spaces).
-[*.{js,jsx,ts,tsx}]
-indent_size = 2
-indent_style = space
-
-# Less
-[*.less]
-indent_size = 2
-indent_style = space
-
-# Sass
-#
-# Style used for http://libsass.com
-[*.s[ac]ss]
-indent_size = 2
-indent_style = space
-
-# YAML
-#
-# Only spaces are allowed.
-[*.yaml]
-indent_size = 2
-indent_style = space

.eslintrc.js

@@ -1,5 +1,11 @@
 module.exports = {
-  extends: ['standard', 'standard-jsx', 'prettier'],
+  extends: [
+    'standard',
+    'standard-jsx',
+    'prettier',
+    'prettier/standard',
+    'prettier/react',
+  ],
   globals: {
     __DEV__: true,
     $Dict: true,
@@ -21,8 +27,5 @@ module.exports = {
     'node/no-extraneous-import': 'error',
     'node/no-extraneous-require': 'error',
     'prefer-const': 'error',
-
-    // See https://github.com/prettier/eslint-config-prettier/issues/65
-    'react/jsx-indent': 'off',
   },
 }

@xen-orchestra/async-map/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/async-map",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/async-map",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

@xen-orchestra/babel-config/package.json

@@ -5,6 +5,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/babel-config",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/babel-config",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   }

@xen-orchestra/cr-seed-cli/index.js

@@ -82,35 +82,26 @@ ${cliName} v${pkg.version}
   )
 
   await Promise.all([
-    srcXapi.setFieldEntries(srcSnapshot, 'other_config', metadata),
-    srcXapi.setFieldEntries(srcSnapshot, 'other_config', {
-      'xo:backup:exported': 'true',
-    }),
-    tgtXapi.setField(
-      tgtVm,
-      'name_label',
-      `${srcVm.name_label} (${srcSnapshot.snapshot_time})`
-    ),
-    tgtXapi.setFieldEntries(tgtVm, 'other_config', metadata),
-    tgtXapi.setFieldEntries(tgtVm, 'other_config', {
+    srcSnapshot.update_other_config(metadata),
+    srcSnapshot.update_other_config('xo:backup:exported', 'true'),
+    tgtVm.set_name_label(`${srcVm.name_label} (${srcSnapshot.snapshot_time})`),
+    tgtVm.update_other_config(metadata),
+    tgtVm.update_other_config({
       'xo:backup:sr': tgtSr.uuid,
       'xo:copy_of': srcSnapshotUuid,
     }),
-    tgtXapi.setFieldEntries(tgtVm, 'blocked_operations', {
-      start:
-        'Start operation for this vm is blocked, clone it if you want to use it.',
-    }),
+    tgtVm.update_blocked_operations(
+      'start',
+      'Start operation for this vm is blocked, clone it if you want to use it.'
+    ),
     Promise.all(
       userDevices.map(userDevice => {
         const srcDisk = srcDisks[userDevice]
         const tgtDisk = tgtDisks[userDevice]
 
-        return tgtXapi.setFieldEntry(
-          tgtDisk,
-          'other_config',
-          'xo:copy_of',
-          srcDisk.uuid
-        )
+        return tgtDisk.update_other_config({
+          'xo:copy_of': srcDisk.uuid,
+        })
       })
     ),
   ])

@xen-orchestra/cr-seed-cli/package.json

@@ -4,6 +4,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/cr-seed-cli",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/cr-seed-cli",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -15,6 +16,6 @@
   },
   "dependencies": {
     "golike-defer": "^0.4.1",
-    "xen-api": "^0.24.1"
+    "xen-api": "^0.24.5"
   }
 }

@xen-orchestra/cron/package.json

@@ -17,6 +17,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/cron",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/cron",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

@xen-orchestra/defined/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/defined",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/defined",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

@xen-orchestra/emit-async/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/emit-async",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/emit-async",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

@xen-orchestra/fs/package.json

@@ -1,12 +1,13 @@
 {
   "name": "@xen-orchestra/fs",
-  "version": "0.6.0",
+  "version": "0.7.1",
   "license": "AGPL-3.0",
   "description": "The File System for Xen Orchestra backups.",
   "keywords": [],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/fs",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -23,6 +24,7 @@
     "@marsaud/smb2": "^0.13.0",
     "@sindresorhus/df": "^2.1.0",
     "@xen-orchestra/async-map": "^0.0.0",
+    "decorator-synchronized": "^0.3.0",
     "execa": "^1.0.0",
     "fs-extra": "^7.0.0",
     "get-stream": "^4.0.0",

@xen-orchestra/fs/src/_mount.js

@@ -1,5 +1,6 @@
 import execa from 'execa'
 import fs from 'fs-extra'
+import { ignoreErrors } from 'promise-toolbox'
 import { join } from 'path'
 import { tmpdir } from 'os'
 
@@ -21,7 +22,13 @@ export default class MountHandler extends LocalHandler {
     super(remote, opts)
 
     this._execa = useSudo ? sudoExeca : execa
-    this._params = params
+    this._keeper = undefined
+    this._params = {
+      ...params,
+      options: [params.options, remote.options]
+        .filter(_ => _ !== undefined)
+        .join(','),
+    }
     this._realPath = join(
       mountsDir,
       remote.id ||
@@ -32,19 +39,20 @@ export default class MountHandler extends LocalHandler {
   }
 
   async _forget() {
-    await this._execa('umount', ['--force', this._getRealPath()], {
-      env: {
-        LANG: 'C',
-      },
-    }).catch(error => {
-      if (
-        error == null ||
-        typeof error.stderr !== 'string' ||
-        !error.stderr.includes('not mounted')
-      ) {
-        throw error
-      }
-    })
+    const keeper = this._keeper
+    if (keeper === undefined) {
+      return
+    }
+    this._keeper = undefined
+    await fs.close(keeper)
+
+    await ignoreErrors.call(
+      this._execa('umount', [this._getRealPath()], {
+        env: {
+          LANG: 'C',
+        },
+      })
+    )
   }
 
   _getRealPath() {
@@ -52,26 +60,49 @@ export default class MountHandler extends LocalHandler {
   }
 
   async _sync() {
-    await fs.ensureDir(this._getRealPath())
-    const { type, device, options, env } = this._params
-    return this._execa(
-      'mount',
-      ['-t', type, device, this._getRealPath(), '-o', options],
-      {
-        env: {
-          LANG: 'C',
-          ...env,
-        },
+    // in case of multiple `sync`s, ensure we properly close previous keeper
+    {
+      const keeper = this._keeper
+      if (keeper !== undefined) {
+        this._keeper = undefined
+        ignoreErrors.call(fs.close(keeper))
       }
-    ).catch(error => {
-      let stderr
-      if (
-        error == null ||
-        typeof (stderr = error.stderr) !== 'string' ||
-        !(stderr.includes('already mounted') || stderr.includes('busy'))
-      ) {
+    }
+
+    const realPath = this._getRealPath()
+
+    await fs.ensureDir(realPath)
+
+    try {
+      const { type, device, options, env } = this._params
+      await this._execa(
+        'mount',
+        ['-t', type, device, realPath, '-o', options],
+        {
+          env: {
+            LANG: 'C',
+            ...env,
+          },
+        }
+      )
+    } catch (error) {
+      try {
+        // the failure may mean it's already mounted, use `findmnt` to check
+        // that's the case
+        await this._execa('findmnt', [realPath], {
+          stdio: 'ignore',
+        })
+      } catch (_) {
         throw error
       }
-    })
+    }
+
+    // keep an open file on the mount to prevent it from being unmounted if used
+    // by another handler/process
+    const keeperPath = `${realPath}/.keeper_${Math.random()
+      .toString(36)
+      .slice(2)}`
+    this._keeper = await fs.open(keeperPath, 'w')
+    ignoreErrors.call(fs.unlink(keeperPath))
   }
 }

@xen-orchestra/fs/src/abstract.js

@@ -5,6 +5,7 @@ import getStream from 'get-stream'
 
 import asyncMap from '@xen-orchestra/async-map'
 import path from 'path'
+import synchronized from 'decorator-synchronized'
 import { fromCallback, fromEvent, ignoreErrors, timeout } from 'promise-toolbox'
 import { parse } from 'xo-remote-parser'
 import { randomBytes } from 'crypto'
@@ -34,18 +35,18 @@ const ignoreEnoent = error => {
 }
 
 class PrefixWrapper {
-  constructor(remote, prefix) {
+  constructor(handler, prefix) {
     this._prefix = prefix
-    this._remote = remote
+    this._handler = handler
   }
 
   get type() {
-    return this._remote.type
+    return this._handler.type
   }
 
   // necessary to remove the prefix from the path with `prependDir` option
   async list(dir, opts) {
-    const entries = await this._remote.list(this._resolve(dir), opts)
+    const entries = await this._handler.list(this._resolve(dir), opts)
     if (opts != null && opts.prependDir) {
       const n = this._prefix.length
       entries.forEach((entry, i, entries) => {
@@ -56,7 +57,7 @@ class PrefixWrapper {
   }
 
   rename(oldPath, newPath) {
-    return this._remote.rename(this._resolve(oldPath), this._resolve(newPath))
+    return this._handler.rename(this._resolve(oldPath), this._resolve(newPath))
   }
 
   _resolve(path) {
@@ -216,6 +217,7 @@ export default class RemoteHandlerAbstract {
   // FIXME: Some handlers are implemented based on system-wide mecanisms (such
   // as mount), forgetting them might breaking other processes using the same
   // remote.
+  @synchronized()
   async forget(): Promise<void> {
     await this._forget()
   }
@@ -354,6 +356,7 @@ export default class RemoteHandlerAbstract {
   // metadata
   //
   // This method MUST ALWAYS be called before using the handler.
+  @synchronized()
   async sync(): Promise<void> {
     await this._sync()
   }
@@ -565,7 +568,7 @@ function createPrefixWrapperMethods() {
       if (arguments.length !== 0 && typeof (path = arguments[0]) === 'string') {
         arguments[0] = this._resolve(path)
       }
-      return value.apply(this._remote, arguments)
+      return value.apply(this._handler, arguments)
     }
 
     defineProperty(pPw, name, descriptor)

@xen-orchestra/fs/src/abstract.spec.js

@@ -16,6 +16,8 @@ class TestHandler extends AbstractHandler {
   }
 }
 
+jest.useFakeTimers()
+
 describe('closeFile()', () => {
   it(`throws in case of timeout`, async () => {
     const testHandler = new TestHandler({

@xen-orchestra/fs/src/nfs.js

@@ -6,12 +6,11 @@ const DEFAULT_NFS_OPTIONS = 'vers=3'
 
 export default class NfsHandler extends MountHandler {
   constructor(remote, opts) {
-    const { host, port, path, options } = parse(remote.url)
+    const { host, port, path } = parse(remote.url)
     super(remote, opts, {
       type: 'nfs',
       device: `${host}${port !== undefined ? ':' + port : ''}:${path}`,
-      options:
-        DEFAULT_NFS_OPTIONS + (options !== undefined ? `,${options}` : ''),
+      options: DEFAULT_NFS_OPTIONS,
     })
   }
 

@xen-orchestra/fs/src/smb-mount.js

@@ -5,19 +5,13 @@ import normalizePath from './_normalizePath'
 
 export default class SmbMountHandler extends MountHandler {
   constructor(remote, opts) {
-    const {
-      domain = 'WORKGROUP',
-      host,
-      options,
-      password,
-      path,
-      username,
-    } = parse(remote.url)
+    const { domain = 'WORKGROUP', host, password, path, username } = parse(
+      remote.url
+    )
     super(remote, opts, {
       type: 'cifs',
       device: '//' + host + normalizePath(path),
-      options:
-        `domain=${domain}` + (options !== undefined ? `,${options}` : ''),
+      options: `domain=${domain}`,
       env: {
         USER: username,
         PASSWD: password,

@xen-orchestra/log/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/log",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/log",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

@xen-orchestra/mixin/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/mixin",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "@xen-orchestra/mixin",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

CHANGELOG.md

@@ -1,9 +1,89 @@
 # ChangeLog
 
-## *next*
+## **5.32.2** (2019-02-28)
+
+### Bug fixes
+
+- Fix XAPI events monitoring on old version (XenServer 7.2)
+
+## **5.32.1** (2019-02-28)
+
+### Bug fixes
+
+- Fix a very short timeout in the monitoring of XAPI events which may lead to unresponsive XenServer hosts
+
+## **5.32.0** (2019-02-28)
 
 ### Enhancements
 
+- [VM migration] Display same-pool hosts first in the selector [#3262](https://github.com/vatesfr/xen-orchestra/issues/3262) (PR [#3890](https://github.com/vatesfr/xen-orchestra/pull/3890))
+- [Home/VM] Sort VM by start time [#3955](https://github.com/vatesfr/xen-orchestra/issues/3955) (PR [#3970](https://github.com/vatesfr/xen-orchestra/pull/3970))
+- [Editable fields] Unfocusing (clicking outside) submits the change instead of canceling (PR [#3980](https://github.com/vatesfr/xen-orchestra/pull/3980))
+- [Network] Dedicated page for network creation [#3895](https://github.com/vatesfr/xen-orchestra/issues/3895) (PR [#3906](https://github.com/vatesfr/xen-orchestra/pull/3906))
+- [Logs] Add button to download the log [#3957](https://github.com/vatesfr/xen-orchestra/issues/3957) (PR [#3985](https://github.com/vatesfr/xen-orchestra/pull/3985))
+- [Continuous Replication] Share full copy between schedules [#3973](https://github.com/vatesfr/xen-orchestra/issues/3973) (PR [#3995](https://github.com/vatesfr/xen-orchestra/pull/3995))
+- [Backup] Ability to backup XO configuration and pool metadata [#808](https://github.com/vatesfr/xen-orchestra/issues/808) [#3501](https://github.com/vatesfr/xen-orchestra/issues/3501) (PR [#3912](https://github.com/vatesfr/xen-orchestra/pull/3912))
+
+### Bug fixes
+
+- [Host] Fix multipathing status for XenServer < 7.5 [#3956](https://github.com/vatesfr/xen-orchestra/issues/3956) (PR [#3961](https://github.com/vatesfr/xen-orchestra/pull/3961))
+- [Home/VM] Show creation date of the VM on if it available [#3953](https://github.com/vatesfr/xen-orchestra/issues/3953) (PR [#3959](https://github.com/vatesfr/xen-orchestra/pull/3959))
+- [Notifications] Fix invalid notifications when not registered (PR [#3966](https://github.com/vatesfr/xen-orchestra/pull/3966))
+- [Import] Fix import of some OVA files [#3962](https://github.com/vatesfr/xen-orchestra/issues/3962) (PR [#3974](https://github.com/vatesfr/xen-orchestra/pull/3974))
+- [Servers] Fix *already connected error* after a server has been removed during connection [#3976](https://github.com/vatesfr/xen-orchestra/issues/3976) (PR [#3977](https://github.com/vatesfr/xen-orchestra/pull/3977))
+- [Backup] Fix random _mount_ issues with NFS/SMB remotes [#3973](https://github.com/vatesfr/xen-orchestra/issues/3973) (PR [#4003](https://github.com/vatesfr/xen-orchestra/pull/4003))
+
+### Released packages
+
+- @xen-orchestra/fs v0.7.0
+- xen-api v0.24.3
+- xoa-updater v0.15.2
+- xo-server v5.36.0
+- xo-web v5.36.0
+
+## **5.31.2** (2019-02-08)
+
+### Enhancements
+
+- [Home] Set description on bulk snapshot [#3925](https://github.com/vatesfr/xen-orchestra/issues/3925) (PR [#3933](https://github.com/vatesfr/xen-orchestra/pull/3933))
+- Work-around the XenServer issue when `VBD#VDI` is an empty string instead of an opaque reference (PR [#3950](https://github.com/vatesfr/xen-orchestra/pull/3950))
+- [VDI migration] Retry when XenServer fails with `TOO_MANY_STORAGE_MIGRATES` (PR [#3940](https://github.com/vatesfr/xen-orchestra/pull/3940))
+- [VM]
+  - [General] The creation date of the VM is now visible [#3932](https://github.com/vatesfr/xen-orchestra/issues/3932) (PR [#3947](https://github.com/vatesfr/xen-orchestra/pull/3947))
+  - [Disks] Display device name [#3902](https://github.com/vatesfr/xen-orchestra/issues/3902) (PR [#3946](https://github.com/vatesfr/xen-orchestra/pull/3946))
+- [VM Snapshotting]
+  - Detect and destroy broken quiesced snapshot left by XenServer [#3936](https://github.com/vatesfr/xen-orchestra/issues/3936) (PR [#3937](https://github.com/vatesfr/xen-orchestra/pull/3937))
+  - Retry twice after a 1 minute delay if quiesce failed [#3938](https://github.com/vatesfr/xen-orchestra/issues/3938) (PR [#3952](https://github.com/vatesfr/xen-orchestra/pull/3952))
+
+### Bug fixes
+
+- [Import] Fix import of big OVA files
+- [Host] Show the host's memory usage instead of the sum of the VMs' memory usage (PR [#3924](https://github.com/vatesfr/xen-orchestra/pull/3924))
+- [SAML] Make `AssertionConsumerServiceURL` matches the callback URL
+- [Backup NG] Correctly delete broken VHD chains [#3875](https://github.com/vatesfr/xen-orchestra/issues/3875) (PR [#3939](https://github.com/vatesfr/xen-orchestra/pull/3939))
+- [Remotes] Don't ignore `mount` options [#3935](https://github.com/vatesfr/xen-orchestra/issues/3935) (PR [#3931](https://github.com/vatesfr/xen-orchestra/pull/3931))
+
+### Released packages
+
+- xen-api v0.24.2
+- @xen-orchestra/fs v0.6.1
+- xo-server-auth-saml v0.5.3
+- xo-server v5.35.0
+- xo-web v5.35.0
+
+## **5.31.0** (2019-01-31)
+
+### Enhancements
+
+- [Backup NG] Restore logs moved to restore tab [#3772](https://github.com/vatesfr/xen-orchestra/issues/3772) (PR [#3802](https://github.com/vatesfr/xen-orchestra/pull/3802))
+- [Remotes] New SMB implementation that provides better stability and performance [#2257](https://github.com/vatesfr/xen-orchestra/issues/2257) (PR [#3708](https://github.com/vatesfr/xen-orchestra/pull/3708))
+- [VM/advanced] ACL management from VM view [#3040](https://github.com/vatesfr/xen-orchestra/issues/3040) (PR [#3774](https://github.com/vatesfr/xen-orchestra/pull/3774))
+- [VM / snapshots] Ability to save the VM memory [#3795](https://github.com/vatesfr/xen-orchestra/issues/3795) (PR [#3812](https://github.com/vatesfr/xen-orchestra/pull/3812))
+- [Backup NG / Health] Show number of lone snapshots in tab label [#3500](https://github.com/vatesfr/xen-orchestra/issues/3500) (PR [#3824](https://github.com/vatesfr/xen-orchestra/pull/3824))
+- [Login] Add autofocus on username input on login page [#3835](https://github.com/vatesfr/xen-orchestra/issues/3835) (PR [#3836](https://github.com/vatesfr/xen-orchestra/pull/3836))
+- [Home/VM] Bulk snapshot: specify snapshots' names [#3778](https://github.com/vatesfr/xen-orchestra/issues/3778) (PR [#3787](https://github.com/vatesfr/xen-orchestra/pull/3787))
+- [Remotes] Show free space and disk usage on remote [#3055](https://github.com/vatesfr/xen-orchestra/issues/3055) (PR [#3767](https://github.com/vatesfr/xen-orchestra/pull/3767))
+- [New SR] Add tooltip for reattach action button [#3845](https://github.com/vatesfr/xen-orchestra/issues/3845) (PR [#3852](https://github.com/vatesfr/xen-orchestra/pull/3852))
 - [VM migration] Display hosts' free memory [#3264](https://github.com/vatesfr/xen-orchestra/issues/3264) (PR [#3832](https://github.com/vatesfr/xen-orchestra/pull/3832))
 - [Plugins] New field to filter displayed plugins (PR [#3832](https://github.com/vatesfr/xen-orchestra/pull/3871))
 - Ability to copy ID of "unknown item"s [#3833](https://github.com/vatesfr/xen-orchestra/issues/3833) (PR [#3856](https://github.com/vatesfr/xen-orchestra/pull/3856))
@@ -24,6 +104,12 @@
 
 ### Bug fixes
 
+- [Self] Display sorted Resource Sets [#3818](https://github.com/vatesfr/xen-orchestra/issues/3818) (PR [#3823](https://github.com/vatesfr/xen-orchestra/pull/3823))
+- [Servers] Correctly report connecting status (PR [#3838](https://github.com/vatesfr/xen-orchestra/pull/3838))
+- [Servers] Fix cannot reconnect to a server after connection has been lost [#3839](https://github.com/vatesfr/xen-orchestra/issues/3839) (PR [#3841](https://github.com/vatesfr/xen-orchestra/pull/3841))
+- [New VM] Fix `NO_HOSTS_AVAILABLE()` error when creating  a VM on a local SR from template on another local SR [#3084](https://github.com/vatesfr/xen-orchestra/issues/3084) (PR [#3827](https://github.com/vatesfr/xen-orchestra/pull/3827))
+- [Backup NG] Fix typo in the form [#3854](https://github.com/vatesfr/xen-orchestra/issues/3854) (PR [#3855](https://github.com/vatesfr/xen-orchestra/pull/3855))
+- [New SR] No warning when creating a NFS SR on a path that is already used as NFS SR [#3844](https://github.com/vatesfr/xen-orchestra/issues/3844) (PR [#3851](https://github.com/vatesfr/xen-orchestra/pull/3851))
 - [New SR] No redirection if the SR creation failed or canceled [#3843](https://github.com/vatesfr/xen-orchestra/issues/3843) (PR [#3853](https://github.com/vatesfr/xen-orchestra/pull/3853))
 - [Home] Fix two tabs opened by middle click in Firefox [#3450](https://github.com/vatesfr/xen-orchestra/issues/3450) (PR [#3825](https://github.com/vatesfr/xen-orchestra/pull/3825))
 - [XOA] Enable downgrade for ending trial (PR [#3867](https://github.com/vatesfr/xen-orchestra/pull/3867))
@@ -38,6 +124,8 @@
 
 ### Released packages
 
+- vhd-cli v0.2.0
+- @xen-orchestra/fs v0.6.0
 - vhd-lib v0.5.1
 - xoa-updater v0.15.0
 - xen-api v0.24.1
@@ -45,38 +133,6 @@
 - xo-server v5.34.0
 - xo-web v5.34.0
 
-## *staging*
-
-### Enhancements
-
-- [Backup NG] Restore logs moved to restore tab [#3772](https://github.com/vatesfr/xen-orchestra/issues/3772) (PR [#3802](https://github.com/vatesfr/xen-orchestra/pull/3802))
-- [Remotes] New SMB implementation that provides better stability and performance [#2257](https://github.com/vatesfr/xen-orchestra/issues/2257) (PR [#3708](https://github.com/vatesfr/xen-orchestra/pull/3708))
-- [VM/advanced] ACL management from VM view [#3040](https://github.com/vatesfr/xen-orchestra/issues/3040) (PR [#3774](https://github.com/vatesfr/xen-orchestra/pull/3774))
-- [VM / snapshots] Ability to save the VM memory [#3795](https://github.com/vatesfr/xen-orchestra/issues/3795) (PR [#3812](https://github.com/vatesfr/xen-orchestra/pull/3812))
-- [Backup NG / Health] Show number of lone snapshots in tab label [#3500](https://github.com/vatesfr/xen-orchestra/issues/3500) (PR [#3824](https://github.com/vatesfr/xen-orchestra/pull/3824))
-- [Login] Add autofocus on username input on login page [#3835](https://github.com/vatesfr/xen-orchestra/issues/3835) (PR [#3836](https://github.com/vatesfr/xen-orchestra/pull/3836))
-- [Home/VM] Bulk snapshot: specify snapshots' names [#3778](https://github.com/vatesfr/xen-orchestra/issues/3778) (PR [#3787](https://github.com/vatesfr/xen-orchestra/pull/3787))
-- [Remotes] Show free space and disk usage on remote [#3055](https://github.com/vatesfr/xen-orchestra/issues/3055) (PR [#3767](https://github.com/vatesfr/xen-orchestra/pull/3767))
-- [New SR] Add tooltip for reattach action button [#3845](https://github.com/vatesfr/xen-orchestra/issues/3845) (PR [#3852](https://github.com/vatesfr/xen-orchestra/pull/3852))
-
-### Bug fixes
-
-- [Self] Display sorted Resource Sets [#3818](https://github.com/vatesfr/xen-orchestra/issues/3818) (PR [#3823](https://github.com/vatesfr/xen-orchestra/pull/3823))
-- [Servers] Correctly report connecting status (PR [#3838](https://github.com/vatesfr/xen-orchestra/pull/3838))
-- [Servers] Fix cannot reconnect to a server after connection has been lost [#3839](https://github.com/vatesfr/xen-orchestra/issues/3839) (PR [#3841](https://github.com/vatesfr/xen-orchestra/pull/3841))
-- [New VM] Fix `NO_HOSTS_AVAILABLE()` error when creating  a VM on a local SR from template on another local SR [#3084](https://github.com/vatesfr/xen-orchestra/issues/3084) (PR [#3827](https://github.com/vatesfr/xen-orchestra/pull/3827))
-- [Backup NG] Fix typo in the form [#3854](https://github.com/vatesfr/xen-orchestra/issues/3854) (PR [#3855](https://github.com/vatesfr/xen-orchestra/pull/3855))
-- [New SR] No warning when creating a NFS SR on a path that is already used as NFS SR [#3844](https://github.com/vatesfr/xen-orchestra/issues/3844) (PR [#3851](https://github.com/vatesfr/xen-orchestra/pull/3851))
-
-### Released packages
-
-- vhd-lib v0.5.0
-- vhd-cli v0.2.0
-- xen-api v0.24.0
-- @xen-orchestra/fs v0.6.0
-- xo-server v5.33.0
-- xo-web v5.33.0
-
 ## **5.30.0** (2018-12-20)
 
 ### Enhancements

CHANGELOG.unreleased.md

@@ -0,0 +1,28 @@
+> This file contains all changes that have not been released yet.
+
+### Enhancements
+
+- [SR/Disk] Disable actions on unmanaged VDIs [#3988](https://github.com/vatesfr/xen-orchestra/issues/3988) (PR [#4000](https://github.com/vatesfr/xen-orchestra/pull/4000))
+- [Pool] Specify automatic networks on a Pool [#3916](https://github.com/vatesfr/xen-orchestra/issues/3916) (PR [#3958](https://github.com/vatesfr/xen-orchestra/pull/3958))
+- [VM/advanced] Manage start delay for VM [#3909](https://github.com/vatesfr/xen-orchestra/issues/3909) (PR [#4002](https://github.com/vatesfr/xen-orchestra/pull/4002))
+- [New/Vm] SR section: Display warning message when the selected SRs aren't in the same host [#3911](https://github.com/vatesfr/xen-orchestra/issues/3911) (PR [#3967](https://github.com/vatesfr/xen-orchestra/pull/3967))
+- Enable compression for HTTP requests (and initial objects fetch)
+- [VDI migration] Display same-pool SRs first in the selector [#3945](https://github.com/vatesfr/xen-orchestra/issues/3945) (PR [#3996](https://github.com/vatesfr/xen-orchestra/pull/3996))
+- [Home] Save the current page in url [#3993](https://github.com/vatesfr/xen-orchestra/issues/3993) (PR [#3999](https://github.com/vatesfr/xen-orchestra/pull/3999))
+- [VDI] Ensure suspend VDI is destroyed when destroying a VM  [#4027](https://github.com/vatesfr/xen-orchestra/issues/4027) (PR [#4038](https://github.com/vatesfr/xen-orchestra/pull/4038))
+- [VM/disk]: Warning when 2 VDIs are on 2 different hosts' local SRs [#3911](https://github.com/vatesfr/xen-orchestra/issues/3911) (PR [#3969](https://github.com/vatesfr/xen-orchestra/pull/3969))
+
+### Bug fixes
+
+- [New network] PIF was wrongly required which prevented from creating a private network (PR [#4010](https://github.com/vatesfr/xen-orchestra/pull/4010))
+- [Google authentication] Migrate to new endpoint
+- [Backup NG] Better handling of huge logs [#4025](https://github.com/vatesfr/xen-orchestra/issues/4025) (PR [#4026](https://github.com/vatesfr/xen-orchestra/pull/4026))
+- [Home/VM] Bulk migration: fixed VM VDIs not migrated to the selected SR [#3986](https://github.com/vatesfr/xen-orchestra/issues/3986) (PR [#3987](https://github.com/vatesfr/xen-orchestra/pull/3987))
+- [Stats] Fix cache usage with simultaneous requests [#4017](https://github.com/vatesfr/xen-orchestra/issues/4017) (PR [#4028](https://github.com/vatesfr/xen-orchestra/pull/4028))
+- [Backup NG] Fix compression displayed for the wrong backup mode (PR [#4021](https://github.com/vatesfr/xen-orchestra/pull/4021))
+
+### Released packages
+
+- xo-server-auth-google v0.2.1
+- xo-server v5.37.0
+- xo-web v5.37.0

PULL_REQUEST_TEMPLATE.md

@@ -4,7 +4,7 @@
 
 - [ ] PR reference the relevant issue (e.g. `Fixes #007`)
 - [ ] if UI changes, a screenshot has been added to the PR
-- [ ] CHANGELOG:
+- [ ] `CHANGELOG.unreleased.md`:
    - enhancement/bug fix entry added
    - list of packages to release updated (`${name} v${new version}`)
 - [ ] documentation updated

docs/SUMMARY.md

@@ -33,6 +33,7 @@
        * [Disaster recovery](disaster_recovery.md)
        * [Smart Backup](smart_backup.md)
        * [File level Restore](file_level_restore.md)
+       * [Metadata Backup](metadata_backup.md)
        * [Backup Concurrency](concurrency.md)
        * [Configure backup reports](backup_reports.md)
        * [Backup troubleshooting](backup_troubleshooting.md)
@@ -51,6 +52,7 @@
    * [Job manager](scheduler.md)
    * [Alerts](alerts.md)
    * [Load balancing](load_balancing.md)
+   * [Emergency Shutdown](emergency_shutdown.md)
    * [Auto scalability](auto_scalability.md)
    * [Forecaster](forecaster.md)
 * [Recipes](recipes.md)

docs/backup_troubleshooting.md

@@ -33,7 +33,7 @@ Just remember this: **coalesce will happen every time a snapshot is removed**.
 
 First check SMlog on the XenServer host for messages relating to VDI corruption or coalesce job failure. For example, by running `cat /var/log/SMlog | grep -i exception` or `cat /var/log/SMlog | grep -i error` on the XenServer host with the affected storage.  
 
-Coalesce jobs can also fail to run if the SR does not have enough free space. Check the problematic SR and make sure it has enough free space, generally 30% or more free is recommended depending on VM size.  
+Coalesce jobs can also fail to run if the SR does not have enough free space. Check the problematic SR and make sure it has enough free space, generally 30% or more free is recommended depending on VM size. You can check if this is the issue by searching `SMlog` with `grep -i coales /var/log/SMlog` (you may have to look at previous logs such as `SMlog.1`).
 
 You can check if a coalesce job is currently active by running `ps axf | grep vhd` on the XenServer host and looking for a VHD process in the results (one of the resulting processes will be the grep command you just ran, ignore that one).  
 

docs/backups.md

@@ -1,5 +1,7 @@
 # Backups
 
+> Watch our [introduction video](https://www.youtube.com/watch?v=FfUqIwT8KzI) (45m) to Backup in Xen Orchestra!
+
 This section is dedicated to all existing methods of rolling back or backing up your VMs in Xen Orchestra.
 
 There are several ways to protect your VMs:
@@ -8,6 +10,7 @@ There are several ways to protect your VMs:
 * [Rolling Snapshots](rolling_snapshots.md) [*Starter Edition*]
 * [Delta Backups](delta_backups.md) (best of both previous ones) [*Enterprise Edition*]
 * [Disaster Recovery](disaster_recovery.md) [*Enterprise Edition*]
+* [Metadata Backups](metadata_backup.md) [*Enterprise Edition*]
 * [Continuous Replication](continuous_replication.md) [*Premium Edition*]
 * [File Level Restore](file_level_restore.md) [*Premium Edition*]
 
@@ -39,7 +42,7 @@ Each backups' job execution is identified by a `runId`. You can find this `runId
 
 All backup types rely on snapshots. But what about data consistency? By default, Xen Orchestra will try to take a **quiesced snapshot** every time a snapshot is done (and fall back to normal snapshots if it's not possible).
 
-Snapshots of Windows VMs can be quiesced (especially MS SQL or Exchange services) after you have installed Xen Tools in your VMs. However, [there is an extra step to install the VSS provider on windows](quiesce). A quiesced snapshot means the operating system will be notified and the cache will be flushed to disks. This way, your backups will always be consistent.
+Snapshots of Windows VMs can be quiesced (especially MS SQL or Exchange services) after you have installed Xen Tools in your VMs. However, [there is an extra step to install the VSS provider on windows](https://xen-orchestra.com/blog/xenserver-quiesce-snapshots/). A quiesced snapshot means the operating system will be notified and the cache will be flushed to disks. This way, your backups will always be consistent.
 
 To see if you have quiesced snapshots for a VM, just go into its snapshot tab, then the "info" icon means it is a quiesced snapshot:
 

docs/cloudinit.md

@@ -1,7 +1,5 @@
 # CloudInit
 
-> CloudInit support is available in the 4.11 release and higher
-
 Cloud-init is a program "that handles the early initialization of a cloud instance"[^n]. In other words, you can, on a "cloud-init"-ready template VM, pass a lot of data at first boot:
 
 * setting the hostname
@@ -18,25 +16,27 @@ So it means very easily customizing your VM when you create it from a compatible
 
 You only need to use a template of a VM with CloudInit installed inside it. [Check this blog post to learn how to install CloudInit](https://xen-orchestra.com/blog/centos-cloud-template-for-xenserver/).
 
+**Note:** In XOA 5.31, we changed the cloud-init config drive type from [OpenStack](https://cloudinit.readthedocs.io/en/latest/topics/datasources/configdrive.html) to the [NoCloud](https://cloudinit.readthedocs.io/en/latest/topics/datasources/nocloud.html) type. This will allow us to pass network configuration to VMs in the future. For 99% of users, including default cloud-init installs, this change will have no effect. However if you have previously modified your cloud-init installation in a VM template to only look for `openstack` drive types (for instance with the `datasource_list` setting in `/etc/cloud/cloud.cfg`) you need to modify it to also look for `nocloud`.
+
 ## Usage
 
 First, select your compatible template (CloudInit ready) and name it:
 
-![](https://xen-orchestra.com/blog/content/images/2015/12/template_choice.png)
+![](./assets/cloud-init-1.png)
 
 Then, activate the config drive and insert your SSH key. Or you can also use a custom CloudInit configuration:
 
-![](https://xen-orchestra.com/blog/content/images/2016/02/CloudInit.png)
+![](./assets/cloud-init-2.png)
 
 > CloudInit configuration examples are [available here](http://cloudinit.readthedocs.org/en/latest/topics/examples.html).
 
-You can extend the disk size (**in this case, the template disk was 8 GiB originally**):
+You can extend the disk size (**in this case, the template disk was 8 GiB originally**). We'll extend it to 20GiB:
 
-![](https://xen-orchestra.com/blog/content/images/2015/12/diskedition.png)
+![](./assets/cloud-init-3.png)
 
 Finally, create the VM:
 
-![](https://xen-orchestra.com/blog/content/images/2015/12/recap.png)
+![](./assets/cloud-init-4.png)
 
 Now start the VM and SSH to its IP:
 

docs/continuous_replication.md

@@ -66,7 +66,7 @@ The XOA backup system requires metadata to correctly associate the source snapsh
 First install the tool (all the following is done from the XOA VM CLI):
 
 ```
-npm i -g xo-cr-seed
+sudo npm i -g --unsafe-perm @xen-orchestra/cr-seed-cli
 ```
 
 Here is an example of how the utility expects the UUIDs and info passed to it:

docs/emergency_shutdown.md

@@ -0,0 +1,27 @@
+# Emergency Shutdown
+
+If you have a UPS for your hosts, and lose power, you may have a limited amount of time to shut down all of your VM infrastructure before the batteries run out. If you find yourself in this situation, or any other situation requiring the fast shutdown of everything, you can use the **Emergency Shutdown** feature.
+
+## How to activate
+On the host view, clicking on this button will trigger the  _Emergency Shutdown_ procedure:
+
+![](./assets/e-shutdown-1.png)
+
+1.  **All running VMs will be suspended**  (think of it like "hibernate" on your laptop: the RAM will be stored in the storage repository).
+2.  Only after this is complete, the host will be halted.
+
+Here, you can see the running VMs are being suspended:
+
+![](./assets/e-shutdown-2.png)
+
+And finally, that's it. They are cleanly shut down with the RAM saved to disk to be resumed later:
+
+![](./assets/e-shutdown-3.png)
+
+Now the host is halted automatically.
+
+## Powering back on
+When the power outage is over, all you need to do is:
+
+1.  Start your host.
+2.  All your VMs can be resumed, your RAM is preserved and therefore your VMs will be in the exact same state as they were before the power outage.

docs/from_the_sources.md

@@ -6,9 +6,9 @@
 
 > Please take time to read this guide carefully.
 
-This installation has been validated against a fresh Debian 8 (Jessie) x64 install. It should be nearly the same on other dpkg systems. For RPM based OS's, it should be close, as most of our dependencies come from NPM and not the OS itself.
+This installation has been validated against a fresh Debian 9 (Stretch) x64 install. It should be nearly the same on other dpkg systems. For RPM based OS's, it should be close, as most of our dependencies come from NPM and not the OS itself.
 
-As you may have seen,in other parts of the documentation, XO is composed of two parts: [xo-server](https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server/) and [xo-web](https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-web/). They can be installed separately, even on different machines, but for the sake of simplicity we will set them up together.
+As you may have seen in other parts of the documentation, XO is composed of two parts: [xo-server](https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server/) and [xo-web](https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-web/). They can be installed separately, even on different machines, but for the sake of simplicity we will set them up together.
 
 ## Packages and Pre-requisites
 
@@ -38,7 +38,7 @@ XO needs the following packages to be installed. Redis is used as a database by
 For example, on Debian:
 
 ```
-apt-get install build-essential redis-server libpng-dev git python-minimal libvhdi-utils lvm2
+apt-get install build-essential redis-server libpng-dev git python-minimal libvhdi-utils lvm2 cifs-utils
 ```
 
 ## Fetching the Code
@@ -49,13 +49,14 @@ You need to use the `git` source code manager to fetch the code. Ideally, you sh
 git clone -b master http://github.com/vatesfr/xen-orchestra
 ```
 
-> Note: xo-server and xo-web have been migrated to the [xen-orchestra](https://github.com/vatesfr/xen-orchestra) mono-repository.
+> Note: xo-server and xo-web have been migrated to the [xen-orchestra](https://github.com/vatesfr/xen-orchestra) mono-repository - so you only need the single clone command above
 
 ## Installing dependencies
 
-Once you have it, use `yarn`, as the non-root (or root) user owning the fetched code, to install the other dependencies. Enter the `xen-orchestra` directory and run the following commands:
+Now that you have the code, you can enter the `xen-orchestra` directory and use `yarn` to install other dependencies. Then finally build it using `yarn build`. Be sure to run `yarn` commands as the same user you will be using to run Xen Orchestra:
 
 ```
+$ cd xen-orchestra
 $ yarn
 $ yarn build
 ```
@@ -86,7 +87,7 @@ WebServer listening on localhost:80
 
 ## Running XO
 
-The only part you need to launch is xo-server which is quite easy to do. From the `xen-orchestra/packages/xo-server` directory, run the following:
+The only part you need to launch is xo-server, which is quite easy to do. From the `xen-orchestra/packages/xo-server` directory, run the following:
 
 ```
 $ yarn start

docs/metadata_backup.md

@@ -0,0 +1,31 @@
+# Metadata backup
+
+> WARNING: Metadata backup is an experimental feature. Restore is not yet available and some unexpected issues may occur.
+
+## Introduction
+
+XCP-ng and Citrix Hypervisor (Xenserver) hosts use a database to store metadata about VMs and their associated resources such as storage and networking. Metadata forms this complete view of all VMs available on your pool. Backing up the metadata of your pool allows you to recover from a physical hardware failure scenario in which you lose your hosts without losing your storage (SAN, NAS...).
+
+In Xen Orchestra, Metadata backup is divided into two different options:
+
+* Pool metadata backup
+* XO configuration backup
+
+### How to use metadata backup
+
+In the backup job section, when creating a new backup job, you will now have a choice between backing up VMs and backing up Metadata.
+![](https://user-images.githubusercontent.com/21563339/53413921-bd636f00-39cd-11e9-8a3c-d4f893135fa4.png)
+
+When you select Metadata backup, you will have a new backup job screen, letting you choose between a pool metadata backup and an XO configuration backup (or both at the same time):
+
+![](https://user-images.githubusercontent.com/21563339/52416838-d2de2b00-2aea-11e9-8da0-340fcb2767db.png)
+
+Define the name and retention for the job.
+
+![](https://user-images.githubusercontent.com/21563339/52471527-65390a00-2b91-11e9-8019-600a4d9eeafb.png)
+
+Once created, the job is displayed with the other classic jobs.
+
+![](https://user-images.githubusercontent.com/21563339/52416802-c0fc8800-2aea-11e9-8ef0-b0c1bd0e48b8.png)
+
+> Restore for metadata backup jobs should be available in XO 5.33

package.json

@@ -4,10 +4,10 @@
     "@babel/register": "^7.0.0",
     "babel-core": "^7.0.0-0",
     "babel-eslint": "^10.0.1",
-    "babel-jest": "^23.0.1",
+    "babel-jest": "^24.1.0",
     "benchmark": "^2.1.4",
     "eslint": "^5.1.0",
-    "eslint-config-prettier": "^3.3.0",
+    "eslint-config-prettier": "^4.1.0",
     "eslint-config-standard": "12.0.0",
     "eslint-config-standard-jsx": "^6.0.2",
     "eslint-plugin-import": "^2.8.0",
@@ -16,10 +16,10 @@
     "eslint-plugin-react": "^7.6.1",
     "eslint-plugin-standard": "^4.0.0",
     "exec-promise": "^0.7.0",
-    "flow-bin": "^0.90.0",
+    "flow-bin": "^0.94.0",
     "globby": "^9.0.0",
     "husky": "^1.2.1",
-    "jest": "^23.0.1",
+    "jest": "^24.1.0",
     "lodash": "^4.17.4",
     "prettier": "^1.10.2",
     "promise-toolbox": "^0.11.0",
@@ -34,7 +34,6 @@
     }
   },
   "jest": {
-    "timers": "fake",
     "collectCoverage": true,
     "projects": [
       "<rootDir>"

packages/complex-matcher/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/complex-matcher",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/complex-matcher",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/value-matcher/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/value-matcher",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/value-matcher",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/vhd-cli/package.json

@@ -1,12 +1,13 @@
 {
   "name": "vhd-cli",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "license": "ISC",
   "description": "",
   "keywords": [],
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-cli",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/vhd-cli",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -26,7 +27,7 @@
     "node": ">=6"
   },
   "dependencies": {
-    "@xen-orchestra/fs": "^0.6.0",
+    "@xen-orchestra/fs": "^0.7.1",
     "cli-progress": "^2.0.0",
     "exec-promise": "^0.7.0",
     "getopts": "^2.2.3",

packages/vhd-cli/src/commands/repl.js

@@ -0,0 +1,33 @@
+import { asCallback, fromCallback, fromEvent } from 'promise-toolbox'
+import { getHandler } from '@xen-orchestra/fs'
+import { relative } from 'path'
+import { start as createRepl } from 'repl'
+import Vhd, * as vhdLib from 'vhd-lib'
+
+export default async args => {
+  const cwd = process.cwd()
+  const handler = getHandler({ url: 'file://' + cwd })
+  await handler.sync()
+  try {
+    const repl = createRepl({
+      prompt: 'vhd> ',
+    })
+    Object.assign(repl.context, vhdLib)
+    repl.context.handler = handler
+    repl.context.open = path => new Vhd(handler, relative(cwd, path))
+
+    // Make the REPL waits for promise completion.
+    repl.eval = (evaluate => (cmd, context, filename, cb) => {
+      asCallback.call(
+        fromCallback(cb => {
+          evaluate.call(repl, cmd, context, filename, cb)
+        }).then(value => (Array.isArray(value) ? Promise.all(value) : value)),
+        cb
+      )
+    })(repl.eval)
+
+    await fromEvent(repl, 'exit')
+  } finally {
+    await handler.forget()
+  }
+}

packages/vhd-cli/src/index.js

@@ -2,6 +2,8 @@
 
 import execPromise from 'exec-promise'
 
+import pkg from '../package.json'
+
 import commands from './commands'
 
 function runCommand(commands, [command, ...args]) {
@@ -18,7 +20,9 @@ function runCommand(commands, [command, ...args]) {
 ${Object.keys(commands)
   .filter(command => command !== 'help')
   .map(command => `    ${this.command} ${command}`)
-  .join('\n\n')}`
+  .join('\n\n')}
+
+vhd-cli ${pkg.version}`
     }
 
     throw `invalid command ${command}` // eslint-disable-line no-throw-literal

packages/vhd-lib/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-lib",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/vhd-lib",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -34,7 +35,7 @@
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
     "@babel/preset-flow": "^7.0.0",
-    "@xen-orchestra/fs": "^0.6.0",
+    "@xen-orchestra/fs": "^0.7.1",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^5.1.3",
     "execa": "^1.0.0",

packages/xapi-explore-sr/package.json

@@ -15,6 +15,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xapi-explore-sr",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xapi-explore-sr",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -40,7 +41,7 @@
     "human-format": "^0.10.0",
     "lodash": "^4.17.4",
     "pw": "^0.0.4",
-    "xen-api": "^0.24.1"
+    "xen-api": "^0.24.5"
   },
   "devDependencies": {
     "@babel/cli": "^7.1.5",

packages/xen-api/README.md

@@ -95,7 +95,7 @@ root@xen1.company.net> xapi.pool.$master.name_label
 To ease searches, `find()` and `findAll()` functions are available:
 
 ```
-root@xen1.company.net> findAll({ $type: 'vm' }).length
+root@xen1.company.net> findAll({ $type: 'VM' }).length
 183
 ```
 

packages/xen-api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xen-api",
-  "version": "0.24.1",
+  "version": "0.24.5",
   "license": "ISC",
   "description": "Connector to the Xen API",
   "keywords": [
@@ -13,6 +13,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xen-api",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xen-api",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -36,9 +37,9 @@
     "debug": "^4.0.1",
     "event-to-promise": "^0.8.0",
     "exec-promise": "^0.7.0",
-    "http-request-plus": "^0.7.1",
-    "iterable-backoff": "^0.0.0",
-    "jest-diff": "^23.5.0",
+    "http-request-plus": "^0.7.2",
+    "iterable-backoff": "^0.1.0",
+    "jest-diff": "^24.0.0",
     "json-rpc-protocol": "^0.13.1",
     "kindof": "^2.0.0",
     "lodash": "^4.17.4",

packages/xen-api/src/index.js

@@ -7,8 +7,8 @@ import { BaseError } from 'make-error'
 import { EventEmitter } from 'events'
 import { fibonacci } from 'iterable-backoff'
 import {
-  filter,
   forEach,
+  forOwn,
   isArray,
   isInteger,
   map,
@@ -37,7 +37,7 @@ const debug = createDebug('xen-api')
 
 // ===================================================================
 
-// in seconds
+// in seconds!
 const EVENT_TIMEOUT = 60
 
 // http://www.gnu.org/software/libc/manual/html_node/Error-Codes.html
@@ -248,6 +248,11 @@ const RESERVED_FIELDS = {
   pool: true,
   ref: true,
   type: true,
+  xapi: true,
+}
+
+function getPool() {
+  return this.$xapi.pool
 }
 
 // -------------------------------------------------------------------
@@ -266,17 +271,14 @@ export class Xapi extends EventEmitter {
     super()
 
     this._allowUnauthorized = opts.allowUnauthorized
-    this._auth = opts.auth
     this._callTimeout = makeCallSetting(opts.callTimeout, 0)
-    this._debounce = opts.debounce == null ? 200 : opts.debounce
     this._pool = null
     this._readOnly = Boolean(opts.readOnly)
     this._RecordsByType = createObject(null)
     this._sessionId = null
-    ;(this._objects = new Collection()).getKey = getKey
-    ;(this._objectsByRef = createObject(null))[NULL_REF] = undefined
-    const url = (this._url = parseUrl(opts.url))
 
+    this._auth = opts.auth
+    const url = (this._url = parseUrl(opts.url))
     if (this._auth === undefined) {
       const user = url.username
       if (user !== undefined) {
@@ -289,10 +291,19 @@ export class Xapi extends EventEmitter {
       }
     }
 
-    // Memoize this function _addObject().
-    this._getPool = () => this._pool
+    ;(this._objects = new Collection()).getKey = getKey
+    this._debounce = opts.debounce == null ? 200 : opts.debounce
+    this._watchedTypes = undefined
+    this._watching = false
 
-    if (opts.watchEvents !== false) {
+    this.on(DISCONNECTED, this._clearObjects)
+    this._clearObjects()
+
+    const { watchEvents } = opts
+    if (watchEvents !== false) {
+      if (Array.isArray(watchEvents)) {
+        this._watchedTypes = watchEvents
+      }
       this.watchEvents()
     }
   }
@@ -300,19 +311,14 @@ export class Xapi extends EventEmitter {
   watchEvents() {
     this._eventWatchers = createObject(null)
 
-    this._fromToken = ''
-
-    this._nTasks = 0
-
     this._taskWatchers = Object.create(null)
 
     if (this.status === CONNECTED) {
-      this._watchEvents()
+      this._watchEventsWrapper()
     }
 
-    this.on('connected', this._watchEvents)
+    this.on('connected', this._watchEventsWrapper)
     this.on('disconnected', () => {
-      this._fromToken = ''
       this._objects.clear()
     })
   }
@@ -401,42 +407,55 @@ export class Xapi extends EventEmitter {
     )
   }
 
-  connect() {
+  async connect() {
     const { status } = this
 
     if (status === CONNECTED) {
-      return Promise.reject(new Error('already connected'))
+      throw new Error('already connected')
     }
 
     if (status === CONNECTING) {
-      return Promise.reject(new Error('already connecting'))
+      throw new Error('already connecting')
     }
 
     const auth = this._auth
     if (auth === undefined) {
-      return Promise.reject(new Error('missing credentials'))
+      throw new Error('missing credentials')
     }
 
     this._sessionId = CONNECTING
 
-    return this._transportCall('session.login_with_password', [
-      auth.user,
-      auth.password,
-    ]).then(
-      async sessionId => {
-        this._sessionId = sessionId
-        this._pool = (await this.getAllRecords('pool'))[0]
+    try {
+      const [methods, sessionId] = await Promise.all([
+        this._transportCall('system.listMethods', []),
+        this._transportCall('session.login_with_password', [
+          auth.user,
+          auth.password,
+        ]),
+      ])
 
-        debug('%s: connected', this._humanId)
+      // Uses introspection to list available types.
+      const types = (this._types = methods
+        .filter(isGetAllRecordsMethod)
+        .map(method => method.slice(0, method.indexOf('.'))))
+      this._lcToTypes = { __proto__: null }
+      types.forEach(type => {
+        const lcType = type.toLowerCase()
+        if (lcType !== type) {
+          this._lcToTypes[lcType] = type
+        }
+      })
 
-        this.emit(CONNECTED)
-      },
-      error => {
-        this._sessionId = null
+      this._sessionId = sessionId
+      this._pool = (await this.getAllRecords('pool'))[0]
 
-        throw error
-      }
-    )
+      debug('%s: connected', this._humanId)
+      this.emit(CONNECTED)
+    } catch (error) {
+      this._sessionId = null
+
+      throw error
+    }
   }
 
   disconnect() {
@@ -499,6 +518,10 @@ export class Xapi extends EventEmitter {
     return promise
   }
 
+  getField(type, ref, field) {
+    return this._sessionCall(`${type}.get_${field}`, [ref])
+  }
+
   // Nice getter which returns the object for a given $id (internal to
   // this lib), UUID (unique identifier that some objects have) or
   // opaque reference (internal to XAPI).
@@ -550,6 +573,10 @@ export class Xapi extends EventEmitter {
     )
   }
 
+  getRecords(type, refs) {
+    return Promise.all(refs.map(ref => this.getRecord(type, ref)))
+  }
+
   async getAllRecords(type) {
     return map(
       await this._sessionCall(`${type}.get_all_records`),
@@ -565,7 +592,7 @@ export class Xapi extends EventEmitter {
   }
 
   @cancelable
-  getResource($cancelToken, pathname, { host, query, task }) {
+  getResource($cancelToken, pathname, { host, query, task } = {}) {
     return this._autoTask(task, `Xapi#getResource ${pathname}`).then(
       taskRef => {
         query = { ...query, session_id: this.sessionId }
@@ -718,41 +745,38 @@ export class Xapi extends EventEmitter {
     )
   }
 
-  setField({ $type, $ref }, field, value) {
-    return this.call(`${$type}.set_${field}`, $ref, value).then(noop)
+  setField(type, ref, field, value) {
+    return this.call(`${type}.set_${field}`, ref, value).then(noop)
   }
 
-  setFieldEntries(record, field, entries) {
+  setFieldEntries(type, ref, field, entries) {
     return Promise.all(
       getKeys(entries).map(entry => {
         const value = entries[entry]
         if (value !== undefined) {
-          return value === null
-            ? this.unsetFieldEntry(record, field, entry)
-            : this.setFieldEntry(record, field, entry, value)
+          return this.setFieldEntry(type, ref, field, entry, value)
         }
       })
     ).then(noop)
   }
 
-  async setFieldEntry({ $type, $ref }, field, entry, value) {
+  async setFieldEntry(type, ref, field, entry, value) {
+    if (value === null) {
+      return this.call(`${type}.remove_from_${field}`, ref, entry).then(noop)
+    }
     while (true) {
       try {
-        await this.call(`${$type}.add_to_${field}`, $ref, entry, value)
+        await this.call(`${type}.add_to_${field}`, ref, entry, value)
         return
       } catch (error) {
         if (error == null || error.code !== 'MAP_DUPLICATE_KEY') {
           throw error
         }
       }
-      await this.unsetFieldEntry({ $type, $ref }, field, entry)
+      await this.call(`${type}.remove_from_${field}`, ref, entry)
     }
   }
 
-  unsetFieldEntry({ $type, $ref }, field, entry) {
-    return this.call(`${$type}.remove_from_${field}`, $ref, entry)
-  }
-
   watchTask(ref) {
     const watchers = this._taskWatchers
     if (watchers === undefined) {
@@ -786,6 +810,15 @@ export class Xapi extends EventEmitter {
     return this._objects
   }
 
+  _clearObjects() {
+    ;(this._objectsByRef = createObject(null))[NULL_REF] = undefined
+    this._nTasks = 0
+    this._objects.clear()
+    this.objectsFetched = new Promise(resolve => {
+      this._resolveObjectsFetched = resolve
+    })
+  }
+
   // return a promise which resolves to a task ref or undefined
   _autoTask(task = this._taskWatchers !== undefined, name) {
     if (task === false) {
@@ -801,7 +834,7 @@ export class Xapi extends EventEmitter {
   }
 
   // Medium level call: handle session errors.
-  _sessionCall(method, args) {
+  _sessionCall(method, args, timeout = this._callTimeout(method, args)) {
     try {
       if (startsWith(method, 'session.')) {
         throw new Error('session.*() methods are disabled from this interface')
@@ -825,7 +858,7 @@ export class Xapi extends EventEmitter {
             return this.connect().then(() => this._sessionCall(method, args))
           }
         ),
-        this._callTimeout(method, args)
+        timeout
       )
     } catch (error) {
       return Promise.reject(error)
@@ -904,7 +937,12 @@ export class Xapi extends EventEmitter {
 
   _processEvents(events) {
     forEach(events, event => {
-      const { class: type, ref } = event
+      let type = event.class
+      const lcToTypes = this._lcToTypes
+      if (type in lcToTypes) {
+        type = lcToTypes[type]
+      }
+      const { ref } = event
       if (event.operation === 'del') {
         this._removeObject(type, ref)
       } else {
@@ -913,34 +951,112 @@ export class Xapi extends EventEmitter {
     })
   }
 
-  _watchEvents() {
-    const loop = () =>
-      this.status === CONNECTED &&
-      pTimeout
-        .call(
-          this._sessionCall('event.from', [
-            ['*'],
-            this._fromToken,
-            EVENT_TIMEOUT + 0.1, // Force float.
-          ]),
-          EVENT_TIMEOUT * 1.1e3 // 10% longer than the XenAPI timeout
+  // - prevent multiple watches
+  // - swallow errors
+  async _watchEventsWrapper() {
+    if (!this._watching) {
+      this._watching = true
+      try {
+        await this._watchEvents()
+      } catch (error) {
+        console.error('_watchEventsWrapper', error)
+      }
+      this._watching = false
+    }
+  }
+
+  // TODO: cancelation
+  async _watchEvents() {
+    this._clearObjects()
+
+    // compute the initial token for the event loop
+    //
+    // we need to do this before the initial fetch to avoid losing events
+    let fromToken
+    try {
+      fromToken = await this._sessionCall('event.inject', [
+        'pool',
+        this._pool.$ref,
+      ])
+    } catch (error) {
+      if (isMethodUnknown(error)) {
+        return this._watchEventsLegacy()
+      }
+    }
+
+    const types = this._watchedTypes || this._types
+
+    // initial fetch
+    const flush = this.objects.bufferEvents()
+    try {
+      await Promise.all(
+        types.map(async type => {
+          try {
+            // FIXME: use _transportCall to avoid auto-reconnection
+            forOwn(
+              await this._sessionCall(`${type}.get_all_records`),
+              (record, ref) => {
+                // we can bypass _processEvents here because they are all *add*
+                // event and all objects are of the same type
+                this._addObject(type, ref, record)
+              }
+            )
+          } catch (error) {
+            // there is nothing ideal to do here, do not interrupt event
+            // handling
+            if (error != null && error.code !== 'MESSAGE_REMOVED') {
+              console.warn('_watchEvents', 'initial fetch', type, error)
+            }
+          }
+        })
+      )
+    } finally {
+      flush()
+    }
+    this._resolveObjectsFetched()
+
+    // event loop
+    const debounce = this._debounce
+    while (true) {
+      if (debounce != null) {
+        await pDelay(debounce)
+      }
+
+      let result
+      try {
+        result = await this._sessionCall(
+          'event.from',
+          [
+            types,
+            fromToken,
+            EVENT_TIMEOUT + 0.1, // must be float for XML-RPC transport
+          ],
+          EVENT_TIMEOUT * 1e3 * 1.1
         )
-        .then(onSuccess, onFailure)
+      } catch (error) {
+        if (error instanceof TimeoutError) {
+          continue
+        }
+        if (areEventsLost(error)) {
+          return this._watchEvents()
+        }
+        throw error
+      }
 
-    const onSuccess = ({ events, token, valid_ref_counts: { task } }) => {
-      this._fromToken = token
-      this._processEvents(events)
+      fromToken = result.token
+      this._processEvents(result.events)
 
-      if (task !== this._nTasks) {
-        this._sessionCall('task.get_all_records')
-          .then(tasks => {
+      // detect and fix disappearing tasks (e.g. when toolstack restarts)
+      if (result.valid_ref_counts.task !== this._nTasks) {
+        await ignoreErrors.call(
+          this._sessionCall('task.get_all_records').then(tasks => {
             const toRemove = new Set()
-            forEach(this.objects.all, object => {
+            forOwn(this.objects.all, object => {
               if (object.$type === 'task') {
                 toRemove.add(object.$ref)
               }
             })
-            forEach(tasks, (task, ref) => {
+            forOwn(tasks, (task, ref) => {
               toRemove.delete(ref)
               this._addObject('task', ref, task)
             })
@@ -948,40 +1064,9 @@ export class Xapi extends EventEmitter {
               this._removeObject('task', ref)
             })
           })
-          .catch(noop)
+        )
       }
-
-      const debounce = this._debounce
-      return debounce != null ? pDelay(debounce).then(loop) : loop()
     }
-    const onFailure = error => {
-      if (error instanceof TimeoutError) {
-        return loop()
-      }
-
-      if (areEventsLost(error)) {
-        this._fromToken = ''
-        this._objects.clear()
-
-        return loop()
-      }
-
-      throw error
-    }
-
-    ignoreErrors.call(
-      pCatch.call(
-        loop(),
-        isMethodUnknown,
-
-        // If the server failed, it is probably due to an excessively
-        // large response.
-        // Falling back to legacy events watch should be enough.
-        error => error && error.res && error.res.statusCode === 500,
-
-        () => this._watchEventsLegacy()
-      )
-    )
   }
 
   // This method watches events using the legacy `event.next` XAPI
@@ -989,17 +1074,13 @@ export class Xapi extends EventEmitter {
   //
   // It also has to manually get all objects first.
   _watchEventsLegacy() {
-    const getAllObjects = () => {
-      return this._sessionCall('system.listMethods').then(methods => {
-        // Uses introspection to determine the methods to use to get
-        // all objects.
-        const getAllRecordsMethods = filter(methods, isGetAllRecordsMethod)
-
-        return Promise.all(
-          map(getAllRecordsMethods, method =>
-            this._sessionCall(method).then(
+    const getAllObjects = async () => {
+      const flush = this.objects.bufferEvents()
+      try {
+        await Promise.all(
+          this._types.map(type =>
+            this._sessionCall(`${type}.get_all_records`).then(
               objects => {
-                const type = method.slice(0, method.indexOf('.')).toLowerCase()
                 forEach(objects, (object, ref) => {
                   this._addObject(type, ref, object)
                 })
@@ -1012,7 +1093,10 @@ export class Xapi extends EventEmitter {
             )
           )
         )
-      })
+      } finally {
+        flush()
+      }
+      this._resolveObjectsFetched()
     }
 
     const watchEvents = () =>
@@ -1048,13 +1132,13 @@ export class Xapi extends EventEmitter {
       const nFields = fields.length
       const xapi = this
 
-      const objectsByRef = this._objectsByRef
-      const getObjectByRef = ref => objectsByRef[ref]
+      const getObjectByRef = ref => this._objectsByRef[ref]
 
       Record = function(ref, data) {
         defineProperties(this, {
           $id: { value: data.uuid || ref },
           $ref: { value: ref },
+          $xapi: { value: xapi },
         })
         for (let i = 0; i < nFields; ++i) {
           const field = fields[i]
@@ -1062,11 +1146,11 @@ export class Xapi extends EventEmitter {
         }
       }
 
-      const getters = { $pool: this._getPool }
+      const getters = { $pool: getPool }
       const props = { $type: type }
       fields.forEach(field => {
         props[`set_${field}`] = function(value) {
-          return xapi.setField(this, field, value)
+          return xapi.setField(this.$type, this.$ref, field, value)
         }
 
         const $field = (field in RESERVED_FIELDS ? '$$' : '$') + field
@@ -1090,16 +1174,21 @@ export class Xapi extends EventEmitter {
             const value = this[field]
             const result = {}
             getKeys(value).forEach(key => {
-              result[key] = objectsByRef[value[key]]
+              result[key] = xapi._objectsByRef[value[key]]
             })
             return result
           }
-          props[`update_${field}`] = function(entries) {
-            return xapi.setFieldEntries(this, field, entries)
+          props[`update_${field}`] = function(entries, value) {
+            return typeof entries === 'string'
+              ? xapi.setFieldEntry(this.$type, this.$ref, field, entries, value)
+              : xapi.setFieldEntries(this.$type, this.$ref, field, entries)
           }
-        } else if (isOpaqueRef(value)) {
+        } else if (value === '' || isOpaqueRef(value)) {
+          // 2019-02-07 - JFT: even if `value` should not be an empty string for
+          // a ref property, an user had the case on XenServer 7.0 on the CD VBD
+          // of a VM created by XenCenter
           getters[$field] = function() {
-            return objectsByRef[this[field]]
+            return xapi._objectsByRef[this[field]]
           }
         }
       })
@@ -1128,17 +1217,25 @@ export class Xapi extends EventEmitter {
 Xapi.prototype._transportCall = reduce(
   [
     function(method, args) {
-      return this._call(method, args).catch(error => {
-        if (!(error instanceof Error)) {
-          error = wrapError(error)
-        }
+      return pTimeout
+        .call(this._call(method, args), HTTP_TIMEOUT)
+        .catch(error => {
+          if (!(error instanceof Error)) {
+            error = wrapError(error)
+          }
 
-        error.call = {
-          method,
-          params: replaceSensitiveValues(args, '* obfuscated *'),
-        }
-        throw error
-      })
+          // do not log the session ID
+          //
+          // TODO: should log at the session level to avoid logging sensitive
+          // values?
+          const params = args[0] === this._sessionId ? args.slice(1) : args
+
+          error.call = {
+            method,
+            params: replaceSensitiveValues(params, '* obfuscated *'),
+          }
+          throw error
+        })
     },
     call =>
       function() {

packages/xen-api/src/transports/json-rpc.js

@@ -3,6 +3,7 @@ import { format, parse } from 'json-rpc-protocol'
 
 import { UnsupportedTransport } from './_utils'
 
+// https://github.com/xenserver/xenadmin/blob/0df39a9d83cd82713f32d24704852a0fd57b8a64/XenModel/XenAPI/Session.cs#L403-L433
 export default ({ allowUnauthorized, url }) => {
   return (method, args) =>
     httpRequestPlus

packages/xo-acl-resolver/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-acl-resolver",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-acl-resolver",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-cli/package.json

@@ -12,6 +12,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-cli",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-cli",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -33,7 +34,7 @@
     "chalk": "^2.2.0",
     "exec-promise": "^0.7.0",
     "fs-promise": "^2.0.3",
-    "http-request-plus": "^0.7.1",
+    "http-request-plus": "^0.7.2",
     "human-format": "^0.10.0",
     "l33teral": "^3.0.3",
     "lodash": "^4.17.4",

packages/xo-collection/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-collection",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-collection",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-common/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-common",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-common",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-import-servers-csv/package.json

@@ -16,6 +16,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-import-servers-csv",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-import-servers-csv",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-lib/package.json

@@ -11,6 +11,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-lib",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-lib",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-remote-parser/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-remote-parser",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-remote-parser",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-auth-github/package.json

@@ -12,6 +12,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-auth-github",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-auth-github",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-auth-google/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-server-auth-google",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "license": "AGPL-3.0",
   "description": "Google authentication plugin for XO-Server",
   "keywords": [
@@ -15,6 +15,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-auth-google",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-auth-google",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -32,7 +33,7 @@
     "node": ">=6"
   },
   "dependencies": {
-    "passport-google-oauth20": "^1.0.0"
+    "passport-google-oauth20": "^2.0.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/xo-server-auth-ldap/package.json

@@ -14,6 +14,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-auth-ldap",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-auth-ldap",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-auth-saml/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-server-auth-saml",
-  "version": "0.5.2",
+  "version": "0.5.3",
   "license": "AGPL-3.0",
   "description": "SAML authentication plugin for XO-Server",
   "keywords": [
@@ -15,6 +15,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-auth-saml",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-auth-saml",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-auth-saml/src/index.js

@@ -42,7 +42,12 @@ class AuthSamlXoPlugin {
 
   configure({ usernameField, ...conf }) {
     this._usernameField = usernameField
-    this._conf = conf
+    this._conf = {
+      ...conf,
+
+      // must match the callback URL
+      path: '/signin/saml/callback',
+    }
   }
 
   load() {

packages/xo-server-backup-reports/package.json

@@ -18,6 +18,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-backup-reports",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-backup-reports",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-backup-reports/src/index.js

@@ -154,6 +154,10 @@ class BackupReportsXoPlugin {
   }
 
   _wrapper(status, job, schedule, runJobId) {
+    if (job.type === 'metadataBackup') {
+      return
+    }
+
     return new Promise(resolve =>
       resolve(
         job.type === 'backup'

packages/xo-server-cloud/package.json

@@ -14,6 +14,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-cloud",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-cloud",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -31,7 +32,7 @@
     "node": ">=6"
   },
   "dependencies": {
-    "http-request-plus": "^0.7.1",
+    "http-request-plus": "^0.7.2",
     "jsonrpc-websocket-client": "^0.4.1"
   },
   "devDependencies": {

packages/xo-server-load-balancer/package.json

@@ -13,6 +13,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-load-balancer",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-load-balancer",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-perf-alert/package.json

@@ -7,6 +7,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-perf-alert",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-perf-alert",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-perf-alert/src/index.js

@@ -673,8 +673,9 @@ ${entry.listItem}
     }
   }
 
-  async getRrd(xoObject, secondsAgo) {
-    const host = xoObject.$type === 'host' ? xoObject : xoObject.$resident_on
+  async getRrd(xapiObject, secondsAgo) {
+    const host =
+      xapiObject.$type === 'host' ? xapiObject : xapiObject.$resident_on
     if (host == null) {
       return null
     }
@@ -685,13 +686,13 @@ ${entry.listItem}
       host,
       query: {
         cf: 'AVERAGE',
-        host: (xoObject.$type === 'host').toString(),
+        host: (xapiObject.$type === 'host').toString(),
         json: 'true',
         start: serverTimestamp - secondsAgo,
       },
     }
-    if (xoObject.$type === 'vm') {
-      payload['vm_uuid'] = xoObject.uuid
+    if (xapiObject.$type === 'VM') {
+      payload['vm_uuid'] = xapiObject.uuid
     }
     // JSON is not well formed, can't use the default node parser
     return JSON5.parse(

packages/xo-server-test-plugin/package.json

@@ -5,6 +5,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-test-plugin",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-test-plugin",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   }

packages/xo-server-transport-email/package.json

@@ -14,6 +14,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-transport-email",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-transport-email",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-transport-nagios/package.json

@@ -14,6 +14,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-transport-nagios",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-transport-nagios",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-transport-slack/package.json

@@ -15,6 +15,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-transport-slack",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-transport-slack",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-transport-xmpp/package.json

@@ -15,6 +15,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-transport-xmpp",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-transport-xmpp",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server-usage-report/package.json

@@ -15,6 +15,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-usage-report",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server-usage-report",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },

packages/xo-server/bin/xo-server

@@ -4,6 +4,11 @@
 
 // ===================================================================
 
+// https://expressjs.com/en/advanced/best-practice-performance.html#set-node_env-to-production
+if (process.env.NODE_ENV === undefined) {
+  process.env.NODE_ENV = 'production'
+}
+
 // Better stack traces if possible.
 require('../better-stacks')
 

packages/xo-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-server",
-  "version": "5.33.0",
+  "version": "5.37.0",
   "license": "AGPL-3.0",
   "description": "Server part of Xen-Orchestra",
   "keywords": [
@@ -12,6 +12,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-server",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
@@ -20,7 +21,7 @@
     "better-stacks.js",
     "bin/",
     "dist/",
-    "config.json",
+    "config.toml",
     "index.js",
     "signin.pug"
   ],
@@ -35,7 +36,7 @@
     "@xen-orchestra/async-map": "^0.0.0",
     "@xen-orchestra/cron": "^1.0.3",
     "@xen-orchestra/emit-async": "^0.0.0",
-    "@xen-orchestra/fs": "^0.6.0",
+    "@xen-orchestra/fs": "^0.7.1",
     "@xen-orchestra/log": "^0.1.4",
     "@xen-orchestra/mixin": "^0.0.0",
     "ajv": "^6.1.1",
@@ -46,12 +47,13 @@
     "blocked": "^1.2.1",
     "bluebird": "^3.5.1",
     "body-parser": "^1.18.2",
+    "compression": "^1.7.3",
     "connect-flash": "^0.1.1",
     "cookie": "^0.3.1",
     "cookie-parser": "^1.4.3",
     "d3-time-format": "^2.1.1",
     "debug": "^4.0.1",
-    "decorator-synchronized": "^0.3.0",
+    "decorator-synchronized": "^0.5.0",
     "deptree": "^1.0.0",
     "escape-string-regexp": "^1.0.5",
     "event-to-promise": "^0.8.0",
@@ -68,12 +70,12 @@
     "helmet": "^3.9.0",
     "highland": "^2.11.1",
     "http-proxy": "^1.16.2",
-    "http-request-plus": "^0.7.1",
+    "http-request-plus": "^0.7.2",
     "http-server-plus": "^0.10.0",
     "human-format": "^0.10.0",
     "is-redirect": "^1.0.0",
-    "iterable-backoff": "^0.0.0",
-    "jest-worker": "^23.0.0",
+    "iterable-backoff": "^0.1.0",
+    "jest-worker": "^24.0.0",
     "js-yaml": "^3.10.0",
     "json-rpc-peer": "^0.15.3",
     "json5": "^2.0.1",
@@ -92,17 +94,18 @@
     "ms": "^2.1.1",
     "multikey-hash": "^1.0.4",
     "ndjson": "^1.5.0",
-    "otplib": "^10.0.1",
+    "otplib": "^11.0.0",
     "parse-pairs": "^0.2.2",
     "partial-stream": "0.0.0",
     "passport": "^0.4.0",
     "passport-local": "^1.0.0",
-    "pretty-format": "^23.0.0",
+    "pretty-format": "^24.0.0",
     "promise-toolbox": "^0.11.0",
     "proxy-agent": "^3.0.0",
     "pug": "^2.0.0-rc.4",
     "pump": "^3.0.0",
     "pw": "^0.0.4",
+    "readable-stream": "^3.2.0",
     "redis": "^2.8.0",
     "schema-inspector": "^1.6.8",
     "semver": "^5.4.1",
@@ -111,14 +114,14 @@
     "stack-chain": "^2.0.0",
     "stoppable": "^1.0.5",
     "struct-fu": "^1.2.0",
-    "tar-stream": "^1.5.5",
+    "tar-stream": "^2.0.1",
     "through2": "^3.0.0",
     "tmp": "^0.0.33",
     "uuid": "^3.0.1",
     "value-matcher": "^0.2.0",
     "vhd-lib": "^0.5.1",
     "ws": "^6.0.0",
-    "xen-api": "^0.24.1",
+    "xen-api": "^0.24.5",
     "xml2js": "^0.4.19",
     "xo-acl-resolver": "^0.4.1",
     "xo-collection": "^0.4.1",

packages/xo-server/sample.config.toml

@@ -44,6 +44,14 @@
 
 #=====================================================================
 
+# Directory containing the database of XO.
+# Currently used for logs.
+#
+# Default: '/var/lib/xo-server/data'
+#datadir = '/var/lib/xo-server/data'
+
+#=====================================================================
+
 # Configuration of the embedded HTTP server.
 [http]
 # If set to true, all HTTP traffic will be redirected to the first HTTPs
@@ -136,12 +144,6 @@ port = 80
 #  del = '3dda29ad-3015-44f9-b13b-fa570de92489'
 #  srem = '3fd758c9-5610-4e9d-a058-dbf4cb6d8bf0'
 
-# Directory containing the database of XO.
-# Currently used for logs.
-#
-# Default: '/var/lib/xo-server/data'
-#datadir = '/var/lib/xo-server/data'
-
 #=====================================================================
 
 # Configuration for remotes

packages/xo-server/src/_createNdJsonStream.js

@@ -0,0 +1,22 @@
+import asyncIteratorToStream from 'async-iterator-to-stream'
+
+function* values(object) {
+  const keys = Object.keys(object)
+  for (let i = 0, n = keys.length; i < n; ++i) {
+    yield object[keys[i]]
+  }
+}
+
+/**
+ * Creates a NDJSON stream of all the values
+ *
+ * @param {(Array|Object)} collection
+ */
+module.exports = asyncIteratorToStream(function*(collection) {
+  for (const value of Array.isArray(collection)
+    ? collection
+    : values(collection)) {
+    yield JSON.stringify(value)
+    yield '\n'
+  }
+})

packages/xo-server/src/_pRetry.js

@@ -0,0 +1,36 @@
+import iteratee from 'lodash/iteratee'
+import pDelay from 'promise-toolbox/delay'
+
+function stopRetry(error) {
+  this.error = error
+  // eslint-disable-next-line no-throw-literal
+  throw this
+}
+
+// do not retry on ReferenceError and TypeError which are programmer errors
+const defaultMatcher = error =>
+  !(error instanceof ReferenceError || error instanceof TypeError)
+
+export default async function pRetry(
+  fn,
+  { delay = 1e3, tries = 10, when } = {}
+) {
+  const container = { error: undefined }
+  const stop = stopRetry.bind(container)
+
+  when = when === undefined ? defaultMatcher : iteratee(when)
+
+  while (true) {
+    try {
+      return await fn(stop)
+    } catch (error) {
+      if (error === container) {
+        throw container.error
+      }
+      if (--tries === 0 || !when(error)) {
+        throw error
+      }
+    }
+    await pDelay(delay)
+  }
+}

packages/xo-server/src/_pRetry.spec.js

@@ -0,0 +1,95 @@
+/* eslint-env jest */
+
+import { forOwn } from 'lodash'
+
+import pRetry from './_pRetry'
+
+describe('pRetry()', () => {
+  it('retries until the function succeeds', async () => {
+    let i = 0
+    expect(
+      await pRetry(
+        () => {
+          if (++i < 3) {
+            throw new Error()
+          }
+          return 'foo'
+        },
+        { delay: 0 }
+      )
+    ).toBe('foo')
+    expect(i).toBe(3)
+  })
+
+  it('returns the last error', async () => {
+    let tries = 5
+    const e = new Error()
+    await expect(
+      pRetry(
+        () => {
+          throw --tries > 0 ? new Error() : e
+        },
+        { delay: 0, tries }
+      )
+    ).rejects.toBe(e)
+  })
+  ;[ReferenceError, TypeError].forEach(ErrorType => {
+    it(`does not retry if a ${ErrorType.name} is thrown`, async () => {
+      let i = 0
+      await expect(
+        pRetry(() => {
+          ++i
+          throw new ErrorType()
+        })
+      ).rejects.toBeInstanceOf(ErrorType)
+      expect(i).toBe(1)
+    })
+  })
+
+  it('does not retry if `stop` callback is called', async () => {
+    const e = new Error()
+    let i = 0
+    await expect(
+      pRetry(stop => {
+        ++i
+        stop(e)
+      })
+    ).rejects.toBe(e)
+    expect(i).toBe(1)
+  })
+
+  describe('`when` option', () => {
+    forOwn(
+      {
+        'with function predicate': _ => _.message === 'foo',
+        'with object predicate': { message: 'foo' },
+      },
+      (when, title) =>
+        describe(title, () => {
+          it('retries when error matches', async () => {
+            let i = 0
+            await pRetry(
+              () => {
+                ++i
+                throw new Error('foo')
+              },
+              { when, tries: 2 }
+            ).catch(Function.prototype)
+            expect(i).toBe(2)
+          })
+
+          it('does not retry when error does not match', async () => {
+            let i = 0
+            await pRetry(
+              () => {
+                ++i
+                throw new Error('bar')
+              },
+              { when, tries: 2 }
+            ).catch(Function.prototype)
+            expect(i).toBe(1)
+          })
+        })
+    )
+  })
+})

packages/xo-server/src/api/backup-ng.js

@@ -1,5 +1,8 @@
 import { basename } from 'path'
+import { fromCallback } from 'promise-toolbox'
+import { pipeline } from 'readable-stream'
 
+import createNdJsonStream from '../_createNdJsonStream'
 import { safeDateFormat } from '../utils'
 
 export function createJob({ schedules, ...job }) {
@@ -150,12 +153,26 @@ runJob.params = {
 
 // -----------------------------------------------------------------------------
 
-export function getAllLogs() {
-  return this.getBackupNgLogs()
+async function handleGetAllLogs(req, res) {
+  const logs = await this.getBackupNgLogs()
+  res.set('Content-Type', 'application/json')
+  return fromCallback(cb => pipeline(createNdJsonStream(logs), res, cb))
+}
+
+export function getAllLogs({ ndjson = false }) {
+  return ndjson
+    ? this.registerHttpRequest(handleGetAllLogs).then($getFrom => ({
+        $getFrom,
+      }))
+    : this.getBackupNgLogs()
 }
 
 getAllLogs.permission = 'admin'
 
+getAllLogs.params = {
+  ndjson: { type: 'boolean', optional: true },
+}
+
 export function getLogs({ after, before, limit, ...filter }) {
   return this.getBackupNgLogsSorted({ after, before, limit, filter })
 }

packages/xo-server/src/api/metadata-backup.js

@@ -0,0 +1,103 @@
+export function createJob({ schedules, ...job }) {
+  job.userId = this.user.id
+  return this.createMetadataBackupJob(job, schedules)
+}
+
+createJob.permission = 'admin'
+createJob.params = {
+  name: {
+    type: 'string',
+    optional: true,
+  },
+  pools: {
+    type: 'object',
+    optional: true,
+  },
+  remotes: {
+    type: 'object',
+  },
+  schedules: {
+    type: 'object',
+  },
+  settings: {
+    type: 'object',
+  },
+  xoMetadata: {
+    type: 'boolean',
+    optional: true,
+  },
+}
+
+export function getAllJobs() {
+  return this.getAllJobs('metadataBackup')
+}
+
+getAllJobs.permission = 'admin'
+
+export function getJob({ id }) {
+  return this.getJob(id, 'metadataBackup')
+}
+
+getJob.permission = 'admin'
+getJob.params = {
+  id: {
+    type: 'string',
+  },
+}
+
+export function deleteJob({ id }) {
+  return this.deleteMetadataBackupJob(id)
+}
+
+deleteJob.permission = 'admin'
+deleteJob.params = {
+  id: {
+    type: 'string',
+  },
+}
+
+export function editJob(props) {
+  return this.updateJob(props)
+}
+
+editJob.permission = 'admin'
+editJob.params = {
+  id: {
+    type: 'string',
+  },
+  name: {
+    type: 'string',
+    optional: true,
+  },
+  pools: {
+    type: ['object', 'null'],
+    optional: true,
+  },
+  settings: {
+    type: 'object',
+    optional: true,
+  },
+  remotes: {
+    type: 'object',
+    optional: true,
+  },
+  xoMetadata: {
+    type: 'boolean',
+    optional: true,
+  },
+}
+
+export async function runJob({ id, schedule }) {
+  return this.runJobSequence([id], await this.getSchedule(schedule))
+}
+
+runJob.permission = 'admin'
+
+runJob.params = {
+  id: {
+    type: 'string',
+  },
+  schedule: {
+    type: 'string',
+  },
+}

packages/xo-server/src/api/network.js

@@ -85,34 +85,35 @@ createBonded.description =
 // ===================================================================
 
 export async function set({
-  network,
-
+  automatic,
+  defaultIsLocked,
   name_description: nameDescription,
   name_label: nameLabel,
-  defaultIsLocked,
-  id,
+  network,
 }) {
   await this.getXapi(network).setNetworkProperties(network._xapiId, {
+    automatic,
+    defaultIsLocked,
     nameDescription,
     nameLabel,
-    defaultIsLocked,
   })
 }
 
 set.params = {
-  id: {
-    type: 'string',
+  automatic: {
+    type: 'boolean',
+    optional: true,
   },
-  name_label: {
-    type: 'string',
+  defaultIsLocked: {
+    type: 'boolean',
     optional: true,
   },
   name_description: {
     type: 'string',
     optional: true,
   },
-  defaultIsLocked: {
-    type: 'boolean',
+  name_label: {
+    type: 'string',
     optional: true,
   },
 }

packages/xo-server/src/api/vm.js

@@ -612,6 +612,8 @@ set.params = {
 
   share: { type: 'boolean', optional: true },
 
+  startDelay: { type: 'integer', optional: true },
+
   // set the VM network interface controller
   nicType: { type: ['string', 'null'], optional: true },
 }
@@ -753,6 +755,7 @@ export const snapshot = defer(async function(
     vm,
     name = `${vm.name_label}_${new Date().toISOString()}`,
     saveMemory = false,
+    description,
   }
 ) {
   await checkPermissionOnSrs.call(this, vm)
@@ -763,6 +766,10 @@ export const snapshot = defer(async function(
     : xapi.snapshotVm(vm._xapiRef, name))
   $defer.onFailure(() => xapi.deleteVm(snapshotId))
 
+  if (description !== undefined) {
+    await xapi.editVm(snapshotId, { name_description: description })
+  }
+
   const { user } = this
   if (user.permission !== 'admin') {
     await this.addAcl(user.id, snapshotId, 'admin')
@@ -771,6 +778,7 @@ export const snapshot = defer(async function(
 })
 
 snapshot.params = {
+  description: { type: 'string', optional: true },
   id: { type: 'string' },
   name: { type: 'string', optional: true },
   saveMemory: { type: 'boolean', optional: true },
@@ -1455,14 +1463,25 @@ getCloudInitConfig.resolve = {
 
 // -------------------------------------------------------------------
 
-export async function createCloudInitConfigDrive({ vm, sr, config, coreos }) {
+export async function createCloudInitConfigDrive({
+  config,
+  coreos,
+  networkConfig,
+  sr,
+  vm,
+}) {
   const xapi = this.getXapi(vm)
   if (coreos) {
     // CoreOS is a special CloudConfig drive created by XS plugin
     await xapi.createCoreOsCloudInitConfigDrive(vm._xapiId, sr._xapiId, config)
   } else {
     // use generic Cloud Init drive
-    await xapi.createCloudInitConfigDrive(vm._xapiId, sr._xapiId, config)
+    await xapi.createCloudInitConfigDrive(
+      vm._xapiId,
+      sr._xapiId,
+      config,
+      networkConfig
+    )
   }
 }
 
@@ -1470,6 +1489,7 @@ createCloudInitConfigDrive.params = {
   vm: { type: 'string' },
   sr: { type: 'string' },
   config: { type: 'string' },
+  networkConfig: { type: 'string', optional: true },
 }
 
 createCloudInitConfigDrive.resolve = {

packages/xo-server/src/api/xo.js

@@ -1,5 +1,8 @@
 import getStream from 'get-stream'
-import { forEach } from 'lodash'
+import { fromCallback } from 'promise-toolbox'
+import { pipeline } from 'readable-stream'
+
+import createNdJsonStream from '../_createNdJsonStream'
 
 // ===================================================================
 
@@ -17,6 +20,7 @@ export async function exportConfig() {
       (req, res) => {
         res.writeHead(200, 'OK', {
           'content-disposition': 'attachment',
+          'content-type': 'application/json',
         })
 
         return this.exportConfig()
@@ -32,11 +36,9 @@ exportConfig.permission = 'admin'
 // -------------------------------------------------------------------
 
 function handleGetAllObjects(req, res, { filter, limit }) {
-  forEach(this.getObjects({ filter, limit }), object => {
-    res.write(JSON.stringify(object))
-    res.write('\n')
-  })
-  res.end()
+  const objects = this.getObjects({ filter, limit })
+  res.set('Content-Type', 'application/json')
+  return fromCallback(cb => pipeline(createNdJsonStream(objects), res, cb))
 }
 
 export function getAllObjects({ filter, limit, ndjson = false }) {

packages/xo-server/src/index.js

@@ -3,6 +3,7 @@ import assert from 'assert'
 import authenticator from 'otplib/authenticator'
 import bind from 'lodash/bind'
 import blocked from 'blocked'
+import compression from 'compression'
 import createExpress from 'express'
 import createLogger from '@xen-orchestra/log'
 import crypto from 'crypto'
@@ -91,6 +92,8 @@ function createExpressApp(config) {
 
   app.use(helmet())
 
+  app.use(compression())
+
   // Registers the cookie-parser and express-session middlewares,
   // necessary for connect-flash.
   app.use(cookieParser(null, config.http.cookies))

packages/xo-server/src/utils.js

@@ -15,6 +15,8 @@ import { dirname, resolve } from 'path'
 import { utcFormat, utcParse } from 'd3-time-format'
 import { fromCallback, pAll, pReflect, promisify } from 'promise-toolbox'
 
+import { type SimpleIdPattern } from './utils'
+
 // ===================================================================
 
 export function camelToSnakeCase(string) {
@@ -417,3 +419,13 @@ export const getFirstPropertyName = object => {
     }
   }
 }
+
+// -------------------------------------------------------------------
+
+export const unboxIdsFromPattern = (pattern?: SimpleIdPattern): string[] => {
+  if (pattern === undefined) {
+    return []
+  }
+  const { id } = pattern
+  return typeof id === 'string' ? [id] : id.__or
+}

packages/xo-server/src/utils.js.flow

@@ -11,3 +11,5 @@ declare export function safeDateFormat(timestamp: number): string
 declare export function serializeError(error: Error): Object
 
 declare export function streamToBuffer(stream: Readable): Promise<Buffer>
+
+export type SimpleIdPattern = {| id: string | {| __or: string[] |}, |}

packages/xo-server/src/xapi-object-to-xo.js

@@ -54,12 +54,9 @@ function toTimestamp(date) {
     return timestamp
   }
 
-  const ms = parseDateTime(date)
-  if (!ms) {
-    return null
-  }
+  const ms = parseDateTime(date)?.getTime()
 
-  return Math.round(ms.getTime() / 1000)
+  return ms === undefined || ms === 0 ? null : Math.round(ms / 1000)
 }
 
 // ===================================================================
@@ -173,7 +170,7 @@ const TRANSFORMS = {
           total: 0,
         }
       })(),
-      multipathing: obj.multipathing,
+      multipathing: otherConfig.multipathing === 'true',
       patches: patches || link(obj, 'patches'),
       powerOnMode: obj.power_on_mode,
       power_state: metrics ? (isRunning ? 'Running' : 'Halted') : 'Unknown',
@@ -350,6 +347,7 @@ const TRANSFORMS = {
       hasVendorDevice: obj.has_vendor_device,
       resourceSet,
       snapshots: link(obj, 'snapshots'),
+      startDelay: +obj.start_delay,
       startTime: metrics && toTimestamp(metrics.start_time),
       tags: obj.tags,
       VIFs: link(obj, 'VIFs'),
@@ -581,6 +579,7 @@ const TRANSFORMS = {
 
   network(obj) {
     return {
+      automatic: obj.other_config?.automatic === 'true',
       bridge: obj.bridge,
       defaultIsLocked: obj.default_locking_mode === 'disabled',
       MTU: +obj.MTU,

packages/xo-server/src/xapi-stats.js

@@ -1,5 +1,6 @@
 import JSON5 from 'json5'
 import limitConcurrency from 'limit-concurrency-decorator'
+import synchronized from 'decorator-synchronized'
 import { BaseError } from 'make-error'
 import {
   endsWith,
@@ -245,6 +246,34 @@ const STATS = {
 
 // -------------------------------------------------------------------
 
+// RRD
+// json: {
+//   meta: {
+//     start: Number,
+//     step: Number,
+//     end: Number,
+//     rows: Number,
+//     columns: Number,
+//     legend: String[rows]
+//   },
+//   data: Item[columns] // Item = { t: Number, values: Number[rows] }
+// }
+
+// Local cache
+// _statsByObject : {
+//   [uuid]: {
+//     [step]: {
+//       endTimestamp: Number, // the timestamp of the last statistic point
+//       interval: Number, // step
+//       stats: {
+//         [metric1]: Number[],
+//         [metric2]: {
+//           [subMetric]: Number[],
+//         }
+//       }
+//     }
+//   }
+// }
 export default class XapiStats {
   constructor() {
     this._statsByObject = {}
@@ -305,6 +334,7 @@ export default class XapiStats {
     }
   }
 
+  @synchronized.withKey((_, { host }) => host.uuid)
   async _getAndUpdateStats(xapi, { host, vmUuid, granularity }) {
     const step =
       granularity === undefined

packages/xo-server/src/xapi/index.js

@@ -36,6 +36,7 @@ import { satisfies as versionSatisfies } from 'semver'
 
 import createSizeStream from '../size-stream'
 import fatfsBuffer, { init as fatfsBufferInit } from '../fatfs-buffer'
+import pRetry from '../_pRetry'
 import {
   camelToSnakeCase,
   ensureArray,
@@ -59,7 +60,6 @@ import {
   asInteger,
   extractOpaqueRef,
   filterUndefineds,
-  getNamespaceForType,
   getVmDisks,
   canSrHaveNewVdiOfSize,
   isVmHvm,
@@ -226,7 +226,7 @@ export default class Xapi extends XapiBase {
 
   _setObjectProperty(object, name, value) {
     return this.call(
-      `${getNamespaceForType(object.$type)}.set_${camelToSnakeCase(name)}`,
+      `${object.$type}.set_${camelToSnakeCase(name)}`,
       object.$ref,
       prepareXapiParam(value)
     )
@@ -235,15 +235,13 @@ export default class Xapi extends XapiBase {
   _setObjectProperties(object, props) {
     const { $ref: ref, $type: type } = object
 
-    const namespace = getNamespaceForType(type)
-
     // TODO: the thrown error should contain the name of the
     // properties that failed to be set.
     return Promise.all(
       mapToArray(props, (value, name) => {
         if (value != null) {
           return this.call(
-            `${namespace}.set_${camelToSnakeCase(name)}`,
+            `${type}.set_${camelToSnakeCase(name)}`,
             ref,
             prepareXapiParam(value)
           )
@@ -257,9 +255,8 @@ export default class Xapi extends XapiBase {
 
     prop = camelToSnakeCase(prop)
 
-    const namespace = getNamespaceForType(type)
-    const add = `${namespace}.add_to_${prop}`
-    const remove = `${namespace}.remove_from_${prop}`
+    const add = `${type}.add_to_${prop}`
+    const remove = `${type}.remove_from_${prop}`
 
     await Promise.all(
       mapToArray(values, (value, name) => {
@@ -308,17 +305,24 @@ export default class Xapi extends XapiBase {
 
   async setNetworkProperties(
     id,
-    { nameLabel, nameDescription, defaultIsLocked }
+    { automatic, defaultIsLocked, nameDescription, nameLabel }
   ) {
     let defaultLockingMode
     if (defaultIsLocked != null) {
       defaultLockingMode = defaultIsLocked ? 'disabled' : 'unlocked'
     }
-    await this._setObjectProperties(this.getObject(id), {
-      nameLabel,
-      nameDescription,
-      defaultLockingMode,
-    })
+    const network = this.getObject(id)
+    await Promise.all([
+      this._setObjectProperties(network, {
+        defaultLockingMode,
+        nameDescription,
+        nameLabel,
+      }),
+      this._updateObjectMapProperty(network, 'other_config', {
+        automatic:
+          automatic === undefined ? undefined : automatic ? 'true' : null,
+      }),
+    ])
   }
 
   // =================================================================
@@ -326,15 +330,13 @@ export default class Xapi extends XapiBase {
   async addTag(id, tag) {
     const { $ref: ref, $type: type } = this.getObject(id)
 
-    const namespace = getNamespaceForType(type)
-    await this.call(`${namespace}.add_tags`, ref, tag)
+    await this.call(`${type}.add_tags`, ref, tag)
   }
 
   async removeTag(id, tag) {
     const { $ref: ref, $type: type } = this.getObject(id)
 
-    const namespace = getNamespaceForType(type)
-    await this.call(`${namespace}.remove_tags`, ref, tag)
+    await this.call(`${type}.remove_tags`, ref, tag)
   }
 
   // =================================================================
@@ -415,10 +417,23 @@ export default class Xapi extends XapiBase {
     await this.call('host.enable', this.getObject(hostId).$ref)
   }
 
+  // Resources:
+  // - Citrix XenServer ® 7.0 Administrator's Guide ch. 5.4
+  // - https://github.com/xcp-ng/xenadmin/blob/60dd70fc36faa0ec91654ec97e24b7af36acff9f/XenModel/Actions/Host/EditMultipathAction.cs
+  // - https://github.com/serencorbett1/xenadmin/blob/1c3fb0c1112e4e316423afc6a028066001d3dea1/XenModel/XenAPI-Extensions/SR.cs
   @deferrable.onError(log.warn)
   async setHostMultipathing($defer, hostId, multipathing) {
     const host = this.getObject(hostId)
 
+    if (host.enabled) {
+      await this.disableHost(hostId)
+      $defer(() => this.enableHost(hostId))
+    }
+
+    // Xen center evacuate running VMs before unplugging the PBDs.
+    // The evacuate method uses the live migration to migrate running VMs
+    // from host to another. It only works when a shared SR is present
+    // in the host. For this reason we chose to show a warning instead.
     const pluggedPbds = host.$PBDs.filter(pbd => pbd.currently_attached)
     await asyncMap(pluggedPbds, async pbd => {
       const ref = pbd.$ref
@@ -426,11 +441,6 @@ export default class Xapi extends XapiBase {
       $defer(() => this.plugPbd(ref))
     })
 
-    if (host.enabled) {
-      await this.disableHost(hostId)
-      $defer(() => this.enableHost(hostId))
-    }
-
     return this._updateObjectMapProperty(
       host,
       'other_config',
@@ -676,17 +686,17 @@ export default class Xapi extends XapiBase {
   }
 
   async _deleteVm(
-    vm,
+    vmOrRef,
     deleteDisks = true,
     force = false,
     forceDeleteDefaultTemplate = false
   ) {
-    log.debug(`Deleting VM ${vm.name_label}`)
-
-    const { $ref } = vm
+    const $ref = typeof vmOrRef === 'string' ? vmOrRef : vmOrRef.$ref
 
     // ensure the vm record is up-to-date
-    vm = await this.barrier($ref)
+    const vm = await this.barrier($ref)
+
+    log.debug(`Deleting VM ${vm.name_label}`)
 
     if (!force && 'destroy' in vm.blocked_operations) {
       throw forbiddenOperation('destroy', vm.blocked_operations.destroy.reason)
@@ -727,6 +737,10 @@ export default class Xapi extends XapiBase {
         this._deleteVm(snapshot)
       )::ignoreErrors(),
 
+      vm.power_state === 'Suspended' &&
+        vm.suspend_VDI !== NULL_REF &&
+        this._deleteVdi(vm.suspend_VDI)::ignoreErrors(),
+
       deleteDisks &&
         asyncMap(disks, ({ $ref: vdiRef }) => {
           let onFailure = () => {
@@ -751,7 +765,7 @@ export default class Xapi extends XapiBase {
             return (
               // Only remove VBDs not attached to other VMs.
               vdi.VBDs.length < 2 || every(vdi.$VBDs, vbd => vbd.VM === $ref)
-                ? this._deleteVdi(vdi)
+                ? this._deleteVdi(vdiRef)
                 : onFailure(vdi)
             )
           }
@@ -921,7 +935,7 @@ export default class Xapi extends XapiBase {
         //
         // The snapshot must not exist otherwise it could break the
         // next export.
-        this._deleteVdi(vdi)::ignoreErrors()
+        this._deleteVdi(vdi.$ref)::ignoreErrors()
         return
       }
 
@@ -1077,7 +1091,7 @@ export default class Xapi extends XapiBase {
         }
 
         newVdi = await this._getOrWaitObject(await this._cloneVdi(baseVdi))
-        $defer.onFailure(() => this._deleteVdi(newVdi))
+        $defer.onFailure(() => this._deleteVdi(newVdi.$ref))
 
         await this._updateObjectMapProperty(newVdi, 'other_config', {
           [TAG_COPY_SRC]: vdi.uuid,
@@ -1092,7 +1106,7 @@ export default class Xapi extends XapiBase {
           },
           sr: mapVdisSrs[vdi.uuid] || srId,
         })
-        $defer.onFailure(() => this._deleteVdi(newVdi))
+        $defer.onFailure(() => this._deleteVdi(newVdi.$ref))
       }
 
       await asyncMap(vbds[vdiId], vbd =>
@@ -1255,7 +1269,7 @@ export default class Xapi extends XapiBase {
     return loop()
   }
 
-  @synchronized
+  @synchronized()
   _callInstallationPlugin(hostRef, vdi) {
     return this.call(
       'host.call_plugin',
@@ -1283,7 +1297,7 @@ export default class Xapi extends XapiBase {
       '[XO] Supplemental pack ISO',
       'small temporary VDI to store a supplemental pack ISO'
     )
-    $defer(() => this._deleteVdi(vdi))
+    $defer(() => this._deleteVdi(vdi.$ref))
 
     await this._callInstallationPlugin(this.getObject(hostId).$ref, vdi.uuid)
   }
@@ -1311,7 +1325,7 @@ export default class Xapi extends XapiBase {
         '[XO] Supplemental pack ISO',
         'small temporary VDI to store a supplemental pack ISO'
       )
-      $defer(() => this._deleteVdi(vdi))
+      $defer(() => this._deleteVdi(vdi.$ref))
 
       // Install pack sequentially to prevent concurrent access to the unique VDI
       for (const host of hosts) {
@@ -1342,7 +1356,7 @@ export default class Xapi extends XapiBase {
             '[XO] Supplemental pack ISO',
             'small temporary VDI to store a supplemental pack ISO'
           )
-          $defer(() => this._deleteVdi(vdi))
+          $defer(() => this._deleteVdi(vdi.$ref))
 
           await this._callInstallationPlugin(host.$ref, vdi.uuid)
         })
@@ -1428,7 +1442,7 @@ export default class Xapi extends XapiBase {
           size: disk.capacity,
           sr: sr.$ref,
         }))
-        $defer.onFailure(() => this._deleteVdi(vdi))
+        $defer.onFailure(() => this._deleteVdi(vdi.$ref))
 
         return this.createVbd({
           userdevice: disk.position,
@@ -1538,7 +1552,12 @@ export default class Xapi extends XapiBase {
 
   @concurrency(2)
   @cancelable
-  async _snapshotVm($cancelToken, vm, nameLabel = vm.name_label) {
+  async _snapshotVm($cancelToken, { $ref: vmRef }, nameLabel) {
+    const vm = await this.getRecord('VM', vmRef)
+    if (nameLabel === undefined) {
+      nameLabel = vm.name_label
+    }
+
     log.debug(
       `Snapshotting VM ${vm.name_label}${
         nameLabel !== vm.name_label ? ` as ${nameLabel}` : ''
@@ -1549,13 +1568,45 @@ export default class Xapi extends XapiBase {
     do {
       if (!vm.tags.includes('xo-disable-quiesce')) {
         try {
-          ref = await this.callAsync(
-            $cancelToken,
-            'VM.snapshot_with_quiesce',
-            vm.$ref,
-            nameLabel
+          ref = await pRetry(
+            async bail => {
+              try {
+                return await this.callAsync(
+                  $cancelToken,
+                  'VM.snapshot_with_quiesce',
+                  vmRef,
+                  nameLabel
+                )
+              } catch (error) {
+                if (error?.code !== 'VM_SNAPSHOT_WITH_QUIESCE_FAILED') {
+                  throw bail(error)
+                }
+
+                // detect and remove new broken snapshots
+                //
+                // see https://github.com/vatesfr/xen-orchestra/issues/3936
+                const prevSnapshotRefs = new Set(vm.snapshots)
+                const snapshotNameLabelPrefix = `Snapshot of ${vm.uuid} [`
+                vm.snapshots = await this.getField('VM', vmRef, 'snapshots')
+                const createdSnapshots = (await this.getRecords(
+                  'VM',
+                  vm.snapshots.filter(_ => !prevSnapshotRefs.has(_))
+                )).filter(_ => _.name_label.startsWith(snapshotNameLabelPrefix))
+
+                // be safe: only delete if there was a single match
+                if (createdSnapshots.length === 1) {
+                  ignoreErrors.call(this._deleteVm(createdSnapshots[0]))
+                }
+
+                throw error
+              }
+            },
+            {
+              delay: 60e3,
+              tries: 3,
+            }
           ).then(extractOpaqueRef)
-          this.addTag(ref, 'quiesce')::ignoreErrors()
+          ignoreErrors.call(this.call('VM.add_tags', ref, 'quiesce'))
 
           break
         } catch (error) {
@@ -1575,19 +1626,14 @@ export default class Xapi extends XapiBase {
       ref = await this.callAsync(
         $cancelToken,
         'VM.snapshot',
-        vm.$ref,
+        vmRef,
         nameLabel
       ).then(extractOpaqueRef)
     } while (false)
 
-    // Convert the template to a VM and wait to have receive the up-
-    // to-date object.
-    const [, snapshot] = await Promise.all([
-      this.call('VM.set_is_a_template', ref, false),
-      this.barrier(ref),
-    ])
+    await this.setField('VM', ref, 'is_a_template', false)
 
-    return snapshot
+    return this.getRecord('VM', ref)
   }
 
   async snapshotVm(vmId, nameLabel = undefined) {
@@ -1664,7 +1710,7 @@ export default class Xapi extends XapiBase {
         find(
           this.objects.all,
           obj =>
-            obj.$type === 'vm' &&
+            obj.$type === 'VM' &&
             obj.is_a_template &&
             obj.name_label === templateNameLabel
         )
@@ -1849,7 +1895,9 @@ export default class Xapi extends XapiBase {
       }`
     )
     try {
-      await this.call('VDI.pool_migrate', vdi.$ref, sr.$ref, {})
+      await pRetry(() => this.call('VDI.pool_migrate', vdi.$ref, sr.$ref, {}), {
+        when: { code: 'TOO_MANY_STORAGE_MIGRATES' },
+      })
     } catch (error) {
       const { code } = error
       if (
@@ -1869,15 +1917,21 @@ export default class Xapi extends XapiBase {
           vdi: newVdi,
         })
       })
-      await this._deleteVdi(vdi)
+      await this._deleteVdi(vdi.$ref)
     }
   }
 
   // TODO: check whether the VDI is attached.
-  async _deleteVdi(vdi) {
-    log.debug(`Deleting VDI ${vdi.name_label}`)
+  async _deleteVdi(vdiRef) {
+    log.debug(`Deleting VDI ${vdiRef}`)
 
-    await this.call('VDI.destroy', vdi.$ref)
+    try {
+      await this.call('VDI.destroy', vdiRef)
+    } catch (error) {
+      if (error?.code !== 'HANDLE_INVALID') {
+        throw error
+      }
+    }
   }
 
   _resizeVdi(vdi, size) {
@@ -1972,7 +2026,7 @@ export default class Xapi extends XapiBase {
   }
 
   async deleteVdi(vdiId) {
-    await this._deleteVdi(this.getObject(vdiId))
+    await this._deleteVdi(this.getObject(vdiId).$ref)
   }
 
   async resizeVdi(vdiId, size) {
@@ -2162,7 +2216,7 @@ export default class Xapi extends XapiBase {
     const physPif = find(
       this.objects.all,
       obj =>
-        obj.$type === 'pif' &&
+        obj.$type === 'PIF' &&
         (obj.physical || !isEmpty(obj.bond_master_of)) &&
         obj.$pool === pif.$pool &&
         obj.device === pif.device
@@ -2324,8 +2378,16 @@ export default class Xapi extends XapiBase {
   }
 
   // Generic Config Drive
+  //
+  // https://cloudinit.readthedocs.io/en/latest/topics/datasources/nocloud.html
   @deferrable
-  async createCloudInitConfigDrive($defer, vmId, srId, config) {
+  async createCloudInitConfigDrive(
+    $defer,
+    vmId,
+    srId,
+    userConfig,
+    networkConfig
+  ) {
     const vm = this.getObject(vmId)
     const sr = this.getObject(srId)
 
@@ -2336,14 +2398,16 @@ export default class Xapi extends XapiBase {
       size: buffer.length,
       sr: sr.$ref,
     })
-    $defer.onFailure(() => this._deleteVdi(vdi))
+    $defer.onFailure(() => this._deleteVdi(vdi.$ref))
 
     // Then, generate a FAT fs
     const fs = promisifyAll(fatfs.createFileSystem(fatfsBuffer(buffer)))
 
     await Promise.all([
       fs.writeFile('meta-data', 'instance-id: ' + vm.uuid + '\n'),
-      fs.writeFile('user-data', config),
+      fs.writeFile('user-data', userConfig),
+      networkConfig !== undefined &&
+        fs.writeFile('network-config', networkConfig),
     ])
 
     // ignore errors, I (JFT) don't understand why they are emitted
@@ -2369,7 +2433,7 @@ export default class Xapi extends XapiBase {
       size: stream.length,
       sr: sr.$ref,
     })
-    $defer.onFailure(() => this._deleteVdi(vdi))
+    $defer.onFailure(() => this._deleteVdi(vdi.$ref))
 
     await this.importVdiContent(vdi.$id, stream, { format: VDI_FORMAT_RAW })
 
@@ -2398,7 +2462,7 @@ export default class Xapi extends XapiBase {
     return find(
       this.objects.all,
       obj =>
-        obj.$type === 'sr' && obj.shared && canSrHaveNewVdiOfSize(obj, minSize)
+        obj.$type === 'SR' && obj.shared && canSrHaveNewVdiOfSize(obj, minSize)
     )
   }
 

packages/xo-server/src/xapi/mixins/patching.js

@@ -166,7 +166,7 @@ export default {
   async _ejectToolsIsos(hostRef) {
     return Promise.all(
       mapFilter(this.objects.all, vm => {
-        if (vm.$type !== 'vm' || (hostRef && vm.resident_on !== hostRef)) {
+        if (vm.$type !== 'VM' || (hostRef && vm.resident_on !== hostRef)) {
           return
         }
 
@@ -300,7 +300,7 @@ export default {
         'small temporary VDI to store a patch ISO'
       )
     }
-    $defer(() => this._deleteVdi(vdi))
+    $defer(() => this._deleteVdi(vdi.$ref))
 
     return vdi
   },

packages/xo-server/src/xapi/mixins/pool.js

@@ -0,0 +1,14 @@
+import { cancelable } from 'promise-toolbox'
+
+export default {
+  @cancelable
+  exportPoolMetadata($cancelToken) {
+    const { pool } = this
+    return this.getResource($cancelToken, '/pool/xmldbdump', {
+      task: this.createTask(
+        'Pool metadata',
+        pool.name_label ?? pool.$master.name_label
+      ),
+    })
+  },
+}

packages/xo-server/src/xapi/mixins/vm.js

@@ -468,6 +468,13 @@ export default {
         return this._updateObjectMapProperty(vm, 'platform', { videoram })
       },
     },
+
+    startDelay: {
+      get: vm => +vm.start_delay,
+      set(startDelay, vm) {
+        return this.call('VM.set_start_delay', vm.$ref, startDelay)
+      },
+    },
   }),
 
   async editVm(id, props, checkLimits) {

packages/xo-server/src/xapi/utils.js

@@ -71,44 +71,6 @@ export const extractOpaqueRef = str => {
 
 // -------------------------------------------------------------------
 
-const TYPE_TO_NAMESPACE = { __proto__: null }
-forEach(
-  [
-    'Bond',
-    'DR_task',
-    'GPU_group',
-    'PBD',
-    'PCI',
-    'PGPU',
-    'PIF',
-    'PIF_metrics',
-    'SM',
-    'SR',
-    'VBD',
-    'VBD_metrics',
-    'VDI',
-    'VGPU',
-    'VGPU_type',
-    'VIF',
-    'VLAN',
-    'VM',
-    'VM_appliance',
-    'VM_guest_metrics',
-    'VM_metrics',
-    'VMPP',
-    'VTPM',
-  ],
-  namespace => {
-    TYPE_TO_NAMESPACE[namespace.toLowerCase()] = namespace
-  }
-)
-
-// Object types given by `xen-api` are always lowercase but the
-// namespaces in the Xen API can have a different casing.
-export const getNamespaceForType = type => TYPE_TO_NAMESPACE[type] || type
-
-// -------------------------------------------------------------------
-
 export const getVmDisks = vm => {
   const disks = { __proto__: null }
   forEach(vm.$VBDs, vbd => {
@@ -288,7 +250,7 @@ export const makeEditObject = specs => {
     const object = this.getObject(id)
 
     const _objectRef = object.$ref
-    const _setMethodPrefix = `${getNamespaceForType(object.$type)}.set_`
+    const _setMethodPrefix = `${object.$type}.set_`
 
     // Context used to execute functions.
     const context = {

packages/xo-server/src/xo-mixins/backups-ng/index.js

@@ -54,6 +54,8 @@ import {
   resolveRelativeFromFile,
   safeDateFormat,
   serializeError,
+  type SimpleIdPattern,
+  unboxIdsFromPattern,
 } from '../../utils'
 
 import { translateLegacyJob } from './migration'
@@ -75,10 +77,6 @@ type Settings = {|
   vmTimeout?: number,
 |}
 
-type SimpleIdPattern = {|
-  id: string | {| __or: string[] |},
-|}
-
 export type BackupJob = {|
   ...$Exact<Job>,
   compression?: 'native' | 'zstd' | '',
@@ -182,7 +180,7 @@ const getJobCompression = ({ compression: c }) =>
 
 const listReplicatedVms = (
   xapi: Xapi,
-  scheduleId: string,
+  scheduleOrJobId: string,
   srId?: string,
   vmUuid?: string
 ): Vm[] => {
@@ -192,11 +190,12 @@ const listReplicatedVms = (
     const object = all[key]
     const oc = object.other_config
     if (
-      object.$type === 'vm' &&
+      object.$type === 'VM' &&
       !object.is_a_snapshot &&
       !object.is_a_template &&
       'start' in object.blocked_operations &&
-      oc['xo:backup:schedule'] === scheduleId &&
+      (oc['xo:backup:job'] === scheduleOrJobId ||
+        oc['xo:backup:schedule'] === scheduleOrJobId) &&
       oc['xo:backup:sr'] === srId &&
       (oc['xo:backup:vm'] === vmUuid ||
         // 2018-03-28, JFT: to catch VMs replicated before this fix
@@ -309,14 +308,6 @@ const parseVmBackupId = (id: string) => {
   }
 }
 
-const unboxIds = (pattern?: SimpleIdPattern): string[] => {
-  if (pattern === undefined) {
-    return []
-  }
-  const { id } = pattern
-  return typeof id === 'string' ? [id] : id.__or
-}
-
 // similar to Promise.all() but do not gather results
 async function waitAll<T>(
   promises: Promise<T>[],
@@ -604,7 +595,7 @@ export default class BackupNg {
           }
         }
         const jobId = job.id
-        const srs = unboxIds(job.srs).map(id => {
+        const srs = unboxIdsFromPattern(job.srs).map(id => {
           const xapi = app.getXapi(id)
           return {
             __proto__: xapi.getObject(id),
@@ -612,7 +603,7 @@ export default class BackupNg {
           }
         })
         const remotes = await Promise.all(
-          unboxIds(job.remotes).map(async id => ({
+          unboxIdsFromPattern(job.remotes).map(async id => ({
             id,
             handler: await app.getRemoteHandler(id),
           }))
@@ -1323,7 +1314,7 @@ export default class BackupNg {
         for (const { $id: srId, xapi } of srs) {
           const replicatedVm = listReplicatedVms(
             xapi,
-            scheduleId,
+            jobId,
             srId,
             vmUuid
           ).find(vm => vm.other_config[TAG_COPY_SRC] === baseSnapshot.uuid)
@@ -1365,10 +1356,9 @@ export default class BackupNg {
 
                   return
                 } catch (error) {
-                  if (
-                    !(error instanceof AssertionError) ||
-                    error?.code === 'ENOENT'
-                  ) {
+                  const corruptedVhdOrMissingParent =
+                    error instanceof AssertionError || error?.code === 'ENOENT'
+                  if (!corruptedVhdOrMissingParent) {
                     throw error
                   }
                 }

packages/xo-server/src/xo-mixins/backups.js

@@ -427,7 +427,7 @@ export default class {
 
       let toRemove = filter(
         targetXapi.objects.all,
-        obj => obj.$type === 'vm' && obj.other_config[TAG_SOURCE_VM] === uuid
+        obj => obj.$type === 'VM' && obj.other_config[TAG_SOURCE_VM] === uuid
       )
       const { length } = toRemove
       const deleteBase = length === 0 // old replications are not captured in toRemove

packages/xo-server/src/xo-mixins/ip-pools.js

@@ -158,7 +158,7 @@ export default class IpPools {
     return countBy(ipPools, ({ id }) => `ipPool:${id}`)
   }
 
-  @synchronized
+  @synchronized()
   allocIpAddresses(vifId, addAddresses, removeAddresses) {
     const updatedIpPools = {}
     const limits = {}

packages/xo-server/src/xo-mixins/metadata-backups.js

@@ -0,0 +1,264 @@
+// @flow
+import asyncMap from '@xen-orchestra/async-map'
+import defer from 'golike-defer'
+import { fromEvent, ignoreErrors } from 'promise-toolbox'
+
+import { type Xapi } from '../xapi'
+import {
+  safeDateFormat,
+  type SimpleIdPattern,
+  unboxIdsFromPattern,
+} from '../utils'
+
+import { type Executor, type Job } from './jobs'
+import { type Schedule } from './scheduling'
+
+const METADATA_BACKUP_JOB_TYPE = 'metadataBackup'
+
+type Settings = {|
+  retentionXoMetadata?: number,
+  retentionPoolMetadata?: number,
+|}
+
+type MetadataBackupJob = {
+  ...$Exact<Job>,
+  pools?: SimpleIdPattern,
+  remotes: SimpleIdPattern,
+  settings: $Dict<Settings>,
+  type: METADATA_BACKUP_JOB_TYPE,
+  xoMetadata?: boolean,
+}
+
+// File structure on remotes:
+//
+// <remote>
+// ├─ xo-config-backups
+// │  └─ <schedule ID>
+// │     └─ <YYYYMMDD>T<HHmmss>
+// │        ├─ metadata.json
+// │        └─ data.json
+// └─ xo-pool-metadata-backups
+//    └─ <schedule ID>
+//       └─ <pool UUID>
+//          └─ <YYYYMMDD>T<HHmmss>
+//             ├─ metadata.json
+//             └─ data
+
+export default class metadataBackup {
+  _app: {
+    createJob: (
+      $Diff<MetadataBackupJob, {| id: string |}>
+    ) => Promise<MetadataBackupJob>,
+    createSchedule: ($Diff<Schedule, {| id: string |}>) => Promise<Schedule>,
+    deleteSchedule: (id: string) => Promise<void>,
+    getXapi: (id: string) => Xapi,
+    getJob: (
+      id: string,
+      ?METADATA_BACKUP_JOB_TYPE
+    ) => Promise<MetadataBackupJob>,
+    updateJob: (
+      $Shape<MetadataBackupJob>,
+      ?boolean
+    ) => Promise<MetadataBackupJob>,
+    removeJob: (id: string) => Promise<void>,
+  }
+
+  constructor(app: any) {
+    this._app = app
+    app.on('start', () => {
+      app.registerJobExecutor(
+        METADATA_BACKUP_JOB_TYPE,
+        this._executor.bind(this)
+      )
+    })
+  }
+
+  async _executor({ cancelToken, job: job_, schedule }): Executor {
+    if (schedule === undefined) {
+      throw new Error('backup job cannot run without a schedule')
+    }
+
+    const job: MetadataBackupJob = (job_: any)
+    const remoteIds = unboxIdsFromPattern(job.remotes)
+    if (remoteIds.length === 0) {
+      throw new Error('metadata backup job cannot run without remotes')
+    }
+
+    const poolIds = unboxIdsFromPattern(job.pools)
+    const isEmptyPools = poolIds.length === 0
+    if (!job.xoMetadata && isEmptyPools) {
+      throw new Error('no metadata mode found')
+    }
+
+    const app = this._app
+    const { retentionXoMetadata, retentionPoolMetadata } =
+      job?.settings[schedule.id] || {}
+
+    const timestamp = Date.now()
+    const formattedTimestamp = safeDateFormat(timestamp)
+    const commonMetadata = {
+      jobId: job.id,
+      jobName: job.name,
+      scheduleId: schedule.id,
+      scheduleName: schedule.name,
+      timestamp,
+    }
+
+    const files = []
+    if (job.xoMetadata && retentionXoMetadata > 0) {
+      const xoMetadataDir = `xo-config-backups/${schedule.id}`
+      const dir = `${xoMetadataDir}/${formattedTimestamp}`
+
+      const data = JSON.stringify(await app.exportConfig(), null, 2)
+      const fileName = `${dir}/data.json`
+
+      const metadata = JSON.stringify(commonMetadata, null, 2)
+      const metaDataFileName = `${dir}/metadata.json`
+
+      files.push({
+        executeBackup: defer(($defer, handler) => {
+          $defer.onFailure(() => handler.rmtree(dir))
+          return Promise.all([
+            handler.outputFile(fileName, data),
+            handler.outputFile(metaDataFileName, metadata),
+          ])
+        }),
+        dir: xoMetadataDir,
+        retention: retentionXoMetadata,
+      })
+    }
+    if (!isEmptyPools && retentionPoolMetadata > 0) {
+      files.push(
+        ...(await Promise.all(
+          poolIds.map(async id => {
+            const poolMetadataDir = `xo-pool-metadata-backups/${
+              schedule.id
+            }/${id}`
+            const dir = `${poolMetadataDir}/${formattedTimestamp}`
+
+            // TODO: export the metadata only once then split the stream between remotes
+            const stream = await app.getXapi(id).exportPoolMetadata(cancelToken)
+            const fileName = `${dir}/data`
+
+            const xapi = this._app.getXapi(id)
+            const metadata = JSON.stringify(
+              {
+                ...commonMetadata,
+                pool: xapi.pool,
+                poolMaster: await xapi.getRecord('host', xapi.pool.master),
+              },
+              null,
+              2
+            )
+            const metaDataFileName = `${dir}/metadata.json`
+
+            return {
+              executeBackup: defer(($defer, handler) => {
+                $defer.onFailure(() => handler.rmtree(dir))
+                return Promise.all([
+                  (async () => {
+                    const outputStream = await handler.createOutputStream(
+                      fileName
+                    )
+                    $defer.onFailure(() => outputStream.destroy())
+
+                    // 'readable-stream/pipeline' not call the callback when an error throws
+                    // from the readable stream
+                    stream.pipe(outputStream)
+                    return fromEvent(stream, 'end').catch(error => {
+                      if (error.message !== 'aborted') {
+                        throw error
+                      }
+                    })
+                  })(),
+                  handler.outputFile(metaDataFileName, metadata),
+                ])
+              }),
+              dir: poolMetadataDir,
+              retention: retentionPoolMetadata,
+            }
+          })
+        ))
+      )
+    }
+
+    if (files.length === 0) {
+      throw new Error('no retentions corresponding to the metadata modes found')
+    }
+
+    cancelToken.throwIfRequested()
+
+    const timestampReg = /^\d{8}T\d{6}Z$/
+    return asyncMap(
+      // TODO: emit a warning task if a remote is broken
+      asyncMap(remoteIds, id => app.getRemoteHandler(id)::ignoreErrors()),
+      async handler => {
+        if (handler === undefined) {
+          return
+        }
+
+        await Promise.all(
+          files.map(async ({ executeBackup, dir, retention }) => {
+            await executeBackup(handler)
+
+            // deleting old backups
+            await handler.list(dir).then(list => {
+              list.sort()
+              list = list
+                .filter(timestampDir => timestampReg.test(timestampDir))
+                .slice(0, -retention)
+              return Promise.all(
+                list.map(timestampDir =>
+                  handler.rmtree(`${dir}/${timestampDir}`)
+                )
+              )
+            })
+          })
+        )
+      }
+    )
+  }
+
+  async createMetadataBackupJob(
+    props: $Diff<MetadataBackupJob, {| id: string |}>,
+    schedules: $Dict<$Diff<Schedule, {| id: string |}>>
+  ): Promise<MetadataBackupJob> {
+    const app = this._app
+
+    const job: MetadataBackupJob = await app.createJob({
+      ...props,
+      type: METADATA_BACKUP_JOB_TYPE,
+    })
+
+    const { id: jobId, settings } = job
+    await asyncMap(schedules, async (schedule, tmpId) => {
+      const { id: scheduleId } = await app.createSchedule({
+        ...schedule,
+        jobId,
+      })
+      settings[scheduleId] = settings[tmpId]
+      delete settings[tmpId]
+    })
+    await app.updateJob({ id: jobId, settings })
+
+    return job
+  }
+
+  async deleteMetadataBackupJob(id: string): Promise<void> {
+    const app = this._app
+    const [schedules] = await Promise.all([
+      app.getAllSchedules(),
+      // it test if the job is of type metadataBackup
+      app.getJob(id, METADATA_BACKUP_JOB_TYPE),
+    ])
+
+    await Promise.all([
+      app.removeJob(id),
+      asyncMap(schedules, schedule => {
+        if (schedule.id === id) {
+          return app.deleteSchedule(id)
+        }
+      }),
+    ])
+  }
+}

packages/xo-server/src/xo-mixins/remotes.js

@@ -96,7 +96,7 @@ export default class {
           handler.getInfo().then(info => {
             this._remotesInfo[remote.id] = info
           }),
-          this._remoteOptions.timeoutInfo
+          5e3
         )
       } catch (_) {}
     })

packages/xo-server/src/xo-mixins/resource-sets.js

@@ -329,7 +329,7 @@ export default class {
         let id
         let set
         if (
-          object.$type !== 'vm' ||
+          object.$type !== 'VM' ||
           object.is_a_snapshot ||
           ('start' in object.blocked_operations &&
             (object.tags.includes('Disaster Recovery') ||

packages/xo-server/src/xo-mixins/xen-servers.js

@@ -30,6 +30,15 @@ class PoolAlreadyConnected extends BaseError {
 
 const log = createLogger('xo:xo-mixins:xen-servers')
 
+// Server is disconnected:
+// - _xapis[server.id] is undefined
+
+// Server is connecting:
+// - _xapis[server.id] is defined
+
+// Server is connected:
+// - _xapis[server.id] id defined
+// - _serverIdsByPool[xapi.pool.$id] is server.id
 export default class {
   constructor(xo, { xapiOptions }) {
     this._objectConflicts = { __proto__: null } // TODO: clean when a server is disconnected.
@@ -267,6 +276,12 @@ export default class {
     try {
       await xapi.connect()
 
+      // requesting disconnection on the connecting server
+      if (this._xapis[server.id] === undefined) {
+        xapi.disconnect()::ignoreErrors()
+        return
+      }
+
       const serverIdsByPool = this._serverIdsByPool
       const poolId = xapi.pool.$id
       if (serverIdsByPool[poolId] !== undefined) {
@@ -390,15 +405,17 @@ export default class {
   }
 
   async disconnectXenServer(id) {
-    const xapi = this._xapis[id]
-    if (!xapi) {
-      throw noSuchObject(id, 'xenServer')
+    const status = this._getXenServerStatus(id)
+    if (status === 'disconnected') {
+      return
     }
 
+    const xapi = this._xapis[id]
     delete this._xapis[id]
-    delete this._serverIdsByPool[xapi.pool.$id]
-
-    xapi.xo.uninstall()
+    if (status === 'connected') {
+      delete this._serverIdsByPool[xapi.pool.$id]
+      xapi.xo.uninstall()
+    }
     return xapi.disconnect()
   }
 
@@ -425,18 +442,22 @@ export default class {
     return xapi
   }
 
+  _getXenServerStatus(id) {
+    const xapi = this._xapis[id]
+    return xapi === undefined
+      ? 'disconnected'
+      : this._serverIdsByPool[(xapi.pool?.$id)] === id
+      ? 'connected'
+      : 'connecting'
+  }
+
   async getAllXenServers() {
     const servers = await this._servers.get()
     const xapis = this._xapis
     forEach(servers, server => {
-      const xapi = xapis[server.id]
-      if (xapi !== undefined) {
-        server.status = xapi.status
-
-        let pool
-        if (server.label === undefined && (pool = xapi.pool) != null) {
-          server.label = pool.name_label
-        }
+      server.status = this._getXenServerStatus(server.id)
+      if (server.status === 'connected' && server.label === undefined) {
+        server.label = xapis[server.id].pool.name_label
       }
 
       // Do not expose password.

packages/xo-vmdk-to-vhd/package.json

@@ -10,6 +10,7 @@
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-vmdk-to-vhd",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
+    "directory": "packages/xo-vmdk-to-vhd",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },