feat(xo-server): 5.35.0

Fixes #3938

.editorconfig

@@ -3,63 +3,12 @@
 # Julien Fontanet's configuration
 # https://gist.github.com/julien-f/8096213
 
-# Top-most EditorConfig file.
 root = true
 
-# Common config.
 [*]
 charset = utf-8
 end_of_line = lf
+indent_size = 2
+indent_style = space
 insert_final_newline = true
 trim_trailing_whitespace = true
-
-# CoffeeScript
-#
-# https://github.com/polarmobile/coffeescript-style-guide/blob/master/README.md
-[*.{,lit}coffee]
-indent_size = 2
-indent_style = space
-
-# Markdown
-[*.{md,mdwn,mdown,markdown}]
-indent_size = 4
-indent_style = space
-
-# Package.json
-#
-# This indentation style is the one used by npm.
-[package.json]
-indent_size = 2
-indent_style = space
-
-# Pug (Jade)
-[*.{jade,pug}]
-indent_size = 2
-indent_style = space
-
-# JavaScript
-#
-# Two spaces seems to be the standard most common style, at least in
-# Node.js (http://nodeguide.com/style.html#tabs-vs-spaces).
-[*.{js,jsx,ts,tsx}]
-indent_size = 2
-indent_style = space
-
-# Less
-[*.less]
-indent_size = 2
-indent_style = space
-
-# Sass
-#
-# Style used for http://libsass.com
-[*.s[ac]ss]
-indent_size = 2
-indent_style = space
-
-# YAML
-#
-# Only spaces are allowed.
-[*.yaml]
-indent_size = 2
-indent_style = space

@xen-orchestra/cr-seed-cli/package.json

@@ -15,6 +15,6 @@
   },
   "dependencies": {
     "golike-defer": "^0.4.1",
-    "xen-api": "^0.23.0"
+    "xen-api": "^0.24.2"
   }
 }

@xen-orchestra/fs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/fs",
-  "version": "0.5.0",
+  "version": "0.6.1",
   "license": "AGPL-3.0",
   "description": "The File System for Xen Orchestra backups.",
   "keywords": [],
@@ -21,6 +21,7 @@
   },
   "dependencies": {
     "@marsaud/smb2": "^0.13.0",
+    "@sindresorhus/df": "^2.1.0",
     "@xen-orchestra/async-map": "^0.0.0",
     "execa": "^1.0.0",
     "fs-extra": "^7.0.0",

@xen-orchestra/fs/src/_mount.js

@@ -0,0 +1,82 @@
+import execa from 'execa'
+import fs from 'fs-extra'
+import { join } from 'path'
+import { tmpdir } from 'os'
+
+import LocalHandler from './local'
+
+const sudoExeca = (command, args, opts) =>
+  execa('sudo', [command, ...args], opts)
+
+export default class MountHandler extends LocalHandler {
+  constructor(
+    remote,
+    {
+      mountsDir = join(tmpdir(), 'xo-fs-mounts'),
+      useSudo = false,
+      ...opts
+    } = {},
+    params
+  ) {
+    super(remote, opts)
+
+    this._execa = useSudo ? sudoExeca : execa
+    this._params = {
+      ...params,
+      options: [params.options, remote.options].filter(
+        _ => _ !== undefined
+      ).join(','),
+    }
+    this._realPath = join(
+      mountsDir,
+      remote.id ||
+        Math.random()
+          .toString(36)
+          .slice(2)
+    )
+  }
+
+  async _forget() {
+    await this._execa('umount', ['--force', this._getRealPath()], {
+      env: {
+        LANG: 'C',
+      },
+    }).catch(error => {
+      if (
+        error == null ||
+        typeof error.stderr !== 'string' ||
+        !error.stderr.includes('not mounted')
+      ) {
+        throw error
+      }
+    })
+  }
+
+  _getRealPath() {
+    return this._realPath
+  }
+
+  async _sync() {
+    await fs.ensureDir(this._getRealPath())
+    const { type, device, options, env } = this._params
+    return this._execa(
+      'mount',
+      ['-t', type, device, this._getRealPath(), '-o', options],
+      {
+        env: {
+          LANG: 'C',
+          ...env,
+        },
+      }
+    ).catch(error => {
+      let stderr
+      if (
+        error == null ||
+        typeof (stderr = error.stderr) !== 'string' ||
+        !(stderr.includes('already mounted') || stderr.includes('busy'))
+      ) {
+        throw error
+      }
+    })
+  }
+}

@xen-orchestra/fs/src/_normalizePath.js

@@ -0,0 +1,9 @@
+import path from 'path'
+
+const { resolve } = path.posix
+
+// normalize the path:
+// - does not contains `.` or `..`  (cannot escape root dir)
+// - always starts with `/`
+const normalizePath = path => resolve('/', path)
+export { normalizePath as default }

@xen-orchestra/fs/src/abstract.js

@@ -1,32 +1,30 @@
 // @flow
 
 // $FlowFixMe
-import asyncMap from '@xen-orchestra/async-map'
 import getStream from 'get-stream'
+
+import asyncMap from '@xen-orchestra/async-map'
 import path from 'path'
 import { fromCallback, fromEvent, ignoreErrors, timeout } from 'promise-toolbox'
 import { parse } from 'xo-remote-parser'
 import { randomBytes } from 'crypto'
 import { type Readable, type Writable } from 'stream'
 
+import normalizePath from './_normalizePath'
 import { createChecksumStream, validChecksumOfReadStream } from './checksum'
 
-const { dirname, resolve } = path.posix
+const { dirname } = path.posix
 
 type Data = Buffer | Readable | string
 type FileDescriptor = {| fd: mixed, path: string |}
 type LaxReadable = Readable & Object
 type LaxWritable = Writable & Object
+type RemoteInfo = { used?: number, size?: number }
 
 type File = FileDescriptor | string
 
 const checksumFile = file => file + '.checksum'
 
-// normalize the path:
-// - does not contains `.` or `..`  (cannot escape root dir)
-// - always starts with `/`
-const normalizePath = path => resolve('/', path)
-
 const DEFAULT_TIMEOUT = 6e5 // 10 min
 
 const ignoreEnoent = error => {
@@ -222,6 +220,10 @@ export default class RemoteHandlerAbstract {
     await this._forget()
   }
 
+  async getInfo(): Promise<RemoteInfo> {
+    return timeout.call(this._getInfo(), this._timeout)
+  }
+
   async getSize(file: File): Promise<number> {
     return timeout.call(
       this._getSize(typeof file === 'string' ? normalizePath(file) : file),
@@ -430,6 +432,10 @@ export default class RemoteHandlerAbstract {
   // called to finalize the remote
   async _forget(): Promise<void> {}
 
+  async _getInfo(): Promise<Object> {
+    return {}
+  }
+
   async _getSize(file: File): Promise<number> {
     throw new Error('Not implemented')
   }

@xen-orchestra/fs/src/abstract.spec.js

@@ -16,6 +16,8 @@ class TestHandler extends AbstractHandler {
   }
 }
 
+jest.useFakeTimers()
+
 describe('closeFile()', () => {
   it(`throws in case of timeout`, async () => {
     const testHandler = new TestHandler({
@@ -52,6 +54,18 @@ describe('createReadStream()', () => {
   })
 })
 
+describe('getInfo()', () => {
+  it('throws in case of timeout', async () => {
+    const testHandler = new TestHandler({
+      getInfo: () => new Promise(() => {}),
+    })
+
+    const promise = testHandler.getInfo()
+    jest.advanceTimersByTime(TIMEOUT)
+    await expect(promise).rejects.toThrowError(TimeoutError)
+  })
+})
+
 describe('getSize()', () => {
   it(`throws in case of timeout`, async () => {
     const testHandler = new TestHandler({

@xen-orchestra/fs/src/fs.spec.js

@@ -116,6 +116,26 @@ handlers.forEach(url => {
       })
     })
 
+    describe('#getInfo()', () => {
+      let info
+      beforeAll(async () => {
+        info = await handler.getInfo()
+      })
+
+      it('should return an object with info', async () => {
+        expect(typeof info).toBe('object')
+      })
+
+      it('should return correct type of attribute', async () => {
+        if (info.size !== undefined) {
+          expect(typeof info.size).toBe('number')
+        }
+        if (info.used !== undefined) {
+          expect(typeof info.used).toBe('number')
+        }
+      })
+    })
+
     describe('#getSize()', () => {
       beforeEach(() => handler.outputFile('file', TEST_DATA))
 

@xen-orchestra/fs/src/index.js

@@ -1,19 +1,27 @@
 // @flow
+import execa from 'execa'
 
 import type RemoteHandler from './abstract'
 import RemoteHandlerLocal from './local'
 import RemoteHandlerNfs from './nfs'
 import RemoteHandlerSmb from './smb'
+import RemoteHandlerSmbMount from './smb-mount'
 
 export type { default as RemoteHandler } from './abstract'
 export type Remote = { url: string }
 
 const HANDLERS = {
   file: RemoteHandlerLocal,
-  smb: RemoteHandlerSmb,
   nfs: RemoteHandlerNfs,
 }
 
+try {
+  execa.sync('mount.cifs', ['-V'])
+  HANDLERS.smb = RemoteHandlerSmbMount
+} catch (_) {
+  HANDLERS.smb = RemoteHandlerSmb
+}
+
 export const getHandler = (remote: Remote, ...rest: any): RemoteHandler => {
   // FIXME: should be done in xo-remote-parser.
   const type = remote.url.split('://')[0]

@xen-orchestra/fs/src/local.js

@@ -1,3 +1,4 @@
+import df from '@sindresorhus/df'
 import fs from 'fs-extra'
 import { fromEvent } from 'promise-toolbox'
 
@@ -46,6 +47,10 @@ export default class LocalHandler extends RemoteHandlerAbstract {
     })
   }
 
+  _getInfo() {
+    return df.file(this._getFilePath('/'))
+  }
+
   async _getSize(file) {
     const stats = await fs.stat(
       this._getFilePath(typeof file === 'string' ? file : file.path)

@xen-orchestra/fs/src/nfs.js

@@ -1,92 +1,20 @@
-import execa from 'execa'
-import fs from 'fs-extra'
-import { join } from 'path'
-import { tmpdir } from 'os'
+import { parse } from 'xo-remote-parser'
 
-import LocalHandler from './local'
+import MountHandler from './_mount'
 
 const DEFAULT_NFS_OPTIONS = 'vers=3'
 
-export default class NfsHandler extends LocalHandler {
-  constructor(
-    remote,
-    { mountsDir = join(tmpdir(), 'xo-fs-mounts'), ...opts } = {}
-  ) {
-    super(remote, opts)
-
-    this._realPath = join(
-      mountsDir,
-      remote.id ||
-        Math.random()
-          .toString(36)
-          .slice(2)
-    )
+export default class NfsHandler extends MountHandler {
+  constructor(remote, opts) {
+    const { host, port, path } = parse(remote.url)
+    super(remote, opts, {
+      type: 'nfs',
+      device: `${host}${port !== undefined ? ':' + port : ''}:${path}`,
+      options: DEFAULT_NFS_OPTIONS,
+    })
   }
 
   get type() {
     return 'nfs'
   }
-
-  _getRealPath() {
-    return this._realPath
-  }
-
-  async _mount() {
-    await fs.ensureDir(this._getRealPath())
-    const { host, path, port, options } = this._remote
-    return execa(
-      'mount',
-      [
-        '-t',
-        'nfs',
-        '-o',
-        DEFAULT_NFS_OPTIONS + (options !== undefined ? `,${options}` : ''),
-        `${host}${port !== undefined ? ':' + port : ''}:${path}`,
-        this._getRealPath(),
-      ],
-      {
-        env: {
-          LANG: 'C',
-        },
-      }
-    ).catch(error => {
-      if (
-        error == null ||
-        typeof error.stderr !== 'string' ||
-        !error.stderr.includes('already mounted')
-      ) {
-        throw error
-      }
-    })
-  }
-
-  async _umount() {
-    await execa('umount', ['--force', this._getRealPath()], {
-      env: {
-        LANG: 'C',
-      },
-    }).catch(error => {
-      if (
-        error == null ||
-        typeof error.stderr !== 'string' ||
-        !error.stderr.includes('not mounted')
-      ) {
-        throw error
-      }
-    })
-  }
-
-  async _forget() {
-    try {
-      await this._umount(this._remote)
-    } catch (_) {
-      // We have to go on...
-    }
-  }
-
-  async _sync() {
-    await this._mount()
-
-    return this._remote
-  }
 }

@xen-orchestra/fs/src/smb-mount.js

@@ -0,0 +1,25 @@
+import { parse } from 'xo-remote-parser'
+
+import MountHandler from './_mount'
+import normalizePath from './_normalizePath'
+
+export default class SmbMountHandler extends MountHandler {
+  constructor(remote, opts) {
+    const { domain = 'WORKGROUP', host, password, path, username } = parse(
+      remote.url
+    )
+    super(remote, opts, {
+      type: 'cifs',
+      device: '//' + host + normalizePath(path),
+      options: `domain=${domain}`,
+      env: {
+        USER: username,
+        PASSWD: password,
+      },
+    })
+  }
+
+  get type() {
+    return 'smb'
+  }
+}

@xen-orchestra/log/README.md

@@ -75,7 +75,7 @@ catchGlobalErrors(transport)
 ```js
 import transportConsole from '@xen-orchestra/log/transports/console'
 
-configure(transports.console())
+configure(transportConsole())
 ```
 
 #### Email

@xen-orchestra/log/src/utils.js

@@ -55,7 +55,8 @@ export const required = name => {
 // -------------------------------------------------------------------
 
 export const serializeError = error => ({
-  ...error,
+  ...error, // Copy enumerable properties.
+  code: error.code,
   message: error.message,
   name: error.name,
   stack: error.stack,

CHANGELOG.md

@@ -1,7 +1,68 @@
 # ChangeLog
 
-## *next*
+## **5.31.0** (2019-01-31)
 
+### Enhancements
+
+- [Backup NG] Restore logs moved to restore tab [#3772](https://github.com/vatesfr/xen-orchestra/issues/3772) (PR [#3802](https://github.com/vatesfr/xen-orchestra/pull/3802))
+- [Remotes] New SMB implementation that provides better stability and performance [#2257](https://github.com/vatesfr/xen-orchestra/issues/2257) (PR [#3708](https://github.com/vatesfr/xen-orchestra/pull/3708))
+- [VM/advanced] ACL management from VM view [#3040](https://github.com/vatesfr/xen-orchestra/issues/3040) (PR [#3774](https://github.com/vatesfr/xen-orchestra/pull/3774))
+- [VM / snapshots] Ability to save the VM memory [#3795](https://github.com/vatesfr/xen-orchestra/issues/3795) (PR [#3812](https://github.com/vatesfr/xen-orchestra/pull/3812))
+- [Backup NG / Health] Show number of lone snapshots in tab label [#3500](https://github.com/vatesfr/xen-orchestra/issues/3500) (PR [#3824](https://github.com/vatesfr/xen-orchestra/pull/3824))
+- [Login] Add autofocus on username input on login page [#3835](https://github.com/vatesfr/xen-orchestra/issues/3835) (PR [#3836](https://github.com/vatesfr/xen-orchestra/pull/3836))
+- [Home/VM] Bulk snapshot: specify snapshots' names [#3778](https://github.com/vatesfr/xen-orchestra/issues/3778) (PR [#3787](https://github.com/vatesfr/xen-orchestra/pull/3787))
+- [Remotes] Show free space and disk usage on remote [#3055](https://github.com/vatesfr/xen-orchestra/issues/3055) (PR [#3767](https://github.com/vatesfr/xen-orchestra/pull/3767))
+- [New SR] Add tooltip for reattach action button [#3845](https://github.com/vatesfr/xen-orchestra/issues/3845) (PR [#3852](https://github.com/vatesfr/xen-orchestra/pull/3852))
+- [VM migration] Display hosts' free memory [#3264](https://github.com/vatesfr/xen-orchestra/issues/3264) (PR [#3832](https://github.com/vatesfr/xen-orchestra/pull/3832))
+- [Plugins] New field to filter displayed plugins (PR [#3832](https://github.com/vatesfr/xen-orchestra/pull/3871))
+- Ability to copy ID of "unknown item"s [#3833](https://github.com/vatesfr/xen-orchestra/issues/3833) (PR [#3856](https://github.com/vatesfr/xen-orchestra/pull/3856))
+- [Cloud-Init] switch config drive type to `nocloud` to prepare for the passing of network config (PR [#3877](https://github.com/vatesfr/xen-orchestra/pull/3877))
+- [UI] Show pool name next to templates' names [#3894](https://github.com/vatesfr/xen-orchestra/issues/3894) (PR [#3896](https://github.com/vatesfr/xen-orchestra/pull/3896))
+- [Backup NG] Support zstd compression for full backups [#3773](https://github.com/vatesfr/xen-orchestra/issues/3773) (PR [#3883](https://github.com/vatesfr/xen-orchestra/pull/3883))
+- [VM] Ability to copy a VM with zstd compression [#3773](https://github.com/vatesfr/xen-orchestra/issues/3773) (PR [#3889](https://github.com/vatesfr/xen-orchestra/pull/3889))
+- [VM & Host] "Pool > Host" breadcrumb at the top of the page (PR [#3898](https://github.com/vatesfr/xen-orchestra/pull/3898))
+- [Hosts] Ability to enable/disable host multipathing [#3659](https://github.com/vatesfr/xen-orchestra/issues/3659) (PR [#3865](https://github.com/vatesfr/xen-orchestra/pull/3865))
+- [Login] Add OTP authentication [#2044](https://github.com/vatesfr/xen-orchestra/issues/2044) (PR [#3879](https://github.com/vatesfr/xen-orchestra/pull/3879))
+- [Notifications] New notification page to provide important information about XOA (PR [#3904](https://github.com/vatesfr/xen-orchestra/pull/3904))
+- [VM] Ability to export a VM with zstd compression [#3773](https://github.com/vatesfr/xen-orchestra/issues/3773) (PR [#3891](https://github.com/vatesfr/xen-orchestra/pull/3891))
+- [Host/network] Display PIF speed [#3887](https://github.com/vatesfr/xen-orchestra/issues/3887) (PR [#3901](https://github.com/vatesfr/xen-orchestra/pull/3901))
+- [SR] Display iscsi paths and mark the SR with a yellow dot if one path is not available. [#3659](https://github.com/vatesfr/xen-orchestra/issues/3659) (PR [#3829](https://github.com/vatesfr/xen-orchestra/pull/3829))
+- [UI] Unifies the Signin buttons (PR [#3913](https://github.com/vatesfr/xen-orchestra/pull/3913))
+- [Settings/remotes] NFS: display default option on placeholder [#3631](https://github.com/vatesfr/xen-orchestra/issues/3631) (PR [#3921](https://github.com/vatesfr/xen-orchestra/pull/3921))
+- [VM/advanced] Ability to pin vCPU to physical cores [#3241](https://github.com/vatesfr/xen-orchestra/issues/3241) (PR [#3254](https://github.com/vatesfr/xen-orchestra/pull/3254))
+
+### Bug fixes
+
+- [Self] Display sorted Resource Sets [#3818](https://github.com/vatesfr/xen-orchestra/issues/3818) (PR [#3823](https://github.com/vatesfr/xen-orchestra/pull/3823))
+- [Servers] Correctly report connecting status (PR [#3838](https://github.com/vatesfr/xen-orchestra/pull/3838))
+- [Servers] Fix cannot reconnect to a server after connection has been lost [#3839](https://github.com/vatesfr/xen-orchestra/issues/3839) (PR [#3841](https://github.com/vatesfr/xen-orchestra/pull/3841))
+- [New VM] Fix `NO_HOSTS_AVAILABLE()` error when creating  a VM on a local SR from template on another local SR [#3084](https://github.com/vatesfr/xen-orchestra/issues/3084) (PR [#3827](https://github.com/vatesfr/xen-orchestra/pull/3827))
+- [Backup NG] Fix typo in the form [#3854](https://github.com/vatesfr/xen-orchestra/issues/3854) (PR [#3855](https://github.com/vatesfr/xen-orchestra/pull/3855))
+- [New SR] No warning when creating a NFS SR on a path that is already used as NFS SR [#3844](https://github.com/vatesfr/xen-orchestra/issues/3844) (PR [#3851](https://github.com/vatesfr/xen-orchestra/pull/3851))
+- [New SR] No redirection if the SR creation failed or canceled [#3843](https://github.com/vatesfr/xen-orchestra/issues/3843) (PR [#3853](https://github.com/vatesfr/xen-orchestra/pull/3853))
+- [Home] Fix two tabs opened by middle click in Firefox [#3450](https://github.com/vatesfr/xen-orchestra/issues/3450) (PR [#3825](https://github.com/vatesfr/xen-orchestra/pull/3825))
+- [XOA] Enable downgrade for ending trial (PR [#3867](https://github.com/vatesfr/xen-orchestra/pull/3867))
+- [OVA import] allow import of big files [#3468](https://github.com/vatesfr/xen-orchestra/issues/3468) (PR [#3504](https://github.com/vatesfr/xen-orchestra/pull/3504))
+- [Backup NG] Smart settings not saved when editing a backup job [#3885](https://github.com/vatesfr/xen-orchestra/issues/3885) (PR [#3886](https://github.com/vatesfr/xen-orchestra/pull/3886))
+- [VM/snapshot] New snapshot with memory: fix "invalid parameters" error (PR [#3903](https://github.com/vatesfr/xen-orchestra/pull/3903))
+- [VM creation] Broken CloudInit config drive when VM created on local SR
+- [Legacy Backup] Fix error when restoring a backup
+- [Home] Fix `user.getAll` error when user is not admin [#3573](https://github.com/vatesfr/xen-orchestra/issues/3573) (PR [#3918](https://github.com/vatesfr/xen-orchestra/pull/3918))
+- [Backup NG] Fix restore issue when a disk has grown [#3910](https://github.com/vatesfr/xen-orchestra/issues/3910)  (PR [#3920](https://github.com/vatesfr/xen-orchestra/pull/3920))
+- [Backup NG] Delete _importing_ VMs due to interrupted CR/DR (PR [#3923](https://github.com/vatesfr/xen-orchestra/pull/3923))
+
+### Released packages
+
+- vhd-cli v0.2.0
+- @xen-orchestra/fs v0.6.0
+- vhd-lib v0.5.1
+- xoa-updater v0.15.0
+- xen-api v0.24.1
+- xo-vmdk-to-vhd v0.1.6
+- xo-server v5.34.0
+- xo-web v5.34.0
+
+## **5.30.0** (2018-12-20)
 
 ### Enhancements
 
@@ -17,6 +78,7 @@
 - [Backup NG] Add a link to the documentation [#3789](https://github.com/vatesfr/xen-orchestra/issues/3789) (PR [#3790](https://github.com/vatesfr/xen-orchestra/pull/3790))
 - [Backup NG] Ability to copy schedule/job id to the clipboard [#3753](https://github.com/vatesfr/xen-orchestra/issues/3753) (PR [#3791](https://github.com/vatesfr/xen-orchestra/pull/3791))
 - [Backup NG / logs] Merge the job log status with the display details button [#3797](https://github.com/vatesfr/xen-orchestra/issues/3797) (PR [#3800](https://github.com/vatesfr/xen-orchestra/pull/3800))
+- [XOA] Notification banner when XOA is not registered [#3803](https://github.com/vatesfr/xen-orchestra/issues/3803) (PR [#3808](https://github.com/vatesfr/xen-orchestra/pull/3808))
 
 ### Bug fixes
 
@@ -34,6 +96,8 @@
 
 ## **5.29.0** (2018-11-29)
 
+### Enhancements
+
 - [Perf alert] Ability to trigger an alarm if a host/VM/SR usage value is below the threshold [#3612](https://github.com/vatesfr/xen-orchestra/issues/3612) (PR [#3675](https://github.com/vatesfr/xen-orchestra/pull/3675))
 - [Home/VMs] Display pool's name [#2226](https://github.com/vatesfr/xen-orchestra/issues/2226) (PR [#3709](https://github.com/vatesfr/xen-orchestra/pull/3709))
 - [Servers] Prevent new connection if pool is already connected [#2238](https://github.com/vatesfr/xen-orchestra/issues/2238) (PR [#3724](https://github.com/vatesfr/xen-orchestra/pull/3724))

CHANGELOG.unreleased.md

@@ -0,0 +1,29 @@
+> This file contains all changes that have not been released yet.
+
+### Enhancements
+
+- [Home] Set description on bulk snapshot [#3925](https://github.com/vatesfr/xen-orchestra/issues/3925) (PR [#3933](https://github.com/vatesfr/xen-orchestra/pull/3933))
+- Work-around the XenServer issue when `VBD#VDI` is an empty string instead of an opaque reference (PR [#3950](https://github.com/vatesfr/xen-orchestra/pull/3950))
+- [VDI migration] Retry when XenServer fails with `TOO_MANY_STORAGE_MIGRATES` (PR [#3940](https://github.com/vatesfr/xen-orchestra/pull/3940))
+- [VM]
+  - [General] The creation date of the VM is now visible [#3932](https://github.com/vatesfr/xen-orchestra/issues/3932) (PR [#3947](https://github.com/vatesfr/xen-orchestra/pull/3947))
+  - [Disks] Display device name [#3902](https://github.com/vatesfr/xen-orchestra/issues/3902) (PR [#3946](https://github.com/vatesfr/xen-orchestra/pull/3946))
+- [VM Snapshotting]
+  - Detect and destroy broken quiesced snapshot left by XenServer [#3936](https://github.com/vatesfr/xen-orchestra/issues/3936) (PR [#3937](https://github.com/vatesfr/xen-orchestra/pull/3937))
+  - Retry twice after a 1 minute delay if quiesce failed [#3938](https://github.com/vatesfr/xen-orchestra/issues/3938) (PR [#3952](https://github.com/vatesfr/xen-orchestra/pull/3952))
+
+### Bug fixes
+
+- [Import] Fix import of big OVA files
+- [Host] Show the host's memory usage instead of the sum of the VMs' memory usage (PR [#3924](https://github.com/vatesfr/xen-orchestra/pull/3924))
+- [SAML] Make `AssertionConsumerServiceURL` matches the callback URL
+- [Backup NG] Correctly delete broken VHD chains [#3875](https://github.com/vatesfr/xen-orchestra/issues/3875) (PR [#3939](https://github.com/vatesfr/xen-orchestra/pull/3939))
+- [Remotes] Don't ignore `mount` options [#3935](https://github.com/vatesfr/xen-orchestra/issues/3935) (PR [#3931](https://github.com/vatesfr/xen-orchestra/pull/3931))
+
+### Released packages
+
+- xen-api v0.24.2
+- @xen-orchestra/fs v0.6.1
+- xo-server-auth-saml v0.5.3
+- xo-server v5.35.0
+- xo-web v5.35.0

PULL_REQUEST_TEMPLATE.md

@@ -4,10 +4,11 @@
 
 - [ ] PR reference the relevant issue (e.g. `Fixes #007`)
 - [ ] if UI changes, a screenshot has been added to the PR
-- [ ] CHANGELOG:
+- [ ] `CHANGELOG.unreleased.md`:
    - enhancement/bug fix entry added
    - list of packages to release updated (`${name} v${new version}`)
 - [ ] documentation updated
+- [ ] **I have tested added/updated features** (and impacted code)
 
 ### Process
 

docs/backup_troubleshooting.md

@@ -33,7 +33,7 @@ Just remember this: **coalesce will happen every time a snapshot is removed**.
 
 First check SMlog on the XenServer host for messages relating to VDI corruption or coalesce job failure. For example, by running `cat /var/log/SMlog | grep -i exception` or `cat /var/log/SMlog | grep -i error` on the XenServer host with the affected storage.  
 
-Coalesce jobs can also fail to run if the SR does not have enough free space. Check the problematic SR and make sure it has enough free space, generally 30% or more free is recommended depending on VM size.  
+Coalesce jobs can also fail to run if the SR does not have enough free space. Check the problematic SR and make sure it has enough free space, generally 30% or more free is recommended depending on VM size. You can check if this is the issue by searching `SMlog` with `grep -i coales /var/log/SMlog` (you may have to look at previous logs such as `SMlog.1`).
 
 You can check if a coalesce job is currently active by running `ps axf | grep vhd` on the XenServer host and looking for a VHD process in the results (one of the resulting processes will be the grep command you just ran, ignore that one).  
 

docs/backups.md

@@ -1,5 +1,7 @@
 # Backups
 
+> Watch our [introduction video](https://www.youtube.com/watch?v=FfUqIwT8KzI) (45m) to Backup in Xen Orchestra!
+
 This section is dedicated to all existing methods of rolling back or backing up your VMs in Xen Orchestra.
 
 There are several ways to protect your VMs:

docs/cloudinit.md

@@ -1,7 +1,5 @@
 # CloudInit
 
-> CloudInit support is available in the 4.11 release and higher
-
 Cloud-init is a program "that handles the early initialization of a cloud instance"[^n]. In other words, you can, on a "cloud-init"-ready template VM, pass a lot of data at first boot:
 
 * setting the hostname
@@ -18,25 +16,27 @@ So it means very easily customizing your VM when you create it from a compatible
 
 You only need to use a template of a VM with CloudInit installed inside it. [Check this blog post to learn how to install CloudInit](https://xen-orchestra.com/blog/centos-cloud-template-for-xenserver/).
 
+**Note:** In XOA 5.31, we changed the cloud-init config drive type from [OpenStack](https://cloudinit.readthedocs.io/en/latest/topics/datasources/configdrive.html) to the [NoCloud](https://cloudinit.readthedocs.io/en/latest/topics/datasources/nocloud.html) type. This will allow us to pass network configuration to VMs in the future. For 99% of users, including default cloud-init installs, this change will have no effect. However if you have previously modified your cloud-init installation in a VM template to only look for `openstack` drive types (for instance with the `datasource_list` setting in `/etc/cloud/cloud.cfg`) you need to modify it to also look for `nocloud`.
+
 ## Usage
 
 First, select your compatible template (CloudInit ready) and name it:
 
-![](https://xen-orchestra.com/blog/content/images/2015/12/template_choice.png)
+![](./assets/cloud-init-1.png)
 
 Then, activate the config drive and insert your SSH key. Or you can also use a custom CloudInit configuration:
 
-![](https://xen-orchestra.com/blog/content/images/2016/02/CloudInit.png)
+![](./assets/cloud-init-2.png)
 
 > CloudInit configuration examples are [available here](http://cloudinit.readthedocs.org/en/latest/topics/examples.html).
 
-You can extend the disk size (**in this case, the template disk was 8 GiB originally**):
+You can extend the disk size (**in this case, the template disk was 8 GiB originally**). We'll extend it to 20GiB:
 
-![](https://xen-orchestra.com/blog/content/images/2015/12/diskedition.png)
+![](./assets/cloud-init-3.png)
 
 Finally, create the VM:
 
-![](https://xen-orchestra.com/blog/content/images/2015/12/recap.png)
+![](./assets/cloud-init-4.png)
 
 Now start the VM and SSH to its IP:
 

docs/configuration.md

@@ -4,42 +4,37 @@ Once Xen Orchestra is installed, you can configure some parameters in the config
 
 ## Configuration
 
-The configuration file is located at `/etc/xo-server/config.yaml`.
-
-**WARNING: YAML is very strict with indentation: use spaces, not tabs.**
+The configuration file is located at `/etc/xo-server/config.toml`.
 
 ### User to run XO-server as
 
 By default, XO-server runs as 'root'. You can change that by uncommenting these lines and choose whatever user/group you want:
 
-```yaml
-user: 'nobody'
-group: 'nogroup'
+```toml
+user = 'nobody'
+group = 'nogroup'
 ```
 
-**Warning!** A non-privileged user:
-
-* can't bind to a port < 1024
-* can't mount NFS shares
+**Warning!** A non-privileged user requires the use of ``sudo`` to mount NFS shares. See [installation from the sources](from_the_sources.md).
 
 ### HTTP listen address and port
 
 By default, XO-server listens on all addresses (0.0.0.0) and runs on port 80. If you need to, you can change this in the `# Basic HTTP` section:
 
-```yaml
-host: '0.0.0.0'
-port: 80
+```toml
+host = '0.0.0.0'
+port = 80
 ```
 
 ### HTTPS
 
 XO-server can also run in HTTPS (you can run HTTP and HTTPS at the same time) - just modify what's needed in the `# Basic HTTPS` section, this time with the certificates/keys you need and their path:
 
-```yaml
-host: '0.0.0.0'
-port: 443
-certificate: './certificate.pem'
-key: './key.pem'
+```toml
+host = '0.0.0.0'
+port = 443
+certificate = './certificate.pem'
+key = './key.pem'
 ```
 
 > If a chain of certificates authorities is needed, you may bundle them directly in the certificate. Note: the order of certificates does matter, your certificate should come first followed by the certificate of the above certificate authority up to the root.
@@ -60,10 +55,9 @@ This should be written just before the `mount` option, inside the `http:` block.
 
 You shouldn't have to change this. It's the path where `xo-web` files are served by `xo-server`.
 
-```yaml
-  mounts:
-    '/':
-      - '../xo-web/dist/'
+```toml
+[http.mounts]
+'/' = '../xo-web/dist/'
 ```
 
 ### Custom certificate authority
@@ -87,8 +81,8 @@ Don't forget to reload `systemd` conf and restart `xo-server`:
 
 By default, XO-server will try to contact Redis server on `localhost`, with the port `6379`. But you can define whatever you want:
 
-```yaml
-  uri: 'tcp://db:password@hostname:port'
+```toml
+uri = 'tcp://db:password@hostname:port'
 ```
 
 ### Proxy for XenServer updates and patches
@@ -101,12 +95,12 @@ To do that behind a corporate proxy, just add the `httpProxy` variable to match
 
 You can add this at the end of your config file:
 
-```yaml
+```toml
 # HTTP proxy configuration used by xo-server to fetch resources on the Internet.
 #
 # See: https://github.com/TooTallNate/node-proxy-agent#maps-proxy-protocols-to-httpagent-implementations
 
-httpProxy: 'http://username:password@proxyAddress:port'
+httpProxy = 'http://username:password@proxyAddress:port'
 ```
 
 ### Log file

docs/continuous_replication.md

@@ -36,38 +36,52 @@ To protect the replication, we removed the possibility to boot your copied VM di
 
 ## Manual initial seed
 
-> This is **only** if you need to make the initial copy without making the whole transfer through your network. Otherwise, **you don't need this**.
+**If you can't transfer the first backup through your network because it's too large**, you can make a seed locally. In order to do this, follow this procedure (until we make it accessible directly in XO).  
 
-**If you can't transfer the first backup through your network**, you can make a seed locally. In order to do this, follow this procedure (until we make it accessible directly in XO):
+> This is **only** if you need to make the initial copy without making the whole transfer through your network. Otherwise, **you don't need this**. These instructions are for Backup-NG jobs, and will not work to seed a legacy backup job. Please migrate any legacy jobs to Backup-NG!
 
-### Preparation
 
-1. create a cont. rep job to a non-distant SR (even the SR where the VM currently is). Do NOT enable the job during creation.
-1. manually start the first replication (only the first)
-1. when finished, export the replicated VM (via XOA or any other means, doesn't matter how you get your XVA file)
-1. import the replicated VM on your distant destination
-1. you can now remove your local replicated copy
+### Job creation
 
-### Modifications
+Create the Continuous Replication backup job, and leave it disabled for now. On the main Backup-NG page, note its identifiers, the main `backupJobId` and the ID of one on the schedules for the job, `backupScheduleId`.
 
-In your source host:
+### Seed creation
 
-1. Get the UUID of the remote destination SR where your VM was imported
-1. On the source host: `xe vm-param-list uuid=<SourceVM_UUID> | grep other-config`.
-  * You should see somewhere in other-config: `xo:base_delta:<SR_UUID>: <VM_snapshot_UUID>;`
-  * Remove this entry with `xe vm-param-remove uuid=<OriginalVM_UUID> param-name=other-config param-key=xo:base_delta:<SR_UUID>`
-  * Recreate the correct param: `xe vm-param-set uuid=<OriginalVM_UUID> other-config:xo:base_delta:<destination_SR_UUID>=<VM_snapshot_UUID>`
+Manually create a snapshot on the VM to backup, and note its UUID as `snapshotUuid` from the snapshot panel for the VM.
 
-In XO:
+> DO NOT ever delete or alter this snapshot, feel free to rename it to make that clear.
 
-1. Edit the replication job and select the new destination SR
+### Seed copy
 
-On the destination host; to avoid data corruption, you need to avoid any VM start:
+Export this snapshot to a file, then import it on the target SR.
+
+Note the UUID of this newly created VM as `targetVmUuid`.
+
+> DO not start this VM or it will break the Continuous Replication job! You can rename this VM to more easily remember this.
+
+### Set up metadata
+
+The XOA backup system requires metadata to correctly associate the source snapshot and the target VM to the backup job. We're going to use the `xo-cr-seed` utility to help us set them up.
+
+First install the tool (all the following is done from the XOA VM CLI):
 
 ```
-xe vm-param-set blocked-operations:start uuid=<DestinationVM_UUID>
+npm i -g xo-cr-seed
 ```
 
-### Enable
+Here is an example of how the utility expects the UUIDs and info passed to it:
 
-Manually run the job the first time to check if everything is OK. Then, enable the job. **Now, only the deltas are sent, your initial seed saved you a LOT of time if you have a slow network.**
+```
+xo-cr-seed
+Usage: xo-cr-seed <source XAPI URL> <source snapshot UUID> <target XAPI URL> <target VM UUID> <backup job id> <backup schedule id>
+
+xo-cr-seed v0.2.0
+```
+Putting it altogether and putting our values and UUID's into the command, it will look like this (it is a long command):
+```
+xo-cr-seed https://root:password@xen1.company.tld 4a21c1cd-e8bd-4466-910a-f7524ecc07b1 https://root:password@xen2.company.tld 5aaf86ca-ae06-4a4e-b6e1-d04f0609e64d 90d11a94-a88f-4a84-b7c1-ed207d3de2f9 369a26f0-da77-41ab-a998-fa6b02c69b9a
+```
+
+### Finished
+
+Your backup job should now be working correctly! Manually run the job the first time to check if everything is OK. Then, enable the job. **Now, only the deltas are sent, your initial seed saved you a LOT of time if you have a slow network.**

docs/contributing.md

@@ -24,10 +24,11 @@ to create a [GitHub pull request](https://help.github.com/articles/using-pull-re
 
 
 1. Create a branch for your work
-2. Create a pull request for this branch against the `master` branch
-3. Push into the branch until the pull request is ready to merge
-4. Avoid unnecessary merges: keep you branch up to date by regularly rebasing `git rebase origin/master`
-5. When ready to merge, clean up the history (reorder commits, squash some of them together, rephrase messages): `git rebase -i origin/master`
+2. Add a summary of your changes to `CHANGELOG.md` under the `next` section, if your changes do not relate to an existing changelog item
+3. Create a pull request for this branch against the `master` branch
+4. Push into the branch until the pull request is ready to merge
+5. Avoid unnecessary merges: keep you branch up to date by regularly rebasing `git rebase origin/master`
+6. When ready to merge, clean up the history (reorder commits, squash some of them together, rephrase messages): `git rebase -i origin/master`
 
 ### Issue triage
 

docs/directpurchase.md

@@ -2,26 +2,50 @@
 
 This is the easiest purchase option: you can buy XOA with your registered email account on `xen-orchestra.com`.
 
+## Choose your edition
+
 You can choose the edition you want in two places:
 
 * [the pricing page](https://xen-orchestra.com/#!/pricing)
-* [your account/member page](https://xen-orchestra.com/#!/member)
+* [your account/purchases page](https://xen-orchestra.com/#!/purchases)
 
 
-> You need to be logged to make a purchase. If you don't have an account, please [register here](https://xen-orchestra.com/#!/signup).
+> You need to be logged in to make a purchase. If you don't have an account, please [register here](https://xen-orchestra.com/#!/signup).
 
 From your account page, click on the purchase menu, then select the edition you need:
 
 ![](./assets/directpurchase.png)
 
-Then you need to fill in your information and select **"buy it for my own use"**:
+## Purchase options
+
+The second step is to select your purchase option: 
+
+- Subscription: only available with a credit card payment. Choose this option for a monthly payment or a yearly payment **renewed automatically** each year.
+
+- Paid period: **check or wire transfer only**. This purchase allows you to subscribe for a one, two or three year period
+
+> A 2 year subscription period grants you 1 month discounted
+> A 3 year subscription period grants you 2 months discounted
+
+Then you need to fill in your information and select **"Buy for my own use"** (direct purchase)
 
 ![](./assets/member_purchase_2.png)
 
-The default payment method is by **credit card**. But you can also choose the "wire transfer" tab (with the "bank" icon):
+## Billing information
+You need to complete all the required information on this page in order to move forward. 
 
-![](./assets/member_purchase3.png)
+> Note: If you are part of the Eurozone, you will need to provide a valid EU VAT number in order to proceed to payment. Transactions between companies inside the Eurozone are VAT free. 
+Transactions outside the Eurozone are VAT free. 
 
-## Wire transfer process
+![](./assets/billing_info.png)
 
-If you select wire transfer, you need to upload a proof of transfer before we can unlock your XOA. If you don't, you'll have to wait for funds to actually be transferred into our account.
+## Select your payment mode
+
+Credit Card, Wire transfer or Bank check are the three payment methods available on our store. Some methods can be unavailable regarding the purchase option you have selected during step one. 
+
+> Wire transfer is not available for monthly and yearly subscription - Credit Card is not available for paid period.
+
+![](./assets/payment_mode.png)
+
+> All required information for wire transfer and Check payment will be available in the last step of the payment AND on your proforma invoice.
+> ⚠ Please, use an explicit reference for your wire transfer in order for us to easily identify your payment.

docs/from_the_sources.md

@@ -38,12 +38,12 @@ XO needs the following packages to be installed. Redis is used as a database by
 For example, on Debian:
 
 ```
-apt-get install build-essential redis-server libpng-dev git python-minimal libvhdi-utils lvm2
+apt-get install build-essential redis-server libpng-dev git python-minimal libvhdi-utils lvm2 cifs-utils
 ```
 
 ## Fetching the Code
 
-You need to use the `git` source code manager to fetch the code. Ideally you should run XO as a non-root user, however if you don't run as root you will not be able to mount NFS remotes. As your chosen non-root (or root) user, run the following:
+You need to use the `git` source code manager to fetch the code. Ideally, you should run XO as a non-root user, and if you choose to, you need to set up `sudo` to be able to mount NFS remotes. As your chosen non-root (or root) user, run the following:
 
 ```
 git clone -b master http://github.com/vatesfr/xen-orchestra
@@ -64,17 +64,15 @@ Now you have to create a config file for `xo-server`:
 
 ```
 $ cd packages/xo-server
-$ cp sample.config.yaml .xo-server.yaml
+$ cp sample.config.toml .xo-server.toml
 ```
 
-Edit and uncomment it to have the right path to serve `xo-web`, because `xo-server` embeds an HTTP server (we assume that `xen-orchestra` and `xo-web` are in the same directory). It's near the end of the file:
+Edit and uncomment it to have the right path to serve `xo-web`, because `xo-server` embeds an HTTP server (we assume that `xen-orchestra` and `xo-web` are in the same directory):
 
-```yaml
-  mounts: '/': '../xo-web/dist/'
+```toml
+[http.mounts]
+'/' = '../xo-web/dist/'
 ```
-> Note this `dist` folder will be created in the next step.
-
-**WARNING: YAML is very strict with indentation: use spaces for it, not tabs**.
 
 In this config file, you can also change default ports (80 and 443) for xo-server. If you are running the server as a non-root user, you will need to set the port to 1024 or higher.
 
@@ -143,9 +141,6 @@ If you need to delete the service:
 forever-service delete orchestra
 ```
 
-
-
-
 ## Troubleshooting
 
 If you have problems during the building phase, follow these steps in your `xen-orchestra` directory:
@@ -187,3 +182,17 @@ Don't forget to start redis if you don't reboot now:
 ```
 service redis start
 ```
+
+## SUDO
+
+If you are running `xo-server` as a non-root user, you need to use `sudo` to be able to mount NFS remotes. You can do this by editing `xo-server/.xo-server.toml` and setting `useSudo = true`. It's near the end of the file:
+
+```
+useSudo = true
+```
+
+You need to configure `sudo` to allow the user of your choice to run mount/umount commands without asking for a password. Depending on your operating system / sudo version, the location of this configuration may change. Regardless, you can use:
+
+```
+username ALL=(root)NOPASSWD: /bin/mount, /bin/umount
+```

docs/through_purchase_department.md

@@ -4,7 +4,7 @@ If you can't purchase using your own account, usually because you need to go thr
 
 Typically, you will provide two contacts:
 
-* The "billing contact" (in general, the purchaser email). This account will have access to invoices. This is the account which makes the purchase and then binds the XO plan to the second contact account, the technical contact.
+* The "billing contact" (in general, the purchaser email). This account will have access to invoices. This is the account doing the purchase. Once purchased, the license needs to be bound to the second contact account, the technical contact.
 * The "technical contact", the email of the system administrator using the solution and making support requests.
 
 ## As "billing contact"
@@ -17,10 +17,10 @@ Typically, you will provide two contacts:
 
 Now, you just have to pick the edition of Xen Orchestra you want to purchase for your IT team.
 
-2. You will then see the payment screen. If your are not purchasing the edition for yourself, you have to pick the **buy for another account** option. 
+2. On the first payment screen, after you choose the plan and the subscription method. You can select the option "Buy for another account"
 
 
-![](./assets/purchase-for-another.png)
+![](./assets/member_purchase_2.png)
 
 3. Once the payment is completed, you will have to bind the plan with the end-user account (technical contact). If the end-user doesn't have an account yet, the system will create one and send an e-mail to your end user. 
 
@@ -29,4 +29,4 @@ Now, you just have to pick the edition of Xen Orchestra you want to purchase for
 
 That's it, you have now completed the purchase. 
 
-**Once you have bound the plan to your end user account, you cannot change it. Double check the spelling of the e-mail before binding the account.**
+**⚠ Once you have bound the plan to your end user account, you cannot change it. Double check the spelling of the e-mail before binding the account.**

package.json

@@ -16,9 +16,9 @@
     "eslint-plugin-react": "^7.6.1",
     "eslint-plugin-standard": "^4.0.0",
     "exec-promise": "^0.7.0",
-    "flow-bin": "^0.87.0",
-    "globby": "^8.0.0",
-    "husky": "^1.0.0-rc.15",
+    "flow-bin": "^0.90.0",
+    "globby": "^9.0.0",
+    "husky": "^1.2.1",
     "jest": "^23.0.1",
     "lodash": "^4.17.4",
     "prettier": "^1.10.2",
@@ -34,7 +34,6 @@
     }
   },
   "jest": {
-    "timers": "fake",
     "collectCoverage": true,
     "projects": [
       "<rootDir>"

packages/vhd-cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vhd-cli",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "license": "ISC",
   "description": "",
   "keywords": [],
@@ -26,11 +26,12 @@
     "node": ">=6"
   },
   "dependencies": {
-    "@xen-orchestra/fs": "^0.5.0",
+    "@xen-orchestra/fs": "^0.6.1",
     "cli-progress": "^2.0.0",
     "exec-promise": "^0.7.0",
+    "getopts": "^2.2.3",
     "struct-fu": "^1.2.0",
-    "vhd-lib": "^0.4.0"
+    "vhd-lib": "^0.5.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/vhd-cli/src/commands/check.js

@@ -1,12 +1,24 @@
-import Vhd from 'vhd-lib'
+import Vhd, { checkVhdChain } from 'vhd-lib'
+import getopts from 'getopts'
 import { getHandler } from '@xen-orchestra/fs'
 import { resolve } from 'path'
 
-export default async args => {
+const checkVhd = (handler, path) => new Vhd(handler, path).readHeaderAndFooter()
+
+export default async rawArgs => {
+  const { chain, _: args } = getopts(rawArgs, {
+    boolean: ['chain'],
+    default: {
+      chain: false,
+    },
+  })
+
+  const check = chain ? checkVhdChain : checkVhd
+
   const handler = getHandler({ url: 'file:///' })
   for (const vhd of args) {
     try {
-      await new Vhd(handler, resolve(vhd)).readHeaderAndFooter()
+      await check(handler, resolve(vhd))
       console.log('ok:', vhd)
     } catch (error) {
       console.error('nok:', vhd, error)

packages/vhd-cli/src/index.js

@@ -2,6 +2,8 @@
 
 import execPromise from 'exec-promise'
 
+import pkg from '../package.json'
+
 import commands from './commands'
 
 function runCommand(commands, [command, ...args]) {
@@ -16,9 +18,11 @@ function runCommand(commands, [command, ...args]) {
       return `Usage:
 
 ${Object.keys(commands)
-        .filter(command => command !== 'help')
-        .map(command => `    ${this.command} ${command}`)
-        .join('\n\n')}`
+  .filter(command => command !== 'help')
+  .map(command => `    ${this.command} ${command}`)
+  .join('\n\n')}
+
+vhd-cli ${pkg.version}`
     }
 
     throw `invalid command ${command}` // eslint-disable-line no-throw-literal

packages/vhd-lib/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vhd-lib",
-  "version": "0.4.0",
+  "version": "0.5.1",
   "license": "AGPL-3.0",
   "description": "Primitives for VHD file handling",
   "keywords": [],
@@ -34,7 +34,7 @@
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
     "@babel/preset-flow": "^7.0.0",
-    "@xen-orchestra/fs": "^0.5.0",
+    "@xen-orchestra/fs": "^0.6.1",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^5.1.3",
     "execa": "^1.0.0",

packages/vhd-lib/src/_resolveRelativeFromFile.js

@@ -0,0 +1,6 @@
+import { dirname, resolve } from 'path'
+
+const resolveRelativeFromFile = (file, path) =>
+  resolve('/', dirname(file), path).slice(1)
+
+export { resolveRelativeFromFile as default }

packages/vhd-lib/src/checkChain.js

@@ -0,0 +1,16 @@
+import Vhd from './vhd'
+import resolveRelativeFromFile from './_resolveRelativeFromFile'
+import { DISK_TYPE_DYNAMIC } from './_constants'
+
+export default async function checkChain(handler, path) {
+  while (true) {
+    const vhd = new Vhd(handler, path)
+    await vhd.readHeaderAndFooter()
+
+    if (vhd.footer.diskType === DISK_TYPE_DYNAMIC) {
+      break
+    }
+
+    path = resolveRelativeFromFile(path, vhd.header.parentUnicodeName)
+  }
+}

packages/vhd-lib/src/createSyntheticStream.js

@@ -1,5 +1,6 @@
 import asyncIteratorToStream from 'async-iterator-to-stream'
-import { dirname, resolve } from 'path'
+
+import resolveRelativeFromFile from './_resolveRelativeFromFile'
 
 import Vhd from './vhd'
 import {
@@ -12,9 +13,6 @@ import {
 import { fuFooter, fuHeader, checksumStruct } from './_structs'
 import { test as mapTestBit } from './_bitmap'
 
-const resolveRelativeFromFile = (file, path) =>
-  resolve('/', dirname(file), path).slice(1)
-
 export default async function createSyntheticStream(handler, path) {
   const fds = []
   const cleanup = () => {

packages/vhd-lib/src/index.js

@@ -3,6 +3,7 @@ import 'core-js/features/symbol/async-iterator'
 
 export { default } from './vhd'
 export { default as chainVhd } from './chain'
+export { default as checkVhdChain } from './checkChain'
 export { default as createContentStream } from './createContentStream'
 export { default as createReadableRawStream } from './createReadableRawStream'
 export {

packages/vhd-lib/src/vhd.js

@@ -216,7 +216,9 @@ export default class Vhd {
 
   // return the first sector (bitmap) of a block
   _getBatEntry(block) {
-    return this.blockTable.readUInt32BE(block * 4)
+    const i = block * 4
+    const { blockTable } = this
+    return i < blockTable.length ? blockTable.readUInt32BE(i) : BLOCK_UNUSED
   }
 
   _readBlock(blockId, onlyBitmap = false) {

packages/xapi-explore-sr/package.json

@@ -40,7 +40,7 @@
     "human-format": "^0.10.0",
     "lodash": "^4.17.4",
     "pw": "^0.0.4",
-    "xen-api": "^0.23.0"
+    "xen-api": "^0.24.2"
   },
   "devDependencies": {
     "@babel/cli": "^7.1.5",

packages/xen-api/examples/export-vdi

@@ -1,30 +1,42 @@
 #!/usr/bin/env node
 
-process.env.DEBUG = '*'
+process.env.DEBUG = 'xen-api'
 
+const createProgress = require('progress-stream')
+const createTop = require('process-top')
 const defer = require('golike-defer').default
-const pump = require('pump')
-const { CancelToken, fromCallback } = require('promise-toolbox')
+const getopts = require('getopts')
+const { CancelToken } = require('promise-toolbox')
 
 const { createClient } = require('../')
 
-const { createOutputStream, resolveRef } = require('./utils')
+const {
+  createOutputStream,
+  formatProgress,
+  pipeline,
+  resolveRecord,
+  throttle,
+} = require('./utils')
 
-defer(async ($defer, args) => {
-  let raw = false
-  if (args[0] === '--raw') {
-    raw = true
-    args.shift()
-  }
+defer(async ($defer, rawArgs) => {
+  const { raw, throttle: bps, _: args } = getopts(rawArgs, {
+    boolean: 'raw',
+    alias: {
+      raw: 'r',
+      throttle: 't',
+    },
+  })
 
   if (args.length < 2) {
-    return console.log('Usage: export-vdi [--raw] <XS URL> <VDI identifier> [<VHD file>]')
+    return console.log(
+      'Usage: export-vdi [--raw] <XS URL> <VDI identifier> [<VHD file>]'
+    )
   }
 
   const xapi = createClient({
     allowUnauthorized: true,
     url: args[0],
-    watchEvents: false
+    watchEvents: false,
   })
 
   await xapi.connect()
@@ -33,21 +45,32 @@ defer(async ($defer, args) => {
   const { cancel, token } = CancelToken.source()
   process.on('SIGINT', cancel)
 
+  const vdi = await resolveRecord(xapi, 'VDI', args[1])
+
   // https://xapi-project.github.io/xen-api/snapshots.html#downloading-a-disk-or-snapshot
   const exportStream = await xapi.getResource(token, '/export_raw_vdi/', {
     query: {
       format: raw ? 'raw' : 'vhd',
-      vdi: await resolveRef(xapi, 'VDI', args[1])
-    }
+      vdi: vdi.$ref,
+    },
   })
 
   console.warn('Export task:', exportStream.headers['task-id'])
 
-  await fromCallback(cb => pump(
+  const top = createTop()
+  const progressStream = createProgress()
+
+  $defer(
+    clearInterval,
+    setInterval(() => {
+      console.warn('\r %s | %s', top.toString(), formatProgress(progressStream.progress()))
+    }, 1e3)
+  )
+
+  await pipeline(
     exportStream,
-    createOutputStream(args[2]),
-    cb
-  ))
-})(process.argv.slice(2)).catch(
-  console.error.bind(console, 'error')
-)
+    progressStream,
+    throttle(bps),
+    createOutputStream(args[2])
+  )
+})(process.argv.slice(2)).catch(console.error.bind(console, 'error'))

packages/xen-api/examples/export-vm

@@ -2,15 +2,25 @@
 
 process.env.DEBUG = '*'
 
+const createProgress = require('progress-stream')
 const defer = require('golike-defer').default
-const pump = require('pump')
-const { CancelToken, fromCallback } = require('promise-toolbox')
+const getopts = require('getopts')
+const { CancelToken } = require('promise-toolbox')
 
 const { createClient } = require('../')
 
-const { createOutputStream, resolveRef } = require('./utils')
+const {
+  createOutputStream,
+  formatProgress,
+  pipeline,
+  resolveRecord,
+} = require('./utils')
+
+defer(async ($defer, rawArgs) => {
+  const { gzip, zstd, _: args } = getopts(rawArgs, {
+    boolean: ['gzip', 'zstd'],
+  })
 
-defer(async ($defer, args) => {
   if (args.length < 2) {
     return console.log('Usage: export-vm <XS URL> <VM identifier> [<XVA file>]')
   }
@@ -18,7 +28,7 @@ defer(async ($defer, args) => {
   const xapi = createClient({
     allowUnauthorized: true,
     url: args[0],
-    watchEvents: false
+    watchEvents: false,
   })
 
   await xapi.connect()
@@ -30,18 +40,16 @@ defer(async ($defer, args) => {
   // https://xapi-project.github.io/xen-api/importexport.html
   const exportStream = await xapi.getResource(token, '/export/', {
     query: {
-      ref: await resolveRef(xapi, 'VM', args[1]),
-      use_compression: 'true'
-    }
+      ref: (await resolveRecord(xapi, 'VM', args[1])).$ref,
+      use_compression: zstd ? 'zstd' : gzip ? 'true' : 'false',
+    },
   })
 
   console.warn('Export task:', exportStream.headers['task-id'])
 
-  await fromCallback(cb => pump(
+  await pipeline(
     exportStream,
-    createOutputStream(args[2]),
-    cb
-  ))
-})(process.argv.slice(2)).catch(
-  console.error.bind(console, 'error')
-)
+    createProgress({ time: 1e3 }, p => console.warn(formatProgress(p))),
+    createOutputStream(args[2])
+  )
+})(process.argv.slice(2)).catch(console.error.bind(console, 'error'))

packages/xen-api/examples/package.json

@@ -1,6 +1,12 @@
 {
   "dependencies": {
+    "getopts": "^2.2.3",
     "golike-defer": "^0.4.1",
-    "pump": "^3.0.0"
+    "human-format": "^0.10.1",
+    "process-top": "^1.0.0",
+    "progress-stream": "^2.0.0",
+    "promise-toolbox": "^0.11.0",
+    "readable-stream": "^3.1.1",
+    "throttle": "^1.0.3"
   }
 }

packages/xen-api/examples/utils.js

@@ -1,5 +1,8 @@
 const { createReadStream, createWriteStream, statSync } = require('fs')
-const { PassThrough } = require('stream')
+const { fromCallback } = require('promise-toolbox')
+const { PassThrough, pipeline } = require('readable-stream')
+const humanFormat = require('human-format')
+const Throttle = require('throttle')
 
 const { isOpaqueRef } = require('../')
 
@@ -26,7 +29,32 @@ exports.createOutputStream = path => {
   return stream
 }
 
-exports.resolveRef = (xapi, type, refOrUuidOrNameLabel) =>
+const formatSizeOpts = { scale: 'binary', unit: 'B' }
+const formatSize = bytes => humanFormat(bytes, formatSizeOpts)
+
+exports.formatProgress = p =>
+  [
+    formatSize(p.transferred),
+    ' / ',
+    formatSize(p.length),
+    ' | ',
+    p.runtime,
+    's / ',
+    p.eta,
+    's | ',
+    formatSize(p.speed),
+    '/s',
+  ].join('')
+
+exports.pipeline = (...streams) => {
+  return fromCallback(cb => {
+    streams = streams.filter(_ => _ != null)
+    streams.push(cb)
+    pipeline.apply(undefined, streams)
+  })
+}
+
+const resolveRef = (xapi, type, refOrUuidOrNameLabel) =>
   isOpaqueRef(refOrUuidOrNameLabel)
     ? refOrUuidOrNameLabel
     : xapi.call(`${type}.get_by_uuid`, refOrUuidOrNameLabel).catch(() =>
@@ -41,3 +69,10 @@ exports.resolveRef = (xapi, type, refOrUuidOrNameLabel) =>
             )
           })
       )
+
+exports.resolveRecord = async (xapi, type, refOrUuidOrNameLabel) =>
+  xapi.getRecord(type, await resolveRef(xapi, type, refOrUuidOrNameLabel))
+
+exports.resolveRef = resolveRef
+
+exports.throttle = opts => (opts != null ? new Throttle(opts) : undefined)

packages/xen-api/examples/yarn.lock

@@ -2,29 +2,178 @@
 # yarn lockfile v1
 
 
-end-of-stream@^1.1.0:
-  version "1.4.1"
-  resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.1.tgz#ed29634d19baba463b6ce6b80a37213eab71ec43"
+core-util-is@~1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
+  integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=
+
+debug@2:
+  version "2.6.9"
+  resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
+  integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==
   dependencies:
-    once "^1.4.0"
+    ms "2.0.0"
+
+event-loop-delay@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/event-loop-delay/-/event-loop-delay-1.0.0.tgz#5af6282549494fd0d868c499cbdd33e027978b8c"
+  integrity sha512-8YtyeIWHXrvTqlAhv+fmtaGGARmgStbvocERYzrZ3pwhnQULe5PuvMUTjIWw/emxssoaftfHZsJtkeY8xjiXCg==
+  dependencies:
+    napi-macros "^1.8.2"
+    node-gyp-build "^3.7.0"
+
+getopts@^2.2.3:
+  version "2.2.3"
+  resolved "https://registry.yarnpkg.com/getopts/-/getopts-2.2.3.tgz#11d229775e2ec2067ed8be6fcc39d9b4bf39cf7d"
+  integrity sha512-viEcb8TpgeG05+Nqo5EzZ8QR0hxdyrYDp6ZSTZqe2M/h53Bk036NmqG38Vhf5RGirC/Of9Xql+v66B2gp256SQ==
 
 golike-defer@^0.4.1:
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/golike-defer/-/golike-defer-0.4.1.tgz#7a1cd435d61e461305805d980b133a0f3db4e1cc"
 
-once@^1.3.1, once@^1.4.0:
-  version "1.4.0"
-  resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
-  dependencies:
-    wrappy "1"
+human-format@^0.10.1:
+  version "0.10.1"
+  resolved "https://registry.yarnpkg.com/human-format/-/human-format-0.10.1.tgz#107793f355912e256148d5b5dcf66a0230187ee9"
+  integrity sha512-UzCHToSw3HI9MxH9tYzMr1JbHJbgzr6o0hZCun7sruv59S1leps21bmgpBkkwEvQon5n/2OWKH1iU7BEko02cg==
 
-pump@^3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64"
-  dependencies:
-    end-of-stream "^1.1.0"
-    once "^1.3.1"
+inherits@^2.0.3, inherits@~2.0.3:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
+  integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=
 
-wrappy@1:
+isarray@~1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
+  integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=
+
+make-error@^1.3.2:
+  version "1.3.5"
+  resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.5.tgz#efe4e81f6db28cadd605c70f29c831b58ef776c8"
+  integrity sha512-c3sIjNUow0+8swNwVpqoH4YCShKNFkMaw6oH1mNS2haDZQqkeZFlHS3dhoeEbKKmJB4vXpJucU6oH75aDYeE9g==
+
+ms@2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
+  integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=
+
+napi-macros@^1.8.2:
+  version "1.8.2"
+  resolved "https://registry.yarnpkg.com/napi-macros/-/napi-macros-1.8.2.tgz#299265c1d8aa401351ad0675107d751228c03eda"
+  integrity sha512-Tr0DNY4RzTaBG2W2m3l7ZtFuJChTH6VZhXVhkGGjF/4cZTt+i8GcM9ozD+30Lmr4mDoZ5Xx34t2o4GJqYWDGcg==
+
+node-gyp-build@^3.7.0:
+  version "3.7.0"
+  resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-3.7.0.tgz#daa77a4f547b9aed3e2aac779eaf151afd60ec8d"
+  integrity sha512-L/Eg02Epx6Si2NXmedx+Okg+4UHqmaf3TNcxd50SF9NQGcJaON3AtU++kax69XV7YWz4tUspqZSAsVofhFKG2w==
+
+prettier-bytes@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/prettier-bytes/-/prettier-bytes-1.0.4.tgz#994b02aa46f699c50b6257b5faaa7fe2557e62d6"
+  integrity sha1-mUsCqkb2mcULYle1+qp/4lV+YtY=
+
+process-nextick-args@~2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa"
+  integrity sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==
+
+process-top@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/process-top/-/process-top-1.0.0.tgz#52892bedb581c5abf0df2d0aa5c429e34275cc7e"
+  integrity sha512-er8iSmBMslOt5cgIHg9m6zilTPsuUqpEb1yfQ4bDmO80zr/e/5hNn+Tay3CJM/FOBnJo8Bt3fFiDDH6GvIgeAg==
+  dependencies:
+    event-loop-delay "^1.0.0"
+    prettier-bytes "^1.0.4"
+
+progress-stream@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/progress-stream/-/progress-stream-2.0.0.tgz#fac63a0b3d11deacbb0969abcc93b214bce19ed5"
+  integrity sha1-+sY6Cz0R3qy7CWmrzJOyFLzhntU=
+  dependencies:
+    speedometer "~1.0.0"
+    through2 "~2.0.3"
+
+promise-toolbox@^0.11.0:
+  version "0.11.0"
+  resolved "https://registry.yarnpkg.com/promise-toolbox/-/promise-toolbox-0.11.0.tgz#9ed928355355395072dace3f879879504e07d1bc"
+  integrity sha512-bjHk0kq+Ke3J3zbkbbJH6kXCyQZbFHwOTrE/Et7vS0uS0tluoV+PLqU/kEyxl8aARM7v04y2wFoDo/wWAEPvjA==
+  dependencies:
+    make-error "^1.3.2"
+
+"readable-stream@>= 0.3.0", readable-stream@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.1.1.tgz#ed6bbc6c5ba58b090039ff18ce670515795aeb06"
+  integrity sha512-DkN66hPyqDhnIQ6Jcsvx9bFjhw214O4poMBcIMgPVpQvNy9a0e0Uhg5SqySyDKAmUlwt8LonTBz1ezOnM8pUdA==
+  dependencies:
+    inherits "^2.0.3"
+    string_decoder "^1.1.1"
+    util-deprecate "^1.0.1"
+
+readable-stream@~2.3.6:
+  version "2.3.6"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf"
+  integrity sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==
+  dependencies:
+    core-util-is "~1.0.0"
+    inherits "~2.0.3"
+    isarray "~1.0.0"
+    process-nextick-args "~2.0.0"
+    safe-buffer "~5.1.1"
+    string_decoder "~1.1.1"
+    util-deprecate "~1.0.1"
+
+safe-buffer@~5.1.0, safe-buffer@~5.1.1:
+  version "5.1.2"
+  resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
+  integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
+
+speedometer@~1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-1.0.0.tgz#cd671cb06752c22bca3370e2f334440be4fc62e2"
+  integrity sha1-zWccsGdSwivKM3Di8zREC+T8YuI=
+
+"stream-parser@>= 0.0.2":
+  version "0.3.1"
+  resolved "https://registry.yarnpkg.com/stream-parser/-/stream-parser-0.3.1.tgz#1618548694420021a1182ff0af1911c129761773"
+  integrity sha1-FhhUhpRCACGhGC/wrxkRwSl2F3M=
+  dependencies:
+    debug "2"
+
+string_decoder@^1.1.1:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.2.0.tgz#fe86e738b19544afe70469243b2a1ee9240eae8d"
+  integrity sha512-6YqyX6ZWEYguAxgZzHGL7SsCeGx3V2TtOTqZz1xSTSWnqsbWwbptafNyvf/ACquZUXV3DANr5BDIwNYe1mN42w==
+  dependencies:
+    safe-buffer "~5.1.0"
+
+string_decoder@~1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"
+  integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==
+  dependencies:
+    safe-buffer "~5.1.0"
+
+throttle@^1.0.3:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/throttle/-/throttle-1.0.3.tgz#8a32e4a15f1763d997948317c5ebe3ad8a41e4b7"
+  integrity sha1-ijLkoV8XY9mXlIMXxevjrYpB5Lc=
+  dependencies:
+    readable-stream ">= 0.3.0"
+    stream-parser ">= 0.0.2"
+
+through2@~2.0.3:
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd"
+  integrity sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==
+  dependencies:
+    readable-stream "~2.3.6"
+    xtend "~4.0.1"
+
+util-deprecate@^1.0.1, util-deprecate@~1.0.1:
   version "1.0.2"
-  resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
+  resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
+  integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=
+
+xtend@~4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"
+  integrity sha1-pcbVMr5lbiPbgg77lDofBJmNY68=

packages/xen-api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xen-api",
-  "version": "0.23.0",
+  "version": "0.24.2",
   "license": "ISC",
   "description": "Connector to the Xen API",
   "keywords": [
@@ -36,7 +36,7 @@
     "debug": "^4.0.1",
     "event-to-promise": "^0.8.0",
     "exec-promise": "^0.7.0",
-    "http-request-plus": "^0.6.0",
+    "http-request-plus": "^0.7.1",
     "iterable-backoff": "^0.0.0",
     "jest-diff": "^23.5.0",
     "json-rpc-protocol": "^0.13.1",

packages/xen-api/src/index.js

@@ -109,6 +109,9 @@ export const wrapError = error => {
   } else {
     code = error.message
     params = error.data
+    if (!isArray(params)) {
+      params = []
+    }
   }
   return new XapiError(code, params)
 }
@@ -253,6 +256,9 @@ const CONNECTED = 'connected'
 const CONNECTING = 'connecting'
 const DISCONNECTED = 'disconnected'
 
+// timeout of XenAPI HTTP connections
+const HTTP_TIMEOUT = 24 * 3600 * 1e3
+
 // -------------------------------------------------------------------
 
 export class Xapi extends EventEmitter {
@@ -301,7 +307,7 @@ export class Xapi extends EventEmitter {
     this._taskWatchers = Object.create(null)
 
     if (this.status === CONNECTED) {
-      ignoreErrors.call(this._watchEvents())
+      this._watchEvents()
     }
 
     this.on('connected', this._watchEvents)
@@ -538,6 +544,8 @@ export class Xapi extends EventEmitter {
 
   async getRecord(type, ref) {
     return this._wrapRecord(
+      type,
+      ref,
       await this._sessionCall(`${type}.get_record`, [ref])
     )
   }
@@ -571,17 +579,20 @@ export class Xapi extends EventEmitter {
           }
         }
 
-        let promise = httpRequest(
-          $cancelToken,
-          this._url,
-          host && {
-            hostname: this.getObject(host).address,
-          },
-          {
-            pathname,
-            query,
-            rejectUnauthorized: !this._allowUnauthorized,
-          }
+        let promise = pTimeout.call(
+          httpRequest(
+            $cancelToken,
+            this._url,
+            host && {
+              hostname: this.getObject(host).address,
+            },
+            {
+              pathname,
+              query,
+              rejectUnauthorized: !this._allowUnauthorized,
+            }
+          ),
+          HTTP_TIMEOUT
         )
 
         if (taskResult !== undefined) {
@@ -629,21 +640,23 @@ export class Xapi extends EventEmitter {
         }
 
         const doRequest = (...opts) =>
-          httpRequest.put(
-            $cancelToken,
-            this._url,
-            host && {
-              hostname: this.getObject(host).address,
-            },
-            {
-              body,
-              headers,
-              query,
-              pathname,
-              maxRedirects: 0,
-              rejectUnauthorized: !this._allowUnauthorized,
-            },
-            ...opts
+          pTimeout.call(
+            httpRequest.put(
+              $cancelToken,
+              this._url,
+              host && {
+                hostname: this.getObject(host).address,
+              },
+              {
+                body,
+                headers,
+                query,
+                pathname,
+                rejectUnauthorized: !this._allowUnauthorized,
+              },
+              ...opts
+            ),
+            HTTP_TIMEOUT
           )
 
         // if a stream, sends a dummy request to probe for a
@@ -654,15 +667,17 @@ export class Xapi extends EventEmitter {
 
               // omit task_id because this request will fail on purpose
               query: 'task_id' in query ? omit(query, 'task_id') : query,
+
+              maxRedirects: 0,
             }).then(
               response => {
-                response.req.abort()
+                response.cancel()
                 return doRequest()
               },
               error => {
                 let response
                 if (error != null && (response = error.response) != null) {
-                  response.req.abort()
+                  response.cancel()
 
                   const {
                     headers: { location },
@@ -690,12 +705,12 @@ export class Xapi extends EventEmitter {
           }
 
           if (req.finished) {
-            req.abort()
+            response.cancel()
             return taskResult
           }
 
           return fromEvents(req, ['close', 'finish']).then(() => {
-            req.abort()
+            response.cancel()
             return taskResult
           })
         })
@@ -954,16 +969,18 @@ export class Xapi extends EventEmitter {
       throw error
     }
 
-    return pCatch.call(
-      loop(),
-      isMethodUnknown,
+    ignoreErrors.call(
+      pCatch.call(
+        loop(),
+        isMethodUnknown,
 
-      // If the server failed, it is probably due to an excessively
-      // large response.
-      // Falling back to legacy events watch should be enough.
-      error => error && error.res && error.res.statusCode === 500,
+        // If the server failed, it is probably due to an excessively
+        // large response.
+        // Falling back to legacy events watch should be enough.
+        error => error && error.res && error.res.statusCode === 500,
 
-      () => this._watchEventsLegacy()
+        () => this._watchEventsLegacy()
+      )
     )
   }
 
@@ -1080,7 +1097,10 @@ export class Xapi extends EventEmitter {
           props[`update_${field}`] = function(entries) {
             return xapi.setFieldEntries(this, field, entries)
           }
-        } else if (isOpaqueRef(value)) {
+        } else if (value === '' || isOpaqueRef(value)) {
+          // 2019-02-07 - JFT: even if `value` should not be an empty string for
+          // a ref property, an user had the case on XenServer 7.0 on the CD VBD
+          // of a VM created by XenCenter
           getters[$field] = function() {
             return objectsByRef[this[field]]
           }

packages/xo-cli/package.json

@@ -33,7 +33,7 @@
     "chalk": "^2.2.0",
     "exec-promise": "^0.7.0",
     "fs-promise": "^2.0.3",
-    "http-request-plus": "^0.6.0",
+    "http-request-plus": "^0.7.1",
     "human-format": "^0.10.0",
     "l33teral": "^3.0.3",
     "lodash": "^4.17.4",

packages/xo-server-auth-github/README.md

@@ -9,11 +9,7 @@ same identifier.
 
 ## Install
 
-Installation of the [npm package](https://npmjs.org/package/xo-server-auth-github):
-
-```
-> npm install --global xo-server-auth-github
-```
+For installing XO and the plugins from the sources, please take a look at [the documentation](https://xen-orchestra.com/docs/from_the_sources.html).
 
 ## Usage
 

packages/xo-server-auth-google/README.md

@@ -9,11 +9,7 @@ same identifier.
 
 ## Install
 
-Installation of the [npm package](https://npmjs.org/package/xo-server-auth-google):
-
-```
-> npm install --global xo-server-auth-google
-```
+For installing XO and the plugins from the sources, please take a look at [the documentation](https://xen-orchestra.com/docs/from_the_sources.html).
 
 ## Usage
 

packages/xo-server-auth-ldap/README.md

@@ -9,11 +9,7 @@ same identifier.
 
 ## Install
 
-Installation of the [npm package](https://npmjs.org/package/xo-server-auth-ldap):
-
-```
-> npm install --global xo-server-auth-ldap
-```
+For installing XO and the plugins from the sources, please take a look at [the documentation](https://xen-orchestra.com/docs/from_the_sources.html).
 
 ## Usage
 

packages/xo-server-auth-ldap/src/prompt-schema.js

@@ -1,5 +1,5 @@
 import { forEach, isFinite, isInteger } from 'lodash'
-import { pforOwn } from 'promise-toolbox'
+import { pForOwn } from 'promise-toolbox'
 import { prompt } from 'inquirer'
 
 // ===================================================================
@@ -160,7 +160,7 @@ const promptByType = {
       }
     }
 
-    await pforOwn.call(schema.properties || {}, promptProperty)
+    await pForOwn.call(schema.properties || {}, promptProperty)
 
     return value
   },

packages/xo-server-auth-saml/README.md

@@ -9,11 +9,7 @@ same identifier.
 
 ## Install
 
-Installation of the [npm package](https://npmjs.org/package/xo-server-auth-saml):
-
-```
-> npm install --global xo-server-auth-saml
-```
+For installing XO and the plugins from the sources, please take a look at [the documentation](https://xen-orchestra.com/docs/from_the_sources.html).
 
 ## Usage
 

packages/xo-server-auth-saml/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-server-auth-saml",
-  "version": "0.5.2",
+  "version": "0.5.3",
   "license": "AGPL-3.0",
   "description": "SAML authentication plugin for XO-Server",
   "keywords": [
@@ -32,7 +32,7 @@
     "node": ">=6"
   },
   "dependencies": {
-    "passport-saml": "^0.35.0"
+    "passport-saml": "^1.0.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/xo-server-auth-saml/src/index.js

@@ -42,7 +42,12 @@ class AuthSamlXoPlugin {
 
   configure({ usernameField, ...conf }) {
     this._usernameField = usernameField
-    this._conf = conf
+    this._conf = {
+      ...conf,
+
+      // must match the callback URL
+      path: '/signin/saml/callback',
+    }
   }
 
   load() {

packages/xo-server-backup-reports/README.md

@@ -6,11 +6,7 @@ XO-Server plugin which sends email reports and Xmpp messages when backup jobs ar
 
 ## Install
 
-Installation of the [npm package](https://npmjs.org/package/xo-server-backup-reports):
-
-```
-> npm install --global xo-server-backup-reports
-```
+For installing XO and the plugins from the sources, please take a look at [the documentation](https://xen-orchestra.com/docs/from_the_sources.html).
 
 ## Usage
 

packages/xo-server-cloud/README.md

@@ -2,11 +2,7 @@
 
 ## Install
 
-Installation of the [npm package](https://npmjs.org/package/xo-server-cloud):
-
-```
-> npm install --global xo-server-cloud
-```
+For installing XO and the plugins from the sources, please take a look at [the documentation](https://xen-orchestra.com/docs/from_the_sources.html).
 
 ## Usage
 

packages/xo-server-cloud/package.json

@@ -31,7 +31,7 @@
     "node": ">=6"
   },
   "dependencies": {
-    "http-request-plus": "^0.6.0",
+    "http-request-plus": "^0.7.1",
     "jsonrpc-websocket-client": "^0.4.1"
   },
   "devDependencies": {

packages/xo-server-load-balancer/README.md

@@ -4,11 +4,7 @@ XO-Server plugin that allows load balancing.
 
 ## Install
 
-Go inside your `xo-server` folder and install it:
-
-```
-> npm install --global xo-server-load-balancer
-```
+For installing XO and the plugins from the sources, please take a look at [the documentation](https://xen-orchestra.com/docs/from_the_sources.html).
 
 ## Usage
 

packages/xo-server-perf-alert/README.md

@@ -4,11 +4,7 @@
 
 ## Install
 
-Installation of the [npm package](https://npmjs.org/package/xo-server-perf-alert):
-
-```
-> npm install --global xo-server-perf-alert
-```
+For installing XO and the plugins from the sources, please take a look at [the documentation](https://xen-orchestra.com/docs/from_the_sources.html).
 
 ## Usage
 

packages/xo-server-transport-email/README.md

@@ -4,11 +4,7 @@
 
 ## Install
 
-Installation of the [npm package](https://npmjs.org/package/xo-server-transport-email):
-
-```
-> npm install --global xo-server-transport-email
-```
+For installing XO and the plugins from the sources, please take a look at [the documentation](https://xen-orchestra.com/docs/from_the_sources.html).
 
 ## Usage
 

packages/xo-server-transport-email/package.json

@@ -31,7 +31,7 @@
     "node": ">=6"
   },
   "dependencies": {
-    "nodemailer": "^4.4.1",
+    "nodemailer": "^5.0.0",
     "nodemailer-markdown": "^1.0.1",
     "promise-toolbox": "^0.11.0"
   },

packages/xo-server-transport-nagios/README.md

@@ -4,11 +4,7 @@
 
 ## Install
 
-Installation of the [npm package](https://npmjs.org/package/xo-server-transport-nagios):
-
-```
-> npm install --global xo-server-transport-nagios
-```
+For installing XO and the plugins from the sources, please take a look at [the documentation](https://xen-orchestra.com/docs/from_the_sources.html).
 
 ## Usage
 

packages/xo-server-transport-nagios/src/index.js

@@ -120,13 +120,9 @@ class XoServerNagios {
 
       const client = new net.Socket()
 
-      client.connect(
-        this._conf.port,
-        this._conf.server,
-        () => {
-          console.log('Successful connection')
-        }
-      )
+      client.connect(this._conf.port, this._conf.server, () => {
+        console.log('Successful connection')
+      })
 
       client.on('data', data => {
         const timestamp = data.readInt32BE(128)

packages/xo-server-transport-slack/README.md

@@ -4,11 +4,7 @@
 
 ## Install
 
-Installation of the [npm package](https://npmjs.org/package/xo-server-transport-slack):
-
-```
-> npm install --global xo-server-transport-slack
-```
+For installing XO and the plugins from the sources, please take a look at [the documentation](https://xen-orchestra.com/docs/from_the_sources.html).
 
 ## Usage
 

packages/xo-server-transport-xmpp/README.md

@@ -4,11 +4,7 @@ XO-Server plugin which sends XMPP messages.
 
 ## Install
 
-Installation of the [npm package](https://npmjs.org/package/xo-server-transport-xmpp):
-
-```
-> npm install --global xo-server-transport-xmpp
-```
+For installing XO and the plugins from the sources, please take a look at [the documentation](https://xen-orchestra.com/docs/from_the_sources.html).
 
 ## Usage
 

packages/xo-server-usage-report/README.md

@@ -4,11 +4,7 @@
 
 ## Install
 
-Installation of the [npm package](https://npmjs.org/package/xo-server-usage-report):
-
-```
-> npm install --save xo-server-usage-report
-```
+For installing XO and the plugins from the sources, please take a look at [the documentation](https://xen-orchestra.com/docs/from_the_sources.html).
 
 ## Usage
 

packages/xo-server/config.json

@@ -1,59 +0,0 @@
-// Vendor config: DO NOT TOUCH!
-//
-// See sample.config.yaml to override.
-{
-  "apiWebSocketOptions": {
-    // https://github.com/websockets/ws#websocket-compression
-    // "perMessageDeflate": {
-    //   "threshold": 524288 // 512kiB
-    // }
-  },
-
-  "http": {
-    "listen": [
-      {
-        "port": 80
-      }
-    ],
-
-    // These options are applied to all listen entries.
-    "listenOptions": {
-      // Ciphers to use.
-      //
-      // These are the default ciphers in Node 4.2.6, we are setting
-      // them explicitly for older Node versions.
-      "ciphers": "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:DHE-RSA-AES256-SHA384:ECDHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA256:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!SRP:!CAMELLIA",
-
-      // Tell Node to respect the cipher order.
-      "honorCipherOrder": true,
-
-      // Specify to use at least TLSv1.1.
-      // See: https://github.com/certsimple/minimum-tls-version
-      "secureOptions": 117440512
-    },
-
-    "mounts": {}
-  },
-
-  "datadir": "/var/lib/xo-server/data",
-
-  // Should users be created on first sign in?
-  //
-  // Necessary for external authentication providers.
-  "createUserOnFirstSignin": true,
-
-  "remoteOptions": {
-    "mountsDir": "/run/xo-server/mounts",
-
-    // timeout in milliseconds (set to 0 to disable)
-    "timeout": 600e3
-  },
-
-  // Whether API logs should contains the full request/response on
-  // errors.
-  //
-  // This is disabled by default for performance (lots of data) and
-  // security concerns (avoiding sensitive data in the logs) but can
-  // be turned for investigation by the administrator.
-  "verboseApiLogsOnErrors": false
-}

packages/xo-server/config.toml

@@ -0,0 +1,51 @@
+# Vendor config: DO NOT TOUCH!
+#
+# See sample.config.toml to override.
+
+datadir = '/var/lib/xo-server/data'
+
+# Should users be created on first sign in?
+#
+# Necessary for external authentication providers.
+createUserOnFirstSignin = true
+
+# Whether API logs should contains the full request/response on
+# errors.
+#
+# This is disabled by default for performance (lots of data) and
+# security concerns (avoiding sensitive data in the logs) but can
+# be turned for investigation by the administrator.
+verboseApiLogsOnErrors = false
+
+# https:#github.com/websockets/ws#websocket-compression
+[apiWebSocketOptions]
+perMessageDeflate = { threshold = 524288 } # 512kiB
+
+[[http.listen]]
+port = 80
+
+# These options are applied to all listen entries.
+[http.listenOptions]
+# Ciphers to use.
+#
+# These are the default ciphers in Node 4.2.6, we are setting
+# them explicitly for older Node versions.
+ciphers = 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:DHE-RSA-AES256-SHA384:ECDHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA256:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!SRP:!CAMELLIA'
+
+# Tell Node to respect the cipher order.
+honorCipherOrder = true
+
+# Specify to use at least TLSv1.1.
+# See: https:#github.com/certsimple/minimum-tls-version
+secureOptions = 117440512
+
+[http.mounts]
+
+[remoteOptions]
+mountsDir = '/run/xo-server/mounts'
+
+# timeout in milliseconds (set to 0 to disable)
+timeout = 600e3
+
+# see https:#github.com/vatesfr/xen-orchestra/issues/3419
+# useSudo = false

packages/xo-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-server",
-  "version": "5.31.1",
+  "version": "5.35.0",
   "license": "AGPL-3.0",
   "description": "Server part of Xen-Orchestra",
   "keywords": [
@@ -20,7 +20,7 @@
     "better-stacks.js",
     "bin/",
     "dist/",
-    "config.json",
+    "config.toml",
     "index.js",
     "signin.pug"
   ],
@@ -31,10 +31,11 @@
     "node": ">=6"
   },
   "dependencies": {
+    "@iarna/toml": "^2.2.1",
     "@xen-orchestra/async-map": "^0.0.0",
     "@xen-orchestra/cron": "^1.0.3",
     "@xen-orchestra/emit-async": "^0.0.0",
-    "@xen-orchestra/fs": "^0.5.0",
+    "@xen-orchestra/fs": "^0.6.1",
     "@xen-orchestra/log": "^0.1.4",
     "@xen-orchestra/mixin": "^0.0.0",
     "ajv": "^6.1.1",
@@ -67,12 +68,12 @@
     "helmet": "^3.9.0",
     "highland": "^2.11.1",
     "http-proxy": "^1.16.2",
-    "http-request-plus": "^0.6.0",
+    "http-request-plus": "^0.7.1",
     "http-server-plus": "^0.10.0",
     "human-format": "^0.10.0",
     "is-redirect": "^1.0.0",
     "iterable-backoff": "^0.0.0",
-    "jest-worker": "^23.0.0",
+    "jest-worker": "^24.0.0",
     "js-yaml": "^3.10.0",
     "json-rpc-peer": "^0.15.3",
     "json5": "^2.0.1",
@@ -91,6 +92,7 @@
     "ms": "^2.1.1",
     "multikey-hash": "^1.0.4",
     "ndjson": "^1.5.0",
+    "otplib": "^10.0.1",
     "parse-pairs": "^0.2.2",
     "partial-stream": "0.0.0",
     "passport": "^0.4.0",
@@ -114,15 +116,15 @@
     "tmp": "^0.0.33",
     "uuid": "^3.0.1",
     "value-matcher": "^0.2.0",
-    "vhd-lib": "^0.4.0",
+    "vhd-lib": "^0.5.1",
     "ws": "^6.0.0",
-    "xen-api": "^0.23.0",
+    "xen-api": "^0.24.2",
     "xml2js": "^0.4.19",
     "xo-acl-resolver": "^0.4.1",
     "xo-collection": "^0.4.1",
     "xo-common": "^0.2.0",
     "xo-remote-parser": "^0.5.0",
-    "xo-vmdk-to-vhd": "^0.1.5",
+    "xo-vmdk-to-vhd": "^0.1.6",
     "yazl": "^2.4.3"
   },
   "devDependencies": {

packages/xo-server/sample.config.toml

@@ -0,0 +1,157 @@
+# Example XO-Server configuration.
+#
+# This file is automatically looking for at the following places:
+# - `$HOME/.config/xo-server/config.toml`
+# - `/etc/xo-server/config.toml`
+#
+# The first entries have priority.
+#
+# Note: paths are relative to the configuration file.
+
+#=====================================================================
+
+# HTTP proxy configuration used by xo-server to fetch resources on the Internet.
+#
+# See: https://github.com/TooTallNate/node-proxy-agent#maps-proxy-protocols-to-httpagent-implementations
+# httpProxy = 'http://jsmith:qwerty@proxy.lan:3128'
+
+#=====================================================================
+
+# It may be necessary to run XO-Server as a privileged user (e.g. `root`) for
+# instance to allow the HTTP server to listen on a
+# [privileged ports](http://www.w3.org/Daemon/User/Installation/PrivilegedPorts.html).
+#
+# To avoid security issues, XO-Server can drop its privileges by changing the
+# user and the group is running with.
+#
+# Note: XO-Server will change them just after reading the configuration.
+
+# User to run XO-Server as.
+#
+# Note: The user can be specified using either its name or its numeric
+# identifier.
+#
+# Default: undefined
+#user = 'nobody'
+
+# Group to run XO-Server as.
+#
+# Note: The group can be specified using either its name or its numeric
+# identifier.
+#
+# Default: undefined
+# group = 'nogroup'
+
+#=====================================================================
+
+# Configuration of the embedded HTTP server.
+[http]
+# If set to true, all HTTP traffic will be redirected to the first HTTPs
+# configuration.
+# redirectToHttps = true
+
+# Settings applied to cookies created by xo-server's embedded HTTP server.
+#
+# See https://www.npmjs.com/package/cookie#options-1
+[http.cookies]
+#sameSite = true
+#secure = true
+
+# Basic HTTP.
+[[http.listen]]
+# Address on which the server is listening on.
+#
+# Sets it to 'localhost' for IP to listen only on the local host.
+#
+# Default: all IPv6 addresses if available, otherwise all IPv4 addresses.
+# hostname = 'localhost'
+
+# Port on which the server is listening on.
+#
+# Default: undefined
+port = 80
+
+# Instead of `host` and `port` a path to a UNIX socket may be specified
+# (overrides `host` and `port`).
+#
+# Default: undefined
+# socket = './http.sock'
+
+# # Basic HTTPS.
+# #
+# # You can find the list of possible options there
+# # https://nodejs.org/docs/latest/api/tls.html#tls.createServer
+# #
+# # The only difference is the presence of the certificate and the key.
+# [[http.listen]]
+# #hostname = '127.0.0.1'
+# port = 443
+#
+# # File containing the certificate (PEM format).
+# #
+# # If a chain of certificates authorities is needed, you may bundle them
+# # directly in the certificate.
+# #
+# # Note: the order of certificates does matter, your certificate should come
+# # first followed by the certificate of the above
+# # certificate authority up to the root.
+# #
+# # Default: undefined
+# cert = './certificate.pem'
+#
+# # File containing the private key (PEM format).
+# #
+# # If the key is encrypted, the passphrase will be asked at
+# # server startup.
+# #
+# # Default: undefined
+# key = './key.pem'
+
+# List of files/directories which will be served.
+[http.mounts]
+#'/' = '/path/to/xo-web/dist/'
+
+# List of proxied URLs (HTTP & WebSockets).
+[http.proxies]
+#'/any/url' = 'http://localhost:54722'
+
+#=====================================================================
+
+# Connection to the Redis server.
+[redis]
+# Unix sockets can be used
+#
+# Default: undefined
+#socket = '/var/run/redis/redis.sock'
+
+# Syntax: redis://[db[:password]@]hostname[:port][/db-number]
+#
+# Default: redis://localhost:6379/0
+#uri = 'redis://redis.company.lan/42'
+
+# List of aliased commands.
+#
+# See http://redis.io/topics/security#disabling-of-specific-commands
+#renameCommands:
+#  del = '3dda29ad-3015-44f9-b13b-fa570de92489'
+#  srem = '3fd758c9-5610-4e9d-a058-dbf4cb6d8bf0'
+
+# Directory containing the database of XO.
+# Currently used for logs.
+#
+# Default: '/var/lib/xo-server/data'
+#datadir = '/var/lib/xo-server/data'
+
+#=====================================================================
+
+# Configuration for remotes
+[remoteOptions]
+# Directory used to mount remotes
+#
+# Default: '/run/xo-server/mounts'
+#mountsDir = '/run/xo-server/mounts'
+
+# Use sudo for mount with non-root user
+#
+# Default: false
+#useSudo = false

packages/xo-server/sample.config.yaml

@@ -1,144 +0,0 @@
-# BE *VERY* CAREFUL WHEN EDITING!
-# YAML FILES ARE SUPER SUPER SENSITIVE TO MISTAKES IN WHITESPACE OR ALIGNMENT!
-# visit http://www.yamllint.com/ to validate this file as needed
-
-#=====================================================================
-
-# Example XO-Server configuration.
-#
-# This file is automatically looking for at the following places:
-# - `$HOME/.config/xo-server/config.yaml`
-# - `/etc/xo-server/config.yaml`
-#
-# The first entries have priority.
-#
-# Note: paths are relative to the configuration file.
-
-#=====================================================================
-
-# It may be necessary to run XO-Server as a privileged user (e.g.
-# `root`) for instance to allow the HTTP server to listen on a
-# [privileged ports](http://www.w3.org/Daemon/User/Installation/PrivilegedPorts.html).
-#
-# To avoid security issues, XO-Server can drop its privileges by
-# changing the user and the group is running with.
-#
-# Note: XO-Server will change them just after reading the
-# configuration.
-
-# User to run XO-Server as.
-#
-# Note: The user can be specified using either its name or its numeric
-# identifier.
-#
-# Default: undefined
-#user: 'nobody'
-
-# Group to run XO-Server as.
-#
-# Note: The group can be specified using either its name or its
-# numeric identifier.
-#
-# Default: undefined
-#group: 'nogroup'
-
-#=====================================================================
-
-# Configuration of the embedded HTTP server.
-http:
-  # Hosts & ports on which to listen.
-  #
-  # By default, the server listens on [::]:80.
-  listen:
-    # Basic HTTP.
-    - # Address on which the server is listening on.
-      #
-      # Sets it to 'localhost' for IP to listen only on the local host.
-      #
-      # Default: all IPv6 addresses if available, otherwise all IPv4
-      # addresses.
-      #hostname: 'localhost'
-
-      # Port on which the server is listening on.
-      #
-      # Default: undefined
-      port: 80
-
-      # Instead of `host` and `port` a path to a UNIX socket may be
-      # specified (overrides `host` and `port`).
-      #
-      # Default: undefined
-      #socket: './http.sock'
-
-    # Basic HTTPS.
-    #
-    # You can find the list of possible options there https://nodejs.org/docs/latest/api/tls.html#tls.createServer
-    # -
-    #   # The only difference is the presence of the certificate and the
-    #   # key.
-    #   #
-    #   #hostname: '127.0.0.1'
-    #   port: 443
-
-    #   # File containing the certificate (PEM format).
-    #
-    #   # If a chain of certificates authorities is needed, you may bundle
-    #   # them directly in the certificate.
-    #   #
-    #   # Note: the order of certificates does matter, your certificate
-    #   # should come first followed by the certificate of the above
-    #   # certificate authority up to the root.
-    #   #
-    #   # Default: undefined
-    #   cert: './certificate.pem'
-
-    #   # File containing the private key (PEM format).
-    #   #
-    #   # If the key is encrypted, the passphrase will be asked at
-    #   # server startup.
-    #   #
-    #   # Default: undefined
-    #   key: './key.pem'
-
-  # If set to true, all HTTP traffic will be redirected to the first
-  # HTTPs configuration.
-  #redirectToHttps: true
-
-  # List of files/directories which will be served.
-  mounts:
-    #'/': '/path/to/xo-web/dist/'
-
-  # List of proxied URLs (HTTP & WebSockets).
-  proxies:
-    # '/any/url': 'http://localhost:54722'
-
-# HTTP proxy configuration used by xo-server to fetch resources on the
-# Internet.
-#
-# See: https://github.com/TooTallNate/node-proxy-agent#maps-proxy-protocols-to-httpagent-implementations
-#httpProxy: 'http://jsmith:qwerty@proxy.lan:3128'
-
-#=====================================================================
-
-# Connection to the Redis server.
-redis:
-  # Unix sockets can be used
-  #
-  # Default: undefined
-  #socket: /var/run/redis/redis.sock
-  # Syntax: redis://[db[:password]@]hostname[:port][/db-number]
-  #
-  # Default: redis://localhost:6379/0
-  #uri: redis://redis.company.lan/42
-  # List of aliased commands.
-  #
-  # See http://redis.io/topics/security#disabling-of-specific-commands
-  #renameCommands:
-  #  del: '3dda29ad-3015-44f9-b13b-fa570de92489'
-  #  srem: '3fd758c9-5610-4e9d-a058-dbf4cb6d8bf0'
-
-# Directory containing the database of XO.
-# Currently used for logs.
-#
-# Default: '/var/lib/xo-server/data'
-#datadir: '/var/lib/xo-server/data'

packages/xo-server/signin.pug

@@ -12,39 +12,60 @@ html
       div.mb-2(style = 'display: flex;')
         img(src = 'assets/logo.png' style = 'margin: auto;')
       h2.text-xs-center.mb-2 Xen Orchestra
-      form(action = 'signin/local' method = 'post')
-        fieldset
-          if error
-            p.text-danger #{error}
-          .input-group.mb-1
-            span.input-group-addon
-              i.xo-icon-user.fa-fw
-            input.form-control(
-              name = 'username'
-              type = 'text'
-              placeholder = 'Username'
-              required
-            )
-          .input-group.mb-1
-            span.input-group-addon
-              i.fa.fa-key.fa-fw
-            input.form-control(
-              name = 'password'
-              type = 'password'
-              placeholder = 'Password'
-              required
-            )
-          .checkbox
-            label
-              input(
-                name = 'remember-me'
-                type = 'checkbox'
+      if error
+        p.text-danger #{error}
+      if otp
+        form(action = 'signin-otp' method = 'post')
+          fieldset
+            .input-group.mb-1
+              span.input-group-addon
+                i.fa.fa-key.fa-fw
+              input.form-control(
+                autocomplete = 'off'
+                autofocus
+                name = 'otp'
+                type = 'text'
+                placeholder = 'Code'
+                required
               )
-              |  
-              | Remember me
-          div
-            button.btn.btn-block.btn-info
-              i.fa.fa-sign-in
-              |  Sign in
-          each label, id in strategies
-            div: a(href = 'signin/' + id) Sign in with #{label}
+            div
+              button.btn.btn-block.btn-info
+                i.fa.fa-sign-in
+                |  Sign in
+      else
+        div.mb-2
+        each label, id in strategies
+          div: a(href = 'signin/' + id).btn.btn-block.btn-primary.mb-1 Sign in with #{label}
+        form(action = 'signin/local' method = 'post')
+          fieldset
+            .input-group.mb-1
+              span.input-group-addon
+                i.xo-icon-user.fa-fw
+              input.form-control(
+                autofocus
+                name = 'username'
+                placeholder = 'Username'
+                required
+                type = 'text'
+              )
+            .input-group.mb-1
+              span.input-group-addon
+                i.fa.fa-key.fa-fw
+              input.form-control(
+                name = 'password'
+                type = 'password'
+                placeholder = 'Password'
+                required
+              )
+            .checkbox
+              label
+                input(
+                  name = 'remember-me'
+                  type = 'checkbox'
+                )
+                |  
+                | Remember me
+            div
+              button.btn.btn-block.btn-info
+                i.fa.fa-sign-in
+                |  Sign in

packages/xo-server/src/_pRetry.js

@@ -0,0 +1,36 @@
+import iteratee from 'lodash/iteratee'
+import pDelay from 'promise-toolbox/delay'
+
+function stopRetry(error) {
+  this.error = error
+  // eslint-disable-next-line no-throw-literal
+  throw this
+}
+
+// do not retry on ReferenceError and TypeError which are programmer errors
+const defaultMatcher = error =>
+  !(error instanceof ReferenceError || error instanceof TypeError)
+
+export default async function pRetry(
+  fn,
+  { delay = 1e3, tries = 10, when } = {}
+) {
+  const container = { error: undefined }
+  const stop = stopRetry.bind(container)
+
+  when = when === undefined ? defaultMatcher : iteratee(when)
+
+  while (true) {
+    try {
+      return await fn(stop)
+    } catch (error) {
+      if (error === container) {
+        throw container.error
+      }
+      if (--tries === 0 || !when(error)) {
+        throw error
+      }
+    }
+    await pDelay(delay)
+  }
+}

packages/xo-server/src/_pRetry.spec.js

@@ -0,0 +1,46 @@
+/* eslint-env jest */
+
+import pRetry from './_pRetry'
+
+describe('pRetry()', () => {
+  it('retries until the function succeeds', async () => {
+    let i = 0
+    expect(
+      await pRetry(
+        () => {
+          if (++i < 3) {
+            throw new Error()
+          }
+          return 'foo'
+        },
+        { delay: 0 }
+      )
+    ).toBe('foo')
+    expect(i).toBe(3)
+  })
+
+  it('returns the last error', async () => {
+    let tries = 5
+    const e = new Error()
+    await expect(
+      pRetry(
+        () => {
+          throw --tries > 0 ? new Error() : e
+        },
+        { delay: 0, tries }
+      )
+    ).rejects.toBe(e)
+  })
+  ;[ReferenceError, TypeError].forEach(ErrorType => {
+    it(`does not retry if a ${ErrorType.name} is thrown`, async () => {
+      let i = 0
+      await expect(
+        pRetry(() => {
+          ++i
+          throw new ErrorType()
+        })
+      ).rejects.toBeInstanceOf(ErrorType)
+      expect(i).toBe(1)
+    })
+  })
+})

packages/xo-server/src/api/backup-ng.js

@@ -1,5 +1,4 @@
 import { basename } from 'path'
-import { isEmpty, pickBy } from 'lodash'
 
 import { safeDateFormat } from '../utils'
 
@@ -11,7 +10,7 @@ export function createJob({ schedules, ...job }) {
 createJob.permission = 'admin'
 createJob.params = {
   compression: {
-    enum: ['', 'native'],
+    enum: ['', 'native', 'zstd'],
     optional: true,
   },
   mode: {
@@ -68,7 +67,7 @@ export function editJob(props) {
 editJob.permission = 'admin'
 editJob.params = {
   compression: {
-    enum: ['', 'native'],
+    enum: ['', 'native', 'zstd'],
     optional: true,
   },
   id: {
@@ -151,13 +150,24 @@ runJob.params = {
 
 // -----------------------------------------------------------------------------
 
-export async function getAllLogs(filter) {
-  const logs = await this.getBackupNgLogs()
-  return isEmpty(filter) ? logs : pickBy(logs, filter)
+export function getAllLogs() {
+  return this.getBackupNgLogs()
 }
 
 getAllLogs.permission = 'admin'
 
+export function getLogs({ after, before, limit, ...filter }) {
+  return this.getBackupNgLogsSorted({ after, before, limit, filter })
+}
+
+getLogs.permission = 'admin'
+
+getLogs.params = {
+  after: { type: ['number', 'string'], optional: true },
+  before: { type: ['number', 'string'], optional: true },
+  limit: { type: 'number', optional: true },
+}
+
 // -----------------------------------------------------------------------------
 
 export function deleteVmBackup({ id }) {

packages/xo-server/src/api/host.js

@@ -2,14 +2,22 @@ import { format } from 'json-rpc-peer'
 
 // ===================================================================
 
-export function set({
+export async function set({
   host,
+  multipathing,
 
   // TODO: use camel case.
   name_label: nameLabel,
   name_description: nameDescription,
 }) {
-  return this.getXapi(host).setHostProperties(host._xapiId, {
+  const xapi = this.getXapi(host)
+  const hostId = host._xapiId
+
+  if (multipathing !== undefined) {
+    await xapi.setHostMultipathing(hostId, multipathing)
+  }
+
+  return xapi.setHostProperties(hostId, {
     nameLabel,
     nameDescription,
   })
@@ -27,6 +35,10 @@ set.params = {
     type: 'string',
     optional: true,
   },
+  multipathing: {
+    type: 'boolean',
+    optional: true,
+  },
 }
 
 set.resolve = {

packages/xo-server/src/api/remote.js

@@ -15,6 +15,13 @@ get.params = {
   id: { type: 'string' },
 }
 
+export async function getAllInfo() {
+  return this.getAllRemotesInfo()
+}
+
+getAllInfo.permission = 'admin'
+getAllInfo.description = 'Gets all info of remote'
+
 export async function test({ id }) {
   return this.testRemote(id)
 }

packages/xo-server/src/api/sr.js

@@ -431,7 +431,7 @@ export async function probeHba({ host }) {
       hba: hbaDevice.hba.trim(),
       path: hbaDevice.path.trim(),
       scsiId: hbaDevice.SCSIid.trim(),
-      size: hbaDevice.size.trim(),
+      size: +hbaDevice.size.trim(),
       vendor: hbaDevice.vendor.trim(),
     })
   })

packages/xo-server/src/api/vm.js

@@ -587,6 +587,8 @@ set.params = {
   // Kernel arguments for PV VM.
   PV_args: { type: 'string', optional: true },
 
+  cpuMask: { type: 'array', optional: true },
+
   cpuWeight: { type: ['integer', 'null'], optional: true },
 
   cpuCap: { type: ['integer', 'null'], optional: true },
@@ -706,7 +708,7 @@ export async function copy({ compress, name: nameLabel, sr, vm }) {
 
 copy.params = {
   compress: {
-    type: 'boolean',
+    type: ['boolean', 'string'],
     optional: true,
   },
   name: {
@@ -747,14 +749,25 @@ export { convertToTemplate as convert }
 // TODO: implement resource sets
 export const snapshot = defer(async function(
   $defer,
-  { vm, name = `${vm.name_label}_${new Date().toISOString()}` }
+  {
+    vm,
+    name = `${vm.name_label}_${new Date().toISOString()}`,
+    saveMemory = false,
+    description,
+  }
 ) {
   await checkPermissionOnSrs.call(this, vm)
 
   const xapi = this.getXapi(vm)
-  const { $id: snapshotId } = await xapi.snapshotVm(vm._xapiRef, name)
+  const { $id: snapshotId } = await (saveMemory
+    ? xapi.checkpointVm(vm._xapiRef, name)
+    : xapi.snapshotVm(vm._xapiRef, name))
   $defer.onFailure(() => xapi.deleteVm(snapshotId))
 
+  if (description !== undefined) {
+    await xapi.editVm(snapshotId, { name_description: description })
+  }
+
   const { user } = this
   if (user.permission !== 'admin') {
     await this.addAcl(user.id, snapshotId, 'admin')
@@ -763,8 +776,10 @@ export const snapshot = defer(async function(
 })
 
 snapshot.params = {
+  description: { type: 'string', optional: true },
   id: { type: 'string' },
   name: { type: 'string', optional: true },
+  saveMemory: { type: 'boolean', optional: true },
 }
 
 snapshot.resolve = {
@@ -1125,7 +1140,7 @@ revert.resolve = {
 
 async function handleExport(req, res, { xapi, id, compress }) {
   const stream = await xapi.exportVm(id, {
-    compress: compress != null ? compress : true,
+    compress,
   })
   res.on('close', () => stream.cancel())
   // Remove the filename as it is already part of the URL.
@@ -1160,7 +1175,7 @@ async function export_({ vm, compress }) {
 
 export_.params = {
   vm: { type: 'string' },
-  compress: { type: 'boolean', optional: true },
+  compress: { type: ['boolean', 'string'], optional: true },
 }
 
 export_.resolve = {

packages/xo-server/src/fatfs-buffer.js

@@ -16,6 +16,7 @@
 //   }
 // })
 
+import assert from 'assert'
 import { boot16 as fat16 } from 'fatfs/structs'
 
 const SECTOR_SIZE = 512
@@ -23,7 +24,10 @@ const SECTOR_SIZE = 512
 const TEN_MIB = 10 * 1024 * 1024
 
 // Creates a 10MB buffer and initializes it as a FAT 16 volume.
-export function init() {
+export function init({ label = 'NO LABEL   ' } = {}) {
+  assert.strictEqual(typeof label, 'string')
+  assert.strictEqual(label.length, 11)
+
   const buf = Buffer.alloc(TEN_MIB)
 
   // https://github.com/natevw/fatfs/blob/master/structs.js
@@ -47,7 +51,7 @@ export function init() {
       Reserved1: 0,
       BootSig: 41,
       VolID: 895111106,
-      VolLab: 'NO NAME    ',
+      VolLab: label,
       FilSysType: 'FAT16   ',
     },
     buf

packages/xo-server/src/index.js

@@ -1,9 +1,11 @@
 import appConf from 'app-conf'
 import assert from 'assert'
+import authenticator from 'otplib/authenticator'
 import bind from 'lodash/bind'
 import blocked from 'blocked'
 import createExpress from 'express'
 import createLogger from '@xen-orchestra/log'
+import crypto from 'crypto'
 import has from 'lodash/has'
 import helmet from 'helmet'
 import includes from 'lodash/includes'
@@ -45,6 +47,11 @@ import { configure } from '@xen-orchestra/log/configure'
 
 // ===================================================================
 
+// https://github.com/yeojz/otplib#using-specific-otp-implementations
+authenticator.options = { crypto }
+
+// ===================================================================
+
 configure([
   {
     filter: process.env.DEBUG,
@@ -79,14 +86,14 @@ async function loadConfiguration() {
 
 // ===================================================================
 
-function createExpressApp() {
+function createExpressApp(config) {
   const app = createExpress()
 
   app.use(helmet())
 
   // Registers the cookie-parser and express-session middlewares,
   // necessary for connect-flash.
-  app.use(cookieParser())
+  app.use(cookieParser(null, config.http.cookies))
   app.use(
     expressSession({
       resave: false,
@@ -140,6 +147,52 @@ async function setUpPassport(express, xo) {
     res.redirect('/')
   })
 
+  express.get('/signin-otp', (req, res, next) => {
+    if (req.session.user === undefined) {
+      return res.redirect('/signin')
+    }
+
+    res.send(
+      signInPage({
+        error: req.flash('error')[0],
+        otp: true,
+        strategies,
+      })
+    )
+  })
+
+  express.post('/signin-otp', (req, res, next) => {
+    const { user } = req.session
+
+    if (user === undefined) {
+      return res.redirect(303, '/signin')
+    }
+
+    if (authenticator.check(req.body.otp, user.preferences.otp)) {
+      setToken(req, res, next)
+    } else {
+      req.flash('error', 'Invalid code')
+      return res.redirect(303, '/signin-otp')
+    }
+  })
+
+  const setToken = async (req, res, next) => {
+    const { user, isPersistent } = req.session
+    const token = (await xo.createAuthenticationToken({ userId: user.id })).id
+
+    // Persistent cookie ? => 1 year
+    // Non-persistent : external provider as Github, Twitter...
+    res.cookie(
+      'token',
+      token,
+      isPersistent ? { maxAge: 1000 * 60 * 60 * 24 * 365 } : undefined
+    )
+
+    delete req.session.isPersistent
+    delete req.session.user
+    res.redirect(303, req.flash('return-url')[0] || '/')
+  }
+
   const SIGNIN_STRATEGY_RE = /^\/signin\/([^/]+)(\/callback)?(:?\?.*)?$/
   express.use(async (req, res, next) => {
     const { url } = req
@@ -153,41 +206,22 @@ async function setUpPassport(express, xo) {
 
         if (!user) {
           req.flash('error', info ? info.message : 'Invalid credentials')
-          return res.redirect('/signin')
+          return res.redirect(303, '/signin')
         }
 
-        // The cookie will be set in via the next request because some
-        // browsers do not save cookies on redirect.
-        req.flash(
-          'token',
-          (await xo.createAuthenticationToken({ userId: user.id })).id
-        )
-
-        // The session is only persistent for internal provider and if 'Remember me' box is checked
-        req.flash(
-          'session-is-persistent',
+        req.session.user = { id: user.id, preferences: user.preferences }
+        req.session.isPersistent =
           matches[1] === 'local' && req.body['remember-me'] === 'on'
-        )
 
-        res.redirect(req.flash('return-url')[0] || '/')
+        if (user.preferences?.otp !== undefined) {
+          return res.redirect(303, '/signin-otp')
+        }
+
+        setToken(req, res, next)
       })(req, res, next)
     }
 
-    const token = req.flash('token')[0]
-
-    if (token) {
-      const isPersistent = req.flash('session-is-persistent')[0]
-
-      if (isPersistent) {
-        // Persistent cookie ? => 1 year
-        res.cookie('token', token, { maxAge: 1000 * 60 * 60 * 24 * 365 })
-      } else {
-        // Non-persistent : external provider as Github, Twitter...
-        res.cookie('token', token)
-      }
-
-      next()
-    } else if (req.cookies.token) {
+    if (req.cookies.token) {
       next()
     } else if (
       /favicon|fontawesome|images|styles|\.(?:css|jpg|png)$/.test(url)
@@ -601,7 +635,7 @@ export default async function main(args) {
   await xo.clean()
 
   // Express is used to manage non WebSocket connections.
-  const express = createExpressApp()
+  const express = createExpressApp(config)
 
   if (config.http.redirectToHttps) {
     let port

packages/xo-server/src/utils.js

@@ -347,9 +347,11 @@ export const streamToArray = (stream, { filter, mapper } = {}) =>
 
 // Create a serializable object from an error.
 export const serializeError = error => ({
-  message: error.message,
-  stack: error.stack,
   ...error, // Copy enumerable properties.
+  code: error.code,
+  message: error.message,
+  name: error.name,
+  stack: error.stack,
 })
 
 // -------------------------------------------------------------------

packages/xo-server/src/xapi-object-to-xo.js

@@ -173,6 +173,7 @@ const TRANSFORMS = {
           total: 0,
         }
       })(),
+      multipathing: obj.multipathing,
       patches: patches || link(obj, 'patches'),
       powerOnMode: obj.power_on_mode,
       power_state: metrics ? (isRunning ? 'Running' : 'Halted') : 'Unknown',
@@ -426,6 +427,7 @@ const TRANSFORMS = {
     let tmp
     if ((tmp = obj.VCPUs_params)) {
       tmp.cap && (vm.cpuCap = +tmp.cap)
+      tmp.mask && (vm.cpuMask = tmp.mask.split(',').map(_ => +_))
       tmp.weight && (vm.cpuWeight = +tmp.weight)
     }
 
@@ -476,6 +478,7 @@ const TRANSFORMS = {
       host: link(obj, 'host'),
       SR: link(obj, 'SR'),
       device_config: obj.device_config,
+      otherConfig: obj.other_config,
     }
   },
 
@@ -505,6 +508,7 @@ const TRANSFORMS = {
       // A physical PIF cannot be unplugged
       physical: Boolean(obj.physical),
       vlan: +obj.VLAN,
+      speed: metrics && +metrics.speed,
       $host: link(obj, 'host'),
       $network: link(obj, 'network'),
     }
@@ -744,7 +748,7 @@ const TRANSFORMS = {
 
 // ===================================================================
 
-export default function xapiObjectToXo(xapiObj, dependents) {
+export default function xapiObjectToXo(xapiObj, dependents = {}) {
   const transform = TRANSFORMS[xapiObj.$type.toLowerCase()]
   if (!transform) {
     return

packages/xo-server/src/xapi/index.js

@@ -36,6 +36,7 @@ import { satisfies as versionSatisfies } from 'semver'
 
 import createSizeStream from '../size-stream'
 import fatfsBuffer, { init as fatfsBufferInit } from '../fatfs-buffer'
+import pRetry from '../_pRetry'
 import {
   camelToSnakeCase,
   ensureArray,
@@ -415,6 +416,36 @@ export default class Xapi extends XapiBase {
     await this.call('host.enable', this.getObject(hostId).$ref)
   }
 
+  @deferrable.onError(log.warn)
+  async setHostMultipathing($defer, hostId, multipathing) {
+    const host = this.getObject(hostId)
+
+    const pluggedPbds = host.$PBDs.filter(pbd => pbd.currently_attached)
+    await asyncMap(pluggedPbds, async pbd => {
+      const ref = pbd.$ref
+      await this.unplugPbd(ref)
+      $defer(() => this.plugPbd(ref))
+    })
+
+    if (host.enabled) {
+      await this.disableHost(hostId)
+      $defer(() => this.enableHost(hostId))
+    }
+
+    return this._updateObjectMapProperty(
+      host,
+      'other_config',
+      multipathing
+        ? {
+            multipathing: 'true',
+            multipathhandle: 'dmp',
+          }
+        : {
+            multipathing: 'false',
+          }
+    )
+  }
+
   async powerOnHost(hostId) {
     await this.call('host.power_on', this.getObject(hostId).$ref)
   }
@@ -509,7 +540,7 @@ export default class Xapi extends XapiBase {
     vmId,
     targetXapi,
     targetSrId,
-    { compress = true, nameLabel = undefined } = {}
+    { compress, nameLabel = undefined } = {}
   ) {
     // Fall back on local copy if possible.
     if (targetXapi === this) {
@@ -753,7 +784,7 @@ export default class Xapi extends XapiBase {
   // Returns a stream to the exported VM.
   @concurrency(2, stream => stream.then(stream => fromEvent(stream, 'end')))
   @cancelable
-  async exportVm($cancelToken, vmId, { compress = true } = {}) {
+  async exportVm($cancelToken, vmId, { compress = false } = {}) {
     const vm = this.getObject(vmId)
     const useSnapshot = isVmRunning(vm)
     const exportedVm = useSnapshot
@@ -763,7 +794,12 @@ export default class Xapi extends XapiBase {
     const promise = this.getResource($cancelToken, '/export/', {
       query: {
         ref: exportedVm.$ref,
-        use_compression: compress ? 'true' : 'false',
+        use_compression:
+          compress === 'zstd'
+            ? 'zstd'
+            : compress === true || compress === 'gzip'
+            ? 'true'
+            : 'false',
       },
       task: this.createTask('VM export', vm.name_label),
     }).catch(error => {
@@ -1510,15 +1546,50 @@ export default class Xapi extends XapiBase {
       }`
     )
 
+    const vmRef = vm.$ref
     let ref
     do {
       if (!vm.tags.includes('xo-disable-quiesce')) {
         try {
-          ref = await this.callAsync(
-            $cancelToken,
-            'VM.snapshot_with_quiesce',
-            vm.$ref,
-            nameLabel
+          vm = await this.barrier(vmRef)
+          ref = await pRetry(
+            async bail => {
+              try {
+                return await this.callAsync(
+                  $cancelToken,
+                  'VM.snapshot_with_quiesce',
+                  vmRef,
+                  nameLabel
+                )
+              } catch (error) {
+                if (error?.code !== 'VM_SNAPSHOT_WITH_QUIESCE_FAILED') {
+                  throw bail(error)
+                }
+
+                // detect and remove new broken snapshots
+                //
+                // see https://github.com/vatesfr/xen-orchestra/issues/3936
+                const prevSnapshotRefs = new Set(vm.snapshots)
+                const snapshotNameLabelPrefix = `Snapshot of ${vm.uuid} [`
+                vm = await this.barrier(vmRef)
+                const createdSnapshots = vm.$snapshots.filter(
+                  _ =>
+                    !prevSnapshotRefs.has(_.$ref) &&
+                    _.name_label.startsWith(snapshotNameLabelPrefix)
+                )
+
+                // be safe: only delete if there was a single match
+                if (createdSnapshots.length === 1) {
+                  ignoreErrors.call(this._deleteVm(createdSnapshots[0]))
+                }
+
+                throw error
+              }
+            },
+            {
+              delay: 60e3,
+              tries: 3,
+            }
           ).then(extractOpaqueRef)
           this.addTag(ref, 'quiesce')::ignoreErrors()
 
@@ -1540,7 +1611,7 @@ export default class Xapi extends XapiBase {
       ref = await this.callAsync(
         $cancelToken,
         'VM.snapshot',
-        vm.$ref,
+        vmRef,
         nameLabel
       ).then(extractOpaqueRef)
     } while (false)
@@ -1814,10 +1885,13 @@ export default class Xapi extends XapiBase {
       }`
     )
     try {
-      await this.call('VDI.pool_migrate', vdi.$ref, sr.$ref, {})
+      await pRetry(() => this.call('VDI.pool_migrate', vdi.$ref, sr.$ref, {}), {
+        when: { code: 'TOO_MANY_STORAGE_MIGRATES' },
+      })
     } catch (error) {
       const { code } = error
       if (
+        code !== 'NO_HOSTS_AVAILABLE' &&
         code !== 'LICENCE_RESTRICTION' &&
         code !== 'VDI_NEEDS_VM_FOR_MIGRATE'
       ) {
@@ -1826,15 +1900,13 @@ export default class Xapi extends XapiBase {
       const newVdi = await this.barrier(
         await this.call('VDI.copy', vdi.$ref, sr.$ref)
       )
-      await asyncMap(vdi.$VBDs, vbd =>
-        Promise.all([
-          this.call('VBD.destroy', vbd.$ref),
-          this.createVbd({
-            ...vbd,
-            vdi: newVdi,
-          }),
-        ])
-      )
+      await asyncMap(vdi.$VBDs, async vbd => {
+        await this.call('VBD.destroy', vbd.$ref)
+        await this.createVbd({
+          ...vbd,
+          vdi: newVdi,
+        })
+      })
       await this._deleteVdi(vdi)
     }
   }
@@ -2296,7 +2368,7 @@ export default class Xapi extends XapiBase {
     const sr = this.getObject(srId)
 
     // First, create a small VDI (10MB) which will become the ConfigDrive
-    const buffer = fatfsBufferInit()
+    const buffer = fatfsBufferInit({ label: 'cidata     ' })
     const vdi = await this.createVdi({
       name_label: 'XO CloudConfigDrive',
       size: buffer.length,
@@ -2307,14 +2379,9 @@ export default class Xapi extends XapiBase {
     // Then, generate a FAT fs
     const fs = promisifyAll(fatfs.createFileSystem(fatfsBuffer(buffer)))
 
-    await fs.mkdir('openstack')
-    await fs.mkdir('openstack/latest')
     await Promise.all([
-      fs.writeFile(
-        'openstack/latest/meta_data.json',
-        '{\n    "uuid": "' + vm.uuid + '"\n}\n'
-      ),
-      fs.writeFile('openstack/latest/user_data', config),
+      fs.writeFile('meta-data', 'instance-id: ' + vm.uuid + '\n'),
+      fs.writeFile('user-data', config),
     ])
 
     // ignore errors, I (JFT) don't understand why they are emitted

packages/xo-server/src/xapi/index.js.flow

@@ -93,7 +93,7 @@ declare export class Xapi {
   exportVm(
     cancelToken: mixed,
     vm: Vm,
-    options ?: Object
+    options?: { compress?: true | false | 'gzip' | 'zstd' }
   ): Promise<AugmentedReadable>;
   getObject(object: Id): XapiObject;
   importDeltaVm(data: DeltaVmImport, options: Object): Promise<{ vm: Vm }>;

packages/xo-server/src/xapi/mixins/patching.js

@@ -29,15 +29,15 @@ export default {
   // FIXME: should be static
   @debounce(24 * 60 * 60 * 1000)
   async _getXenUpdates() {
-    const { readAll, statusCode } = await this.xo.httpRequest(
+    const response = await this.xo.httpRequest(
       'http://updates.xensource.com/XenServer/updates.xml'
     )
 
-    if (statusCode !== 200) {
+    if (response.statusCode !== 200) {
       throw new Error('cannot fetch patches list from Citrix')
     }
 
-    const data = parseXml(await readAll()).patchdata
+    const data = parseXml(await response.readAll()).patchdata
 
     const patches = { __proto__: null }
     forEach(data.patches.patch, patch => {

packages/xo-server/src/xapi/mixins/vm.js

@@ -5,13 +5,36 @@ import { NULL_REF } from 'xen-api'
 
 import { forEach, mapToArray, parseSize } from '../../utils'
 
-import { isVmHvm, isVmRunning, makeEditObject } from '../utils'
+import {
+  extractOpaqueRef,
+  isVmHvm,
+  isVmRunning,
+  makeEditObject,
+} from '../utils'
 
 // According to: https://xenserver.org/blog/entry/vga-over-cirrus-in-xenserver-6-2.html.
 const XEN_VGA_VALUES = ['std', 'cirrus']
 const XEN_VIDEORAM_VALUES = [1, 2, 4, 8, 16]
 
 export default {
+  // https://xapi-project.github.io/xen-api/classes/vm.html#checkpoint
+  async checkpointVm(vmId, nameLabel) {
+    const vm = this.getObject(vmId)
+    try {
+      const ref = await this.callAsync(
+        'VM.checkpoint',
+        vm.$ref,
+        nameLabel != null ? nameLabel : vm.name_label
+      ).then(extractOpaqueRef)
+      return this.barrier(ref)
+    } catch (error) {
+      if (error.code === 'VM_BAD_POWER_STATE') {
+        return this._snapshotVm(vm, nameLabel)
+      }
+      throw error
+    }
+  },
+
   // TODO: clean up on error.
   @deferrable
   async createVm(
@@ -309,6 +332,15 @@ export default {
       },
     },
 
+    cpuMask: {
+      get: vm => vm.VCPUs_params.mask && vm.VCPUs_params.mask.split(','),
+      set(cpuMask, vm) {
+        return this._updateObjectMapProperty(vm, 'VCPUs_params', {
+          mask: cpuMask == null ? cpuMask : cpuMask.join(','),
+        })
+      },
+    },
+
     cpusMax: 'cpusStaticMax',
     cpusStaticMax: {
       constraints: {

packages/xo-server/src/xo-mixins/api.js

@@ -210,6 +210,7 @@ export default class Api {
   }
 
   async callApiMethod(session, name, params = {}) {
+    const xo = this._xo
     const startTime = Date.now()
 
     const method = this._methods[name]
@@ -219,7 +220,7 @@ export default class Api {
 
     // FIXME: it can cause issues if there any property assignments in
     // XO methods called from the API.
-    const context = Object.create(this._xo, {
+    const context = Object.create(xo, {
       api: {
         // Used by system.*().
         value: this,
@@ -231,9 +232,24 @@ export default class Api {
 
     // Fetch and inject the current user.
     const userId = session.get('user_id', undefined)
-    context.user = userId && (await this._xo.getUser(userId))
+    context.user = userId && (await xo.getUser(userId))
     const userName = context.user ? context.user.email : '(unknown user)'
 
+    const data = {
+      userId,
+      method: name,
+      params: sensitiveValues.replace(params, '* obfuscated *'),
+    }
+
+    const callId = Math.random()
+      .toString(36)
+      .slice(2)
+
+    xo.emit('xo:preCall', {
+      ...data,
+      callId,
+    })
+
     try {
       await checkPermission.call(context, method)
 
@@ -271,23 +287,33 @@ export default class Api {
         )}] ==> ${kindOf(result)}`
       )
 
+      xo.emit('xo:postCall', {
+        callId,
+        method: name,
+        result,
+      })
+
       return result
     } catch (error) {
-      params = sensitiveValues.replace(params, '* obfuscated *')
-      const data = {
-        userId,
+      const serializedError = serializeError(error)
+
+      xo.emit('xo:postCall', {
+        callId,
+        error: serializedError,
         method: name,
-        params,
+      })
+
+      const message = `${userName} | ${name}(${JSON.stringify(
+        data.params
+      )}) [${ms(Date.now() - startTime)}] =!> ${error}`
+
+      this._logger.error(message, {
+        ...data,
         duration: Date.now() - startTime,
-        error: serializeError(error),
-      }
-      const message = `${userName} | ${name}(${JSON.stringify(params)}) [${ms(
-        Date.now() - startTime
-      )}] =!> ${error}`
+        error: serializedError,
+      })
 
-      this._logger.error(message, data)
-
-      if (this._xo._config.verboseLogsOnErrors) {
+      if (xo._config.verboseLogsOnErrors) {
         log.warn(message, { error })
       } else {
         log.warn(
@@ -301,7 +327,7 @@ export default class Api {
       if (xoError) {
         throw xoError(error.params, ref => {
           try {
-            return this._xo.getObject(ref).id
+            return xo.getObject(ref).id
           } catch (e) {
             return ref
           }

packages/xo-server/src/xo-mixins/backups-ng-logs.js

@@ -1,4 +1,5 @@
-import { forEach } from 'lodash'
+import ms from 'ms'
+import { forEach, isEmpty, iteratee, sortedIndexBy } from 'lodash'
 import { noSuchObject } from 'xo-common/api-errors'
 
 const isSkippedError = error =>
@@ -32,6 +33,10 @@ const computeStatusAndSortTasks = (status, tasks) => {
   return status
 }
 
+function getPropertyValue(key) {
+  return this[key]
+}
+
 const taskTimeComparator = ({ start: s1, end: e1 }, { start: s2, end: e2 }) => {
   if (e1 !== undefined) {
     if (e2 !== undefined) {
@@ -170,4 +175,48 @@ export default {
 
     return runId === undefined ? consolidated : consolidated[runId]
   },
+
+  async getBackupNgLogsSorted({ after, before, filter, limit }) {
+    let logs = await this.getBackupNgLogs()
+
+    // convert to array
+    logs = Object.keys(logs).map(getPropertyValue, logs)
+
+    if (!isEmpty(filter)) {
+      logs = logs.filter(iteratee(filter))
+    }
+
+    logs.sort((a, b) => a.start - b.start)
+
+    // only extract the range we are interested in
+    const i =
+      after === undefined
+        ? 0
+        : sortedIndexBy(
+            logs,
+            {
+              start: typeof after === 'number' ? after : Date.now() - ms(after),
+            },
+            'start'
+          )
+    let j =
+      before === undefined
+        ? logs.length
+        : sortedIndexBy(
+            logs,
+            {
+              start:
+                typeof before === 'number' ? before : Date.now() - ms(before),
+            },
+            'start'
+          )
+
+    limit += i
+    if (limit < j) {
+      j = limit
+    }
+    logs = logs.slice(i, j)
+
+    return logs
+  },
 }

packages/xo-server/src/xo-mixins/backups-ng/index.js

@@ -32,6 +32,7 @@ import {
 } from 'promise-toolbox'
 import Vhd, {
   chainVhd,
+  checkVhdChain,
   createSyntheticStream as createVhdReadStream,
 } from 'vhd-lib'
 
@@ -80,7 +81,7 @@ type SimpleIdPattern = {|
 
 export type BackupJob = {|
   ...$Exact<Job>,
-  compression?: 'native',
+  compression?: 'native' | 'zstd' | '',
   mode: Mode,
   remotes?: SimpleIdPattern,
   settings: $Dict<Settings>,
@@ -176,10 +177,13 @@ const isMetadataFile = (filename: string) => filename.endsWith('.json')
 const isVhd = (filename: string) => filename.endsWith('.vhd')
 const isXva = (filename: string) => filename.endsWith('.xva')
 
+const getJobCompression = ({ compression: c }) =>
+  c === undefined || c === '' ? false : c === 'native' ? 'gzip' : 'zstd'
+
 const listReplicatedVms = (
   xapi: Xapi,
   scheduleId: string,
-  srId: string,
+  srId?: string,
   vmUuid?: string
 ): Vm[] => {
   const { all } = xapi.objects
@@ -476,13 +480,26 @@ const disableVmHighAvailability = async (xapi: Xapi, vm: Vm) => {
 //      ├─ <YYYYMMDD>T<HHmmss>.xva
 //      └─ <YYYYMMDD>T<HHmmss>.xva.checksum
 //
+// Attributes on created VM snapshots:
+//
+// - `other_config`:
+//    - `xo:backup:datetime` = snapshot.snapshot_time (allow sorting replicated VMs)
+//    - `xo:backup:job` = job.id
+//    - `xo:backup:schedule` = schedule.id
+//    - `xo:backup:vm` = vm.uuid
+//    - `xo:backup:exported` = 'true' (added at the end of the backup)
+//
 // Attributes of created VMs:
 //
-// - name: `${original name} - ${job name} - (${safeDateFormat(backup timestamp)})`
+// - all snapshots attributes (see above)
+// - `name_label`: `${original name} - ${job name} - (${safeDateFormat(backup timestamp)})`
 // - tag:
 //    - copy in delta mode: `Continuous Replication`
 //    - copy in full mode: `Disaster Recovery`
 //    - imported from backup: `restored from backup`
+// - `blocked_operations.start`: message
+// - for copies/replications only, added after complete transfer
+//    - `other_config[xo:backup:sr]` = sr.uuid
 //
 // Task logs emitted in a backup execution:
 //
@@ -901,6 +918,7 @@ export default class BackupNg {
   // - [x] possibility to (re-)run a single VM in a backup?
   // - [x] validate VHDs after exports and before imports, how?
   // - [x] check merge/transfert duration/size are what we want for delta
+  // - [x] delete interrupted *importing* VMs
   @defer
   async _backupVm(
     $defer: any,
@@ -1118,7 +1136,7 @@ export default class BackupNg {
           parentId: taskId,
         },
         xapi.exportVm($cancelToken, snapshot, {
-          compress: job.compression === 'native',
+          compress: getJobCompression(job),
         })
       )
       const exportTask = xva.task
@@ -1219,6 +1237,14 @@ export default class BackupNg {
 
                 const { $id: srId, xapi } = sr
 
+                // delete previous interrupted copies
+                ignoreErrors.call(
+                  this._deleteVms(
+                    xapi,
+                    listReplicatedVms(xapi, scheduleId, undefined, vmUuid)
+                  )
+                )
+
                 const oldVms = getOldEntries(
                   copyRetention - 1,
                   listReplicatedVms(xapi, scheduleId, srId, vmUuid)
@@ -1276,29 +1302,6 @@ export default class BackupNg {
         $defer.onSuccess.call(xapi, 'deleteVm', baseSnapshot)
       }
 
-      // JFT: TODO: remove when enough time has passed (~2018-09)
-      //
-      // Fix VHDs UUID (= VDI.uuid), which was not done before 2018-06-16.
-      await asyncMap(remotes, async ({ handler }) =>
-        asyncMap(
-          this._listVmBackups(handler, vmUuid, _ => _.mode === 'delta'),
-          ({ _filename, vdis, vhds }) => {
-            const vmDir = dirname(_filename)
-            return asyncMap(vhds, async (vhdPath, vdiId) => {
-              const uuid = parseUuid(vdis[vdiId].uuid)
-
-              const vhd = new Vhd(handler, `${vmDir}/${vhdPath}`)
-              await vhd.readHeaderAndFooter()
-              if (!vhd.footer.uuid.equals(uuid)) {
-                vhd.footer.uuid = uuid
-                await vhd.readBlockAllocationTable()
-                await vhd.writeFooter()
-              }
-            })
-          }
-        )
-      )
-
       let fullVdisRequired
       await (async () => {
         if (baseSnapshot === undefined) {
@@ -1351,16 +1354,20 @@ export default class BackupNg {
             await asyncMap(files, async file => {
               if (file[0] !== '.') {
                 try {
-                  const vhd = new Vhd(handler, `${dir}/${file}`)
+                  const path = `${dir}/${file}`
+                  const vhd = new Vhd(handler, path)
                   await vhd.readHeaderAndFooter()
 
                   if (vhd.footer.uuid.equals(parseUuid(vdi.uuid))) {
+                    await checkVhdChain(handler, path)
                     full = false
                   }
 
                   return
                 } catch (error) {
-                  if (!(error instanceof AssertionError)) {
+                  const corruptedVhdOrMissingParent =
+                    error instanceof AssertionError || error?.code === 'ENOENT'
+                  if (!corruptedVhdOrMissingParent) {
                     throw error
                   }
                 }
@@ -1573,6 +1580,14 @@ export default class BackupNg {
 
                 const { $id: srId, xapi } = sr
 
+                // delete previous interrupted copies
+                ignoreErrors.call(
+                  this._deleteVms(
+                    xapi,
+                    listReplicatedVms(xapi, scheduleId, undefined, vmUuid)
+                  )
+                )
+
                 const oldVms = getOldEntries(
                   copyRetention - 1,
                   listReplicatedVms(xapi, scheduleId, srId, vmUuid)

packages/xo-server/src/xo-mixins/remotes.js

@@ -1,7 +1,8 @@
+import asyncMap from '@xen-orchestra/async-map'
 import synchronized from 'decorator-synchronized'
 import { format, parse } from 'xo-remote-parser'
 import { getHandler } from '@xen-orchestra/fs'
-import { ignoreErrors } from 'promise-toolbox'
+import { ignoreErrors, timeout } from 'promise-toolbox'
 import { noSuchObject } from 'xo-common/api-errors'
 
 import * as sensitiveValues from '../sensitive-values'
@@ -18,13 +19,14 @@ const obfuscateRemote = ({ url, ...remote }) => {
 
 export default class {
   constructor(xo, { remoteOptions }) {
+    this._handlers = { __proto__: null }
     this._remoteOptions = remoteOptions
     this._remotes = new Remotes({
       connection: xo._redis,
       prefix: 'xo:remote',
       indexes: ['enabled'],
     })
-    this._handlers = { __proto__: null }
+    this._remotesInfo = {}
 
     xo.on('clean', () => this._remotes.rebuildIndexes())
     xo.on('start', async () => {
@@ -84,6 +86,23 @@ export default class {
     return handler.test()
   }
 
+  async getAllRemotesInfo() {
+    const remotes = await this._remotes.get()
+
+    await asyncMap(remotes, async remote => {
+      try {
+        const handler = await this.getRemoteHandler(remote.id)
+        await timeout.call(
+          handler.getInfo().then(info => {
+            this._remotesInfo[remote.id] = info
+          }),
+          5e3
+        )
+      } catch (_) {}
+    })
+    return this._remotesInfo
+  }
+
   async getAllRemotes() {
     return (await this._remotes.get()).map(_ => obfuscateRemote(_))
   }

packages/xo-server/src/xo-mixins/xen-servers.js

@@ -250,7 +250,7 @@ export default class {
   async connectXenServer(id) {
     const server = (await this._getXenServer(id)).properties
 
-    const xapi = new Xapi({
+    const xapi = (this._xapis[server.id] = new Xapi({
       allowUnauthorized: Boolean(server.allowUnauthorized),
       readOnly: Boolean(server.readOnly),
 
@@ -262,7 +262,7 @@ export default class {
       },
       url: server.host,
       watchEvents: false,
-    })
+    }))
 
     try {
       await xapi.connect()
@@ -278,7 +278,6 @@ export default class {
       }
 
       serverIdsByPool[poolId] = server.id
-      this._xapis[server.id] = xapi
 
       xapi.xo = (() => {
         const conId = server.id
@@ -376,7 +375,14 @@ export default class {
       xapi.watchEvents()
 
       this.updateXenServer(id, { error: null })::ignoreErrors()
+
+      xapi.once('disconnected', () => {
+        xapi.xo.uninstall()
+        delete this._xapis[server.id]
+        delete this._serverIdsByPool[poolId]
+      })
     } catch (error) {
+      delete this._xapis[server.id]
       xapi.disconnect()::ignoreErrors()
       this.updateXenServer(id, { error: serializeError(error) })::ignoreErrors()
       throw error

packages/xo-vmdk-to-vhd/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-vmdk-to-vhd",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "license": "AGPL-3.0",
   "description": "JS lib streaming a vmdk file to a vhd",
   "keywords": [
@@ -28,7 +28,7 @@
     "pipette": "^0.9.3",
     "promise-toolbox": "^0.11.0",
     "tmp": "^0.0.33",
-    "vhd-lib": "^0.4.0"
+    "vhd-lib": "^0.5.1"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/xo-vmdk-to-vhd/src/vmdk-read-table.js

@@ -54,10 +54,9 @@ export default async function readVmdkGrainTable(fileAccessor) {
   const grainSize =
     getLongLong(headerBuffer, GRAIN_SIZE_OFFSET, 'grain size') * SECTOR_SIZE
   const grainCount = Math.ceil(capacity / grainSize)
-  const numGTEsPerGT = getLongLong(
-    headerBuffer,
+  const numGTEsPerGT = new DataView(headerBuffer).getUint32(
     NUM_GTE_PER_GT_OFFSET,
-    'num GTE per GT'
+    true
   )
   const grainTablePhysicalSize = numGTEsPerGT * 4
   const grainDirectoryEntries = Math.ceil(grainCount / numGTEsPerGT)