feat(xen-api): rewrite from scratch

- never stop event watching no matter what's happening
- never disconnect unless explicit requested
- better handling of critical sections

@xen-orchestra/babel-config/index.js

@@ -46,6 +46,12 @@ const getConfig = (key, ...args) => {
     : config
 }
 
+// some plugins must be used in a specific order
+const pluginsOrder = [
+  '@babel/plugin-proposal-decorators',
+  '@babel/plugin-proposal-class-properties',
+]
+
 module.exports = function(pkg, plugins, presets) {
   plugins === undefined && (plugins = {})
   presets === undefined && (presets = {})
@@ -61,7 +67,13 @@ module.exports = function(pkg, plugins, presets) {
   return {
     comments: !__PROD__,
     ignore: __TEST__ ? undefined : [/\.spec\.js$/],
-    plugins: Object.keys(plugins).map(plugin => [plugin, plugins[plugin]]),
+    plugins: Object.keys(plugins)
+      .map(plugin => [plugin, plugins[plugin]])
+      .sort(([a], [b]) => {
+        const oA = pluginsOrder.indexOf(a)
+        const oB = pluginsOrder.indexOf(b)
+        return oA !== -1 && oB !== -1 ? oA - oB : a < b ? -1 : 1
+      }),
     presets: Object.keys(presets).map(preset => [preset, presets[preset]]),
   }
 }

@xen-orchestra/cr-seed-cli/package.json

@@ -16,6 +16,6 @@
   },
   "dependencies": {
     "golike-defer": "^0.4.1",
-    "xen-api": "^0.24.5"
+    "xen-api": "^0.25.0"
   }
 }

@xen-orchestra/fs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xen-orchestra/fs",
-  "version": "0.7.1",
+  "version": "0.8.0",
   "license": "AGPL-3.0",
   "description": "The File System for Xen Orchestra backups.",
   "keywords": [],

CHANGELOG.md

@@ -1,6 +1,6 @@
 # ChangeLog
 
-## Next (2019-03-19)
+## **5.33.0** (2019-03-29)
 
 ### Enhancements
 
@@ -13,6 +13,29 @@
 - [Home] Save the current page in url [#3993](https://github.com/vatesfr/xen-orchestra/issues/3993) (PR [#3999](https://github.com/vatesfr/xen-orchestra/pull/3999))
 - [VDI] Ensure suspend VDI is destroyed when destroying a VM  [#4027](https://github.com/vatesfr/xen-orchestra/issues/4027) (PR [#4038](https://github.com/vatesfr/xen-orchestra/pull/4038))
 - [VM/disk]: Warning when 2 VDIs are on 2 different hosts' local SRs [#3911](https://github.com/vatesfr/xen-orchestra/issues/3911) (PR [#3969](https://github.com/vatesfr/xen-orchestra/pull/3969))
+- [Remotes] Benchmarks (read and write rate speed) added when remote is tested [#3991](https://github.com/vatesfr/xen-orchestra/issues/3991) (PR [#4015](https://github.com/vatesfr/xen-orchestra/pull/4015))
+- [Cloud Config] Support both NoCloud and Config Drive 2 datasources for maximum compatibility (PR [#4053](https://github.com/vatesfr/xen-orchestra/pull/4053))
+- [Advanced] Configurable cookie validity (PR [#4059](https://github.com/vatesfr/xen-orchestra/pull/4059))
+- [Plugins] Display number of installed plugins [#4008](https://github.com/vatesfr/xen-orchestra/issues/4008) (PR [#4050](https://github.com/vatesfr/xen-orchestra/pull/4050))
+- [Continuous Replication] Opt-in mode to guess VHD size, should help with XenServer 7.1 CU2 and various `VDI_IO_ERROR` errors (PR [#3726](https://github.com/vatesfr/xen-orchestra/pull/3726))
+- [VM/Snapshots] Always delete broken quiesced snapshots [#4074](https://github.com/vatesfr/xen-orchestra/issues/4074) (PR [#4075](https://github.com/vatesfr/xen-orchestra/pull/4075))
+- [Settings/Servers] Display link to pool [#4041](https://github.com/vatesfr/xen-orchestra/issues/4041) (PR [#4045](https://github.com/vatesfr/xen-orchestra/pull/4045))
+- [Import] Change wording of drop zone (PR [#4020](https://github.com/vatesfr/xen-orchestra/pull/4020))
+- [Backup NG] Ability to set the interval of the full backups [#1783](https://github.com/vatesfr/xen-orchestra/issues/1783) (PR [#4083](https://github.com/vatesfr/xen-orchestra/pull/4083))
+- [Hosts] Display a warning icon if you have XenServer license restrictions [#4091](https://github.com/vatesfr/xen-orchestra/issues/4091) (PR [#4094](https://github.com/vatesfr/xen-orchestra/pull/4094))
+- [Restore] Ability to restore a metadata backup [#4004](https://github.com/vatesfr/xen-orchestra/issues/4004) (PR [#4023](https://github.com/vatesfr/xen-orchestra/pull/4023))
+- Improve connection to XCP-ng/XenServer hosts:
+  - never disconnect by itself even in case of errors
+  - never stop watching events
+
+### Released packages
+
+- xen-api v0.25.0
+- vhd-lib v0.6.0
+- @xen-orchestra/fs v0.8.0
+- xo-server-usage-report v0.7.2
+- xo-server v5.38.1
+- xo-web v5.38.0
 
 ### Bug fixes
 
@@ -22,6 +45,16 @@
 - [Home/VM] Bulk migration: fixed VM VDIs not migrated to the selected SR [#3986](https://github.com/vatesfr/xen-orchestra/issues/3986) (PR [#3987](https://github.com/vatesfr/xen-orchestra/pull/3987))
 - [Stats] Fix cache usage with simultaneous requests [#4017](https://github.com/vatesfr/xen-orchestra/issues/4017) (PR [#4028](https://github.com/vatesfr/xen-orchestra/pull/4028))
 - [Backup NG] Fix compression displayed for the wrong backup mode (PR [#4021](https://github.com/vatesfr/xen-orchestra/pull/4021))
+- [Home] Always sort the items by their names as a secondary sort criteria [#3983](https://github.com/vatesfr/xen-orchestra/issues/3983) (PR [#4047](https://github.com/vatesfr/xen-orchestra/pull/4047))
+- [Remotes] Fixes `spawn mount EMFILE` error during backup
+- Properly redirect to sign in page instead of being stuck in a refresh loop
+- [Backup-ng] No more false positives when list matching VMs on Home page [#4078](https://github.com/vatesfr/xen-orchestra/issues/4078) (PR [#4085](https://github.com/vatesfr/xen-orchestra/pull/4085))
+- [Plugins] Properly remove optional settings when unchecking _Fill information_ (PR [#4076](https://github.com/vatesfr/xen-orchestra/pull/4076))
+- [Patches] (PR [#4077](https://github.com/vatesfr/xen-orchestra/pull/4077))
+  - Add a host to a pool: fixes the auto-patching of the host on XenServer < 7.2 [#3783](https://github.com/vatesfr/xen-orchestra/issues/3783)
+  - Add a host to a pool: homogenizes both the host and **pool**'s patches [#2188](https://github.com/vatesfr/xen-orchestra/issues/2188)
+  - Safely install a subset of patches on a pool [#3777](https://github.com/vatesfr/xen-orchestra/issues/3777)
+  - XCP-ng: no longer requires to run `yum install xcp-ng-updater` when it's already installed [#3934](https://github.com/vatesfr/xen-orchestra/issues/3934)
 
 ## **5.32.2** (2019-02-28)
 

CHANGELOG.unreleased.md

@@ -2,19 +2,9 @@
 
 ### Enhancements
 
-- [Remotes] Benchmarks (read and write rate speed) added when remote is tested [#3991](https://github.com/vatesfr/xen-orchestra/issues/3991) (PR [#4015](https://github.com/vatesfr/xen-orchestra/pull/4015))
-- [Cloud Config] Support both NoCloud and Config Drive 2 datasources for maximum compatibility (PR [#4053](https://github.com/vatesfr/xen-orchestra/pull/4053))
-- [Advanced] Configurable cookie validity (PR [#4059](https://github.com/vatesfr/xen-orchestra/pull/4059))
-- [Plugins] Display number of installed plugins [#4008](https://github.com/vatesfr/xen-orchestra/issues/4008) (PR [#4050](https://github.com/vatesfr/xen-orchestra/pull/4050))
-
 ### Bug fixes
 
-- [Home] Always sort the items by their names as a secondary sort criteria [#3983](https://github.com/vatesfr/xen-orchestra/issues/3983) (PR [#4047](https://github.com/vatesfr/xen-orchestra/pull/4047))
-- [Remotes] Fixes `spawn mount EMFILE` error during backup
-- Properly redirect to sign in page instead of being stuck in a refresh loop
-
 ### Released packages
 
-- @xen-orchestra/fs v0.8.0
-- xo-server v5.38.0
-- xo-web v5.38.0
+- xo-server v5.39.0
+- xo-web v5.39.0

packages/vhd-cli/package.json

@@ -27,12 +27,12 @@
     "node": ">=6"
   },
   "dependencies": {
-    "@xen-orchestra/fs": "^0.7.1",
+    "@xen-orchestra/fs": "^0.8.0",
     "cli-progress": "^2.0.0",
     "exec-promise": "^0.7.0",
     "getopts": "^2.2.3",
     "struct-fu": "^1.2.0",
-    "vhd-lib": "^0.5.1"
+    "vhd-lib": "^0.6.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/vhd-lib/merge.integ.spec.js

@@ -1,38 +1,40 @@
 /* eslint-env jest */
 
+import asyncIteratorToStream from 'async-iterator-to-stream'
 import execa from 'execa'
 import fs from 'fs-extra'
 import getStream from 'get-stream'
 import rimraf from 'rimraf'
 import tmp from 'tmp'
-import { fromEvent, pFromCallback } from 'promise-toolbox'
 import { getHandler } from '@xen-orchestra/fs'
+import { pFromCallback } from 'promise-toolbox'
+import { pipeline } from 'readable-stream'
 import { randomBytes } from 'crypto'
 
 import Vhd, { chainVhd, createSyntheticStream, mergeVhd as vhdMerge } from './'
 
 import { SECTOR_SIZE } from './src/_constants'
 
-const initialDir = process.cwd()
+let tempDir = null
 
 jest.setTimeout(60000)
 
 beforeEach(async () => {
-  const dir = await pFromCallback(cb => tmp.dir(cb))
-  process.chdir(dir)
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
 })
 
 afterEach(async () => {
-  const tmpDir = process.cwd()
-  process.chdir(initialDir)
-  await pFromCallback(cb => rimraf(tmpDir, cb))
+  await pFromCallback(cb => rimraf(tempDir, cb))
 })
 
-async function createRandomFile(name, sizeMb) {
-  await execa('bash', [
-    '-c',
-    `< /dev/urandom tr -dc "\\t\\n [:alnum:]" | head -c ${sizeMb}M >${name}`,
-  ])
+async function createRandomFile(name, sizeMB) {
+  const createRandomStream = asyncIteratorToStream(function*(size) {
+    while (size-- > 0) {
+      yield Buffer.from([Math.floor(Math.random() * 256)])
+    }
+  })
+  const input = createRandomStream(sizeMB * 1024 * 1024)
+  await pFromCallback(cb => pipeline(input, fs.createWriteStream(name), cb))
 }
 
 async function checkFile(vhdName) {
@@ -53,31 +55,35 @@ async function convertFromRawToVhd(rawName, vhdName) {
 
 test('blocks can be moved', async () => {
   const initalSize = 4
-  await createRandomFile('randomfile', initalSize)
-  await convertFromRawToVhd('randomfile', 'randomfile.vhd')
-  const handler = getHandler({ url: 'file://' + process.cwd() })
-  const originalSize = await handler.getSize('randomfile')
-  const newVhd = new Vhd(handler, 'randomfile.vhd')
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  const handler = getHandler({ url: 'file://' })
+  const originalSize = await handler.getSize(rawFileName)
+  const newVhd = new Vhd(handler, vhdFileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   await newVhd._freeFirstBlockSpace(8000000)
-  await recoverRawContent('randomfile.vhd', 'recovered', originalSize)
-  expect(await fs.readFile('recovered')).toEqual(
-    await fs.readFile('randomfile')
+  const recoveredFileName = `${tempDir}/recovered`
+  await recoverRawContent(vhdFileName, recoveredFileName, originalSize)
+  expect(await fs.readFile(recoveredFileName)).toEqual(
+    await fs.readFile(rawFileName)
   )
 })
 
 test('the BAT MSB is not used for sign', async () => {
   const randomBuffer = await pFromCallback(cb => randomBytes(SECTOR_SIZE, cb))
-  await execa('qemu-img', ['create', '-fvpc', 'empty.vhd', '1.8T'])
-  const handler = getHandler({ url: 'file://' + process.cwd() })
-  const vhd = new Vhd(handler, 'empty.vhd')
+  const emptyFileName = `${tempDir}/empty.vhd`
+  await execa('qemu-img', ['create', '-fvpc', emptyFileName, '1.8T'])
+  const handler = getHandler({ url: 'file://' })
+  const vhd = new Vhd(handler, emptyFileName)
   await vhd.readHeaderAndFooter()
   await vhd.readBlockAllocationTable()
   // we want the bit 31 to be on, to prove it's not been used for sign
   const hugeWritePositionSectors = Math.pow(2, 31) + 200
   await vhd.writeData(hugeWritePositionSectors, randomBuffer)
-  await checkFile('empty.vhd')
+  await checkFile(emptyFileName)
   // here we are moving the first sector very far in the VHD to prove the BAT doesn't use signed int32
   const hugePositionBytes = hugeWritePositionSectors * SECTOR_SIZE
   await vhd._freeFirstBlockSpace(hugePositionBytes)
@@ -85,9 +91,10 @@ test('the BAT MSB is not used for sign', async () => {
   // we recover the data manually for speed reasons.
   // fs.write() with offset is way faster than qemu-img when there is a 1.5To
   // hole before the block of data
-  const recoveredFile = await fs.open('recovered', 'w')
+  const recoveredFileName = `${tempDir}/recovered`
+  const recoveredFile = await fs.open(recoveredFileName, 'w')
   try {
-    const vhd2 = new Vhd(handler, 'empty.vhd')
+    const vhd2 = new Vhd(handler, emptyFileName)
     await vhd2.readHeaderAndFooter()
     await vhd2.readBlockAllocationTable()
     for (let i = 0; i < vhd.header.maxTableEntries; i++) {
@@ -107,7 +114,7 @@ test('the BAT MSB is not used for sign', async () => {
     fs.close(recoveredFile)
   }
   const recovered = await getStream.buffer(
-    await fs.createReadStream('recovered', {
+    await fs.createReadStream(recoveredFileName, {
       start: hugePositionBytes,
       end: hugePositionBytes + randomBuffer.length - 1,
     })
@@ -117,27 +124,33 @@ test('the BAT MSB is not used for sign', async () => {
 
 test('writeData on empty file', async () => {
   const mbOfRandom = 3
-  await createRandomFile('randomfile', mbOfRandom)
-  await execa('qemu-img', ['create', '-fvpc', 'empty.vhd', mbOfRandom + 'M'])
-  const randomData = await fs.readFile('randomfile')
-  const handler = getHandler({ url: 'file://' + process.cwd() })
-  const originalSize = await handler.getSize('randomfile')
-  const newVhd = new Vhd(handler, 'empty.vhd')
+  const rawFileName = `${tempDir}/randomfile`
+  const emptyFileName = `${tempDir}/empty.vhd`
+  await createRandomFile(rawFileName, mbOfRandom)
+  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 'M'])
+  const randomData = await fs.readFile(rawFileName)
+  const handler = getHandler({ url: 'file://' })
+  const originalSize = await handler.getSize(rawFileName)
+  const newVhd = new Vhd(handler, emptyFileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   await newVhd.writeData(0, randomData)
-  await recoverRawContent('empty.vhd', 'recovered', originalSize)
-  expect(await fs.readFile('recovered')).toEqual(randomData)
+  const recoveredFileName = `${tempDir}/recovered`
+  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
+  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
 })
 
 test('writeData in 2 non-overlaping operations', async () => {
   const mbOfRandom = 3
-  await createRandomFile('randomfile', mbOfRandom)
-  await execa('qemu-img', ['create', '-fvpc', 'empty.vhd', mbOfRandom + 'M'])
-  const randomData = await fs.readFile('randomfile')
-  const handler = getHandler({ url: 'file://' + process.cwd() })
-  const originalSize = await handler.getSize('randomfile')
-  const newVhd = new Vhd(handler, 'empty.vhd')
+  const rawFileName = `${tempDir}/randomfile`
+  const emptyFileName = `${tempDir}/empty.vhd`
+  const recoveredFileName = `${tempDir}/recovered`
+  await createRandomFile(rawFileName, mbOfRandom)
+  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 'M'])
+  const randomData = await fs.readFile(rawFileName)
+  const handler = getHandler({ url: 'file://' })
+  const originalSize = await handler.getSize(rawFileName)
+  const newVhd = new Vhd(handler, emptyFileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   const splitPointSectors = 2
@@ -146,18 +159,21 @@ test('writeData in 2 non-overlaping operations', async () => {
     splitPointSectors,
     randomData.slice(splitPointSectors * 512)
   )
-  await recoverRawContent('empty.vhd', 'recovered', originalSize)
-  expect(await fs.readFile('recovered')).toEqual(randomData)
+  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
+  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
 })
 
 test('writeData in 2 overlaping operations', async () => {
   const mbOfRandom = 3
-  await createRandomFile('randomfile', mbOfRandom)
-  await execa('qemu-img', ['create', '-fvpc', 'empty.vhd', mbOfRandom + 'M'])
-  const randomData = await fs.readFile('randomfile')
-  const handler = getHandler({ url: 'file://' + process.cwd() })
-  const originalSize = await handler.getSize('randomfile')
-  const newVhd = new Vhd(handler, 'empty.vhd')
+  const rawFileName = `${tempDir}/randomfile`
+  const emptyFileName = `${tempDir}/empty.vhd`
+  const recoveredFileName = `${tempDir}/recovered`
+  await createRandomFile(rawFileName, mbOfRandom)
+  await execa('qemu-img', ['create', '-fvpc', emptyFileName, mbOfRandom + 'M'])
+  const randomData = await fs.readFile(rawFileName)
+  const handler = getHandler({ url: 'file://' })
+  const originalSize = await handler.getSize(rawFileName)
+  const newVhd = new Vhd(handler, emptyFileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   const endFirstWrite = 3
@@ -167,119 +183,138 @@ test('writeData in 2 overlaping operations', async () => {
     startSecondWrite,
     randomData.slice(startSecondWrite * 512)
   )
-  await recoverRawContent('empty.vhd', 'recovered', originalSize)
-  expect(await fs.readFile('recovered')).toEqual(randomData)
+  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
+  expect(await fs.readFile(recoveredFileName)).toEqual(randomData)
 })
 
 test('BAT can be extended and blocks moved', async () => {
   const initalSize = 4
-  await createRandomFile('randomfile', initalSize)
-  await convertFromRawToVhd('randomfile', 'randomfile.vhd')
-  const handler = getHandler({ url: 'file://' + process.cwd() })
-  const originalSize = await handler.getSize('randomfile')
-  const newVhd = new Vhd(handler, 'randomfile.vhd')
+  const rawFileName = `${tempDir}/randomfile`
+  const recoveredFileName = `${tempDir}/recovered`
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await createRandomFile(rawFileName, initalSize)
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  const handler = getHandler({ url: 'file://' })
+  const originalSize = await handler.getSize(rawFileName)
+  const newVhd = new Vhd(handler, vhdFileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   await newVhd.ensureBatSize(2000)
-  await recoverRawContent('randomfile.vhd', 'recovered', originalSize)
-  expect(await fs.readFile('recovered')).toEqual(
-    await fs.readFile('randomfile')
+  await recoverRawContent(vhdFileName, recoveredFileName, originalSize)
+  expect(await fs.readFile(recoveredFileName)).toEqual(
+    await fs.readFile(rawFileName)
   )
 })
 
 test('coalesce works with empty parent files', async () => {
   const mbOfRandom = 2
-  await createRandomFile('randomfile', mbOfRandom)
-  await convertFromRawToVhd('randomfile', 'randomfile.vhd')
+  const rawFileName = `${tempDir}/randomfile`
+  const emptyFileName = `${tempDir}/empty.vhd`
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  const recoveredFileName = `${tempDir}/recovered`
+  await createRandomFile(rawFileName, mbOfRandom)
+  await convertFromRawToVhd(rawFileName, vhdFileName)
   await execa('qemu-img', [
     'create',
     '-fvpc',
-    'empty.vhd',
+    emptyFileName,
     mbOfRandom + 1 + 'M',
   ])
-  await checkFile('randomfile.vhd')
-  await checkFile('empty.vhd')
-  const handler = getHandler({ url: 'file://' + process.cwd() })
-  const originalSize = await handler._getSize('randomfile')
-  await chainVhd(handler, 'empty.vhd', handler, 'randomfile.vhd', true)
-  await checkFile('randomfile.vhd')
-  await checkFile('empty.vhd')
-  await vhdMerge(handler, 'empty.vhd', handler, 'randomfile.vhd')
-  await recoverRawContent('empty.vhd', 'recovered', originalSize)
-  expect(await fs.readFile('recovered')).toEqual(
-    await fs.readFile('randomfile')
+  await checkFile(vhdFileName)
+  await checkFile(emptyFileName)
+  const handler = getHandler({ url: 'file://' })
+  const originalSize = await handler._getSize(rawFileName)
+  await chainVhd(handler, emptyFileName, handler, vhdFileName, true)
+  await checkFile(vhdFileName)
+  await checkFile(emptyFileName)
+  await vhdMerge(handler, emptyFileName, handler, vhdFileName)
+  await recoverRawContent(emptyFileName, recoveredFileName, originalSize)
+  expect(await fs.readFile(recoveredFileName)).toEqual(
+    await fs.readFile(rawFileName)
   )
 })
 
 test('coalesce works in normal cases', async () => {
   const mbOfRandom = 5
-  await createRandomFile('randomfile', mbOfRandom)
-  await createRandomFile('small_randomfile', Math.ceil(mbOfRandom / 2))
+  const randomFileName = `${tempDir}/randomfile`
+  const random2FileName = `${tempDir}/randomfile2`
+  const smallRandomFileName = `${tempDir}/small_randomfile`
+  const parentFileName = `${tempDir}/parent.vhd`
+  const child1FileName = `${tempDir}/child1.vhd`
+  const child2FileName = `${tempDir}/child2.vhd`
+  const recoveredFileName = `${tempDir}/recovered`
+  await createRandomFile(randomFileName, mbOfRandom)
+  await createRandomFile(smallRandomFileName, Math.ceil(mbOfRandom / 2))
   await execa('qemu-img', [
     'create',
     '-fvpc',
-    'parent.vhd',
+    parentFileName,
     mbOfRandom + 1 + 'M',
   ])
-  await convertFromRawToVhd('randomfile', 'child1.vhd')
-  const handler = getHandler({ url: 'file://' + process.cwd() })
-  await execa('vhd-util', ['snapshot', '-n', 'child2.vhd', '-p', 'child1.vhd'])
-  const vhd = new Vhd(handler, 'child2.vhd')
+  await convertFromRawToVhd(randomFileName, child1FileName)
+  const handler = getHandler({ url: 'file://' })
+  await execa('vhd-util', [
+    'snapshot',
+    '-n',
+    child2FileName,
+    '-p',
+    child1FileName,
+  ])
+  const vhd = new Vhd(handler, child2FileName)
   await vhd.readHeaderAndFooter()
   await vhd.readBlockAllocationTable()
   vhd.footer.creatorApplication = 'xoa'
   await vhd.writeFooter()
 
-  const originalSize = await handler._getSize('randomfile')
-  await chainVhd(handler, 'parent.vhd', handler, 'child1.vhd', true)
-  await execa('vhd-util', ['check', '-t', '-n', 'child1.vhd'])
-  await chainVhd(handler, 'child1.vhd', handler, 'child2.vhd', true)
-  await execa('vhd-util', ['check', '-t', '-n', 'child2.vhd'])
-  const smallRandom = await fs.readFile('small_randomfile')
-  const newVhd = new Vhd(handler, 'child2.vhd')
+  const originalSize = await handler._getSize(randomFileName)
+  await chainVhd(handler, parentFileName, handler, child1FileName, true)
+  await execa('vhd-util', ['check', '-t', '-n', child1FileName])
+  await chainVhd(handler, child1FileName, handler, child2FileName, true)
+  await execa('vhd-util', ['check', '-t', '-n', child2FileName])
+  const smallRandom = await fs.readFile(smallRandomFileName)
+  const newVhd = new Vhd(handler, child2FileName)
   await newVhd.readHeaderAndFooter()
   await newVhd.readBlockAllocationTable()
   await newVhd.writeData(5, smallRandom)
-  await checkFile('child2.vhd')
-  await checkFile('child1.vhd')
-  await checkFile('parent.vhd')
-  await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd')
-  await checkFile('parent.vhd')
-  await chainVhd(handler, 'parent.vhd', handler, 'child2.vhd', true)
-  await checkFile('child2.vhd')
-  await vhdMerge(handler, 'parent.vhd', handler, 'child2.vhd')
-  await checkFile('parent.vhd')
-  await recoverRawContent(
-    'parent.vhd',
-    'recovered_from_coalescing',
-    originalSize
-  )
-  await execa('cp', ['randomfile', 'randomfile2'])
-  const fd = await fs.open('randomfile2', 'r+')
+  await checkFile(child2FileName)
+  await checkFile(child1FileName)
+  await checkFile(parentFileName)
+  await vhdMerge(handler, parentFileName, handler, child1FileName)
+  await checkFile(parentFileName)
+  await chainVhd(handler, parentFileName, handler, child2FileName, true)
+  await checkFile(child2FileName)
+  await vhdMerge(handler, parentFileName, handler, child2FileName)
+  await checkFile(parentFileName)
+  await recoverRawContent(parentFileName, recoveredFileName, originalSize)
+  await execa('cp', [randomFileName, random2FileName])
+  const fd = await fs.open(random2FileName, 'r+')
   try {
     await fs.write(fd, smallRandom, 0, smallRandom.length, 5 * SECTOR_SIZE)
   } finally {
     await fs.close(fd)
   }
-  expect(await fs.readFile('recovered_from_coalescing')).toEqual(
-    await fs.readFile('randomfile2')
+  expect(await fs.readFile(recoveredFileName)).toEqual(
+    await fs.readFile(random2FileName)
   )
 })
 
-test('createSyntheticStream passes vhd-util check', async () => {
+test.only('createSyntheticStream passes vhd-util check', async () => {
   const initalSize = 4
-  const expectedVhdSize = 4197888
-  await createRandomFile('randomfile', initalSize)
-  await convertFromRawToVhd('randomfile', 'randomfile.vhd')
-  const handler = getHandler({ url: 'file://' + process.cwd() })
-  const stream = await createSyntheticStream(handler, 'randomfile.vhd')
-  expect(stream.length).toEqual(expectedVhdSize)
-  await fromEvent(
-    stream.pipe(await fs.createWriteStream('recovered.vhd')),
-    'finish'
+  const rawFileName = `${tempDir}/randomfile`
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  const recoveredVhdFileName = `${tempDir}/recovered.vhd`
+  await createRandomFile(rawFileName, initalSize)
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  await checkFile(vhdFileName)
+  const handler = getHandler({ url: 'file://' })
+  const stream = await createSyntheticStream(handler, vhdFileName)
+  const expectedVhdSize = (await fs.stat(vhdFileName)).size
+  expect(stream.length).toEqual((await fs.stat(vhdFileName)).size)
+  await pFromCallback(cb =>
+    pipeline(stream, fs.createWriteStream(recoveredVhdFileName), cb)
   )
-  await checkFile('recovered.vhd')
-  const stats = await fs.stat('recovered.vhd')
+  await checkFile(recoveredVhdFileName)
+  const stats = await fs.stat(recoveredVhdFileName)
   expect(stats.size).toEqual(expectedVhdSize)
-  await execa('qemu-img', ['compare', 'recovered.vhd', 'randomfile'])
+  await execa('qemu-img', ['compare', recoveredVhdFileName, rawFileName])
 })

packages/vhd-lib/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vhd-lib",
-  "version": "0.5.1",
+  "version": "0.6.0",
   "license": "AGPL-3.0",
   "description": "Primitives for VHD file handling",
   "keywords": [],
@@ -35,13 +35,14 @@
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
     "@babel/preset-flow": "^7.0.0",
-    "@xen-orchestra/fs": "^0.7.1",
+    "@xen-orchestra/fs": "^0.8.0",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^5.1.3",
     "execa": "^1.0.0",
     "fs-promise": "^2.0.0",
     "get-stream": "^4.0.0",
     "index-modules": "^0.3.0",
+    "readable-stream": "^3.0.6",
     "rimraf": "^2.6.2",
     "tmp": "^0.0.33"
   },

packages/vhd-lib/src/_checkFooter.js

@@ -0,0 +1,20 @@
+import assert from 'assert'
+
+import {
+  DISK_TYPE_DIFFERENCING,
+  DISK_TYPE_DYNAMIC,
+  FILE_FORMAT_VERSION,
+  FOOTER_COOKIE,
+  FOOTER_SIZE,
+} from './_constants'
+
+export default footer => {
+  assert.strictEqual(footer.cookie, FOOTER_COOKIE)
+  assert.strictEqual(footer.dataOffset, FOOTER_SIZE)
+  assert.strictEqual(footer.fileFormatVersion, FILE_FORMAT_VERSION)
+  assert(footer.originalSize <= footer.currentSize)
+  assert(
+    footer.diskType === DISK_TYPE_DIFFERENCING ||
+      footer.diskType === DISK_TYPE_DYNAMIC
+  )
+}

packages/vhd-lib/src/_checkHeader.js

@@ -0,0 +1,14 @@
+import assert from 'assert'
+
+import { HEADER_COOKIE, HEADER_VERSION, SECTOR_SIZE } from './_constants'
+
+export default (header, footer) => {
+  assert.strictEqual(header.cookie, HEADER_COOKIE)
+  assert.strictEqual(header.dataOffset, undefined)
+  assert.strictEqual(header.headerVersion, HEADER_VERSION)
+  assert(Number.isInteger(Math.log2(header.blockSize / SECTOR_SIZE)))
+
+  if (footer !== undefined) {
+    assert(header.maxTableEntries >= footer.currentSize / header.blockSize)
+  }
+}

packages/vhd-lib/src/_getFirstAndLastBlocks.js

@@ -0,0 +1,47 @@
+import assert from 'assert'
+
+import { BLOCK_UNUSED } from './_constants'
+
+// get the identifiers and first sectors of the first and last block
+// in the file
+export default bat => {
+  const n = bat.length
+  assert.notStrictEqual(n, 0)
+  assert.strictEqual(n % 4, 0)
+
+  let i = 0
+  let j = 0
+  let first, firstSector, last, lastSector
+
+  // get first allocated block for initialization
+  while ((firstSector = bat.readUInt32BE(j)) === BLOCK_UNUSED) {
+    i += 1
+    j += 4
+
+    if (j === n) {
+      const error = new Error('no allocated block found')
+      error.noBlock = true
+      throw error
+    }
+  }
+  lastSector = firstSector
+  first = last = i
+
+  while (j < n) {
+    const sector = bat.readUInt32BE(j)
+    if (sector !== BLOCK_UNUSED) {
+      if (sector < firstSector) {
+        first = i
+        firstSector = sector
+      } else if (sector > lastSector) {
+        last = i
+        lastSector = sector
+      }
+    }
+
+    i += 1
+    j += 4
+  }
+
+  return { first, firstSector, last, lastSector }
+}

packages/vhd-lib/src/_readChunk.js

@@ -0,0 +1,50 @@
+export default async function readChunk(stream, n) {
+  if (n === 0) {
+    return Buffer.alloc(0)
+  }
+  return new Promise((resolve, reject) => {
+    const chunks = []
+    let i = 0
+
+    function clean() {
+      stream.removeListener('readable', onReadable)
+      stream.removeListener('end', onEnd)
+      stream.removeListener('error', onError)
+    }
+
+    function resolve2() {
+      clean()
+      resolve(Buffer.concat(chunks, i))
+    }
+
+    function onEnd() {
+      resolve2()
+      clean()
+    }
+
+    function onError(error) {
+      reject(error)
+      clean()
+    }
+
+    function onReadable() {
+      const chunk = stream.read(n - i)
+      if (chunk === null) {
+        return // wait for more data
+      }
+      i += chunk.length
+      chunks.push(chunk)
+      if (i >= n) {
+        resolve2()
+      }
+    }
+
+    stream.on('end', onEnd)
+    stream.on('error', onError)
+    stream.on('readable', onReadable)
+
+    if (stream.readable) {
+      onReadable()
+    }
+  })
+}

packages/vhd-lib/src/createVhdStreamWithLength.integ.spec.js

@@ -0,0 +1,93 @@
+/* eslint-env jest */
+
+import asyncIteratorToStream from 'async-iterator-to-stream'
+import execa from 'execa'
+import fs from 'fs-extra'
+import rimraf from 'rimraf'
+import getStream from 'get-stream'
+import tmp from 'tmp'
+import { createReadStream, createWriteStream } from 'fs'
+import { pFromCallback } from 'promise-toolbox'
+import { pipeline } from 'readable-stream'
+
+import { createVhdStreamWithLength } from '.'
+import { FOOTER_SIZE } from './_constants'
+
+let tempDir = null
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+async function convertFromRawToVhd(rawName, vhdName) {
+  await execa('qemu-img', ['convert', '-f', 'raw', '-Ovpc', rawName, vhdName])
+}
+
+async function createRandomFile(name, size) {
+  const createRandomStream = asyncIteratorToStream(function*(size) {
+    while (size-- > 0) {
+      yield Buffer.from([Math.floor(Math.random() * 256)])
+    }
+  })
+  const input = await createRandomStream(size)
+  await pFromCallback(cb => pipeline(input, fs.createWriteStream(name), cb))
+}
+
+test('createVhdStreamWithLength can extract length', async () => {
+  const initialSize = 4 * 1024
+  const rawFileName = `${tempDir}/randomfile`
+  const vhdName = `${tempDir}/randomfile.vhd`
+  const outputVhdName = `${tempDir}/output.vhd`
+  await createRandomFile(rawFileName, initialSize)
+  await convertFromRawToVhd(rawFileName, vhdName)
+  const vhdSize = fs.statSync(vhdName).size
+  const result = await createVhdStreamWithLength(
+    await createReadStream(vhdName)
+  )
+  expect(result.length).toEqual(vhdSize)
+  const outputFileStream = await createWriteStream(outputVhdName)
+  await pFromCallback(cb => pipeline(result, outputFileStream, cb))
+  const outputSize = fs.statSync(outputVhdName).size
+  expect(outputSize).toEqual(vhdSize)
+})
+
+test('createVhdStreamWithLength can skip blank after last block and before footer', async () => {
+  const initialSize = 4 * 1024
+  const rawFileName = `${tempDir}/randomfile`
+  const vhdName = `${tempDir}/randomfile.vhd`
+  const outputVhdName = `${tempDir}/output.vhd`
+  await createRandomFile(rawFileName, initialSize)
+  await convertFromRawToVhd(rawFileName, vhdName)
+  const vhdSize = fs.statSync(vhdName).size
+  // read file footer
+  const footer = await getStream.buffer(
+    createReadStream(vhdName, { start: vhdSize - FOOTER_SIZE })
+  )
+
+  // we'll override the footer
+  const endOfFile = await createWriteStream(vhdName, {
+    flags: 'r+',
+    start: vhdSize - FOOTER_SIZE,
+  })
+  // write a blank over the previous footer
+  await pFromCallback(cb => endOfFile.write(Buffer.alloc(FOOTER_SIZE), cb))
+  // write the footer after the new blank
+  await pFromCallback(cb => endOfFile.end(footer, cb))
+  const longerSize = fs.statSync(vhdName).size
+  // check input file has been lengthened
+  expect(longerSize).toEqual(vhdSize + FOOTER_SIZE)
+  const result = await createVhdStreamWithLength(
+    await createReadStream(vhdName)
+  )
+  expect(result.length).toEqual(vhdSize)
+  const outputFileStream = await createWriteStream(outputVhdName)
+  await pFromCallback(cb => pipeline(result, outputFileStream, cb))
+  const outputSize = fs.statSync(outputVhdName).size
+  // check out file has been shortened again
+  expect(outputSize).toEqual(vhdSize)
+  await execa('qemu-img', ['compare', outputVhdName, vhdName])
+})

packages/vhd-lib/src/createVhdStreamWithLength.js

@@ -0,0 +1,80 @@
+import assert from 'assert'
+import { pipeline, Transform } from 'readable-stream'
+
+import checkFooter from './_checkFooter'
+import checkHeader from './_checkHeader'
+import noop from './_noop'
+import getFirstAndLastBlocks from './_getFirstAndLastBlocks'
+import readChunk from './_readChunk'
+import { FOOTER_SIZE, HEADER_SIZE, SECTOR_SIZE } from './_constants'
+import { fuFooter, fuHeader } from './_structs'
+
+class EndCutterStream extends Transform {
+  constructor(footerOffset, footerBuffer) {
+    super()
+    this._footerOffset = footerOffset
+    this._footerBuffer = footerBuffer
+    this._position = 0
+    this._done = false
+  }
+
+  _transform(data, encoding, callback) {
+    if (!this._done) {
+      if (this._position + data.length >= this._footerOffset) {
+        this._done = true
+        const difference = this._footerOffset - this._position
+        data = data.slice(0, difference)
+        this.push(data)
+        this.push(this._footerBuffer)
+      } else {
+        this.push(data)
+      }
+      this._position += data.length
+    }
+    callback()
+  }
+}
+
+export default async function createVhdStreamWithLength(stream) {
+  const readBuffers = []
+  let streamPosition = 0
+
+  async function readStream(length) {
+    const chunk = await readChunk(stream, length)
+    assert.strictEqual(chunk.length, length)
+    streamPosition += chunk.length
+    readBuffers.push(chunk)
+    return chunk
+  }
+
+  const footerBuffer = await readStream(FOOTER_SIZE)
+  const footer = fuFooter.unpack(footerBuffer)
+  checkFooter(footer)
+
+  const header = fuHeader.unpack(await readStream(HEADER_SIZE))
+  checkHeader(header, footer)
+
+  await readStream(header.tableOffset - streamPosition)
+
+  const table = await readStream(header.maxTableEntries * 4)
+
+  readBuffers.reverse()
+  for (const buf of readBuffers) {
+    stream.unshift(buf)
+  }
+
+  const footerOffset =
+    getFirstAndLastBlocks(table).lastSector * SECTOR_SIZE +
+    Math.ceil(header.blockSize / SECTOR_SIZE / 8 / SECTOR_SIZE) * SECTOR_SIZE +
+    header.blockSize
+
+  // ignore any data after footerOffset and push footerBuffer
+  //
+  // this is necessary to ignore any blank space between the last block and the
+  // final footer which would invalidate the size we computed
+  const newStream = new EndCutterStream(footerOffset, footerBuffer)
+  pipeline(stream, newStream, noop)
+
+  newStream.length = footerOffset + FOOTER_SIZE
+  return newStream
+}

packages/vhd-lib/src/index.js

@@ -11,3 +11,6 @@ export {
 } from './createReadableSparseStream'
 export { default as createSyntheticStream } from './createSyntheticStream'
 export { default as mergeVhd } from './merge'
+export {
+  default as createVhdStreamWithLength,
+} from './createVhdStreamWithLength'

packages/vhd-lib/src/vhd.js

@@ -1,19 +1,16 @@
 import assert from 'assert'
 import { fromEvent } from 'promise-toolbox'
 
+import checkFooter from './_checkFooter'
+import checkHeader from './_checkHeader'
 import constantStream from './_constant-stream'
+import getFirstAndLastBlocks from './_getFirstAndLastBlocks'
 import { fuFooter, fuHeader, checksumStruct, unpackField } from './_structs'
 import { set as mapSetBit, test as mapTestBit } from './_bitmap'
 import {
   BLOCK_UNUSED,
-  DISK_TYPE_DIFFERENCING,
-  DISK_TYPE_DYNAMIC,
-  FILE_FORMAT_VERSION,
-  FOOTER_COOKIE,
   FOOTER_SIZE,
-  HEADER_COOKIE,
   HEADER_SIZE,
-  HEADER_VERSION,
   PARENT_LOCATOR_ENTRIES,
   PLATFORM_NONE,
   PLATFORM_W2KU,
@@ -170,21 +167,10 @@ export default class Vhd {
     }
 
     const footer = (this.footer = fuFooter.unpack(bufFooter))
-    assert.strictEqual(footer.cookie, FOOTER_COOKIE, 'footer cookie')
-    assert.strictEqual(footer.dataOffset, FOOTER_SIZE)
-    assert.strictEqual(footer.fileFormatVersion, FILE_FORMAT_VERSION)
-    assert(footer.originalSize <= footer.currentSize)
-    assert(
-      footer.diskType === DISK_TYPE_DIFFERENCING ||
-        footer.diskType === DISK_TYPE_DYNAMIC
-    )
+    checkFooter(footer)
 
     const header = (this.header = fuHeader.unpack(bufHeader))
-    assert.strictEqual(header.cookie, HEADER_COOKIE)
-    assert.strictEqual(header.dataOffset, undefined)
-    assert.strictEqual(header.headerVersion, HEADER_VERSION)
-    assert(header.maxTableEntries >= footer.currentSize / header.blockSize)
-    assert(Number.isInteger(Math.log2(header.blockSize / SECTOR_SIZE)))
+    checkHeader(header, footer)
 
     // Compute the number of sectors in one block.
     // Default: One block contains 4096 sectors of 512 bytes.
@@ -242,49 +228,6 @@ export default class Vhd {
     )
   }
 
-  // get the identifiers and first sectors of the first and last block
-  // in the file
-  //
-  _getFirstAndLastBlocks() {
-    const n = this.header.maxTableEntries
-    const bat = this.blockTable
-    let i = 0
-    let j = 0
-    let first, firstSector, last, lastSector
-
-    // get first allocated block for initialization
-    while ((firstSector = bat.readUInt32BE(j)) === BLOCK_UNUSED) {
-      i += 1
-      j += 4
-
-      if (i === n) {
-        const error = new Error('no allocated block found')
-        error.noBlock = true
-        throw error
-      }
-    }
-    lastSector = firstSector
-    first = last = i
-
-    while (i < n) {
-      const sector = bat.readUInt32BE(j)
-      if (sector !== BLOCK_UNUSED) {
-        if (sector < firstSector) {
-          first = i
-          firstSector = sector
-        } else if (sector > lastSector) {
-          last = i
-          lastSector = sector
-        }
-      }
-
-      i += 1
-      j += 4
-    }
-
-    return { first, firstSector, last, lastSector }
-  }
-
   // =================================================================
   // Write functions.
   // =================================================================
@@ -311,7 +254,9 @@ export default class Vhd {
 
   async _freeFirstBlockSpace(spaceNeededBytes) {
     try {
-      const { first, firstSector, lastSector } = this._getFirstAndLastBlocks()
+      const { first, firstSector, lastSector } = getFirstAndLastBlocks(
+        this.blockTable
+      )
       const tableOffset = this.header.tableOffset
       const { batSize } = this
       const newMinSector = Math.ceil(

packages/vhd-lib/vhd.integ.spec.js

@@ -4,22 +4,20 @@ import rimraf from 'rimraf'
 import tmp from 'tmp'
 import { createWriteStream, readFile } from 'fs-promise'
 import { fromEvent, pFromCallback } from 'promise-toolbox'
+import { pipeline } from 'readable-stream'
 
 import { createReadableRawStream, createReadableSparseStream } from './'
 
 import { createFooter } from './src/_createFooterHeader'
 
-const initialDir = process.cwd()
+let tempDir = null
 
 beforeEach(async () => {
-  const dir = await pFromCallback(cb => tmp.dir(cb))
-  process.chdir(dir)
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
 })
 
 afterEach(async () => {
-  const tmpDir = process.cwd()
-  process.chdir(initialDir)
-  await pFromCallback(cb => rimraf(tmpDir, cb))
+  await pFromCallback(cb => rimraf(tempDir, cb))
 })
 
 test('createFooter() does not crash', () => {
@@ -55,9 +53,10 @@ test('ReadableRawVHDStream does not crash', async () => {
   }
   const fileSize = 1000
   const stream = createReadableRawStream(fileSize, mockParser)
-  const pipe = stream.pipe(createWriteStream('output.vhd'))
-  await fromEvent(pipe, 'finish')
-  await execa('vhd-util', ['check', '-t', '-i', '-n', 'output.vhd'])
+  await pFromCallback(cb =>
+    pipeline(stream, createWriteStream(`${tempDir}/output.vhd`), cb)
+  )
+  await execa('vhd-util', ['check', '-t', '-i', '-n', `${tempDir}/output.vhd`])
 })
 
 test('ReadableRawVHDStream detects when blocks are out of order', async () => {
@@ -87,9 +86,9 @@ test('ReadableRawVHDStream detects when blocks are out of order', async () => {
     new Promise((resolve, reject) => {
       const stream = createReadableRawStream(100000, mockParser)
       stream.on('error', reject)
-      const pipe = stream.pipe(createWriteStream('outputStream'))
-      pipe.on('finish', resolve)
-      pipe.on('error', reject)
+      pipeline(stream, createWriteStream(`${tempDir}/outputStream`), err =>
+        err ? reject(err) : resolve()
+      )
     })
   ).rejects.toThrow('Received out of order blocks')
 })
@@ -114,19 +113,19 @@ test('ReadableSparseVHDStream can handle a sparse file', async () => {
     blocks
   )
   expect(stream.length).toEqual(4197888)
-  const pipe = stream.pipe(createWriteStream('output.vhd'))
+  const pipe = stream.pipe(createWriteStream(`${tempDir}/output.vhd`))
   await fromEvent(pipe, 'finish')
-  await execa('vhd-util', ['check', '-t', '-i', '-n', 'output.vhd'])
+  await execa('vhd-util', ['check', '-t', '-i', '-n', `${tempDir}/output.vhd`])
   await execa('qemu-img', [
     'convert',
     '-f',
     'vpc',
     '-O',
     'raw',
-    'output.vhd',
-    'out1.raw',
+    `${tempDir}/output.vhd`,
+    `${tempDir}/out1.raw`,
   ])
-  const out1 = await readFile('out1.raw')
+  const out1 = await readFile(`${tempDir}/out1.raw`)
   const expected = Buffer.alloc(fileSize)
   blocks.forEach(b => {
     b.data.copy(expected, b.offsetBytes)

packages/xapi-explore-sr/package.json

@@ -41,7 +41,7 @@
     "human-format": "^0.10.0",
     "lodash": "^4.17.4",
     "pw": "^0.0.4",
-    "xen-api": "^0.24.5"
+    "xen-api": "^0.25.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.1.5",

packages/xen-api/examples/utils.js

@@ -4,7 +4,7 @@ const { PassThrough, pipeline } = require('readable-stream')
 const humanFormat = require('human-format')
 const Throttle = require('throttle')
 
-const { isOpaqueRef } = require('../')
+const isOpaqueRef = require('../dist/_isOpaqueRef').default
 
 exports.createInputStream = path => {
   if (path === undefined || path === '-') {

packages/xen-api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xen-api",
-  "version": "0.24.5",
+  "version": "0.25.0",
   "license": "ISC",
   "description": "Connector to the Xen API",
   "keywords": [
@@ -38,7 +38,6 @@
     "event-to-promise": "^0.8.0",
     "exec-promise": "^0.7.0",
     "http-request-plus": "^0.8.0",
-    "iterable-backoff": "^0.1.0",
     "jest-diff": "^24.0.0",
     "json-rpc-protocol": "^0.13.1",
     "kindof": "^2.0.0",
@@ -54,7 +53,10 @@
   "devDependencies": {
     "@babel/cli": "^7.0.0",
     "@babel/core": "^7.0.0",
+    "@babel/plugin-proposal-class-properties": "^7.3.4",
     "@babel/plugin-proposal-decorators": "^7.0.0",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.2.0",
+    "@babel/plugin-proposal-optional-chaining": "^7.2.0",
     "@babel/preset-env": "^7.0.0",
     "babel-plugin-lodash": "^3.3.2",
     "cross-env": "^5.1.3",

packages/xen-api/src/_XapiError.js

@@ -0,0 +1,30 @@
+import { BaseError } from 'make-error'
+
+export default class XapiError extends BaseError {
+  static wrap(error) {
+    let code, params
+    if (Array.isArray(error)) {
+      // < XenServer 7.3
+      ;[code, ...params] = error
+    } else {
+      code = error.message
+      params = error.data
+      if (!Array.isArray(params)) {
+        params = []
+      }
+    }
+    return new XapiError(code, params)
+  }
+
+  constructor(code, params) {
+    super(`${code}(${params.join(', ')})`)
+
+    this.code = code
+    this.params = params
+
+    // slots than can be assigned later
+    this.call = undefined
+    this.url = undefined
+    this.task = undefined
+  }
+}

packages/xen-api/src/_coalesceCalls.js

@@ -0,0 +1,15 @@
+// decorates fn so that more than one concurrent calls will be coalesced
+export default function coalesceCalls(fn) {
+  let promise
+  const clean = () => {
+    promise = undefined
+  }
+  return function() {
+    if (promise !== undefined) {
+      return promise
+    }
+    promise = fn.apply(this, arguments)
+    promise.then(clean, clean)
+    return promise
+  }
+}

packages/xen-api/src/_coalesceCalls.spec.js

@@ -0,0 +1,26 @@
+/* eslint-env jest */
+
+import pDefer from 'promise-toolbox/defer'
+
+import coalesceCalls from './_coalesceCalls'
+
+describe('coalesceCalls', () => {
+  it('decorates an async function', async () => {
+    const fn = coalesceCalls(promise => promise)
+
+    const defer1 = pDefer()
+    const promise1 = fn(defer1.promise)
+    const defer2 = pDefer()
+    const promise2 = fn(defer2.promise)
+
+    defer1.resolve('foo')
+    expect(await promise1).toBe('foo')
+    expect(await promise2).toBe('foo')
+
+    const defer3 = pDefer()
+    const promise3 = fn(defer3.promise)
+
+    defer3.resolve('bar')
+    expect(await promise3).toBe('bar')
+  })
+})

packages/xen-api/src/_debug.js

@@ -0,0 +1,3 @@
+import debug from 'debug'
+
+export default debug('xen-api')

packages/xen-api/src/_getTaskResult.js

@@ -0,0 +1,22 @@
+import { Cancel } from 'promise-toolbox'
+
+import XapiError from './_XapiError'
+
+export default task => {
+  const { status } = task
+  if (status === 'cancelled') {
+    return Promise.reject(new Cancel('task canceled'))
+  }
+  if (status === 'failure') {
+    const error = XapiError.wrap(task.error_info)
+    error.task = task
+    return Promise.reject(error)
+  }
+  if (status === 'success') {
+    // the result might be:
+    // - empty string
+    // - an opaque reference
+    // - an XML-RPC value
+    return Promise.resolve(task.result)
+  }
+}

packages/xen-api/src/_isGetAllRecordsMethod.js

@@ -0,0 +1,3 @@
+const SUFFIX = '.get_all_records'
+
+export default method => method.endsWith(SUFFIX)

packages/xen-api/src/_isOpaqueRef.js

@@ -0,0 +1,3 @@
+const PREFIX = 'OpaqueRef:'
+
+export default value => typeof value === 'string' && value.startsWith(PREFIX)

packages/xen-api/src/_isReadOnlyCall.js

@@ -0,0 +1,4 @@
+const RE = /^[^.]+\.get_/
+
+export default (method, args) =>
+  args.length === 1 && typeof args[0] === 'string' && RE.test(method)

packages/xen-api/src/_makeCallSetting.js

@@ -0,0 +1,8 @@
+export default (setting, defaultValue) =>
+  setting === undefined
+    ? () => defaultValue
+    : typeof setting === 'function'
+    ? setting
+    : typeof setting === 'object'
+    ? method => setting[method] ?? setting['*'] ?? defaultValue
+    : () => setting

packages/xen-api/src/_parseUrl.js

@@ -0,0 +1,18 @@
+const URL_RE = /^(?:(https?:)\/*)?(?:([^:]+):([^@]+)@)?([^/]+?)(?::([0-9]+))?\/?$/
+
+export default url => {
+  const matches = URL_RE.exec(url)
+  if (matches === null) {
+    throw new Error('invalid URL: ' + url)
+  }
+
+  const [, protocol = 'https:', username, password, hostname, port] = matches
+  const parsedUrl = { protocol, hostname, port }
+  if (username !== undefined) {
+    parsedUrl.username = decodeURIComponent(username)
+  }
+  if (password !== undefined) {
+    parsedUrl.password = decodeURIComponent(password)
+  }
+  return parsedUrl
+}

packages/xen-api/src/transports/_UnsupportedTransport.js

@@ -0,0 +1,3 @@
+import makeError from 'make-error'
+
+export default makeError('UnsupportedTransport')

packages/xen-api/src/transports/_prepareXmlRpcParams.js

@@ -0,0 +1,25 @@
+// Prepare values before passing them to the XenAPI:
+//
+// - cast integers to strings
+export default function prepare(param) {
+  if (Number.isInteger(param)) {
+    return String(param)
+  }
+
+  if (typeof param !== 'object' || param === null) {
+    return param
+  }
+
+  if (Array.isArray(param)) {
+    return param.map(prepare)
+  }
+
+  const values = {}
+  Object.keys(param).forEach(key => {
+    const value = param[key]
+    if (value !== undefined) {
+      values[key] = prepare(value)
+    }
+  })
+  return values
+}

packages/xen-api/src/transports/_utils.js

@@ -1,3 +0,0 @@
-import makeError from 'make-error'
-
-export const UnsupportedTransport = makeError('UnsupportedTransport')

packages/xen-api/src/transports/auto.js

@@ -1,7 +1,7 @@
 import jsonRpc from './json-rpc'
+import UnsupportedTransport from './_UnsupportedTransport'
 import xmlRpc from './xml-rpc'
 import xmlRpcJson from './xml-rpc-json'
-import { UnsupportedTransport } from './_utils'
 
 const factories = [jsonRpc, xmlRpcJson, xmlRpc]
 const { length } = factories

packages/xen-api/src/transports/json-rpc.js

@@ -1,7 +1,7 @@
 import httpRequestPlus from 'http-request-plus'
 import { format, parse } from 'json-rpc-protocol'
 
-import { UnsupportedTransport } from './_utils'
+import UnsupportedTransport from './_UnsupportedTransport'
 
 // https://github.com/xenserver/xenadmin/blob/0df39a9d83cd82713f32d24704852a0fd57b8a64/XenModel/XenAPI/Session.cs#L403-L433
 export default ({ allowUnauthorized, url }) => {

packages/xen-api/src/transports/xml-rpc-json.js

@@ -1,7 +1,8 @@
 import { createClient, createSecureClient } from 'xmlrpc'
 import { promisify } from 'promise-toolbox'
 
-import { UnsupportedTransport } from './_utils'
+import prepareXmlRpcParams from './_prepareXmlRpcParams'
+import UnsupportedTransport from './_UnsupportedTransport'
 
 const logError = error => {
   if (error.res) {
@@ -71,10 +72,7 @@ const parseResult = result => {
   throw new UnsupportedTransport()
 }
 
-export default ({
-  allowUnauthorized,
-  url: { hostname, path, port, protocol },
-}) => {
+export default ({ allowUnauthorized, url: { hostname, port, protocol } }) => {
   const client = (protocol === 'https:' ? createSecureClient : createClient)({
     host: hostname,
     path: '/json',
@@ -83,5 +81,6 @@ export default ({
   })
   const call = promisify(client.methodCall, client)
 
-  return (method, args) => call(method, args).then(parseResult, logError)
+  return (method, args) =>
+    call(method, prepareXmlRpcParams(args)).then(parseResult, logError)
 }

packages/xen-api/src/transports/xml-rpc.js

@@ -1,6 +1,8 @@
 import { createClient, createSecureClient } from 'xmlrpc'
 import { promisify } from 'promise-toolbox'
 
+import prepareXmlRpcParams from './_prepareXmlRpcParams'
+
 const logError = error => {
   if (error.res) {
     console.error(
@@ -30,10 +32,7 @@ const parseResult = result => {
   return result.Value
 }
 
-export default ({
-  allowUnauthorized,
-  url: { hostname, path, port, protocol },
-}) => {
+export default ({ allowUnauthorized, url: { hostname, port, protocol } }) => {
   const client = (protocol === 'https:' ? createSecureClient : createClient)({
     host: hostname,
     port,
@@ -41,5 +40,6 @@ export default ({
   })
   const call = promisify(client.methodCall, client)
 
-  return (method, args) => call(method, args).then(parseResult, logError)
+  return (method, args) =>
+    call(method, prepareXmlRpcParams(args)).then(parseResult, logError)
 }

packages/xo-collection/src/collection.js

@@ -1,7 +1,7 @@
 import kindOf from 'kindof'
 import { BaseError } from 'make-error'
 import { EventEmitter } from 'events'
-import { forEach } from 'lodash'
+import { forOwn } from 'lodash'
 
 import isEmpty from './is-empty'
 import isObject from './is-object'
@@ -10,6 +10,7 @@ import isObject from './is-object'
 
 const {
   create: createObject,
+  keys,
   prototype: { hasOwnProperty },
 } = Object
 
@@ -63,6 +64,16 @@ export class NoSuchItem extends BaseError {
 
 // -------------------------------------------------------------------
 
+const assertValidKey = key => {
+  if (!isValidKey(key)) {
+    throw new InvalidKey(key)
+  }
+}
+
+const isValidKey = key => typeof key === 'number' || typeof key === 'string'
+
+// -------------------------------------------------------------------
+
 export default class Collection extends EventEmitter {
   constructor() {
     super()
@@ -71,7 +82,7 @@ export default class Collection extends EventEmitter {
     this._buffering = 0
     this._indexes = createObject(null)
     this._indexedItems = createObject(null)
-    this._items = {} // createObject(null)
+    this._items = createObject(null)
     this._size = 0
   }
 
@@ -113,7 +124,7 @@ export default class Collection extends EventEmitter {
   }
 
   clear() {
-    forEach(this._items, (_, key) => this._remove(key))
+    keys(this._items).forEach(this._remove, this)
   }
 
   remove(keyOrObjectWithId) {
@@ -176,8 +187,7 @@ export default class Collection extends EventEmitter {
       return defaultValue
     }
 
-    // Throws a NoSuchItem.
-    this._assertHas(key)
+    throw new NoSuchItem(key)
   }
 
   has(key) {
@@ -189,7 +199,7 @@ export default class Collection extends EventEmitter {
   // -----------------------------------------------------------------
 
   createIndex(name, index) {
-    const { _indexes: indexes } = this
+    const indexes = this._indexes
     if (hasOwnProperty.call(indexes, name)) {
       throw new DuplicateIndex(name)
     }
@@ -201,7 +211,7 @@ export default class Collection extends EventEmitter {
   }
 
   deleteIndex(name) {
-    const { _indexes: indexes } = this
+    const indexes = this._indexes
     if (!hasOwnProperty.call(indexes, name)) {
       throw new NoSuchIndex(name)
     }
@@ -218,7 +228,7 @@ export default class Collection extends EventEmitter {
   // -----------------------------------------------------------------
 
   *[Symbol.iterator]() {
-    const { _items: items } = this
+    const items = this._items
 
     for (const key in items) {
       yield [key, items[key]]
@@ -226,7 +236,7 @@ export default class Collection extends EventEmitter {
   }
 
   *keys() {
-    const { _items: items } = this
+    const items = this._items
 
     for (const key in items) {
       yield key
@@ -234,7 +244,7 @@ export default class Collection extends EventEmitter {
   }
 
   *values() {
-    const { _items: items } = this
+    const items = this._items
 
     for (const key in items) {
       yield items[key]
@@ -255,11 +265,11 @@ export default class Collection extends EventEmitter {
       }
       called = true
 
-      if (--this._buffering) {
+      if (--this._buffering !== 0) {
         return
       }
 
-      const { _buffer: buffer } = this
+      const buffer = this._buffer
 
       // Due to deduplication there could be nothing in the buffer.
       if (isEmpty(buffer)) {
@@ -276,7 +286,7 @@ export default class Collection extends EventEmitter {
         data[buffer[key]][key] = this._items[key]
       }
 
-      forEach(data, (items, action) => {
+      forOwn(data, (items, action) => {
         if (!isEmpty(items)) {
           this.emit(action, items)
         }
@@ -306,16 +316,6 @@ export default class Collection extends EventEmitter {
     }
   }
 
-  _assertValidKey(key) {
-    if (!this._isValidKey(key)) {
-      throw new InvalidKey(key)
-    }
-  }
-
-  _isValidKey(key) {
-    return typeof key === 'number' || typeof key === 'string'
-  }
-
   _remove(key) {
     delete this._items[key]
     this._size--
@@ -324,17 +324,17 @@ export default class Collection extends EventEmitter {
 
   _resolveItem(keyOrObjectWithId, valueIfKey = undefined) {
     if (valueIfKey !== undefined) {
-      this._assertValidKey(keyOrObjectWithId)
+      assertValidKey(keyOrObjectWithId)
 
       return [keyOrObjectWithId, valueIfKey]
     }
 
-    if (this._isValidKey(keyOrObjectWithId)) {
+    if (isValidKey(keyOrObjectWithId)) {
       return [keyOrObjectWithId]
     }
 
     const key = this.getKey(keyOrObjectWithId)
-    this._assertValidKey(key)
+    assertValidKey(key)
 
     return [key, keyOrObjectWithId]
   }
@@ -347,7 +347,7 @@ export default class Collection extends EventEmitter {
     }
 
     if (action === ACTION_ADD) {
-      this._buffer[key] = this._buffer[key] ? ACTION_UPDATE : ACTION_ADD
+      this._buffer[key] = key in this._buffer ? ACTION_UPDATE : ACTION_ADD
     } else if (action === ACTION_REMOVE) {
       if (this._buffer[key] === ACTION_ADD) {
         delete this._buffer[key]
@@ -356,7 +356,7 @@ export default class Collection extends EventEmitter {
       }
     } else {
       // update
-      if (!this._buffer[key]) {
+      if (!(key in this._buffer)) {
         this._buffer[key] = ACTION_UPDATE
       }
     }

packages/xo-server-usage-report/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-server-usage-report",
-  "version": "0.7.1",
+  "version": "0.7.2",
   "license": "AGPL-3.0",
   "description": "",
   "keywords": [

packages/xo-server-usage-report/src/index.js

@@ -494,7 +494,7 @@ async function getHostsMissingPatches({ runningHosts, xo }) {
     map(runningHosts, async host => {
       let hostsPatches = await xo
         .getXapi(host)
-        .listMissingPoolPatchesOnHost(host._xapiId)
+        .listMissingPatches(host._xapiId)
         .catch(error => {
           console.error(
             '[WARN] error on fetching hosts missing patches:',

packages/xo-server/config.toml

@@ -9,6 +9,18 @@ datadir = '/var/lib/xo-server/data'
 # Necessary for external authentication providers.
 createUserOnFirstSignin = true
 
+# XAPI does not support chunked encoding in HTTP requests which is necessary
+# when the content length is not know which is the case for many backup related
+# operations in XO.
+#
+# It's possible to work-around this for VHDs  because it's possible to guess
+# their size just by looking at the beginning of the stream.
+#
+# But it is a guess, not a certainty, it depends on how the VHDs are formatted
+# by XenServer, therefore it's disabled for the moment but can be enabled
+# specifically for a user if necessary.
+guessVhdSizeOnImport = false
+
 # Whether API logs should contains the full request/response on
 # errors.
 #
@@ -33,6 +45,10 @@ maxTokenValidity = '0.5 year'
 # https://developer.mozilla.org/fr/docs/Web/HTTP/Headers/Set-Cookie#Session_cookie
 #sessionCookieValidity = '10 hours'
 
+[backup]
+# Delay for which backups listing on a remote is cached
+listingDebounce = '1 min'
+
 [[http.listen]]
 port = 80
 

packages/xo-server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "xo-server",
-  "version": "5.37.0",
+  "version": "5.38.1",
   "license": "AGPL-3.0",
   "description": "Server part of Xen-Orchestra",
   "keywords": [
@@ -37,7 +37,7 @@
     "@xen-orchestra/cron": "^1.0.3",
     "@xen-orchestra/defined": "^0.0.0",
     "@xen-orchestra/emit-async": "^0.0.0",
-    "@xen-orchestra/fs": "^0.7.1",
+    "@xen-orchestra/fs": "^0.8.0",
     "@xen-orchestra/log": "^0.1.4",
     "@xen-orchestra/mixin": "^0.0.0",
     "ajv": "^6.1.1",
@@ -120,9 +120,9 @@
     "tmp": "^0.0.33",
     "uuid": "^3.0.1",
     "value-matcher": "^0.2.0",
-    "vhd-lib": "^0.5.1",
+    "vhd-lib": "^0.6.0",
     "ws": "^6.0.0",
-    "xen-api": "^0.24.5",
+    "xen-api": "^0.25.0",
     "xml2js": "^0.4.19",
     "xo-acl-resolver": "^0.4.1",
     "xo-collection": "^0.4.1",

packages/xo-server/src/api/host.js

@@ -199,59 +199,6 @@ forget.resolve = {
 
 // -------------------------------------------------------------------
 
-// Returns an array of missing new patches in the host
-// Returns an empty array if up-to-date
-// Throws an error if the host is not running the latest XS version
-export function listMissingPatches({ host }) {
-  return this.getXapi(host).listMissingPoolPatchesOnHost(host._xapiId)
-}
-
-listMissingPatches.description =
-  'return an array of missing new patches in the host'
-
-listMissingPatches.params = {
-  host: { type: 'string' },
-}
-
-listMissingPatches.resolve = {
-  host: ['host', 'host', 'view'],
-}
-
-// -------------------------------------------------------------------
-
-export function installPatch({ host, patch: patchUuid }) {
-  return this.getXapi(host).installPoolPatchOnHost(patchUuid, host._xapiId)
-}
-
-installPatch.description = 'install a patch on an host'
-
-installPatch.params = {
-  host: { type: 'string' },
-  patch: { type: 'string' },
-}
-
-installPatch.resolve = {
-  host: ['host', 'host', 'administrate'],
-}
-
-// -------------------------------------------------------------------
-
-export function installAllPatches({ host }) {
-  return this.getXapi(host).installAllPoolPatchesOnHost(host._xapiId)
-}
-
-installAllPatches.description = 'install all the missing patches on a host'
-
-installAllPatches.params = {
-  host: { type: 'string' },
-}
-
-installAllPatches.resolve = {
-  host: ['host', 'host', 'administrate'],
-}
-
-// -------------------------------------------------------------------
-
 export function emergencyShutdownHost({ host }) {
   return this.getXapi(host).emergencyShutdownHost(host._xapiId)
 }

packages/xo-server/src/api/metadata-backup.js

@@ -101,3 +101,42 @@ runJob.params = {
     type: 'string',
   },
 }
+
+export async function list({ remotes }) {
+  return this.listMetadataBackups(remotes)
+}
+
+list.permission = 'admin'
+
+list.params = {
+  remotes: {
+    type: 'array',
+    items: {
+      type: 'string',
+    },
+  },
+}
+
+export function restore({ id }) {
+  return this.restoreMetadataBackup(id)
+}
+
+restore.permission = 'admin'
+
+restore.params = {
+  id: {
+    type: 'string',
+  },
+}
+
+function delete_({ id }) {
+  return this.deleteMetadataBackup(id)
+}
+delete_.permission = 'admin'
+
+delete_.params = {
+  id: {
+    type: 'string',
+  },
+}
+export { delete_ as delete }

packages/xo-server/src/api/pool.js

@@ -1,6 +1,4 @@
 import { format } from 'json-rpc-peer'
-import { differenceBy } from 'lodash'
-import { mapToArray } from '../utils'
 
 // ===================================================================
 
@@ -75,40 +73,43 @@ setPoolMaster.resolve = {
 
 // -------------------------------------------------------------------
 
-export async function installPatch({ pool, patch: patchUuid }) {
-  await this.getXapi(pool).installPoolPatchOnAllHosts(patchUuid)
+// Returns an array of missing new patches in the host
+// Returns an empty array if up-to-date
+export function listMissingPatches({ host }) {
+  return this.getXapi(host).listMissingPatches(host._xapiId)
 }
 
-installPatch.params = {
-  pool: {
-    type: 'string',
-  },
-  patch: {
-    type: 'string',
-  },
+listMissingPatches.description =
+  'return an array of missing new patches in the host'
+
+listMissingPatches.params = {
+  host: { type: 'string' },
 }
 
-installPatch.resolve = {
-  pool: ['pool', 'pool', 'administrate'],
+listMissingPatches.resolve = {
+  host: ['host', 'host', 'view'],
 }
+
 // -------------------------------------------------------------------
 
-export async function installAllPatches({ pool }) {
-  await this.getXapi(pool).installAllPoolPatchesOnAllHosts()
+export async function installPatches({ pool, patches, hosts }) {
+  await this.getXapi(hosts === undefined ? pool : hosts[0]).installPatches({
+    patches,
+    hosts,
+  })
 }
 
-installAllPatches.params = {
-  pool: {
-    type: 'string',
-  },
+installPatches.params = {
+  pool: { type: 'string', optional: true },
+  patches: { type: 'array', optional: true },
+  hosts: { type: 'array', optional: true },
 }
 
-installAllPatches.resolve = {
+installPatches.resolve = {
   pool: ['pool', 'pool', 'administrate'],
 }
 
-installAllPatches.description =
-  'Install automatically all patches for every hosts of a pool'
+installPatches.description = 'Install patches on hosts'
 
 // -------------------------------------------------------------------
 
@@ -144,6 +145,22 @@ export { uploadPatch as patch }
 
 // -------------------------------------------------------------------
 
+export async function getPatchesDifference({ source, target }) {
+  return this.getPatchesDifference(target.id, source.id)
+}
+
+getPatchesDifference.params = {
+  source: { type: 'string' },
+  target: { type: 'string' },
+}
+
+getPatchesDifference.resolve = {
+  source: ['source', 'host', 'view'],
+  target: ['target', 'host', 'view'],
+}
+
+// -------------------------------------------------------------------
+
 export async function mergeInto({ source, target, force }) {
   const sourceHost = this.getObject(source.master)
   const targetHost = this.getObject(target.master)
@@ -156,21 +173,21 @@ export async function mergeInto({ source, target, force }) {
     )
   }
 
-  const sourcePatches = sourceHost.patches
-  const targetPatches = targetHost.patches
-  const counterDiff = differenceBy(sourcePatches, targetPatches, 'name')
-
+  const counterDiff = this.getPatchesDifference(source.master, target.master)
   if (counterDiff.length > 0) {
-    throw new Error('host has patches that are not applied on target pool')
+    const targetXapi = this.getXapi(target)
+    await targetXapi.installPatches({
+      patches: await targetXapi.findPatches(counterDiff),
+    })
   }
 
-  const diff = differenceBy(targetPatches, sourcePatches, 'name')
-
-  // TODO: compare UUIDs
-  await this.getXapi(source).installSpecificPatchesOnHost(
-    mapToArray(diff, 'name'),
-    sourceHost._xapiId
-  )
+  const diff = this.getPatchesDifference(target.master, source.master)
+  if (diff.length > 0) {
+    const sourceXapi = this.getXapi(source)
+    await sourceXapi.installPatches({
+      patches: await sourceXapi.findPatches(diff),
+    })
+  }
 
   await this.mergeXenPools(source._xapiId, target._xapiId, force)
 }

packages/xo-server/src/xapi-object-to-xo.js

@@ -102,11 +102,10 @@ const TRANSFORMS = {
     } = obj
 
     const isRunning = isHostRunning(obj)
-    let supplementalPacks, patches
+    let supplementalPacks
 
     if (useUpdateSystem(obj)) {
       supplementalPacks = []
-      patches = []
 
       forEach(obj.$updates, update => {
         const formattedUpdate = {
@@ -121,7 +120,7 @@ const TRANSFORMS = {
         }
 
         if (startsWith(update.name_label, 'XS')) {
-          patches.push(formattedUpdate)
+          // It's a patch update but for homogeneity, we're still using pool_patches
         } else {
           supplementalPacks.push(formattedUpdate)
         }
@@ -171,7 +170,7 @@ const TRANSFORMS = {
         }
       })(),
       multipathing: otherConfig.multipathing === 'true',
-      patches: patches || link(obj, 'patches'),
+      patches: link(obj, 'patches'),
       powerOnMode: obj.power_on_mode,
       power_state: metrics ? (isRunning ? 'Running' : 'Halted') : 'Unknown',
       startTime: toTimestamp(otherConfig.boot_time),
@@ -625,10 +624,18 @@ const TRANSFORMS = {
   // -----------------------------------------------------------------
 
   host_patch(obj) {
+    const poolPatch = obj.$pool_patch
     return {
+      type: 'patch',
+
       applied: Boolean(obj.applied),
+      enforceHomogeneity: poolPatch.pool_applied,
+      description: poolPatch.name_description,
+      name: poolPatch.name_label,
+      pool_patch: poolPatch.$ref,
+      size: poolPatch.size,
+      guidance: poolPatch.after_apply_guidance,
       time: toTimestamp(obj.timestamp_applied),
-      pool_patch: link(obj, 'pool_patch', '$ref'),
 
       $host: link(obj, 'host'),
     }
@@ -640,12 +647,15 @@ const TRANSFORMS = {
     return {
       id: obj.$ref,
 
-      applied: Boolean(obj.pool_applied),
+      dataUuid: obj.uuid, // UUID of the patch file as stated in Citrix's XML file
       description: obj.name_description,
       guidance: obj.after_apply_guidance,
       name: obj.name_label,
       size: +obj.size,
-      uuid: obj.uuid,
+      uuid: obj.$ref,
+
+      // TODO: means that the patch must be applied on every host
+      // applied: Boolean(obj.pool_applied),
 
       // TODO: what does it mean, should we handle it?
       // version: obj.version,

packages/xo-server/src/xapi/index.js

@@ -68,6 +68,7 @@ import {
   parseDateTime,
   prepareXapiParam,
 } from './utils'
+import { createVhdStreamWithLength } from 'vhd-lib'
 
 const log = createLogger('xo:xapi')
 
@@ -93,8 +94,10 @@ export const IPV6_CONFIG_MODES = ['None', 'DHCP', 'Static', 'Autoconf']
 
 @mixin(mapToArray(mixins))
 export default class Xapi extends XapiBase {
-  constructor(...args) {
-    super(...args)
+  constructor({ guessVhdSizeOnImport, ...opts }) {
+    super(opts)
+
+    this._guessVhdSizeOnImport = guessVhdSizeOnImport
 
     // Patch getObject to resolve _xapiId property.
     this.getObject = (getObject => (...args) => {
@@ -1564,47 +1567,28 @@ export default class Xapi extends XapiBase {
       }`
     )
 
+    // see https://github.com/vatesfr/xen-orchestra/issues/4074
+    const snapshotNameLabelPrefix = `Snapshot of ${vm.uuid} [`
+    ignoreErrors.call(
+      Promise.all(
+        vm.snapshots.map(async ref => {
+          const nameLabel = await this.getField('VM', ref, 'name_label')
+          if (nameLabel.startsWith(snapshotNameLabelPrefix)) {
+            return this._deleteVm(ref)
+          }
+        })
+      )
+    )
+
     let ref
     do {
       if (!vm.tags.includes('xo-disable-quiesce')) {
         try {
-          ref = await pRetry(
-            async bail => {
-              try {
-                return await this.callAsync(
-                  $cancelToken,
-                  'VM.snapshot_with_quiesce',
-                  vmRef,
-                  nameLabel
-                )
-              } catch (error) {
-                if (error?.code !== 'VM_SNAPSHOT_WITH_QUIESCE_FAILED') {
-                  throw bail(error)
-                }
-
-                // detect and remove new broken snapshots
-                //
-                // see https://github.com/vatesfr/xen-orchestra/issues/3936
-                const prevSnapshotRefs = new Set(vm.snapshots)
-                const snapshotNameLabelPrefix = `Snapshot of ${vm.uuid} [`
-                vm.snapshots = await this.getField('VM', vmRef, 'snapshots')
-                const createdSnapshots = (await this.getRecords(
-                  'VM',
-                  vm.snapshots.filter(_ => !prevSnapshotRefs.has(_))
-                )).filter(_ => _.name_label.startsWith(snapshotNameLabelPrefix))
-
-                // be safe: only delete if there was a single match
-                if (createdSnapshots.length === 1) {
-                  ignoreErrors.call(this._deleteVm(createdSnapshots[0]))
-                }
-
-                throw error
-              }
-            },
-            {
-              delay: 60e3,
-              tries: 3,
-            }
+          ref = await this.callAsync(
+            $cancelToken,
+            'VM.snapshot_with_quiesce',
+            vmRef,
+            nameLabel
           ).then(extractOpaqueRef)
           ignoreErrors.call(this.call('VM.add_tags', ref, 'quiesce'))
 
@@ -2095,11 +2079,16 @@ export default class Xapi extends XapiBase {
   // -----------------------------------------------------------------
 
   async _importVdiContent(vdi, body, format = VDI_FORMAT_VHD) {
-    if (__DEV__ && body.length == null) {
-      throw new Error(
-        'Trying to import a VDI without a length field. Please report this error to Xen Orchestra.'
-      )
+    if (typeof body.pipe === 'function' && body.length === undefined) {
+      if (this._guessVhdSizeOnImport && format === VDI_FORMAT_VHD) {
+        body = await createVhdStreamWithLength(body)
+      } else if (__DEV__) {
+        throw new Error(
+          'Trying to import a VDI without a length field. Please report this error to Xen Orchestra.'
+        )
+      }
     }
+
     await Promise.all([
       body.task,
       body.checksumVerified,
@@ -2483,6 +2472,15 @@ export default class Xapi extends XapiBase {
     )
   }
 
+  // Main purpose: upload update on VDI
+  // Is a local SR on a non master host OK?
+  findAvailableSr(minSize) {
+    return find(
+      this.objects.all,
+      obj => obj.$type === 'SR' && canSrHaveNewVdiOfSize(obj, minSize)
+    )
+  }
+
   async _assertConsistentHostServerTime(hostRef) {
     const delta =
       parseDateTime(await this.call('host.get_servertime', hostRef)).getTime() -

packages/xo-server/src/xapi/mixins/patching.js

@@ -1,16 +1,8 @@
 import asyncMap from '@xen-orchestra/async-map'
 import createLogger from '@xen-orchestra/log'
 import deferrable from 'golike-defer'
-import every from 'lodash/every'
-import filter from 'lodash/filter'
-import find from 'lodash/find'
-import includes from 'lodash/includes'
-import isObject from 'lodash/isObject'
-import pickBy from 'lodash/pickBy'
-import some from 'lodash/some'
-import sortBy from 'lodash/sortBy'
-import assign from 'lodash/assign'
 import unzip from 'julien-f-unzip'
+import { filter, find, pickBy, some } from 'lodash'
 
 import ensureArray from '../../_ensureArray'
 import { debounce } from '../../decorators'
@@ -18,9 +10,35 @@ import { forEach, mapFilter, mapToArray, parseXml } from '../../utils'
 
 import { extractOpaqueRef, useUpdateSystem } from '../utils'
 
+// TOC -------------------------------------------------------------------------
+
+// # HELPERS
+//    _isXcp
+//    _ejectToolsIsos
+//    _getXenUpdates           Map of Objects
+// # LIST
+//    _listXcpUpdates          XCP available updates - Array of Objects
+//    _listPatches             XS patches (installed or not) - Map of Objects
+//    _listInstalledPatches    XS installed patches on the host - Map of Booleans
+//    _listInstallablePatches  XS (host, requested patches) → sorted patches that are not installed and not conflicting - Array of Objects
+//    listMissingPatches       HL: installable patches (XS) or updates (XCP) - Array of Objects
+//    findPatches              HL: get XS patches IDs from names
+// # INSTALL
+//    _xcpUpdate               XCP yum update
+//    _legacyUploadPatch       XS legacy upload
+//    _uploadPatch             XS upload on a dedicated VDI
+//    installPatches           HL: install patches (XS) or yum update (XCP) on hosts
+
+// HELPERS ---------------------------------------------------------------------
+
 const log = createLogger('xo:xapi')
 
+const _isXcp = host => host.software_version.product_brand === 'XCP-ng'
+
+// =============================================================================
+
 export default {
+  // raw { uuid: patch } map translated from updates.xensource.com/XenServer/updates.xml
   // FIXME: should be static
   @debounce(24 * 60 * 60 * 1000)
   async _getXenUpdates() {
@@ -43,13 +61,16 @@ export default {
         guidance: patch['after-apply-guidance'],
         name: patch['name-label'],
         url: patch['patch-url'],
+        id: patch.uuid,
         uuid: patch.uuid,
-        conflicts: mapToArray(ensureArray(patch.conflictingpatches), patch => {
-          return patch.conflictingpatch.uuid
-        }),
-        requirements: mapToArray(ensureArray(patch.requiredpatches), patch => {
-          return patch.requiredpatch.uuid
-        }),
+        conflicts: mapToArray(
+          ensureArray(patch.conflictingpatches),
+          patch => patch.conflictingpatch.uuid
+        ),
+        requirements: mapToArray(
+          ensureArray(patch.requiredpatches),
+          patch => patch.requiredpatch.uuid
+        ),
         paid: patch['update-stream'] === 'premium',
         upgrade: /^XS\d{2,}$/.test(patch['name-label']),
         // TODO: what does it mean, should we handle it?
@@ -96,72 +117,12 @@ export default {
     }
   },
 
-  // =================================================================
-
-  // Returns installed and not installed patches for a given host.
-  async _getPoolPatchesForHost(host) {
-    const versions = (await this._getXenUpdates()).versions
-
-    const hostVersions = host.software_version
-    const version =
-      versions[hostVersions.product_version] ||
-      versions[hostVersions.product_version_text]
-
-    return version ? version.patches : []
-  },
-
-  _getInstalledPoolPatchesOnHost(host) {
-    const installed = { __proto__: null }
-
-    // platform_version < 2.1.1
-    forEach(host.$patches, hostPatch => {
-      installed[hostPatch.$pool_patch.uuid] = true
-    })
-
-    // platform_version >= 2.1.1
-    forEach(host.$updates, update => {
-      installed[update.uuid] = true // TODO: ignore packs
-    })
-
-    return installed
-  },
-
-  async _listMissingPoolPatchesOnHost(host) {
-    const all = await this._getPoolPatchesForHost(host)
-    const installed = this._getInstalledPoolPatchesOnHost(host)
-
-    const installable = { __proto__: null }
-    forEach(all, (patch, uuid) => {
-      if (installed[uuid]) {
-        return
-      }
-
-      for (const uuid of patch.conflicts) {
-        if (uuid in installed) {
-          return
-        }
-      }
-
-      installable[uuid] = patch
-    })
-
-    return installable
-  },
-
-  async listMissingPoolPatchesOnHost(hostId) {
-    const host = this.getObject(hostId)
-    // Returns an array to not break compatibility.
-    return mapToArray(
-      await (host.software_version.product_brand === 'XCP-ng'
-        ? this._xcpListHostUpdates(host)
-        : this._listMissingPoolPatchesOnHost(host))
-    )
-  },
-
+  // eject all ISOs from all the host's VMs when installing patches
+  // if hostRef is not specified: eject ISOs on all the pool's VMs
   async _ejectToolsIsos(hostRef) {
     return Promise.all(
       mapFilter(this.objects.all, vm => {
-        if (vm.$type !== 'VM' || (hostRef && vm.resident_on !== hostRef)) {
+        if (vm.$type !== 'vm' || (hostRef && vm.resident_on !== hostRef)) {
           return
         }
 
@@ -178,54 +139,235 @@ export default {
     )
   },
 
-  // -----------------------------------------------------------------
+  // LIST ----------------------------------------------------------------------
 
-  _isPoolPatchInstallableOnHost(patchUuid, host) {
-    const installed = this._getInstalledPoolPatchesOnHost(host)
+  // list all yum updates available for a XCP-ng host
+  // (hostObject) → { uuid: patchObject }
+  async _listXcpUpdates(host) {
+    return JSON.parse(
+      await this.call(
+        'host.call_plugin',
+        host.$ref,
+        'updater.py',
+        'check_update',
+        {}
+      )
+    )
+  },
 
-    if (installed[patchUuid]) {
-      return false
+  // list all patches provided by Citrix for this host version regardless
+  // of if they're installed or not
+  // ignores upgrade patches
+  // (hostObject) → { uuid: patchObject }
+  async _listPatches(host) {
+    const versions = (await this._getXenUpdates()).versions
+
+    const hostVersions = host.software_version
+    const version =
+      versions[hostVersions.product_version] ||
+      versions[hostVersions.product_version_text]
+
+    return version ? pickBy(version.patches, patch => !patch.upgrade) : {}
+  },
+
+  // list patches installed on the host
+  // (hostObject) → { uuid: boolean }
+  _listInstalledPatches(host) {
+    const installed = { __proto__: null }
+
+    // Legacy XS patches
+    if (!useUpdateSystem(host)) {
+      forEach(host.$patches, hostPatch => {
+        installed[hostPatch.$pool_patch.uuid] = true
+      })
+      return installed
     }
+    // ----------
 
-    let installable = true
-
-    forEach(installed, patch => {
-      if (includes(patch.conflicts, patchUuid)) {
-        installable = false
-
-        return false
+    forEach(host.$updates, update => {
+      // ignore packs
+      if (update.name_label.startsWith('XS')) {
+        installed[update.uuid] = true
       }
     })
 
+    return installed
+  },
+
+  // TODO: handle upgrade patches
+  // (hostObject, [ patchId ]) → [ patchObject ]
+  async _listInstallablePatches(host, requestedPatches) {
+    const all = await this._listPatches(host)
+    const installed = this._listInstalledPatches(host)
+
+    let getAll = false
+    if (requestedPatches === undefined) {
+      getAll = true
+      requestedPatches = Object.keys(all)
+    }
+    const freeHost = this.pool.$master.license_params.sku_type === 'free'
+    // We assume:
+    // - no conflict transitivity (If A conflicts with B and B with C, Citrix should tell us explicitly that A conflicts with C)
+    // - no requirements transitivity (If A requires B and B requires C, Citrix should tell us explicitly that A requires C)
+    // - sorted requirements (If A requires B and C, then C cannot require B)
+    // For each requested patch:
+    // - throw if not found
+    // - throw if already installed
+    // - ignore if already in installable (may have been added because of requirements)
+    // - if paid patch on free host: either ignore (listing all the patches) or throw (patch is requested)
+    // - throw if conflicting patches installed
+    // - throw if conflicting patches in installable
+    // - throw if one of the requirements is not found
+    // - push its required patches in installable
+    // - push it in installable
+    const installable = []
+    forEach(requestedPatches, id => {
+      const patch = all[id]
+      if (patch === undefined) {
+        throw new Error(`patch not found: ${id}`)
+      }
+
+      if (installed[id] !== undefined) {
+        if (getAll) {
+          return
+        }
+        throw new Error(`patch already installed: ${patch.name} (${id})`)
+      }
+
+      if (find(installable, { id }) !== undefined) {
+        return
+      }
+
+      if (patch.paid && freeHost) {
+        if (getAll) {
+          return
+        }
+        throw new Error(
+          `requested patch ${patch.name} (${id}) requires a XenServer license`
+        )
+      }
+
+      let conflictId
+      if (
+        (conflictId = find(
+          patch.conflicts,
+          conflictId => installed[conflictId] !== undefined
+        )) !== undefined
+      ) {
+        if (getAll) {
+          log(
+            `patch ${
+              patch.name
+            } (${id}) conflicts with installed patch ${conflictId}`
+          )
+          return
+        }
+        throw new Error(
+          `patch ${
+            patch.name
+          } (${id}) conflicts with installed patch ${conflictId}`
+        )
+      }
+
+      if (
+        (conflictId = find(patch.conflicts, conflictId =>
+          find(installable, { id: conflictId })
+        )) !== undefined
+      ) {
+        if (getAll) {
+          log(`patches ${id} and ${conflictId} conflict with eachother`)
+          return
+        }
+        throw new Error(
+          `patches ${id} and ${conflictId} conflict with eachother`
+        )
+      }
+
+      // add requirements
+      forEach(patch.requirements, id => {
+        const requiredPatch = all[id]
+        if (requiredPatch === undefined) {
+          throw new Error(`required patch ${id} not found`)
+        }
+        if (!installed[id] && find(installable, { id }) === undefined) {
+          if (requiredPatch.paid && freeHost) {
+            throw new Error(
+              `required patch ${
+                requiredPatch.name
+              } (${id}) requires a XenServer license`
+            )
+          }
+          installable.push(requiredPatch)
+        }
+      })
+
+      // add itself
+      installable.push(patch)
+    })
+
     return installable
   },
 
-  _isPoolPatchInstallableOnPool(patchUuid) {
-    return every(
-      this.objects.all,
-      obj =>
-        obj.$type !== 'host' ||
-        this._isPoolPatchInstallableOnHost(patchUuid, obj)
+  // high level
+  listMissingPatches(hostId) {
+    const host = this.getObject(hostId)
+    return _isXcp(host)
+      ? this._listXcpUpdates(host)
+      : // TODO: list paid patches of free hosts as well so the UI can show them
+        this._listInstallablePatches(host)
+  },
+
+  // convenient method to find which patches should be installed from a
+  // list of patch names
+  // e.g.: compare the installed patches of 2 hosts by their
+  // names (XS..E...) then find the patches global ID
+  // [ names ] → [ IDs ]
+  async findPatches(names) {
+    const all = await this._listPatches(this.pool.$master)
+    return filter(all, patch => names.includes(patch.name)).map(
+      patch => patch.id
     )
   },
 
-  // -----------------------------------------------------------------
+  // INSTALL -------------------------------------------------------------------
 
-  // platform_version < 2.1.1 ----------------------------------------
-  async uploadPoolPatch(stream, patchName) {
-    const patchRef = await this.putResource(stream, '/pool_patch_upload', {
-      task: this.createTask('Patch upload', patchName),
-    }).then(extractOpaqueRef)
+  _xcpUpdate(hosts) {
+    if (hosts === undefined) {
+      hosts = filter(this.objects.all, { $type: 'host' })
+    } else {
+      hosts = filter(
+        this.objects.all,
+        obj => obj.$type === 'host' && hosts.includes(obj.$id)
+      )
+    }
 
-    return this._getOrWaitObject(patchRef)
+    return asyncMap(hosts, async host => {
+      const update = await this.call(
+        'host.call_plugin',
+        host.$ref,
+        'updater.py',
+        'update',
+        {}
+      )
+
+      if (JSON.parse(update).exit !== 0) {
+        throw new Error('Update install failed')
+      } else {
+        await this._updateObjectMapProperty(host, 'other_config', {
+          rpm_patch_installation_time: String(Date.now() / 1000),
+        })
+      }
+    })
   },
 
-  async _getOrUploadPoolPatch(uuid) {
+  // Legacy XS patches: upload a patch on a pool before installing it
+  async _legacyUploadPatch(uuid) {
+    // check if the patch has already been uploaded
     try {
       return this.getObjectByUuid(uuid)
-    } catch (error) {}
+    } catch (e) {}
 
-    log.debug(`downloading patch ${uuid}`)
+    log.debug(`legacy downloading patch ${uuid}`)
 
     const patchInfo = (await this._getXenUpdates()).patches[uuid]
     if (!patchInfo) {
@@ -248,16 +390,21 @@ export default {
         .on('error', reject)
     })
 
-    return this.uploadPoolPatch(stream, patchInfo.name)
+    const patchRef = await this.putResource(stream, '/pool_patch_upload', {
+      task: this.createTask('Patch upload', patchInfo.name),
+    }).then(extractOpaqueRef)
+
+    return this._getOrWaitObject(patchRef)
   },
+  // ----------
 
-  // patform_version >= 2.1.1 ----------------------------------------
-  async _getUpdateVdi($defer, patchUuid, hostId) {
-    log.debug(`downloading patch ${patchUuid}`)
+  // upload patch on a VDI on a shared SR
+  async _uploadPatch($defer, uuid) {
+    log.debug(`downloading patch ${uuid}`)
 
-    const patchInfo = (await this._getXenUpdates()).patches[patchUuid]
+    const patchInfo = (await this._getXenUpdates()).patches[uuid]
     if (!patchInfo) {
-      throw new Error('no such patch ' + patchUuid)
+      throw new Error('no such patch ' + uuid)
     }
 
     let stream = await this.xo.httpRequest(patchInfo.url)
@@ -271,315 +418,104 @@ export default {
         .on('error', reject)
     })
 
-    let vdi
-
-    // If no hostId provided, try and find a shared SR
-    if (!hostId) {
-      const sr = this.findAvailableSharedSr(stream.length)
-
-      if (!sr) {
-        return
-      }
-
-      vdi = await this.createTemporaryVdiOnSr(
-        stream,
-        sr,
-        '[XO] Patch ISO',
-        'small temporary VDI to store a patch ISO'
-      )
-    } else {
-      vdi = await this.createTemporaryVdiOnHost(
-        stream,
-        hostId,
-        '[XO] Patch ISO',
-        'small temporary VDI to store a patch ISO'
-      )
+    const sr = this.findAvailableSr(stream.length)
+    if (sr === undefined) {
+      return
     }
+
+    const vdi = await this.createTemporaryVdiOnSr(
+      stream,
+      sr,
+      '[XO] Patch ISO',
+      'small temporary VDI to store a patch ISO'
+    )
     $defer(() => this._deleteVdi(vdi.$ref))
 
     return vdi
   },
 
-  // -----------------------------------------------------------------
+  _poolWideInstall: deferrable(async function($defer, patches) {
+    // Legacy XS patches
+    if (!useUpdateSystem(this.pool.$master)) {
+      // for each patch: pool_patch.pool_apply
+      for (const p of patches) {
+        const [patch] = await Promise.all([
+          this._legacyUploadPatch(p.uuid),
+          this._ejectToolsIsos(this.pool.$master.$ref),
+        ])
 
-  // patform_version < 2.1.1 -----------------------------------------
-  async _installPoolPatchOnHost(patchUuid, host) {
-    const [patch] = await Promise.all([
-      this._getOrUploadPoolPatch(patchUuid),
-      this._ejectToolsIsos(host.$ref),
-    ])
-
-    await this.call('pool_patch.apply', patch.$ref, host.$ref)
-  },
-
-  // patform_version >= 2.1.1
-  _installPatchUpdateOnHost: deferrable(async function(
-    $defer,
-    patchUuid,
-    host
-  ) {
-    await this._assertConsistentHostServerTime(host.$ref)
-
-    const [vdi] = await Promise.all([
-      this._getUpdateVdi($defer, patchUuid, host.$id),
-      this._ejectToolsIsos(host.$ref),
-    ])
-
-    const updateRef = await this.call('pool_update.introduce', vdi.$ref)
-    // TODO: check update status
-    // const precheck = await this.call('pool_update.precheck', updateRef, host.$ref)
-    // - ok_livepatch_complete     An applicable live patch exists for every required component
-    // - ok_livepatch_incomplete   An applicable live patch exists but it is not sufficient
-    // - ok                        There is no applicable live patch
-    return this.call('pool_update.apply', updateRef, host.$ref)
-  }),
-
-  // -----------------------------------------------------------------
-
-  async installPoolPatchOnHost(patchUuid, host) {
-    log.debug(`installing patch ${patchUuid}`)
-    if (!isObject(host)) {
-      host = this.getObject(host)
-    }
-
-    return useUpdateSystem(host)
-      ? this._installPatchUpdateOnHost(patchUuid, host)
-      : this._installPoolPatchOnHost(patchUuid, host)
-  },
-
-  // -----------------------------------------------------------------
-
-  // platform_version < 2.1.1
-  async _installPoolPatchOnAllHosts(patchUuid) {
-    const [patch] = await Promise.all([
-      this._getOrUploadPoolPatch(patchUuid),
-      this._ejectToolsIsos(),
-    ])
-
-    await this.call('pool_patch.pool_apply', patch.$ref)
-  },
-
-  // platform_version >= 2.1.1
-  _installPatchUpdateOnAllHosts: deferrable(async function($defer, patchUuid) {
-    await this._assertConsistentHostServerTime(this.pool.master)
-
-    let [vdi] = await Promise.all([
-      this._getUpdateVdi($defer, patchUuid),
-      this._ejectToolsIsos(),
-    ])
-    if (vdi == null) {
-      vdi = await this._getUpdateVdi($defer, patchUuid, this.pool.master)
-    }
-
-    return this.call(
-      'pool_update.pool_apply',
-      await this.call('pool_update.introduce', vdi.$ref)
-    )
-  }),
-
-  async installPoolPatchOnAllHosts(patchUuid) {
-    log.debug(`installing patch ${patchUuid} on all hosts`)
-
-    return useUpdateSystem(this.pool.$master)
-      ? this._installPatchUpdateOnAllHosts(patchUuid)
-      : this._installPoolPatchOnAllHosts(patchUuid)
-  },
-
-  // -----------------------------------------------------------------
-
-  // If no host is provided, install on pool
-  async _installPoolPatchAndRequirements(patch, patchesByUuid, host) {
-    if (
-      host == null
-        ? !this._isPoolPatchInstallableOnPool(patch.uuid)
-        : !this._isPoolPatchInstallableOnHost(patch.uuid, host)
-    ) {
+        await this.call('pool_patch.pool_apply', patch.$ref)
+      }
       return
     }
+    // ----------
 
-    const { requirements } = patch
-
-    if (requirements.length) {
-      for (const requirementUuid of requirements) {
-        const requirement = patchesByUuid[requirementUuid]
-
-        if (requirement != null) {
-          await this._installPoolPatchAndRequirements(
-            requirement,
-            patchesByUuid,
-            host
-          )
-          host = host && this.getObject(host.$id)
-        }
+    // for each patch: pool_update.introduce → pool_update.pool_apply
+    for (const p of patches) {
+      const [vdi] = await Promise.all([
+        this._uploadPatch($defer, p.uuid),
+        this._ejectToolsIsos(),
+      ])
+      if (vdi === undefined) {
+        throw new Error('patch could not be uploaded')
       }
-    }
 
-    return host == null
-      ? this.installPoolPatchOnAllHosts(patch.uuid)
-      : this.installPoolPatchOnHost(patch.uuid, host)
-  },
-
-  async installSpecificPatchesOnHost(patchNames, hostId) {
-    const host = this.getObject(hostId)
-    const missingPatches = await this._listMissingPoolPatchesOnHost(host)
-
-    const patchesToInstall = []
-    const addPatchesToList = patches => {
-      forEach(patches, patch => {
-        addPatchesToList(mapToArray(patch.requirements, { uuid: patch.uuid }))
-
-        if (!find(patchesToInstall, { name: patch.name })) {
-          patchesToInstall.push(patch)
-        }
-      })
-    }
-    addPatchesToList(
-      mapToArray(patchNames, name => find(missingPatches, { name }))
-    )
-
-    for (let i = 0, n = patchesToInstall.length; i < n; i++) {
-      await this._installPoolPatchAndRequirements(
-        patchesToInstall[i],
-        missingPatches,
-        host
-      )
-    }
-  },
-
-  async installAllPoolPatchesOnHost(hostId) {
-    const host = this.getObject(hostId)
-    if (host.software_version.product_brand === 'XCP-ng') {
-      return this._xcpInstallHostUpdates(host)
-    }
-    return this._installAllPoolPatchesOnHost(host)
-  },
-
-  async _installAllPoolPatchesOnHost(host) {
-    const installableByUuid =
-      host.license_params.sku_type !== 'free'
-        ? pickBy(await this._listMissingPoolPatchesOnHost(host), {
-            upgrade: false,
-          })
-        : pickBy(await this._listMissingPoolPatchesOnHost(host), {
-            paid: false,
-            upgrade: false,
-          })
-
-    // List of all installable patches sorted from the newest to the
-    // oldest.
-    const installable = sortBy(
-      installableByUuid,
-      patch => -Date.parse(patch.date)
-    )
-
-    for (let i = 0, n = installable.length; i < n; ++i) {
-      const patch = installable[i]
-
-      if (this._isPoolPatchInstallableOnHost(patch.uuid, host)) {
-        await this._installPoolPatchAndRequirements(
-          patch,
-          installableByUuid,
-          host
-        ).catch(error => {
-          if (
-            error.code !== 'PATCH_ALREADY_APPLIED' &&
-            error.code !== 'UPDATE_ALREADY_APPLIED'
-          ) {
-            throw error
-          }
-        })
-        host = this.getObject(host.$id)
-      }
-    }
-  },
-
-  async installAllPoolPatchesOnAllHosts() {
-    if (this.pool.$master.software_version.product_brand === 'XCP-ng') {
-      return this._xcpInstallAllPoolUpdatesOnHost()
-    }
-    return this._installAllPoolPatchesOnAllHosts()
-  },
-
-  async _installAllPoolPatchesOnAllHosts() {
-    const installableByUuid = assign(
-      {},
-      ...(await Promise.all(
-        mapFilter(this.objects.all, host => {
-          if (host.$type === 'host') {
-            return this._listMissingPoolPatchesOnHost(host).then(patches =>
-              host.license_params.sku_type !== 'free'
-                ? pickBy(patches, { upgrade: false })
-                : pickBy(patches, { paid: false, upgrade: false })
-            )
-          }
-        })
-      ))
-    )
-
-    // List of all installable patches sorted from the newest to the
-    // oldest.
-    const installable = sortBy(
-      installableByUuid,
-      patch => -Date.parse(patch.date)
-    )
-
-    for (let i = 0, n = installable.length; i < n; ++i) {
-      const patch = installable[i]
-
-      await this._installPoolPatchAndRequirements(
-        patch,
-        installableByUuid
-      ).catch(error => {
-        if (
-          error.code !== 'PATCH_ALREADY_APPLIED' &&
-          error.code !== 'UPDATE_ALREADY_APPLIED_IN_POOL'
-        ) {
-          throw error
-        }
-      })
-    }
-  },
-
-  // ----------------------------------
-  // XCP-ng dedicated zone for patching
-  // ----------------------------------
-
-  // list all yum updates available for a XCP-ng host
-  async _xcpListHostUpdates(host) {
-    return JSON.parse(
+      log.debug(`installing patch ${p.uuid}`)
       await this.call(
-        'host.call_plugin',
-        host.$ref,
-        'updater.py',
-        'check_update',
-        {}
+        'pool_update.pool_apply',
+        await this.call('pool_update.introduce', vdi.$ref)
       )
-    )
-  },
-
-  // install all yum updates for a XCP-ng host
-  async _xcpInstallHostUpdates(host) {
-    const update = await this.call(
-      'host.call_plugin',
-      host.$ref,
-      'updater.py',
-      'update',
-      {}
-    )
-
-    if (JSON.parse(update).exit !== 0) {
-      throw new Error('Update install failed')
-    } else {
-      await this._updateObjectMapProperty(host, 'other_config', {
-        rpm_patch_installation_time: String(Date.now() / 1000),
-      })
     }
+  }),
+
+  async _hostInstall(patches, host) {
+    throw new Error('single host install not implemented')
+    // Legacy XS patches
+    // for each patch: pool_patch.apply
+    // ----------
+    // for each patch: pool_update.introduce → pool_update.apply
   },
 
-  // install all yum updates for all XCP-ng hosts in a give pool
-  async _xcpInstallAllPoolUpdatesOnHost() {
-    await asyncMap(filter(this.objects.all, { $type: 'host' }), host =>
-      this._xcpInstallHostUpdates(host)
-    )
+  // high level
+  // install specified patches on specified hosts
+  //
+  // no hosts specified: pool-wide install (only the pool master installed patches will be considered)
+  // no patches specified: install either the pool master's missing patches (no hosts specified) or each host's missing patches
+  //
+  // patches will be ignored for XCP (always updates completely)
+  // patches that are already installed will be ignored (XS only)
+  //
+  // XS pool-wide optimization only works when no hosts are specified
+  // it may install more patches that specified if some of them require other patches
+  async installPatches({ patches, hosts }) {
+    // XCP
+    if (_isXcp(this.pool.$master)) {
+      return this._xcpUpdate(hosts)
+    }
+
+    // XS
+    // TODO: assert consistent time
+    const poolWide = hosts === undefined
+    if (poolWide) {
+      log.debug('patches that were requested to be installed', patches)
+      const installablePatches = await this._listInstallablePatches(
+        this.pool.$master,
+        patches
+      )
+
+      log.debug(
+        'patches that will actually be installed',
+        installablePatches.map(patch => patch.uuid)
+      )
+
+      return this._poolWideInstall(installablePatches)
+    }
+
+    // for each host
+    // get installable patches
+    // filter patches that should be installed
+    // sort patches
+    // host-by-host install
+    throw new Error('non pool-wide install not implemented')
   },
 }

packages/xo-server/src/xapi/mixins/pool.js

@@ -1,14 +1,25 @@
 import { cancelable } from 'promise-toolbox'
 
+const PATH_DB_DUMP = '/pool/xmldbdump'
+
 export default {
   @cancelable
   exportPoolMetadata($cancelToken) {
-    const { pool } = this
-    return this.getResource($cancelToken, '/pool/xmldbdump', {
-      task: this.createTask(
-        'Pool metadata',
-        pool.name_label ?? pool.$master.name_label
-      ),
+    return this.getResource($cancelToken, PATH_DB_DUMP, {
+      task: this.createTask('Export pool metadata'),
+    })
+  },
+
+  // Restore the XAPI database from an XML backup
+  //
+  // See https://github.com/xapi-project/xen-api/blob/405b02e72f1ccc4f4b456fd52db30876faddcdd8/ocaml/xapi/pool_db_backup.ml#L170-L205
+  @cancelable
+  importPoolMetadata($cancelToken, stream, force = false) {
+    return this.putResource($cancelToken, stream, PATH_DB_DUMP, {
+      query: {
+        dry_run: String(!force),
+      },
+      task: this.createTask('Import pool metadata'),
     })
   },
 }

packages/xo-server/src/xo-mixins/backups-ng-logs.js

@@ -53,14 +53,27 @@ const taskTimeComparator = ({ start: s1, end: e1 }, { start: s2, end: e2 }) => {
   return 1
 }
 
+// type Task = {
+//   data: any,
+//   end?: number,
+//   id: string,
+//   jobId?: string,
+//   jobName?: string,
+//   message?:  'backup' | 'metadataRestore' | 'restore',
+//   scheduleId?: string,
+//   start: number,
+//   status: 'pending' | 'failure' | 'interrupted' | 'skipped' | 'success',
+//   tasks?: Task[],
+// }
 export default {
   async getBackupNgLogs(runId?: string) {
-    const [jobLogs, restoreLogs] = await Promise.all([
+    const [jobLogs, restoreLogs, restoreMetadataLogs] = await Promise.all([
       this.getLogs('jobs'),
       this.getLogs('restore'),
+      this.getLogs('metadataRestore'),
     ])
 
-    const { runningJobs, runningRestores } = this
+    const { runningJobs, runningRestores, runningMetadataRestores } = this
     const consolidated = {}
     const started = {}
 
@@ -77,6 +90,7 @@ export default {
             id,
             jobId,
             jobName: data.jobName,
+            message: 'backup',
             scheduleId,
             start: time,
             status: runningJobs[jobId] === id ? 'pending' : 'interrupted',
@@ -105,7 +119,8 @@ export default {
         if (parentId === undefined && (runId === undefined || runId === id)) {
           // top level task
           task.status =
-            message === 'restore' && !runningRestores.has(id)
+            (message === 'restore' && !runningRestores.has(id)) ||
+            (message === 'metadataRestore' && !runningMetadataRestores.has(id))
               ? 'interrupted'
               : 'pending'
           consolidated[id] = started[id] = task
@@ -172,6 +187,7 @@ export default {
 
     forEach(jobLogs, handleLog)
     forEach(restoreLogs, handleLog)
+    forEach(restoreMetadataLogs, handleLog)
 
     return runId === undefined ? consolidated : consolidated[runId]
   },

packages/xo-server/src/xo-mixins/backups-ng/index.js

@@ -140,6 +140,7 @@ const defaultSettings: Settings = {
   concurrency: 0,
   deleteFirst: false,
   exportRetention: 0,
+  fullInterval: 0,
   offlineSnapshot: false,
   reportWhen: 'failure',
   snapshotRetention: 0,
@@ -475,10 +476,11 @@ const disableVmHighAvailability = async (xapi: Xapi, vm: Vm) => {
 //
 // - `other_config`:
 //    - `xo:backup:datetime` = snapshot.snapshot_time (allow sorting replicated VMs)
+//    - `xo:backup:deltaChainLength` = n (number of delta copies/replicated since a full)
+//    - `xo:backup:exported` = 'true' (added at the end of the backup)
 //    - `xo:backup:job` = job.id
 //    - `xo:backup:schedule` = schedule.id
 //    - `xo:backup:vm` = vm.uuid
-//    - `xo:backup:exported` = 'true' (added at the end of the backup)
 //
 // Attributes of created VMs:
 //
@@ -937,6 +939,7 @@ export default class BackupNg {
         },
         xapi._updateObjectMapProperty(vm, 'other_config', {
           'xo:backup:datetime': null,
+          'xo:backup:deltaChainLength': null,
           'xo:backup:exported': null,
           'xo:backup:job': null,
           'xo:backup:schedule': null,
@@ -1293,12 +1296,31 @@ export default class BackupNg {
         $defer.onSuccess.call(xapi, 'deleteVm', baseSnapshot)
       }
 
+      let deltaChainLength = 0
       let fullVdisRequired
       await (async () => {
         if (baseSnapshot === undefined) {
           return
         }
 
+        let prevDeltaChainLength = +baseSnapshot.other_config[
+          'xo:backup:deltaChainLength'
+        ]
+        if (Number.isNaN(prevDeltaChainLength)) {
+          prevDeltaChainLength = 0
+        }
+        deltaChainLength = prevDeltaChainLength + 1
+
+        const fullInterval = getSetting(settings, 'fullInterval', [
+          vmUuid,
+          scheduleId,
+          '',
+        ])
+        if (fullInterval !== 0 && fullInterval <= deltaChainLength) {
+          baseSnapshot = undefined
+          return
+        }
+
         const fullRequired = { __proto__: null }
         const vdis: $Dict<Vdi> = getVmDisks(baseSnapshot)
 
@@ -1626,6 +1648,15 @@ export default class BackupNg {
         ],
         noop // errors are handled in logs
       )
+
+      if (!isFull) {
+        ignoreErrors.call(
+          snapshot.update_other_config(
+            'xo:backup:deltaChainLength',
+            String(deltaChainLength)
+          )
+        )
+      }
     } else {
       throw new Error(`no exporter for backup mode ${mode}`)
     }

packages/xo-server/src/xo-mixins/metadata-backups.js

@@ -1,11 +1,15 @@
 // @flow
 import asyncMap from '@xen-orchestra/async-map'
+import createLogger from '@xen-orchestra/log'
 import defer from 'golike-defer'
 import { fromEvent, ignoreErrors } from 'promise-toolbox'
 
+import debounceWithKey from '../_pDebounceWithKey'
+import parseDuration from '../_parseDuration'
 import { type Xapi } from '../xapi'
 import {
   safeDateFormat,
+  serializeError,
   type SimpleIdPattern,
   unboxIdsFromPattern,
 } from '../utils'
@@ -13,8 +17,14 @@ import {
 import { type Executor, type Job } from './jobs'
 import { type Schedule } from './scheduling'
 
+const log = createLogger('xo:xo-mixins:metadata-backups')
+
+const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
+const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
 const METADATA_BACKUP_JOB_TYPE = 'metadataBackup'
 
+const compareTimestamp = (a, b) => a.timestamp - b.timestamp
+
 type Settings = {|
   retentionXoMetadata?: number,
   retentionPoolMetadata?: number,
@@ -29,6 +39,26 @@ type MetadataBackupJob = {
   xoMetadata?: boolean,
 }
 
+const createSafeReaddir = (handler, methodName) => (path, options) =>
+  handler.list(path, options).catch(error => {
+    if (error?.code !== 'ENOENT') {
+      log.warn(`${methodName} ${path}`, { error })
+    }
+    return []
+  })
+
+// metadata.json
+//
+// {
+//   jobId: String,
+//   jobName: String,
+//   scheduleId: String,
+//   scheduleName: String,
+//   timestamp: number,
+//   pool?: <Pool />
+//   poolMaster?: <Host />
+// }
+//
 // File structure on remotes:
 //
 // <remote>
@@ -43,7 +73,6 @@ type MetadataBackupJob = {
 //          └─ <YYYYMMDD>T<HHmmss>
 //             ├─ metadata.json
 //             └─ data
-
 export default class metadataBackup {
   _app: {
     createJob: (
@@ -63,9 +92,30 @@ export default class metadataBackup {
     removeJob: (id: string) => Promise<void>,
   }
 
-  constructor(app: any) {
+  get runningMetadataRestores() {
+    return this._runningMetadataRestores
+  }
+
+  constructor(app: any, { backup }) {
     this._app = app
-    app.on('start', () => {
+    this._logger = undefined
+    this._runningMetadataRestores = new Set()
+
+    const debounceDelay = parseDuration(backup.listingDebounce)
+    this._listXoMetadataBackups = debounceWithKey(
+      this._listXoMetadataBackups,
+      debounceDelay,
+      remoteId => remoteId
+    )
+    this.__listPoolMetadataBackups = debounceWithKey(
+      this._listPoolMetadataBackups,
+      debounceDelay,
+      remoteId => remoteId
+    )
+
+    app.on('start', async () => {
+      this._logger = await app.getLogger('metadataRestore')
+
       app.registerJobExecutor(
         METADATA_BACKUP_JOB_TYPE,
         this._executor.bind(this)
@@ -106,7 +156,7 @@ export default class metadataBackup {
 
     const files = []
     if (job.xoMetadata && retentionXoMetadata > 0) {
-      const xoMetadataDir = `xo-config-backups/${schedule.id}`
+      const xoMetadataDir = `${DIR_XO_CONFIG_BACKUPS}/${schedule.id}`
       const dir = `${xoMetadataDir}/${formattedTimestamp}`
 
       const data = JSON.stringify(await app.exportConfig(), null, 2)
@@ -131,7 +181,7 @@ export default class metadataBackup {
       files.push(
         ...(await Promise.all(
           poolIds.map(async id => {
-            const poolMetadataDir = `xo-pool-metadata-backups/${
+            const poolMetadataDir = `${DIR_XO_POOL_METADATA_BACKUPS}/${
               schedule.id
             }/${id}`
             const dir = `${poolMetadataDir}/${formattedTimestamp}`
@@ -261,4 +311,210 @@ export default class metadataBackup {
       }),
     ])
   }
+
+  // xoBackups
+  // [{
+  //   id: `${remoteId}/folderPath`,
+  //   jobId,
+  //   jobName,
+  //   scheduleId,
+  //   scheduleName,
+  //   timestamp
+  // }]
+  async _listXoMetadataBackups(remoteId, handler) {
+    const safeReaddir = createSafeReaddir(handler, 'listXoMetadataBackups')
+
+    const backups = []
+    await asyncMap(
+      safeReaddir(DIR_XO_CONFIG_BACKUPS, { prependDir: true }),
+      scheduleDir =>
+        asyncMap(
+          safeReaddir(scheduleDir, { prependDir: true }),
+          async backupDir => {
+            try {
+              backups.push({
+                id: `${remoteId}${backupDir}`,
+                ...JSON.parse(
+                  String(await handler.readFile(`${backupDir}/metadata.json`))
+                ),
+              })
+            } catch (error) {
+              log.warn(`listXoMetadataBackups ${backupDir}`, { error })
+            }
+          }
+        )
+    )
+
+    return backups.sort(compareTimestamp)
+  }
+
+  // poolBackups
+  // {
+  //   [<Pool ID>]: [{
+  //     id: `${remoteId}/folderPath`,
+  //     jobId,
+  //     jobName,
+  //     scheduleId,
+  //     scheduleName,
+  //     timestamp,
+  //     pool,
+  //     poolMaster,
+  //   }]
+  // }
+  async _listPoolMetadataBackups(remoteId, handler) {
+    const safeReaddir = createSafeReaddir(handler, 'listXoMetadataBackups')
+
+    const backupsByPool = {}
+    await asyncMap(
+      safeReaddir(DIR_XO_POOL_METADATA_BACKUPS, { prependDir: true }),
+      scheduleDir =>
+        asyncMap(safeReaddir(scheduleDir), poolId => {
+          const backups = backupsByPool[poolId] ?? (backupsByPool[poolId] = [])
+          return asyncMap(
+            safeReaddir(`${scheduleDir}/${poolId}`, { prependDir: true }),
+            async backupDir => {
+              try {
+                backups.push({
+                  id: `${remoteId}${backupDir}`,
+                  ...JSON.parse(
+                    String(await handler.readFile(`${backupDir}/metadata.json`))
+                  ),
+                })
+              } catch (error) {
+                log.warn(`listPoolMetadataBackups ${backupDir}`, {
+                  error,
+                })
+              }
+            }
+          )
+        })
+    )
+
+    // delete empty entries and sort backups
+    Object.keys(backupsByPool).forEach(poolId => {
+      const backups = backupsByPool[poolId]
+      if (backups.length === 0) {
+        delete backupsByPool[poolId]
+      } else {
+        backups.sort(compareTimestamp)
+      }
+    })
+
+    return backupsByPool
+  }
+
+  //  {
+  //    xo: {
+  //      [remote ID]: xoBackups
+  //    },
+  //    pool: {
+  //      [remote ID]: poolBackups
+  //    }
+  //  }
+  async listMetadataBackups(remoteIds: string[]) {
+    const app = this._app
+
+    const xo = {}
+    const pool = {}
+    await Promise.all(
+      remoteIds.map(async remoteId => {
+        try {
+          const handler = await app.getRemoteHandler(remoteId)
+
+          const [xoList, poolList] = await Promise.all([
+            this._listXoMetadataBackups(remoteId, handler),
+            this._listPoolMetadataBackups(remoteId, handler),
+          ])
+          if (xoList.length !== 0) {
+            xo[remoteId] = xoList
+          }
+          if (Object.keys(poolList).length !== 0) {
+            pool[remoteId] = poolList
+          }
+        } catch (error) {
+          log.warn(`listMetadataBackups for remote ${remoteId}`, { error })
+        }
+      })
+    )
+
+    return {
+      xo,
+      pool,
+    }
+  }
+
+  // Task logs emitted in a restore execution:
+  //
+  // task.start(message: 'restore', data: <Metadata />)
+  // └─ task.end
+  async restoreMetadataBackup(id: string) {
+    const app = this._app
+    const logger = this._logger
+    const message = 'metadataRestore'
+    const [remoteId, dir, ...path] = id.split('/')
+    const handler = await app.getRemoteHandler(remoteId)
+    const metadataFolder = `${dir}/${path.join('/')}`
+
+    const taskId = logger.notice(message, {
+      event: 'task.start',
+      data: JSON.parse(
+        String(await handler.readFile(`${metadataFolder}/metadata.json`))
+      ),
+    })
+    try {
+      this._runningMetadataRestores.add(taskId)
+
+      let result
+      if (dir === DIR_XO_CONFIG_BACKUPS) {
+        result = await app.importConfig(
+          JSON.parse(
+            String(await handler.readFile(`${metadataFolder}/data.json`))
+          )
+        )
+      } else {
+        result = await app
+          .getXapi(path[1])
+          .importPoolMetadata(
+            await handler.createReadStream(`${metadataFolder}/data`),
+            true
+          )
+      }
+
+      logger.notice(message, {
+        event: 'task.end',
+        result,
+        status: 'success',
+        taskId,
+      })
+    } catch (error) {
+      logger.error(message, {
+        event: 'task.end',
+        result: serializeError(error),
+        status: 'failure',
+        taskId,
+      })
+      throw error
+    } finally {
+      this._runningMetadataRestores.delete(taskId)
+    }
+  }
+
+  async deleteMetadataBackup(id: string) {
+    const uuidReg = '\\w{8}(-\\w{4}){3}-\\w{12}'
+    const metadataDirReg = 'xo-(config|pool-metadata)-backups'
+    const timestampReg = '\\d{8}T\\d{6}Z'
+
+    const regexp = new RegExp(
+      `^/?${uuidReg}/${metadataDirReg}/${uuidReg}(/${uuidReg})?/${timestampReg}`
+    )
+
+    if (!regexp.test(id)) {
+      throw new Error(`The id (${id}) not correspond to a metadata folder`)
+    }
+    const app = this._app
+    const [remoteId, ...path] = id.split('/')
+
+    const handler = await app.getRemoteHandler(remoteId)
+    return handler.rmtree(path.join('/'))
+  }
 }

packages/xo-server/src/xo-mixins/patches.js

@@ -0,0 +1,18 @@
+import { differenceBy } from 'lodash'
+
+export default class {
+  constructor(xo) {
+    this._xo = xo
+  }
+
+  getPatchesDifference(hostA, hostB) {
+    const patchesA = this._xo
+      .getObject(hostA)
+      .patches.map(patchId => this._xo.getObject(patchId))
+    const patchesB = this._xo
+      .getObject(hostB)
+      .patches.map(patchId => this._xo.getObject(patchId))
+
+    return differenceBy(patchesA, patchesB, 'name').map(patch => patch.name)
+  }
+}

packages/xo-server/src/xo-mixins/xen-servers.js

@@ -40,7 +40,7 @@ const log = createLogger('xo:xo-mixins:xen-servers')
 // - _xapis[server.id] id defined
 // - _serverIdsByPool[xapi.pool.$id] is server.id
 export default class {
-  constructor(xo, { xapiOptions }) {
+  constructor(xo, { guessVhdSizeOnImport, xapiOptions }) {
     this._objectConflicts = { __proto__: null } // TODO: clean when a server is disconnected.
     const serversDb = (this._servers = new Servers({
       connection: xo._redis,
@@ -49,7 +49,10 @@ export default class {
     }))
     this._serverIdsByPool = { __proto__: null }
     this._stats = new XapiStats()
-    this._xapiOptions = xapiOptions
+    this._xapiOptions = {
+      guessVhdSizeOnImport,
+      ...xapiOptions,
+    }
     this._xapis = { __proto__: null }
     this._xo = xo
 
@@ -456,8 +459,8 @@ export default class {
     const xapis = this._xapis
     forEach(servers, server => {
       server.status = this._getXenServerStatus(server.id)
-      if (server.status === 'connected' && server.label === undefined) {
-        server.label = xapis[server.id].pool.name_label
+      if (server.status === 'connected') {
+        server.poolId = xapis[server.id].pool.uuid
       }
 
       // Do not expose password.

packages/xo-vmdk-to-vhd/package.json

@@ -29,7 +29,7 @@
     "pipette": "^0.9.3",
     "promise-toolbox": "^0.12.1",
     "tmp": "^0.0.33",
-    "vhd-lib": "^0.5.1"
+    "vhd-lib": "^0.6.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",

packages/xo-web/package.json

@@ -1,7 +1,7 @@
 {
   "private": false,
   "name": "xo-web",
-  "version": "5.37.0",
+  "version": "5.38.0",
   "license": "AGPL-3.0",
   "description": "Web interface client for Xen-Orchestra",
   "keywords": [

packages/xo-web/src/common/hosts-patches-table.js

@@ -15,11 +15,7 @@ import {
   createFilter,
   createSelector,
 } from './selectors'
-import {
-  installAllHostPatches,
-  installAllPatchesOnPool,
-  subscribeHostMissingPatches,
-} from './xo'
+import { installAllPatchesOnPool, subscribeHostMissingPatches } from './xo'
 
 // ===================================================================
 
@@ -43,17 +39,6 @@ const MISSING_PATCHES_COLUMNS = [
     ),
     sortCriteria: (host, { missingPatches }) => missingPatches[host.id],
   },
-  {
-    name: _('patchUpdateButton'),
-    itemRenderer: (host, { installAllHostPatches }) => (
-      <ActionButton
-        btnStyle='primary'
-        handler={installAllHostPatches}
-        handlerParam={host}
-        icon='host-patch-update'
-      />
-    ),
-  },
 ]
 
 const POOLS_MISSING_PATCHES_COLUMNS = [
@@ -115,7 +100,9 @@ class HostsPatchesTable extends Component {
       pools[host.$pool] = true
     })
 
-    return Promise.all(map(keys(pools), installAllPatchesOnPool))
+    return Promise.all(
+      map(keys(pools), pool => installAllPatchesOnPool({ pool }))
+    )
   }
 
   componentDidMount() {
@@ -162,7 +149,6 @@ class HostsPatchesTable extends Component {
                 : MISSING_PATCHES_COLUMNS
             }
             userData={{
-              installAllHostPatches,
               missingPatches: this.state.missingPatches,
               pools,
             }}

packages/xo-web/src/common/intl/locales/fr.js

@@ -2643,9 +2643,9 @@ export default {
   // Original text: "New"
   resourceSetNew: 'Nouvelle',
 
-  // Original text: "Try dropping some VMs files here, or click to select VMs to upload. Accept only .xva/.ova files."
+  // Original text: "Drop OVA or XVA files here to import Virtual Machines."
   importVmsList:
-    'Essayez de déposer des fichiers de VMs ici, ou bien cliquez pour sélectionner des VMs à téléverser. Seuls les fichiers .xva/.ova sont acceptés.',
+    'Déposez ici vos fichiers OVA ou XVA pour importer des machines virtuelles.',
 
   // Original text: "No selected VMs."
   noSelectedVms: 'Pas de VM sélectionnée.',

packages/xo-web/src/common/intl/messages.js

@@ -39,6 +39,13 @@ const messages = {
   hasInactivePath: 'Has an inactive path',
   pools: 'Pools',
   remotes: 'Remotes',
+  type: 'Type',
+  restore: 'Restore',
+  delete: 'Delete',
+  vms: 'VMs',
+  metadata: 'Metadata',
+  chooseBackup: 'Choose a backup',
+  clickToShowError: 'Click to show error',
 
   // ----- Modals -----
   alertOk: 'OK',
@@ -443,6 +450,7 @@ const messages = {
   offlineSnapshotInfo: 'Shutdown VMs before snapshotting them',
   timeout: 'Timeout',
   timeoutInfo: 'Number of hours after which a job is considered failed',
+  fullBackupInterval: 'Full backup interval',
   timeoutUnit: 'in hours',
   dbAndDrRequireEnterprisePlan: 'Delta Backup and DR require Enterprise plan',
   crRequiresPremiumPlan: 'CR requires Premium plan',
@@ -713,6 +721,14 @@ const messages = {
   displayAllHosts: 'Display all hosts of this pool',
   displayAllStorages: 'Display all storages of this pool',
   displayAllVMs: 'Display all VMs of this pool',
+  licenseRestrictions: 'License restrictions',
+  licenseRestrictionsModalTitle:
+    'Warning: you are using a Free XenServer license',
+  actionsRestricted: 'Some actions will be restricted.',
+  counterRestrictionsOptions: 'You can:',
+  counterRestrictionsOptionsXcp:
+    'upgrade to XCP-ng for free to get rid of these restrictions',
+  counterRestrictionsOptionsXsLicense: 'or get a commercial Citrix license',
   // ----- Pool tabs -----
   hostsTabName: 'Hosts',
   vmsTabName: 'Vms',
@@ -756,10 +772,12 @@ const messages = {
   addSrLabel: 'Add SR',
   addVmLabel: 'Add VM',
   addHostLabel: 'Add Host',
-  hostNeedsPatchUpdate:
-    'This host needs to install {patches, number} patch{patches, plural, one {} other {es}} before it can be added to the pool. This operation may be long.',
-  hostNeedsPatchUpdateNoInstall:
-    "This host cannot be added to the pool because it's missing some patches.",
+  missingPatchesPool:
+    'The pool needs to install {nMissingPatches, number} patch{nMissingPatches, plural, one {} other {es}}. This operation may be long.',
+  missingPatchesHost:
+    'This host needs to install {nMissingPatches, number} patch{nMissingPatches, plural, one {} other {es}}. This operation may be long.',
+  patchUpdateNoInstall:
+    'This host cannot be added to the pool because the patches are not homogeneous.',
   addHostErrorTitle: 'Adding host failed',
   addHostNotHomogeneousErrorMessage: 'Host patches could not be homogenized.',
   disconnectServer: 'Disconnect',
@@ -886,14 +904,14 @@ const messages = {
   hostAppliedPatches: 'Applied patches',
   hostMissingPatches: 'Missing patches',
   hostUpToDate: 'Host up-to-date!',
-  installPatchWarningTitle: 'Non-recommended patch install',
-  installPatchWarningContent:
-    'This will install a patch only on this host. This is NOT the recommended way: please go into the Pool patch view and follow instructions there. If you are sure about this, you can continue anyway',
-  installPatchWarningReject: 'Go to pool',
-  installPatchWarningResolve: 'Install',
+  installAllPatchesTitle: 'Install all patches',
+  installAllPatchesContent: 'To install all patches go to pool.',
+  installAllPatchesRedirect: 'Go to pool',
+  installAllPatchesOnHostContent:
+    'Are you sure you want to install all patches on this host?',
   patchRelease: 'Release',
   updatePluginNotInstalled:
-    'Update plugin is not installed on this host. Please run `yum install xcp-ng-updater` first.',
+    'An error occurred while fetching the patches. Please make sure the updater plugin is installed by running `yum install xcp-ng-updater` on the host.',
   showChangelog: 'Show changelog',
   changelog: 'Changelog',
   changelogPatch: 'Patch',
@@ -902,6 +920,10 @@ const messages = {
   changelogDescription: 'Description',
   // ----- Pool patch tabs -----
   refreshPatches: 'Refresh patches',
+  install: 'Install',
+  installPatchesTitle: 'Install patch{nPatches, plural, one {} other {es}}',
+  installPatchesContent:
+    'Are you sure you want to install {nPatches, number} patch{nPatches, plural, one {} other {es}}?',
   installPoolPatches: 'Install pool patches',
   confirmPoolPatch:
     'Are you sure you want to install all the patches on this pool?',
@@ -1362,8 +1384,7 @@ const messages = {
   resourceSetNew: 'New',
 
   // ---- VM import ---
-  importVmsList:
-    'Try dropping some VMs files here, or click to select VMs to upload. Accept only .xva/.ova files.',
+  importVmsList: 'Drop OVA or XVA files here to import Virtual Machines.',
   noSelectedVms: 'No selected VMs.',
   vmImportToPool: 'To Pool:',
   vmImportToSr: 'To SR:',
@@ -1420,6 +1441,7 @@ const messages = {
   simpleBackup: 'simple',
   delta: 'delta',
   restoreBackups: 'Restore Backups',
+  noBackups: 'There are no backups!',
   restoreBackupsInfo: 'Click on a VM to display restore options',
   restoreDeltaBackupsInfo:
     'Only the files of Delta Backup which are not on a SMB remote can be restored',
@@ -1460,10 +1482,16 @@ const messages = {
   restoreVmBackupsStart:
     'Start VM{nVms, plural, one {} other {s}} after restore',
   restoreVmBackupsBulkErrorTitle: 'Multi-restore error',
+  restoreMetadataBackupTitle: 'Restore {item}',
+  bulkRestoreMetadataBackupTitle:
+    'Restore {nMetadataBackups, number} metadata backup{nMetadataBackups, plural, one {} other {s}}',
+  bulkRestoreMetadataBackupMessage:
+    'Restore {nMetadataBackups, number} metadata backup{nMetadataBackups, plural, one {} other {s}} from {nMetadataBackups, plural, one {its} other {their}} {oldestOrLatest} backup',
+  deleteMetadataBackupTitle: 'Delete {item} backup',
   restoreVmBackupsBulkErrorMessage: 'You need to select a destination SR',
   deleteVmBackups: 'Delete backups…',
   deleteVmBackupsTitle: 'Delete {vm} backups',
-  deleteVmBackupsSelect: 'Select backups to delete:',
+  deleteBackupsSelect: 'Select backups to delete:',
   deleteVmBackupsSelectAll: 'All',
   deleteVmBackupsBulkTitle: 'Delete backups',
   deleteVmBackupsBulkMessage:
@@ -1471,6 +1499,11 @@ const messages = {
   deleteVmBackupsBulkConfirmText:
     'delete {nBackups} backup{nBackups, plural, one {} other {s}}',
   unknownJob: 'Unknown job',
+  bulkDeleteMetadataBackupsTitle: 'Delete metadata backups',
+  bulkDeleteMetadataBackupsMessage:
+    'Are you sure you want to delete all the backups from {nMetadataBackups, number} metadata backup{nMetadataBackups, plural, one {} other {s}}?',
+  bulkDeleteMetadataBackupsConfirmText:
+    'delete {nMetadataBackups} metadata backup{nMetadataBackups, plural, one {} other {s}}',
 
   // ----- Restore files view -----
   listRemoteBackups: 'List remote backups',
@@ -1905,6 +1938,7 @@ const messages = {
   logsJobName: 'Job name',
   logsBackupTime: 'Backup time',
   logsRestoreTime: 'Restore time',
+  copyLogToClipboard: 'Copy log to clipboard',
   logsVmNotFound: 'VM not found!',
   logsMissingVms: 'Missing VMs skipped ({ vms })',
   logsFailedRestoreError: 'Click to show error',

packages/xo-web/src/common/json-schema-input/object-input.js

@@ -6,6 +6,7 @@ import { keyBy, map } from 'lodash'
 
 import _ from '../intl'
 import Component from '../base-component'
+import getEventValue from '../get-event-value'
 import { EMPTY_OBJECT } from '../utils'
 
 import GenericInput from './generic-input'
@@ -33,6 +34,14 @@ export default class ObjectInput extends Component {
     })
   }
 
+  _onUseChange = event => {
+    const use = getEventValue(event)
+    if (!use) {
+      this.props.onChange()
+    }
+    this.setState({ use })
+  }
+
   _getRequiredProps = createSelector(
     () => this.props.schema.required,
     required => (required ? keyBy(required) : EMPTY_OBJECT)
@@ -67,7 +76,7 @@ export default class ObjectInput extends Component {
               <input
                 checked={use}
                 disabled={disabled}
-                onChange={this.linkState('use')}
+                onChange={this._onUseChange}
                 type='checkbox'
               />{' '}
               {_('fillOptionalInformations')}

packages/xo-web/src/common/selectors.js

@@ -332,8 +332,8 @@ export const createSortForType = invoke(() => {
   const iterateesByType = {
     message: message => message.time,
     PIF: pif => pif.device,
+    patch: patch => patch.name,
     pool: pool => pool.name_label,
-    pool_patch: patch => patch.name,
     tag: tag => tag,
     VBD: vbd => vbd.position,
     'VDI-snapshot': snapshot => snapshot.snapshot_time,
@@ -494,37 +494,18 @@ export const createGetObjectMessages = objectSelector =>
 export const getObject = createGetObject((_, id) => id)
 
 export const createDoesHostNeedRestart = hostSelector => {
-  // XS < 7.1
-  const patchRequiresReboot = createGetObjectsOfType('pool_patch')
-    .pick(
-      // Returns the first patch of the host which requires it to be
-      // restarted.
-      create(
-        createGetObjectsOfType('host_patch')
-          .pick((state, props) => {
-            const host = hostSelector(state, props)
-            return host && host.patches
-          })
-          .filter(
-            create(
-              (state, props) => {
-                const host = hostSelector(state, props)
-                return host && host.startTime
-              },
-              startTime => patch => patch.time > startTime
-            )
-          ),
-        hostPatches => map(hostPatches, hostPatch => hostPatch.pool_patch)
+  const patchRequiresReboot = createGetObjectsOfType('patch')
+    .pick(create(hostSelector, host => host.patches))
+    .find(
+      create(hostSelector, host => ({ guidance, time, upgrade }) =>
+        time > host.startTime &&
+        (upgrade ||
+          some(
+            guidance,
+            action => action === 'restartHost' || action === 'restartXapi'
+          ))
       )
     )
-    .find([
-      ({ guidance, upgrade }) =>
-        upgrade ||
-        find(
-          guidance,
-          action => action === 'restartHost' || action === 'restartXapi'
-        ),
-    ])
 
   return create(
     hostSelector,

packages/xo-web/src/common/smart-backup/index.js

@@ -1,5 +1,5 @@
 import * as CM from 'complex-matcher'
-import { get, identity, isEmpty } from 'lodash'
+import { escapeRegExp, get, identity, isEmpty } from 'lodash'
 
 import { EMPTY_OBJECT } from './../utils'
 
@@ -59,7 +59,7 @@ export const constructSmartPattern = (
 
 const valueToComplexMatcher = pattern => {
   if (typeof pattern === 'string') {
-    return new CM.String(pattern)
+    return new CM.RegExpNode(`^${escapeRegExp(pattern)}$`)
   }
 
   if (Array.isArray(pattern)) {

packages/xo-web/src/common/utils.js

@@ -643,3 +643,10 @@ export const createCompare = criterias => (...items) => {
   })
   return res
 }
+
+// ===================================================================
+
+export const hasLicenseRestrictions = host =>
+  host.productBrand !== 'XCP-ng' &&
+  versionSatisfies(host.version, '>=7.3.0') &&
+  host.license_params.sku_type === 'free'

packages/xo-web/src/common/xo/add-host-modal/index.js

@@ -9,10 +9,10 @@ import {
   createCollectionWrapper,
   createGetObjectsOfType,
   createSelector,
-  createGetObject,
 } from 'selectors'
+import { forEach } from 'lodash'
+import { getPatchesDifference } from 'xo'
 import { SelectHost } from 'select-objects'
-import { differenceBy, forEach } from 'lodash'
 
 @connectStore(
   () => ({
@@ -38,20 +38,20 @@ import { differenceBy, forEach } from 'lodash'
         return singleHosts
       })
     ),
-    poolMasterPatches: createSelector(
-      createGetObject((_, props) => props.pool.master),
-      ({ patches }) => patches
-    ),
   }),
   { withRef: true }
 )
 export default class AddHostModal extends BaseComponent {
   get value() {
-    if (process.env.XOA_PLAN < 2 && this.state.nMissingPatches) {
+    const { nHostMissingPatches, nPoolMissingPatches } = this.state
+    if (
+      process.env.XOA_PLAN < 2 &&
+      (nHostMissingPatches > 0 || nPoolMissingPatches > 0)
+    ) {
       return {}
     }
 
-    return this.state
+    return { host: this.state.host }
   }
 
   _getHostPredicate = createSelector(
@@ -59,18 +59,29 @@ export default class AddHostModal extends BaseComponent {
     singleHosts => host => singleHosts[host.id]
   )
 
-  _onChangeHost = host => {
+  _onChangeHost = async host => {
+    if (host === null) {
+      this.setState({
+        host,
+        nHostMissingPatches: undefined,
+        nPoolMissingPatches: undefined,
+      })
+      return
+    }
+
+    const { master } = this.props.pool
+    const hostMissingPatches = await getPatchesDifference(host.id, master)
+    const poolMissingPatches = await getPatchesDifference(master, host.id)
+
     this.setState({
       host,
-      nMissingPatches: host
-        ? differenceBy(this.props.poolMasterPatches, host.patches, 'name')
-            .length
-        : undefined,
+      nHostMissingPatches: hostMissingPatches.length,
+      nPoolMissingPatches: poolMissingPatches.length,
     })
   }
 
   render() {
-    const { nMissingPatches } = this.state
+    const { nHostMissingPatches, nPoolMissingPatches } = this.state
 
     return (
       <div>
@@ -85,17 +96,39 @@ export default class AddHostModal extends BaseComponent {
           </Col>
         </SingleLineRow>
         <br />
-        {nMissingPatches > 0 && (
-          <SingleLineRow>
-            <Col>
-              <span className='text-danger'>
-                <Icon icon='error' />{' '}
-                {process.env.XOA_PLAN > 1
-                  ? _('hostNeedsPatchUpdate', { patches: nMissingPatches })
-                  : _('hostNeedsPatchUpdateNoInstall')}
-              </span>
-            </Col>
-          </SingleLineRow>
+        {(nHostMissingPatches > 0 || nPoolMissingPatches > 0) && (
+          <div>
+            {process.env.XOA_PLAN > 1 ? (
+              <div>
+                {nPoolMissingPatches > 0 && (
+                  <SingleLineRow>
+                    <Col>
+                      <span className='text-danger'>
+                        <Icon icon='error' />{' '}
+                        {_('missingPatchesPool', {
+                          nMissingPatches: nPoolMissingPatches,
+                        })}
+                      </span>
+                    </Col>
+                  </SingleLineRow>
+                )}
+                {nHostMissingPatches > 0 && (
+                  <SingleLineRow>
+                    <Col>
+                      <span className='text-danger'>
+                        <Icon icon='error' />{' '}
+                        {_('missingPatchesHost', {
+                          nMissingPatches: nHostMissingPatches,
+                        })}
+                      </span>
+                    </Col>
+                  </SingleLineRow>
+                )}
+              </div>
+            ) : (
+              _('patchUpdateNoInstall')
+            )}
+          </div>
         )}
       </div>
     )

packages/xo-web/src/common/xo/index.js

@@ -556,6 +556,12 @@ export const removeServer = server =>
 export const editPool = (pool, props) =>
   _call('pool.set', { id: resolveId(pool), ...props })
 
+export const getPatchesDifference = (source, target) =>
+  _call('pool.getPatchesDifference', {
+    source: resolveId(source),
+    target: resolveId(target),
+  })
+
 import AddHostModalBody from './add-host-modal' // eslint-disable-line import/first
 export const addHostToPool = (pool, host) => {
   if (host) {
@@ -739,23 +745,18 @@ export const enableHost = host => _call('host.enable', { id: resolveId(host) })
 export const disableHost = host =>
   _call('host.disable', { id: resolveId(host) })
 
-const missingUpdatePluginByHost = { __proto__: null }
 export const getHostMissingPatches = async host => {
   const hostId = resolveId(host)
   if (host.productBrand !== 'XCP-ng') {
-    const patches = await _call('host.listMissingPatches', { host: hostId })
+    const patches = await _call('pool.listMissingPatches', { host: hostId })
     // Hide paid patches to XS-free users
     return host.license_params.sku_type !== 'free'
       ? patches
       : filter(patches, { paid: false })
   }
-  if (missingUpdatePluginByHost[hostId]) {
-    return null
-  }
   try {
-    return await _call('host.listMissingPatches', { host: hostId })
+    return await _call('pool.listMissingPatches', { host: hostId })
   } catch (_) {
-    missingUpdatePluginByHost[hostId] = true
     return null
   }
 }
@@ -776,18 +777,30 @@ export const emergencyShutdownHosts = hosts => {
   }).then(() => map(hosts, host => emergencyShutdownHost(host)), noop)
 }
 
-export const installHostPatch = (host, { uuid }) =>
-  _call('host.installPatch', { host: resolveId(host), patch: uuid })::tap(() =>
-    subscribeHostMissingPatches.forceRefresh(host)
+// for XCP-ng now
+export const installAllPatchesOnHost = ({ host }) =>
+  confirm({
+    body: _('installAllPatchesOnHostContent'),
+    title: _('installAllPatchesTitle'),
+  }).then(() =>
+    _call('pool.installPatches', { hosts: [resolveId(host)] })::tap(() =>
+      subscribeHostMissingPatches.forceRefresh(host)
+    )
   )
 
-export const installAllHostPatches = host =>
-  _call('host.installAllPatches', { host: resolveId(host) })::tap(() =>
-    subscribeHostMissingPatches.forceRefresh(host)
+export const installPatches = (patches, pool) =>
+  confirm({
+    body: _('installPatchesContent', { nPatches: patches.length }),
+    title: _('installPatchesTitle', { nPatches: patches.length }),
+  }).then(() =>
+    _call('pool.installPatches', {
+      pool: resolveId(pool),
+      patches: resolveIds(patches),
+    })::tap(() => subscribeHostMissingPatches.forceRefresh())
   )
 
 import InstallPoolPatchesModalBody from './install-pool-patches-modal' // eslint-disable-line import/first
-export const installAllPatchesOnPool = pool => {
+export const installAllPatchesOnPool = ({ pool }) => {
   const poolId = resolveId(pool)
   return confirm({
     body: <InstallPoolPatchesModalBody pool={poolId} />,
@@ -795,7 +808,7 @@ export const installAllPatchesOnPool = pool => {
     icon: 'host-patch-update',
   }).then(
     () =>
-      _call('pool.installAllPatches', { pool: poolId })::tap(() =>
+      _call('pool.installPatches', { pool: poolId })::tap(() =>
         subscribeHostMissingPatches.forceRefresh()
       ),
     noop
@@ -2011,6 +2024,27 @@ export const editMetadataBackupJob = props =>
 export const runMetadataBackupJob = params =>
   _call('metadataBackup.runJob', params)
 
+export const listMetadataBackups = remotes =>
+  _call('metadataBackup.list', { remotes: resolveIds(remotes) })
+
+export const restoreMetadataBackup = backup =>
+  _call('metadataBackup.restore', {
+    id: resolveId(backup),
+  })::tap(subscribeBackupNgLogs.forceRefresh)
+
+export const deleteMetadataBackup = backup =>
+  _call('metadataBackup.delete', {
+    id: resolveId(backup),
+  })
+
+export const deleteMetadataBackups = async (backups = []) => {
+  // delete sequentially from newest to oldest
+  backups = backups.slice().sort((b1, b2) => b2.timestamp - b1.timestamp)
+  for (let i = 0, n = backups.length; i < n; ++i) {
+    await deleteMetadataBackup(backups[i])
+  }
+}
+
 // Plugins -----------------------------------------------------------
 
 export const loadPlugin = async id =>

packages/xo-web/src/xo-app/backup-ng/_getSettingsWithNonDefaultValue.js

@@ -5,6 +5,7 @@ const DEFAULTS = {
 
   compression: '',
   concurrency: 0,
+  fullInterval: 0,
   offlineSnapshot: false,
   timeout: 0,
 }

packages/xo-web/src/xo-app/backup-ng/index.js

@@ -42,7 +42,7 @@ import FileRestore from './file-restore'
 import getSettingsWithNonDefaultValue from './_getSettingsWithNonDefaultValue'
 import Health from './health'
 import NewVmBackup, { NewMetadataBackup } from './new'
-import Restore from './restore'
+import Restore, { RestoreMetadata } from './restore'
 import { destructPattern } from './utils'
 
 const Ul = props => <ul {...props} style={{ listStyleType: 'none' }} />
@@ -249,6 +249,7 @@ class JobsTable extends React.Component {
           const {
             compression,
             concurrency,
+            fullInterval,
             offlineSnapshot,
             reportWhen,
             timeout,
@@ -268,6 +269,9 @@ class JobsTable extends React.Component {
               {timeout !== undefined && (
                 <Li>{_.keyValue(_('timeout'), timeout / 3600e3)} hours</Li>
               )}
+              {fullInterval !== undefined && (
+                <Li>{_.keyValue(_('fullBackupInterval'), fullInterval)}</Li>
+              )}
               {offlineSnapshot !== undefined && (
                 <Li>
                   {_.keyValue(
@@ -423,6 +427,7 @@ export default routes('overview', {
   'new/metadata': NewMetadataBackup,
   overview: Overview,
   restore: Restore,
+  'restore/metadata': RestoreMetadata,
   'file-restore': FileRestore,
   health: Health,
 })(({ children }) => (

packages/xo-web/src/xo-app/backup-ng/new/index.js

@@ -520,6 +520,12 @@ export default decorate([
           value: value && value * 3600e3,
         })
       },
+      setFullInterval({ setGlobalSettings }, value) {
+        setGlobalSettings({
+          name: 'fullInterval',
+          value,
+        })
+      },
       setOfflineSnapshot: (
         { setGlobalSettings },
         { target: { checked: value } }
@@ -534,6 +540,7 @@ export default decorate([
       compressionId: generateId,
       formId: generateId,
       inputConcurrencyId: generateId,
+      inputFullIntervalId: generateId,
       inputReportWhenId: generateId,
       inputTimeoutId: generateId,
 
@@ -631,9 +638,14 @@ export default decorate([
   ({ state, effects, remotes, srsById, job = {}, intl }) => {
     const { formatMessage } = intl
     const { propSettings, settings = propSettings } = state
-    const { concurrency, reportWhen = 'failure', offlineSnapshot, timeout } =
-      settings.get('') || {}
     const compression = defined(state.compression, job.compression, '')
+    const {
+      concurrency,
+      fullInterval,
+      offlineSnapshot,
+      reportWhen = 'failure',
+      timeout,
+    } = settings.get('') || {}
 
     if (state.needUpdateParams) {
       effects.updateParams()
@@ -923,6 +935,16 @@ export default decorate([
                           placeholder={formatMessage(messages.timeoutUnit)}
                         />
                       </FormGroup>
+                      <FormGroup>
+                        <label htmlFor={state.inputFullIntervalId}>
+                          <strong>{_('fullBackupInterval')}</strong>
+                        </label>{' '}
+                        <Number
+                          id={state.inputFullIntervalId}
+                          onChange={effects.setFullInterval}
+                          value={fullInterval}
+                        />
+                      </FormGroup>
                       {state.isFull && (
                         <FormGroup>
                           <label htmlFor={state.compressionId}>

packages/xo-web/src/xo-app/backup-ng/restore/delete-backups-modal-body.js

@@ -49,7 +49,7 @@ export default class DeleteBackupsModalBody extends Component {
   render() {
     return (
       <div>
-        <div>{_('deleteVmBackupsSelect')}</div>
+        <div>{_('deleteBackupsSelect')}</div>
         <div className='list-group'>
           {map(this._getBackups(), backup => (
             <button

packages/xo-web/src/xo-app/backup-ng/restore/delete-metadata-backups-modal-body.js

@@ -0,0 +1,52 @@
+import _ from 'intl'
+import Component from 'base-component'
+import PropTypes from 'prop-types'
+import React from 'react'
+import SingleLineRow from 'single-line-row'
+import { Container, Col } from 'grid'
+import { FormattedDate } from 'react-intl'
+import { Select } from 'form'
+
+export default class DeleteMetadataBackupModalBody extends Component {
+  static propTypes = {
+    backups: PropTypes.array,
+  }
+
+  get value() {
+    return this.state.backups
+  }
+
+  _optionRenderer = ({ timestamp }) => (
+    <FormattedDate
+      value={new Date(timestamp)}
+      month='long'
+      day='numeric'
+      year='numeric'
+      hour='2-digit'
+      minute='2-digit'
+      second='2-digit'
+    />
+  )
+
+  render() {
+    return (
+      <Container>
+        <SingleLineRow>
+          <Col size={6}>{_('deleteBackupsSelect')}</Col>
+          <Col size={6}>
+            <Select
+              labelKey='timestamp'
+              multi
+              onChange={this.linkState('backups')}
+              optionRenderer={this._optionRenderer}
+              options={this.props.backups}
+              required
+              value={this.state.backups}
+              valueKey='id'
+            />
+          </Col>
+        </SingleLineRow>
+      </Container>
+    )
+  }
+}

packages/xo-web/src/xo-app/backup-ng/restore/index.js

@@ -1,6 +1,8 @@
 import _ from 'intl'
 import ActionButton from 'action-button'
+import ButtonLink from 'button-link'
 import Component from 'base-component'
+import Icon from 'icon'
 import React from 'react'
 import SortedTable from 'sorted-table'
 import Upgrade from 'xoa-upgrade'
@@ -37,6 +39,8 @@ import RestoreLegacy from '../restore-legacy'
 
 import Logs from '../../logs/restore'
 
+export RestoreMetadata from './metadata'
+
 // -----------------------------------------------------------------------------
 
 const BACKUPS_COLUMNS = [
@@ -265,7 +269,10 @@ export default class Restore extends Component {
               icon='refresh'
             >
               {_('restoreResfreshList')}
-            </ActionButton>
+            </ActionButton>{' '}
+            <ButtonLink to='backup-ng/restore/metadata'>
+              <Icon icon='database' /> {_('metadata')}
+            </ButtonLink>
           </div>
           <SortedTable
             actions={this._actions}

packages/xo-web/src/xo-app/backup-ng/restore/metadata.js

@@ -0,0 +1,280 @@
+import _ from 'intl'
+import addSubscriptions from 'add-subscriptions'
+import ButtonLink from 'button-link'
+import Copiable from 'copiable'
+import decorate from 'apply-decorators'
+import Icon from 'icon'
+import NoObjects from 'no-objects'
+import React from 'react'
+import SortedTable from 'sorted-table'
+import Upgrade from 'xoa-upgrade'
+import { confirm } from 'modal'
+import { error } from 'notification'
+import { flatMap, forOwn, reduce, toArray } from 'lodash'
+import { FormattedDate } from 'react-intl'
+import { injectState, provideState } from 'reaclette'
+import { noop } from 'utils'
+import {
+  deleteMetadataBackups,
+  listMetadataBackups,
+  restoreMetadataBackup,
+  subscribeRemotes,
+} from 'xo'
+
+import Logs from '../../logs/restore-metadata'
+
+import DeleteMetadataBackupModalBody from './delete-metadata-backups-modal-body'
+import RestoreMetadataBackupModalBody, {
+  RestoreMetadataBackupsBulkModalBody,
+} from './restore-metadata-backups-modal-body'
+
+// Actions -------------------------------------------------------------------
+
+const restore = entry =>
+  confirm({
+    title: _('restoreMetadataBackupTitle', {
+      item: `${entry.type} (${entry.label})`,
+    }),
+    body: <RestoreMetadataBackupModalBody backups={entry.backups} />,
+    icon: 'restore',
+  }).then(backup => {
+    if (backup === undefined) {
+      error(_('backupRestoreErrorTitle'), _('chooseBackup'))
+      return
+    }
+    return restoreMetadataBackup(backup)
+  }, noop)
+
+const bulkRestore = entries => {
+  const nMetadataBackups = entries.length
+  return confirm({
+    title: _('bulkRestoreMetadataBackupTitle', { nMetadataBackups }),
+    body: (
+      <RestoreMetadataBackupsBulkModalBody
+        nMetadataBackups={nMetadataBackups}
+      />
+    ),
+    icon: 'restore',
+  }).then(
+    latest =>
+      Promise.all(
+        entries.map(({ first, last }) =>
+          restoreMetadataBackup(latest ? last : first)
+        )
+      ),
+    noop
+  )
+}
+
+const delete_ = entry =>
+  confirm({
+    title: _('deleteMetadataBackupTitle', {
+      item: `${entry.type} (${entry.label})`,
+    }),
+    body: <DeleteMetadataBackupModalBody backups={entry.backups} />,
+    icon: 'delete',
+  }).then(deleteMetadataBackups, noop)
+
+const bulkDelete = entries => {
+  confirm({
+    title: _('bulkDeleteMetadataBackupsTitle'),
+    body: (
+      <p>
+        {_('bulkDeleteMetadataBackupsMessage', {
+          nMetadataBackups: entries.length,
+        })}
+      </p>
+    ),
+    icon: 'delete',
+    strongConfirm: {
+      messageId: 'bulkDeleteMetadataBackupsConfirmText',
+      values: {
+        nMetadataBackups: reduce(
+          entries,
+          (sum, entry) => sum + entry.backups.length,
+          0
+        ),
+      },
+    },
+  }).then(
+    () => deleteMetadataBackups(flatMap(entries, ({ backups }) => backups)),
+    noop
+  )
+}
+
+// ---------------------------------------------------------------------------
+
+const ACTIONS = [
+  {
+    handler: bulkRestore,
+    icon: 'restore',
+    individualHandler: restore,
+    label: _('restore'),
+    level: 'primary',
+  },
+  {
+    handler: bulkDelete,
+    icon: 'delete',
+    individualHandler: delete_,
+    label: _('delete'),
+    level: 'danger',
+  },
+]
+
+const COLUMNS = [
+  {
+    name: _('type'),
+    valuePath: 'type',
+  },
+  {
+    name: _('item'),
+    itemRenderer: ({ item }) => item,
+    sortCriteria: 'id',
+  },
+  {
+    name: _('firstBackupColumn'),
+    itemRenderer: ({ first }) => (
+      <FormattedDate
+        value={new Date(first.timestamp)}
+        month='long'
+        day='numeric'
+        year='numeric'
+        hour='2-digit'
+        minute='2-digit'
+        second='2-digit'
+      />
+    ),
+    sortCriteria: 'first',
+    sortOrder: 'desc',
+  },
+  {
+    name: _('lastBackupColumn'),
+    itemRenderer: ({ last }) => (
+      <FormattedDate
+        value={new Date(last.timestamp)}
+        month='long'
+        day='numeric'
+        year='numeric'
+        hour='2-digit'
+        minute='2-digit'
+        second='2-digit'
+      />
+    ),
+    sortCriteria: 'last',
+    default: true,
+    sortOrder: 'desc',
+  },
+  {
+    name: _('availableBackupsColumn'),
+    valuePath: 'available',
+  },
+]
+
+export default decorate([
+  addSubscriptions({
+    remotes: cb =>
+      subscribeRemotes(remotes => {
+        cb(toArray(remotes))
+      }),
+  }),
+  provideState({
+    computed: {
+      // {
+      //   [jobId | poolId]: {
+      //     available: Number,
+      //     backups: Array,
+      //     first: Number,
+      //     id: jobId | poolId, // required by the SortedTable
+      //     item: Node | String,
+      //     label: String,
+      //     last: Number,
+      //     type: 'XO' | 'pool',
+      //   }
+      // }
+      async backups(_, { remotes = [] }) {
+        if (remotes.length === 0) {
+          return
+        }
+
+        const { xo: xoType, pool: poolType } = await listMetadataBackups(
+          remotes
+        )
+
+        const collection = {}
+        forOwn(xoType, entries =>
+          entries.forEach(entry => {
+            const { jobName, jobId } = entry
+            let backup = collection[jobId]
+            if (backup === undefined) {
+              backup = collection[jobId] = {
+                backups: [],
+                id: jobId,
+                item: `Xen Orchestra (${jobName})`,
+                label: jobName,
+                type: 'XO',
+              }
+            }
+
+            backup.backups.push(entry)
+          })
+        )
+        forOwn(poolType, entriesByPool =>
+          forOwn(entriesByPool, (poolEntry, poolId) => {
+            let backup = collection[poolId]
+            if (backup === undefined) {
+              const { pool, poolMaster } = poolEntry[0]
+              const label = pool.name_label || poolMaster.name_label
+              backup = collection[poolId] = {
+                backups: [],
+                id: poolId,
+                item: (
+                  <Copiable data={poolId} tagName='p'>
+                    {label || poolId}
+                  </Copiable>
+                ),
+                label,
+                type: 'pool',
+              }
+            }
+            poolEntry.forEach(entry => {
+              backup.backups.push(entry)
+            })
+          })
+        )
+
+        forOwn(collection, entry => {
+          const backups = entry.backups
+          const size = backups.length
+
+          backups.sort((a, b) => a.timestamp - b.timestamp)
+          entry.first = backups[0]
+          entry.last = backups[size - 1]
+          entry.available = size
+        })
+
+        return collection
+      },
+    },
+  }),
+  injectState,
+  ({ state, effects }) => (
+    <Upgrade place='restoreMetadataBackup' available={3}>
+      <div>
+        <div className='mb-1'>
+          <ButtonLink to='backup-ng/restore'>
+            <Icon icon='backup' /> {_('vms')}
+          </ButtonLink>
+        </div>
+        <NoObjects
+          actions={ACTIONS}
+          collection={state.backups}
+          columns={COLUMNS}
+          component={SortedTable}
+          emptyMessage={_('noBackups')}
+        />
+        <br />
+        <Logs />
+      </div>
+    </Upgrade>
+  ),
+])

packages/xo-web/src/xo-app/backup-ng/restore/restore-metadata-backups-modal-body.js

@@ -0,0 +1,82 @@
+import _ from 'intl'
+import Component from 'base-component'
+import PropTypes from 'prop-types'
+import React from 'react'
+import SingleLineRow from 'single-line-row'
+import StateButton from 'state-button'
+import { Container, Col } from 'grid'
+import { FormattedDate } from 'react-intl'
+import { Select } from 'form'
+
+export default class RestoreMetadataBackupModalBody extends Component {
+  static propTypes = {
+    backups: PropTypes.array,
+  }
+
+  get value() {
+    return this.state.backup
+  }
+
+  _optionRenderer = ({ timestamp }) => (
+    <FormattedDate
+      value={new Date(timestamp)}
+      month='long'
+      day='numeric'
+      year='numeric'
+      hour='2-digit'
+      minute='2-digit'
+      second='2-digit'
+    />
+  )
+
+  render() {
+    return (
+      <Container>
+        <SingleLineRow>
+          <Col size={6}>{_('chooseBackup')}</Col>
+          <Col size={6}>
+            <Select
+              labelKey='timestamp'
+              onChange={this.linkState('backup')}
+              optionRenderer={this._optionRenderer}
+              options={this.props.backups}
+              required
+              value={this.state.backup}
+              valueKey='id'
+            />
+          </Col>
+        </SingleLineRow>
+      </Container>
+    )
+  }
+}
+
+export class RestoreMetadataBackupsBulkModalBody extends Component {
+  static propTypes = {
+    nMetadataBackups: PropTypes.number,
+  }
+
+  state = { latest: true }
+
+  get value() {
+    return this.state.latest
+  }
+
+  render() {
+    return (
+      <div>
+        {_('bulkRestoreMetadataBackupMessage', {
+          nMetadataBackups: this.props.nMetadataBackups,
+          oldestOrLatest: (
+            <StateButton
+              disabledLabel={_('oldest')}
+              enabledLabel={_('latest')}
+              handler={this.toggleState('latest')}
+              state={this.state.latest}
+            />
+          ),
+        })}
+      </div>
+    )
+  }
+}

packages/xo-web/src/xo-app/home/host-item.js

@@ -19,7 +19,12 @@ import {
   startHost,
   stopHost,
 } from 'xo'
-import { connectStore, formatSizeShort, osFamily } from 'utils'
+import {
+  connectStore,
+  formatSizeShort,
+  hasLicenseRestrictions,
+  osFamily,
+} from 'utils'
 import {
   createDoesHostNeedRestart,
   createGetObject,
@@ -28,6 +33,7 @@ import {
 } from 'selectors'
 
 import MiniStats from './mini-stats'
+import LicenseWarning from '../host/license-warning'
 import styles from './index.css'
 
 @connectStore(() => ({
@@ -121,6 +127,8 @@ export default class HostItem extends Component {
                     </Link>
                   </Tooltip>
                 )}
+                &nbsp;
+                {hasLicenseRestrictions(host) && <LicenseWarning />}
               </EllipsisContainer>
             </Col>
             <Col mediumSize={3} className='hidden-lg-down'>

packages/xo-web/src/xo-app/host/index.js

@@ -20,7 +20,7 @@ import {
   createGetObjectsOfType,
   createSelector,
 } from 'selectors'
-import { assign, isEmpty, isString, map, pick, sortBy } from 'lodash'
+import { assign, isEmpty, map, pick, sortBy } from 'lodash'
 
 import TabAdvanced from './tab-advanced'
 import TabConsole from './tab-console'
@@ -101,19 +101,11 @@ const isRunning = host => host && host.power_state === 'Running'
     )
   )
 
-  const getHostPatches = createSelector(
-    createGetObjectsOfType('pool_patch'),
-    createGetObjectsOfType('host_patch').pick(
-      createSelector(
-        getHost,
-        host => (isString(host.patches[0]) ? host.patches : [])
-      )
-    ),
-    (poolsPatches, hostsPatches) =>
-      map(hostsPatches, hostPatch => ({
-        ...hostPatch,
-        poolPatch: poolsPatches[hostPatch.pool_patch],
-      }))
+  const getHostPatches = createGetObjectsOfType('patch').pick(
+    createSelector(
+      getHost,
+      host => host.patches
+    )
   )
 
   const doesNeedRestart = createDoesHostNeedRestart(getHost)

packages/xo-web/src/xo-app/host/license-warning.js

@@ -0,0 +1,40 @@
+import _ from 'intl'
+import React from 'react'
+import Icon from 'icon'
+import Tooltip from 'tooltip'
+import { alert } from 'modal'
+
+const showInfo = () =>
+  alert(
+    _('licenseRestrictionsModalTitle'),
+    <span>
+      <a
+        href='https://xcp-ng.com/pricing.html#xcpngvsxenserver'
+        target='_blank'
+      >
+        {_('actionsRestricted')}
+      </a>{' '}
+      {_('counterRestrictionsOptions')}
+      <ul>
+        <li>
+          <a
+            href='https://github.com/xcp-ng/xcp/wiki/Upgrade-from-XenServer'
+            target='_blank'
+          >
+            {_('counterRestrictionsOptionsXcp')}
+          </a>
+        </li>
+        <li>{_('counterRestrictionsOptionsXsLicense')}</li>
+      </ul>
+    </span>
+  )
+
+const LicenseWarning = ({ iconSize = 'sm' }) => (
+  <Tooltip content={_('licenseRestrictions')}>
+    <a className='text-danger' style={{ cursor: 'pointer' }} onClick={showInfo}>
+      <Icon icon='alarm' size={iconSize} />
+    </a>
+  </Tooltip>
+)
+
+export default LicenseWarning

packages/xo-web/src/xo-app/host/tab-general.js

@@ -10,7 +10,7 @@ import { addTag, removeTag } from 'xo'
 import { BlockLink } from 'link'
 import { Container, Row, Col } from 'grid'
 import { FormattedRelative } from 'react-intl'
-import { formatSize, formatSizeShort } from 'utils'
+import { formatSize, formatSizeShort, hasLicenseRestrictions } from 'utils'
 import Usage, { UsageElement } from 'usage'
 import { getObject } from 'selectors'
 import {
@@ -20,6 +20,8 @@ import {
   LoadSparkLines,
 } from 'xo-sparklines'
 
+import LicenseWarning from './license-warning'
+
 export default ({ statsOverview, host, nVms, vmController, vms }) => {
   const pool = getObject(store.getState(), host.$pool)
   const vmsFilter = encodeURIComponent(
@@ -82,7 +84,7 @@ export default ({ statsOverview, host, nVms, vmController, vms }) => {
             {host.productBrand !== 'XCP-ng'
               ? host.license_params.sku_type
               : 'GPLv2'}
-            )
+            ) {hasLicenseRestrictions(host) && <LicenseWarning iconSize='lg' />}
           </p>
         </Col>
         <Col mediumSize={3}>

packages/xo-web/src/xo-app/host/tab-patches.js

@@ -7,10 +7,10 @@ import Upgrade from 'xoa-upgrade'
 import { alert, chooseAction } from 'modal'
 import { connectStore, formatSize } from 'utils'
 import { Container, Row, Col } from 'grid'
-import { createDoesHostNeedRestart, createSelector } from 'selectors'
+import { createDoesHostNeedRestart } from 'selectors'
 import { FormattedRelative, FormattedTime } from 'react-intl'
-import { restartHost, installAllHostPatches, installHostPatch } from 'xo'
-import { isEmpty, isString } from 'lodash'
+import { installAllPatchesOnHost, restartHost } from 'xo'
+import { isEmpty } from 'lodash'
 
 const MISSING_PATCH_COLUMNS = [
   {
@@ -124,13 +124,13 @@ const INDIVIDUAL_ACTIONS_XCP = [
 const INSTALLED_PATCH_COLUMNS = [
   {
     name: _('patchNameLabel'),
-    itemRenderer: patch => patch.poolPatch.name,
-    sortCriteria: patch => patch.poolPatch.name,
+    itemRenderer: patch => patch.name,
+    sortCriteria: patch => patch.name,
   },
   {
     name: _('patchDescription'),
-    itemRenderer: patch => patch.poolPatch.description,
-    sortCriteria: patch => patch.poolPatch.description,
+    itemRenderer: patch => patch.description,
+    sortCriteria: patch => patch.description,
   },
   {
     default: true,
@@ -152,26 +152,6 @@ const INSTALLED_PATCH_COLUMNS = [
     sortCriteria: patch => patch.time,
     sortOrder: 'desc',
   },
-  {
-    name: _('patchSize'),
-    itemRenderer: patch => formatSize(patch.poolPatch.size),
-    sortCriteria: patch => patch.poolPatch.size,
-  },
-]
-
-// support for software_version.platform_version ^2.1.1
-const INSTALLED_PATCH_COLUMNS_2 = [
-  {
-    default: true,
-    name: _('patchNameLabel'),
-    itemRenderer: patch => patch.name,
-    sortCriteria: patch => patch.name,
-  },
-  {
-    name: _('patchDescription'),
-    itemRenderer: patch => patch.description,
-    sortCriteria: patch => patch.description,
-  },
   {
     name: _('patchSize'),
     itemRenderer: patch => formatSize(patch.size),
@@ -225,40 +205,8 @@ class XcpPatches extends Component {
   needsRestart: createDoesHostNeedRestart((_, props) => props.host),
 }))
 class XenServerPatches extends Component {
-  _getPatches = createSelector(
-    () => this.props.host,
-    () => this.props.hostPatches,
-    (host, hostPatches) => {
-      if (isEmpty(host.patches) && isEmpty(hostPatches)) {
-        return { patches: null }
-      }
-
-      if (isString(host.patches[0])) {
-        return {
-          patches: hostPatches,
-          columns: INSTALLED_PATCH_COLUMNS,
-        }
-      }
-
-      return {
-        patches: host.patches,
-        columns: INSTALLED_PATCH_COLUMNS_2,
-      }
-    }
-  )
-
-  _individualActions = [
-    {
-      name: _('patchAction'),
-      level: 'primary',
-      handler: this.props.installPatch,
-      icon: 'host-patch-update',
-    },
-  ]
-
   render() {
-    const { host, missingPatches, installAllPatches } = this.props
-    const { patches, columns } = this._getPatches()
+    const { host, missingPatches, installAllPatches, hostPatches } = this.props
     const hasMissingPatches = !isEmpty(missingPatches)
     return (
       <Container>
@@ -287,7 +235,6 @@ class XenServerPatches extends Component {
             <Col>
               <h3>{_('hostMissingPatches')}</h3>
               <SortedTable
-                individualActions={this._individualActions}
                 collection={missingPatches}
                 columns={MISSING_PATCH_COLUMNS}
               />
@@ -296,14 +243,11 @@ class XenServerPatches extends Component {
         )}
         <Row>
           <Col>
-            {patches ? (
-              <span>
-                <h3>{_('hostAppliedPatches')}</h3>
-                <SortedTable collection={patches} columns={columns} />
-              </span>
-            ) : (
-              <h4 className='text-xs-center'>{_('patchNothing')}</h4>
-            )}
+            <h3>{_('hostAppliedPatches')}</h3>
+            <SortedTable
+              collection={hostPatches}
+              columns={INSTALLED_PATCH_COLUMNS}
+            />
           </Col>
         </Row>
       </Container>
@@ -316,31 +260,22 @@ export default class TabPatches extends Component {
     router: PropTypes.object,
   }
 
-  _chooseActionPatch = async doInstall => {
-    const choice = await chooseAction({
-      body: <p>{_('installPatchWarningContent')}</p>,
-      buttons: [
-        {
-          label: _('installPatchWarningResolve'),
-          value: 'install',
-          btnStyle: 'primary',
-        },
-        { label: _('installPatchWarningReject'), value: 'goToPool' },
-      ],
-      title: _('installPatchWarningTitle'),
-    })
+  _installAllPatches = () => {
+    const { host } = this.props
+    const { $pool: pool, productBrand } = host
 
-    return choice === 'install'
-      ? doInstall()
-      : this.context.router.push(`/pools/${this.props.host.$pool}/patches`)
+    if (productBrand === 'XCP-ng') {
+      return installAllPatchesOnHost({ host })
+    }
+
+    return chooseAction({
+      body: <p>{_('installAllPatchesContent')}</p>,
+      buttons: [{ label: _('installAllPatchesRedirect'), value: 'goToPool' }],
+      icon: 'host-patch-update',
+      title: _('installAllPatchesTitle'),
+    }).then(() => this.context.router.push(`/pools/${pool}/patches`))
   }
 
-  _installAllPatches = () =>
-    this._chooseActionPatch(() => installAllHostPatches(this.props.host))
-
-  _installPatch = patch =>
-    this._chooseActionPatch(() => installHostPatch(this.props.host, patch))
-
   render() {
     if (process.env.XOA_PLAN < 2) {
       return (
@@ -352,14 +287,10 @@ export default class TabPatches extends Component {
     if (this.props.missingPatches === null) {
       return <em>{_('updatePluginNotInstalled')}</em>
     }
-    return this.props.host.productBrand === 'XCP-ng' ? (
-      <XcpPatches {...this.props} installAllPatches={this._installAllPatches} />
-    ) : (
-      <XenServerPatches
-        {...this.props}
-        installAllPatches={this._installAllPatches}
-        installPatch={this._installPatch}
-      />
+    const Patches =
+      this.props.host.productBrand === 'XCP-ng' ? XcpPatches : XenServerPatches
+    return (
+      <Patches {...this.props} installAllPatches={this._installAllPatches} />
     )
   }
 }

packages/xo-web/src/xo-app/logs/backup-ng/index.js

@@ -153,7 +153,7 @@ export default decorate([
   addSubscriptions({
     logs: cb =>
       subscribeBackupNgLogs(logs =>
-        cb(logs && filter(logs, log => log.message !== 'restore'))
+        cb(logs && filter(logs, log => log.message === 'backup'))
       ),
     jobs: cb => subscribeBackupNgJobs(jobs => cb(keyBy(jobs, 'id'))),
     metadataJobs: cb =>

packages/xo-web/src/xo-app/logs/restore-metadata.js

@@ -0,0 +1,145 @@
+import _, { FormattedDuration } from 'intl'
+import ActionButton from 'action-button'
+import addSubscriptions from 'add-subscriptions'
+import Copiable from 'copiable'
+import copy from 'copy-to-clipboard'
+import decorate from 'apply-decorators'
+import Icon from 'icon'
+import NoObjects from 'no-objects'
+import React from 'react'
+import SortedTable from 'sorted-table'
+import { alert } from 'modal'
+import { Card, CardHeader, CardBlock } from 'card'
+import { connectStore, downloadLog } from 'utils'
+import { createGetObjectsOfType } from 'selectors'
+import { filter } from 'lodash'
+import { Pool } from 'render-xo-item'
+import { subscribeBackupNgLogs } from 'xo'
+
+import { STATUS_LABELS, LOG_FILTERS, LogDate } from './utils'
+
+const showError = error =>
+  alert(
+    _('logError'),
+    <pre>{JSON.stringify(error, null, 2).replace(/\\n/g, '\n')}</pre>
+  )
+
+const COLUMNS = [
+  {
+    name: _('job'),
+    itemRenderer: ({ data }) => (
+      <Copiable data={data.jobId} tagName='div'>
+        {data.jobName || data.jobId.slice(4, 8)}
+      </Copiable>
+    ),
+    sortCriteria: 'data.jobId',
+  },
+  {
+    name: _('item'),
+    itemRenderer: ({ data }, { pools }) =>
+      data.pool === undefined ? (
+        'Xen Orchestra'
+      ) : pools[data.pool.uuid] !== undefined ? (
+        <Pool id={data.pool.uuid} link newTab />
+      ) : (
+        <Copiable data={data.pool.uuid} tagName='div'>
+          {data.pool.name_label || data.poolMaster.name_label}
+        </Copiable>
+      ),
+    sortCriteria: ({ data }) =>
+      data.pool !== undefined ? data.pool.uuid : data.jobId,
+  },
+  {
+    name: _('logsBackupTime'),
+    itemRenderer: ({ data: { timestamp } }) => <LogDate time={timestamp} />,
+    sortCriteria: 'data.timestamp',
+  },
+  {
+    default: true,
+    name: _('logsRestoreTime'),
+    itemRenderer: task => <LogDate time={task.start} />,
+    sortCriteria: 'start',
+    sortOrder: 'desc',
+  },
+  {
+    name: _('jobDuration'),
+    itemRenderer: task =>
+      task.end !== undefined && (
+        <FormattedDuration duration={task.end - task.start} />
+      ),
+    sortCriteria: task => task.end - task.start,
+  },
+  {
+    name: _('jobStatus'),
+    itemRenderer: task => {
+      const { className, label } = STATUS_LABELS[task.status]
+
+      // failed task
+      if (task.status !== 'success' && task.status !== 'pending') {
+        return (
+          <ActionButton
+            btnStyle={className}
+            handler={showError}
+            handlerParam={task.result}
+            icon='preview'
+            size='small'
+            tooltip={_('clickToShowError')}
+          >
+            {_(label)}
+          </ActionButton>
+        )
+      }
+
+      return <span className={`tag tag-${className}`}>{_(label)}</span>
+    },
+    sortCriteria: 'status',
+  },
+]
+
+const INDIVIDUAL_ACTIONS = [
+  {
+    icon: 'download',
+    label: _('logDownload'),
+    handler: task =>
+      downloadLog({
+        log: JSON.stringify(task, null, 2),
+        date: task.start,
+        type: 'Metadata restore',
+      }),
+  },
+  {
+    icon: 'clipboard',
+    label: _('copyLogToClipboard'),
+    handler: task => copy(JSON.stringify(task, null, 2)),
+  },
+]
+
+export default decorate([
+  connectStore({
+    pools: createGetObjectsOfType('pool'),
+  }),
+  addSubscriptions({
+    logs: cb =>
+      subscribeBackupNgLogs(logs =>
+        cb(logs && filter(logs, log => log.message === 'metadataRestore'))
+      ),
+  }),
+  ({ logs, pools }) => (
+    <Card>
+      <CardHeader>
+        <Icon icon='logs' /> {_('logTitle')}
+      </CardHeader>
+      <CardBlock>
+        <NoObjects
+          collection={logs}
+          columns={COLUMNS}
+          component={SortedTable}
+          data-pools={pools}
+          emptyMessage={_('noLogs')}
+          filters={LOG_FILTERS}
+          individualActions={INDIVIDUAL_ACTIONS}
+        />
+      </CardBlock>
+    </Card>
+  ),
+])

packages/xo-web/src/xo-app/pool/tab-patches.js

@@ -1,43 +1,238 @@
-import Component from 'base-component'
-import HostsPatchesTable from 'hosts-patches-table'
-import React from 'react'
+import _ from 'intl'
+import React, { Component } from 'react'
+import SortedTable from 'sorted-table'
+import TabButton from 'tab-button'
 import Upgrade from 'xoa-upgrade'
-import { connectStore } from 'utils'
-import { Container, Row, Col } from 'grid'
+import { addSubscriptions, connectStore, formatSize } from 'utils'
+import { alert } from 'modal'
+import { Col, Container, Row } from 'grid'
 import { createGetObjectsOfType } from 'selectors'
+import { FormattedRelative, FormattedTime } from 'react-intl'
+import {
+  installAllPatchesOnPool,
+  installPatches,
+  subscribeHostMissingPatches,
+} from 'xo'
+import { isEmpty } from 'lodash'
 
-// ===================================================================
+const MISSING_PATCH_COLUMNS = [
+  {
+    name: _('patchNameLabel'),
+    itemRenderer: _ => _.name,
+    sortCriteria: 'name',
+  },
+  {
+    name: _('patchDescription'),
+    itemRenderer: ({ description, documentationUrl }) => (
+      <a href={documentationUrl} target='_blank'>
+        {description}
+      </a>
+    ),
+    sortCriteria: 'description',
+  },
+  {
+    name: _('patchReleaseDate'),
+    itemRenderer: ({ date }) => (
+      <span>
+        <FormattedTime value={date} day='numeric' month='long' year='numeric' />{' '}
+        (<FormattedRelative value={date} />)
+      </span>
+    ),
+    sortCriteria: 'date',
+    sortOrder: 'desc',
+  },
+  {
+    name: _('patchGuidance'),
+    itemRenderer: _ => _.guidance,
+    sortCriteria: 'guidance',
+  },
+]
 
-@connectStore(() => {
-  const getHosts = createGetObjectsOfType('host').filter((_, props) => host =>
-    props.pool.id === host.$pool
-  )
+const ACTIONS = [
+  {
+    handler: (patches, { pool }) => installPatches(patches, pool),
+    icon: 'host-patch-update',
+    label: _('install'),
+    level: 'primary',
+  },
+]
 
-  return {
-    hosts: getHosts,
-  }
+const MISSING_PATCH_COLUMNS_XCP = [
+  {
+    name: _('patchNameLabel'),
+    itemRenderer: _ => _.name,
+    sortCriteria: 'name',
+  },
+  {
+    name: _('patchDescription'),
+    itemRenderer: _ => _.description,
+    sortCriteria: 'description',
+  },
+  {
+    name: _('patchVersion'),
+    itemRenderer: _ => _.version,
+  },
+  {
+    name: _('patchRelease'),
+    itemRenderer: _ => _.release,
+  },
+  {
+    name: _('patchSize'),
+    itemRenderer: _ => formatSize(_.size),
+    sortCriteria: 'size',
+  },
+]
+
+const INDIVIDUAL_ACTIONS_XCP = [
+  {
+    disabled: _ => _.changelog === null,
+    handler: ({ name, changelog: { author, date, description } }) =>
+      alert(
+        _('changelog'),
+        <Container>
+          <Row className='mb-1'>
+            <Col size={3}>
+              <strong>{_('changelogPatch')}</strong>
+            </Col>
+            <Col size={9}>{name}</Col>
+          </Row>
+          <Row className='mb-1'>
+            <Col size={3}>
+              <strong>{_('changelogDate')}</strong>
+            </Col>
+            <Col size={9}>
+              <FormattedTime
+                value={date * 1000}
+                day='numeric'
+                month='long'
+                year='numeric'
+              />
+            </Col>
+          </Row>
+          <Row className='mb-1'>
+            <Col size={3}>
+              <strong>{_('changelogAuthor')}</strong>
+            </Col>
+            <Col size={9}>{author}</Col>
+          </Row>
+          <Row>
+            <Col size={3}>
+              <strong>{_('changelogDescription')}</strong>
+            </Col>
+            <Col size={9}>{description}</Col>
+          </Row>
+        </Container>
+      ),
+    icon: 'preview',
+    label: _('showChangelog'),
+  },
+]
+
+const INSTALLED_PATCH_COLUMNS = [
+  {
+    name: _('patchNameLabel'),
+    itemRenderer: _ => _.name,
+    sortCriteria: 'name',
+  },
+  {
+    name: _('patchDescription'),
+    itemRenderer: _ => _.description,
+    sortCriteria: 'description',
+  },
+  {
+    default: true,
+    name: _('patchApplied'),
+    itemRenderer: patch => {
+      const time = patch.time * 1000
+      return (
+        <span>
+          <FormattedTime
+            value={time}
+            day='numeric'
+            month='long'
+            year='numeric'
+          />{' '}
+          (<FormattedRelative value={time} />)
+        </span>
+      )
+    },
+    sortCriteria: 'time',
+    sortOrder: 'desc',
+  },
+  {
+    name: _('patchSize'),
+    itemRenderer: _ => formatSize(_.size),
+    sortCriteria: 'size',
+  },
+]
+
+@addSubscriptions(({ master }) => ({
+  missingPatches: cb => subscribeHostMissingPatches(master, cb),
+}))
+@connectStore({
+  hostPatches: createGetObjectsOfType('patch').pick(
+    (_, { master }) => master.patches
+  ),
 })
 export default class TabPatches extends Component {
-  _getContainer = () => this.refs.container
-
   render() {
+    const {
+      hostPatches,
+      missingPatches = [],
+      pool,
+      master: { productBrand },
+    } = this.props
+
     return (
       <Upgrade place='poolPatches' required={2}>
         <Container>
           <Row>
             <Col className='text-xs-right'>
-              <div ref='container' />
-            </Col>
-          </Row>
-          <Row>
-            <Col>
-              <HostsPatchesTable
-                buttonsGroupContainer={this._getContainer}
-                hosts={this.props.hosts}
-                useTabButton
+              <TabButton
+                btnStyle='primary'
+                data-pool={pool}
+                disabled={isEmpty(missingPatches)}
+                handler={installAllPatchesOnPool}
+                icon='host-patch-update'
+                labelId='installPoolPatches'
               />
             </Col>
           </Row>
+          {productBrand === 'XCP-ng' ? (
+            <Row>
+              <Col>
+                <h3>{_('hostMissingPatches')}</h3>
+                <SortedTable
+                  columns={MISSING_PATCH_COLUMNS_XCP}
+                  collection={missingPatches}
+                  individualActions={INDIVIDUAL_ACTIONS_XCP}
+                />
+              </Col>
+            </Row>
+          ) : (
+            <div>
+              <Row>
+                <Col>
+                  <h3>{_('hostMissingPatches')}</h3>
+                  <SortedTable
+                    actions={ACTIONS}
+                    collection={missingPatches}
+                    columns={MISSING_PATCH_COLUMNS}
+                    data-pool={pool}
+                  />
+                </Col>
+              </Row>
+              <Row>
+                <Col>
+                  <h3>{_('hostAppliedPatches')}</h3>
+                  <SortedTable
+                    collection={hostPatches}
+                    columns={INSTALLED_PATCH_COLUMNS}
+                  />
+                </Col>
+              </Row>
+            </div>
+          )}
         </Container>
       </Upgrade>
     )

packages/xo-web/src/xo-app/settings/servers/index.js

@@ -11,6 +11,7 @@ import { alert, confirm } from 'modal'
 import { Container } from 'grid'
 import { Password as EditablePassword, Text } from 'editable'
 import { Password, Toggle } from 'form'
+import { Pool } from 'render-xo-item'
 import { injectIntl } from 'react-intl'
 import { noop } from 'lodash'
 import {
@@ -155,6 +156,11 @@ const COLUMNS = [
     ),
     sortCriteria: _ => !!_.allowUnauthorized,
   },
+  {
+    itemRenderer: ({ poolId }) =>
+      poolId !== undefined && <Pool id={poolId} link />,
+    name: _('pool'),
+  },
 ]
 const INDIVIDUAL_ACTIONS = [
   {

scripts/travis-tests

@@ -32,11 +32,7 @@ if (process.env.TRAVIS_PULL_REQUEST !== 'false') {
   if (files.length !== 0) {
     run(
       './node_modules/.bin/jest',
-      [
-        '--testRegex=^(?!.*.integ.spec.js$).*.spec.js$',
-        '--findRelatedTests',
-        '--passWithNoTests',
-      ].concat(files)
+      ['--findRelatedTests', '--passWithNoTests'].concat(files)
     )
   }
 } else {

yarn.lock

@@ -256,6 +256,14 @@
     "@babel/helper-remap-async-to-generator" "^7.1.0"
     "@babel/plugin-syntax-async-generators" "^7.2.0"
 
+"@babel/plugin-proposal-class-properties@^7.3.4":
+  version "7.4.0"
+  resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.4.0.tgz#d70db61a2f1fd79de927eea91f6411c964e084b8"
+  integrity sha512-t2ECPNOXsIeK1JxJNKmgbzQtoG27KIlVE61vTqX0DKR9E9sZlVVxWUtEW9D5FlZ8b8j7SBNCHY47GgPKCKlpPg==
+  dependencies:
+    "@babel/helper-create-class-features-plugin" "^7.4.0"
+    "@babel/helper-plugin-utils" "^7.0.0"
+
 "@babel/plugin-proposal-decorators@^7.0.0", "@babel/plugin-proposal-decorators@^7.1.6":
   version "7.4.0"
   resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-decorators/-/plugin-proposal-decorators-7.4.0.tgz#8e1bfd83efa54a5f662033afcc2b8e701f4bb3a9"
@@ -297,7 +305,7 @@
     "@babel/helper-plugin-utils" "^7.0.0"
     "@babel/plugin-syntax-json-strings" "^7.2.0"
 
-"@babel/plugin-proposal-nullish-coalescing-operator@^7.0.0":
+"@babel/plugin-proposal-nullish-coalescing-operator@^7.0.0", "@babel/plugin-proposal-nullish-coalescing-operator@^7.2.0":
   version "7.2.0"
   resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.2.0.tgz#c3fda766187b2f2162657354407247a758ee9cf9"
   integrity sha512-QXj/YjFuFJd68oDvoc1e8aqLr2wz7Kofzvp6Ekd/o7MWZl+nZ0/cpStxND+hlZ7DpRWAp7OmuyT2areZ2V3YUA==
@@ -321,7 +329,7 @@
     "@babel/helper-plugin-utils" "^7.0.0"
     "@babel/plugin-syntax-optional-catch-binding" "^7.2.0"
 
-"@babel/plugin-proposal-optional-chaining@^7.0.0":
+"@babel/plugin-proposal-optional-chaining@^7.0.0", "@babel/plugin-proposal-optional-chaining@^7.2.0":
   version "7.2.0"
   resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.2.0.tgz#ae454f4c21c6c2ce8cb2397dc332ae8b420c5441"
   integrity sha512-ea3Q6edZC/55wEBVZAEz42v528VulyO0eir+7uky/sT4XRcdkWJcFi1aPtitTlwUzGnECWJNExWww1SStt+yWw==