Add vms config (cpus, ram size, disk size)

The following modification have been made!
* separate CIDR in two fields: range and prefix
* choose k8s master's hostname
* choose a pattern for k8s nodes' names and hostnames
* choose an IP for the k8s master
* choose the RAM and disk sizes and number of CPU for the cluster's VMs

.eslintrc.js

@@ -1,5 +1,7 @@
 module.exports = {
   extends: [
+    'plugin:eslint-comments/recommended',
+
     'standard',
     'standard-jsx',
     'prettier',
@@ -16,6 +18,16 @@ module.exports = {
     $PropertyType: true,
     $Shape: true,
   },
+
+  overrides: [
+    {
+      files: ['cli.js', '*-cli.js', '**/*cli*/**/*.js'],
+      rules: {
+        'no-console': 'off',
+      },
+    },
+  ],
+
   parser: 'babel-eslint',
   parserOptions: {
     ecmaFeatures: {
@@ -23,6 +35,19 @@ module.exports = {
     },
   },
   rules: {
+    // disabled because XAPI objects are using camel case
+    camelcase: ['off'],
+
+    'react/jsx-handler-names': 'off',
+
+    // disabled because not always relevant, we might reconsider in the future
+    //
+    // enabled by https://github.com/standard/eslint-config-standard/commit/319b177750899d4525eb1210686f6aca96190b2f
+    //
+    // example: https://github.com/vatesfr/xen-orchestra/blob/31ed3767c67044ca445658eb6b560718972402f2/packages/xen-api/src/index.js#L156-L157
+    'lines-between-class-members': 'off',
+
+    'no-console': ['error', { allow: ['warn', 'error'] }],
     'no-var': 'error',
     'node/no-extraneous-import': 'error',
     'node/no-extraneous-require': 'error',

@xen-orchestra/async-map/README.md

@@ -35,7 +35,7 @@ Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/async
 
 ## Contributions
 
-Contributions are *very* welcomed, either on the documentation or on
+Contributions are _very_ welcomed, either on the documentation or on
 the code.
 
 You may:

@xen-orchestra/async-map/package.json

@@ -1,4 +1,5 @@
 {
+  "private": false,
   "name": "@xen-orchestra/async-map",
   "version": "0.0.0",
   "license": "ISC",
@@ -36,8 +37,8 @@
     "@babel/preset-env": "^7.0.0",
     "@babel/preset-flow": "^7.0.0",
     "babel-plugin-lodash": "^3.3.2",
-    "cross-env": "^5.1.3",
-    "rimraf": "^2.6.2"
+    "cross-env": "^6.0.3",
+    "rimraf": "^3.0.0"
   },
   "scripts": {
     "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
@@ -46,6 +47,7 @@
     "prebuild": "yarn run clean",
     "predev": "yarn run prebuild",
     "prepare": "yarn run build",
-    "prepublishOnly": "yarn run build"
+    "prepublishOnly": "yarn run build",
+    "postversion": "npm publish"
   }
 }

@xen-orchestra/audit-core/.npmignore

@@ -0,0 +1,24 @@
+/benchmark/
+/benchmarks/
+*.bench.js
+*.bench.js.map
+
+/examples/
+example.js
+example.js.map
+*.example.js
+*.example.js.map
+
+/fixture/
+/fixtures/
+*.fixture.js
+*.fixture.js.map
+*.fixtures.js
+*.fixtures.js.map
+
+/test/
+/tests/
+*.spec.js
+*.spec.js.map
+
+__snapshots__/

@xen-orchestra/audit-core/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@xen-orchestra/audit-core",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/audit-core",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@xen-orchestra/audit-core",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "version": "0.1.1",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "main": "dist/",
+  "scripts": {
+    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
+    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
+    "postversion": "npm publish --access public",
+    "prebuild": "rimraf dist/",
+    "predev": "yarn run prebuild",
+    "prepublishOnly": "yarn run build"
+  },
+  "devDependencies": {
+    "@babel/cli": "^7.7.4",
+    "@babel/core": "^7.7.4",
+    "@babel/plugin-proposal-decorators": "^7.8.0",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.8.0",
+    "@babel/preset-env": "^7.7.4",
+    "@babel/preset-typescript": "^7.7.4",
+    "cross": "^1.0.0",
+    "rimraf": "^3.0.0"
+  },
+  "dependencies": {
+    "core-js": "^3.6.4",
+    "golike-defer": "^0.4.1",
+    "lodash": "^4.17.15",
+    "object-hash": "^2.0.1"
+  },
+  "private": false
+}

@xen-orchestra/audit-core/src/index.js

@@ -0,0 +1,142 @@
+// see https://github.com/babel/babel/issues/8450
+import 'core-js/features/symbol/async-iterator'
+
+import assert from 'assert'
+import defer from 'golike-defer'
+import hash from 'object-hash'
+
+export class Storage {
+  constructor() {
+    this._lock = Promise.resolve()
+  }
+
+  async acquireLock() {
+    const lock = this._lock
+    let releaseLock
+    this._lock = new Promise(resolve => {
+      releaseLock = resolve
+    })
+    await lock
+    return releaseLock
+  }
+}
+
+// Format: $<algorithm>$<salt>$<encrypted>
+//
+// http://man7.org/linux/man-pages/man3/crypt.3.html#NOTES
+const ID_TO_ALGORITHM = {
+  '5': 'sha256',
+}
+
+export class AlteredRecordError extends Error {
+  constructor(id, nValid, record) {
+    super('altered record')
+
+    this.id = id
+    this.nValid = nValid
+    this.record = record
+  }
+}
+
+export class MissingRecordError extends Error {
+  constructor(id, nValid) {
+    super('missing record')
+
+    this.id = id
+    this.nValid = nValid
+  }
+}
+
+export const NULL_ID = 'nullId'
+
+const HASH_ALGORITHM_ID = '5'
+const createHash = (data, algorithmId = HASH_ALGORITHM_ID) =>
+  `$${algorithmId}$$${hash(data, {
+    algorithm: ID_TO_ALGORITHM[algorithmId],
+    excludeKeys: key => key === 'id',
+  })}`
+
+export class AuditCore {
+  constructor(storage) {
+    assert.notStrictEqual(storage, undefined)
+    this._storage = storage
+  }
+
+  @defer
+  async add($defer, subject, event, data) {
+    const time = Date.now()
+    const storage = this._storage
+    $defer(await storage.acquireLock())
+
+    // delete "undefined" properties and normalize data with JSON.stringify
+    const record = JSON.parse(
+      JSON.stringify({
+        data,
+        event,
+        previousId: (await storage.getLastId()) ?? NULL_ID,
+        subject,
+        time,
+      })
+    )
+    record.id = createHash(record)
+    await storage.put(record)
+    await storage.setLastId(record.id)
+    return record
+  }
+
+  async checkIntegrity(oldest, newest) {
+    const storage = this._storage
+
+    // handle separated chains case
+    if (newest !== (await storage.getLastId())) {
+      let isNewestAccessible = false
+      for await (const { id } of this.getFrom()) {
+        if (id === newest) {
+          isNewestAccessible = true
+          break
+        }
+      }
+      if (!isNewestAccessible) {
+        throw new MissingRecordError(newest, 0)
+      }
+    }
+
+    let nValid = 0
+    while (newest !== oldest) {
+      const record = await storage.get(newest)
+      if (record === undefined) {
+        throw new MissingRecordError(newest, nValid)
+      }
+      if (
+        newest !== createHash(record, newest.slice(1, newest.indexOf('$', 1)))
+      ) {
+        throw new AlteredRecordError(newest, nValid, record)
+      }
+      newest = record.previousId
+      nValid++
+    }
+    return nValid
+  }
+
+  async *getFrom(newest) {
+    const storage = this._storage
+
+    let id = newest ?? (await storage.getLastId())
+    if (id === undefined) {
+      return
+    }
+
+    let record
+    while ((record = await storage.get(id)) !== undefined) {
+      yield record
+      id = record.previousId
+    }
+  }
+
+  async deleteFrom(newest) {
+    assert.notStrictEqual(newest, undefined)
+    for await (const { id } of this.getFrom(newest)) {
+      await this._storage.del(id)
+    }
+  }
+}

@xen-orchestra/audit-core/src/index.spec.js

@@ -0,0 +1,126 @@
+/* eslint-env jest */
+
+import {
+  AlteredRecordError,
+  AuditCore,
+  MissingRecordError,
+  NULL_ID,
+  Storage,
+} from '.'
+
+const asyncIteratorToArray = async asyncIterator => {
+  const array = []
+  for await (const entry of asyncIterator) {
+    array.push(entry)
+  }
+  return array
+}
+
+class DB extends Storage {
+  constructor() {
+    super()
+
+    this._db = new Map()
+    this._lastId = undefined
+  }
+
+  async put(record) {
+    this._db.set(record.id, record)
+  }
+
+  async setLastId(id) {
+    this._lastId = id
+  }
+
+  async getLastId() {
+    return this._lastId
+  }
+
+  async del(id) {
+    this._db.delete(id)
+  }
+
+  async get(id) {
+    return this._db.get(id)
+  }
+
+  _clear() {
+    return this._db.clear()
+  }
+}
+
+const DATA = [
+  [
+    {
+      name: 'subject0',
+    },
+    'event0',
+    {},
+  ],
+  [
+    {
+      name: 'subject1',
+    },
+    'event1',
+    {},
+  ],
+  [
+    {
+      name: 'subject2',
+    },
+    'event2',
+    {},
+  ],
+]
+
+const db = new DB()
+const auditCore = new AuditCore(db)
+const storeAuditRecords = async () => {
+  await Promise.all(DATA.map(data => auditCore.add(...data)))
+  const records = await asyncIteratorToArray(auditCore.getFrom())
+  expect(records.length).toBe(DATA.length)
+  return records
+}
+
+describe('auditCore', () => {
+  afterEach(() => db._clear())
+
+  it('detects that a record is missing', async () => {
+    const [newestRecord, deletedRecord] = await storeAuditRecords()
+
+    const nValidRecords = await auditCore.checkIntegrity(
+      NULL_ID,
+      newestRecord.id
+    )
+    expect(nValidRecords).toBe(DATA.length)
+
+    await db.del(deletedRecord.id)
+    await expect(
+      auditCore.checkIntegrity(NULL_ID, newestRecord.id)
+    ).rejects.toEqual(new MissingRecordError(deletedRecord.id, 1))
+  })
+
+  it('detects that a record has been altered', async () => {
+    const [newestRecord, alteredRecord] = await storeAuditRecords()
+
+    alteredRecord.event = ''
+    await db.put(alteredRecord)
+
+    await expect(
+      auditCore.checkIntegrity(NULL_ID, newestRecord.id)
+    ).rejects.toEqual(
+      new AlteredRecordError(alteredRecord.id, 1, alteredRecord)
+    )
+  })
+
+  it('confirms interval integrity after deletion of records outside of the interval', async () => {
+    const [thirdRecord, secondRecord, firstRecord] = await storeAuditRecords()
+
+    await auditCore.deleteFrom(secondRecord.id)
+
+    expect(await db.get(firstRecord.id)).toBe(undefined)
+    expect(await db.get(secondRecord.id)).toBe(undefined)
+
+    await auditCore.checkIntegrity(secondRecord.id, thirdRecord.id)
+  })
+})

@xen-orchestra/audit-core/src/specification.ts

@@ -0,0 +1,25 @@
+class Storage {
+  acquire: () => Promise<() => undefined>
+  del: (id: string) => Promise<void>
+  get: (id: string) => Promise<Record | void>
+  getLastId: () => Promise<string | void>
+  put: (record: Record) => Promise<void>
+  setLastId: (id: string) => Promise<void>
+}
+
+interface Record {
+  data: object
+  event: string
+  id: string
+  previousId: string
+  subject: object
+  time: number
+}
+
+export class AuditCore {
+  constructor(storage: Storage) {}
+  public add(subject: any, event: string, data: any): Promise<Record> {}
+  public checkIntegrity(oldest: string, newest: string): Promise<number> {}
+  public getFrom(newest?: string): AsyncIterator {}
+  public deleteFrom(newest: string): Promise<void> {}
+}

@xen-orchestra/babel-config/index.js

@@ -46,6 +46,12 @@ const getConfig = (key, ...args) => {
     : config
 }
 
+// some plugins must be used in a specific order
+const pluginsOrder = [
+  '@babel/plugin-proposal-decorators',
+  '@babel/plugin-proposal-class-properties',
+]
+
 module.exports = function(pkg, plugins, presets) {
   plugins === undefined && (plugins = {})
   presets === undefined && (presets = {})
@@ -61,7 +67,13 @@ module.exports = function(pkg, plugins, presets) {
   return {
     comments: !__PROD__,
     ignore: __TEST__ ? undefined : [/\.spec\.js$/],
-    plugins: Object.keys(plugins).map(plugin => [plugin, plugins[plugin]]),
+    plugins: Object.keys(plugins)
+      .map(plugin => [plugin, plugins[plugin]])
+      .sort(([a], [b]) => {
+        const oA = pluginsOrder.indexOf(a)
+        const oB = pluginsOrder.indexOf(b)
+        return oA !== -1 && oB !== -1 ? oA - oB : a < b ? -1 : 1
+      }),
     presets: Object.keys(presets).map(preset => [preset, presets[preset]]),
   }
 }

@xen-orchestra/babel-config/package.json

@@ -8,5 +8,8 @@
     "directory": "@xen-orchestra/babel-config",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "engines": {
+    "node": ">=6"
   }
 }

@xen-orchestra/backups-cli/_asyncMap.js

@@ -0,0 +1,7 @@
+const curryRight = require('lodash/curryRight')
+
+module.exports = curryRight((iterable, fn) =>
+  Promise.all(
+    Array.isArray(iterable) ? iterable.map(fn) : Array.from(iterable, fn)
+  )
+)

@xen-orchestra/backups-cli/_composeCommands.js

@@ -0,0 +1,32 @@
+const getopts = require('getopts')
+
+const { version } = require('./package.json')
+
+module.exports = commands =>
+  async function(args, prefix) {
+    const opts = getopts(args, {
+      alias: {
+        help: 'h',
+      },
+      boolean: ['help'],
+      stopEarly: true,
+    })
+
+    const commandName = opts.help || args.length === 0 ? 'help' : args[0]
+    const command = commands[commandName]
+    if (command === undefined) {
+      process.stdout.write(`Usage:
+
+${Object.keys(commands)
+  .filter(command => command !== 'help')
+  .map(command => `    ${prefix} ${command} ${commands[command].usage || ''}`)
+  .join('\n\n')}
+
+xo-backups v${version}
+`)
+      process.exitCode = commandName === 'help' ? 0 : 1
+      return
+    }
+
+    return command.main(args.slice(1), prefix + ' ' + commandName)
+  }

@xen-orchestra/backups-cli/_fs.js

@@ -0,0 +1,57 @@
+const { dirname } = require('path')
+
+const fs = require('promise-toolbox/promisifyAll')(require('fs'))
+module.exports = fs
+
+fs.mktree = async function mkdirp(path) {
+  try {
+    await fs.mkdir(path)
+  } catch (error) {
+    const { code } = error
+    if (code === 'EEXIST') {
+      await fs.readdir(path)
+      return
+    }
+    if (code === 'ENOENT') {
+      await mkdirp(dirname(path))
+      return mkdirp(path)
+    }
+    throw error
+  }
+}
+
+// - easier:
+//   - single param for direct use in `Array#map`
+//   - files are prefixed with directory path
+// - safer: returns empty array if path is missing or not a directory
+fs.readdir2 = path =>
+  fs.readdir(path).then(
+    entries => {
+      entries.forEach((entry, i) => {
+        entries[i] = `${path}/${entry}`
+      })
+
+      return entries
+    },
+    error => {
+      if (
+        error != null &&
+        (error.code === 'ENOENT' || error.code === 'ENOTDIR')
+      ) {
+        console.warn('WARN: readdir(%s)', path, error)
+        return []
+      }
+      throw error
+    }
+  )
+
+fs.symlink2 = async (target, path) => {
+  try {
+    await fs.symlink(target, path)
+  } catch (error) {
+    if (error.code === 'EEXIST' && (await fs.readlink(path)) === target) {
+      return
+    }
+    throw error
+  }
+}

@xen-orchestra/backups-cli/commands/clean-vms.js

@@ -0,0 +1,308 @@
+#!/usr/bin/env node
+
+// assigned when options are parsed by the main function
+let force
+
+// -----------------------------------------------------------------------------
+
+const assert = require('assert')
+const flatten = require('lodash/flatten')
+const getopts = require('getopts')
+const isValidXva = require('@xen-orchestra/backups/isValidXva')
+const lockfile = require('proper-lockfile')
+const pipe = require('promise-toolbox/pipe')
+const { default: Vhd } = require('vhd-lib')
+const { dirname, resolve } = require('path')
+const { DISK_TYPE_DIFFERENCING } = require('vhd-lib/dist/_constants')
+
+const asyncMap = require('../_asyncMap')
+const fs = require('../_fs')
+
+const handler = require('@xen-orchestra/fs').getHandler({ url: 'file://' })
+
+// -----------------------------------------------------------------------------
+
+// chain is an array of VHDs from child to parent
+//
+// the whole chain will be merged into parent, parent will be renamed to child
+// and all the others will deleted
+async function mergeVhdChain(chain) {
+  assert(chain.length >= 2)
+
+  const child = chain[0]
+  const parent = chain[chain.length - 1]
+  const children = chain.slice(0, -1).reverse()
+
+  console.warn('Unused parents of VHD', child)
+  chain
+    .slice(1)
+    .reverse()
+    .forEach(parent => {
+      console.warn('  ', parent)
+    })
+  force && console.warn('  merging…')
+  console.warn('')
+  if (force) {
+    // `mergeVhd` does not work with a stream, either
+    // - make it accept a stream
+    // - or create synthetic VHD which is not a stream
+    return console.warn('TODO: implement merge')
+    // await mergeVhd(
+    //   handler,
+    //   parent,
+    //   handler,
+    //   children.length === 1
+    //     ? child
+    //     : await createSyntheticStream(handler, children)
+    // )
+  }
+
+  await Promise.all([
+    force && fs.rename(parent, child),
+    asyncMap(children.slice(0, -1), child => {
+      console.warn('Unused VHD', child)
+      force && console.warn('  deleting…')
+      console.warn('')
+      return force && handler.unlink(child)
+    }),
+  ])
+}
+
+const listVhds = pipe([
+  vmDir => vmDir + '/vdis',
+  fs.readdir2,
+  asyncMap(fs.readdir2),
+  flatten,
+  asyncMap(fs.readdir2),
+  flatten,
+  _ => _.filter(_ => _.endsWith('.vhd')),
+])
+
+async function handleVm(vmDir) {
+  const vhds = new Set()
+  const vhdParents = { __proto__: null }
+  const vhdChildren = { __proto__: null }
+
+  // remove broken VHDs
+  await asyncMap(await listVhds(vmDir), async path => {
+    try {
+      const vhd = new Vhd(handler, path)
+      await vhd.readHeaderAndFooter()
+      vhds.add(path)
+      if (vhd.footer.diskType === DISK_TYPE_DIFFERENCING) {
+        const parent = resolve(dirname(path), vhd.header.parentUnicodeName)
+        vhdParents[path] = parent
+        if (parent in vhdChildren) {
+          const error = new Error(
+            'this script does not support multiple VHD children'
+          )
+          error.parent = parent
+          error.child1 = vhdChildren[parent]
+          error.child2 = path
+          throw error // should we throw?
+        }
+        vhdChildren[parent] = path
+      }
+    } catch (error) {
+      console.warn('Error while checking VHD', path)
+      console.warn('  ', error)
+      if (error != null && error.code === 'ERR_ASSERTION') {
+        force && console.warn('  deleting…')
+        console.warn('')
+        force && (await handler.unlink(path))
+      }
+    }
+  })
+
+  // remove VHDs with missing ancestors
+  {
+    const deletions = []
+
+    // return true if the VHD has been deleted or is missing
+    const deleteIfOrphan = vhd => {
+      const parent = vhdParents[vhd]
+      if (parent === undefined) {
+        return
+      }
+
+      // no longer needs to be checked
+      delete vhdParents[vhd]
+
+      deleteIfOrphan(parent)
+
+      if (!vhds.has(parent)) {
+        vhds.delete(vhd)
+
+        console.warn('Error while checking VHD', vhd)
+        console.warn('  missing parent', parent)
+        force && console.warn('  deleting…')
+        console.warn('')
+        force && deletions.push(handler.unlink(vhd))
+      }
+    }
+
+    // > A property that is deleted before it has been visited will not be
+    // > visited later.
+    // >
+    // > -- https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for...in#Deleted_added_or_modified_properties
+    for (const child in vhdParents) {
+      deleteIfOrphan(child)
+    }
+
+    await Promise.all(deletions)
+  }
+
+  const [jsons, xvas] = await fs
+    .readdir2(vmDir)
+    .then(entries => [
+      entries.filter(_ => _.endsWith('.json')),
+      new Set(entries.filter(_ => _.endsWith('.xva'))),
+    ])
+
+  await asyncMap(xvas, async path => {
+    // check is not good enough to delete the file, the best we can do is report
+    // it
+    if (!(await isValidXva(path))) {
+      console.warn('Potential broken XVA', path)
+      console.warn('')
+    }
+  })
+
+  const unusedVhds = new Set(vhds)
+  const unusedXvas = new Set(xvas)
+
+  // compile the list of unused XVAs and VHDs, and remove backup metadata which
+  // reference a missing XVA/VHD
+  await asyncMap(jsons, async json => {
+    const metadata = JSON.parse(await fs.readFile(json))
+    const { mode } = metadata
+    if (mode === 'full') {
+      const linkedXva = resolve(vmDir, metadata.xva)
+
+      if (xvas.has(linkedXva)) {
+        unusedXvas.delete(linkedXva)
+      } else {
+        console.warn('Error while checking backup', json)
+        console.warn('  missing file', linkedXva)
+        force && console.warn('  deleting…')
+        console.warn('')
+        force && (await handler.unlink(json))
+      }
+    } else if (mode === 'delta') {
+      const linkedVhds = (() => {
+        const { vhds } = metadata
+        return Object.keys(vhds).map(key => resolve(vmDir, vhds[key]))
+      })()
+
+      // FIXME: find better approach by keeping as much of the backup as
+      // possible (existing disks) even if one disk is missing
+      if (linkedVhds.every(_ => vhds.has(_))) {
+        linkedVhds.forEach(_ => unusedVhds.delete(_))
+      } else {
+        console.warn('Error while checking backup', json)
+        const missingVhds = linkedVhds.filter(_ => !vhds.has(_))
+        console.warn(
+          '  %i/%i missing VHDs',
+          missingVhds.length,
+          linkedVhds.length
+        )
+        missingVhds.forEach(vhd => {
+          console.warn('  ', vhd)
+        })
+        force && console.warn('  deleting…')
+        console.warn('')
+        force && (await handler.unlink(json))
+      }
+    }
+  })
+
+  // TODO: parallelize by vm/job/vdi
+  const unusedVhdsDeletion = []
+  {
+    // VHD chains (as list from child to ancestor) to merge indexed by last
+    // ancestor
+    const vhdChainsToMerge = { __proto__: null }
+
+    const toCheck = new Set(unusedVhds)
+
+    const getUsedChildChainOrDelete = vhd => {
+      if (vhd in vhdChainsToMerge) {
+        const chain = vhdChainsToMerge[vhd]
+        delete vhdChainsToMerge[vhd]
+        return chain
+      }
+
+      if (!unusedVhds.has(vhd)) {
+        return [vhd]
+      }
+
+      // no longer needs to be checked
+      toCheck.delete(vhd)
+
+      const child = vhdChildren[vhd]
+      if (child !== undefined) {
+        const chain = getUsedChildChainOrDelete(child)
+        if (chain !== undefined) {
+          chain.push(vhd)
+          return chain
+        }
+      }
+
+      console.warn('Unused VHD', vhd)
+      force && console.warn('  deleting…')
+      console.warn('')
+      force && unusedVhdsDeletion.push(handler.unlink(vhd))
+    }
+
+    toCheck.forEach(vhd => {
+      vhdChainsToMerge[vhd] = getUsedChildChainOrDelete(vhd)
+    })
+
+    Object.keys(vhdChainsToMerge).forEach(key => {
+      const chain = vhdChainsToMerge[key]
+      if (chain !== undefined) {
+        unusedVhdsDeletion.push(mergeVhdChain(chain))
+      }
+    })
+  }
+
+  await Promise.all([
+    unusedVhdsDeletion,
+    asyncMap(unusedXvas, path => {
+      console.warn('Unused XVA', path)
+      force && console.warn('  deleting…')
+      console.warn('')
+      return force && handler.unlink(path)
+    }),
+  ])
+}
+
+// -----------------------------------------------------------------------------
+
+module.exports = async function main(args) {
+  const opts = getopts(args, {
+    alias: {
+      force: 'f',
+    },
+    boolean: ['force'],
+    default: {
+      force: false,
+    },
+  })
+
+  ;({ force } = opts)
+  await asyncMap(opts._, async vmDir => {
+    vmDir = resolve(vmDir)
+
+    // TODO: implement this in `xo-server`, not easy because not compatible with
+    // `@xen-orchestra/fs`.
+    const release = await lockfile.lock(vmDir)
+    try {
+      await handleVm(vmDir)
+    } catch (error) {
+      console.error('handleVm', vmDir, error)
+    } finally {
+      await release()
+    }
+  })
+}

@xen-orchestra/backups-cli/commands/create-symlink-index.js

@@ -0,0 +1,28 @@
+const filenamify = require('filenamify')
+const get = require('lodash/get')
+const { dirname, join, relative } = require('path')
+
+const asyncMap = require('../_asyncMap')
+const { mktree, readdir2, readFile, symlink2 } = require('../_fs')
+
+module.exports = async function createSymlinkIndex([backupDir, fieldPath]) {
+  const indexDir = join(backupDir, 'indexes', filenamify(fieldPath))
+  await mktree(indexDir)
+
+  await asyncMap(await readdir2(backupDir), async vmDir =>
+    asyncMap(
+      (await readdir2(vmDir)).filter(_ => _.endsWith('.json')),
+      async json => {
+        const metadata = JSON.parse(await readFile(json))
+        const value = get(metadata, fieldPath)
+        if (value !== undefined) {
+          const target = relative(indexDir, dirname(json))
+          const path = join(indexDir, filenamify(String(value)))
+          await symlink2(target, path).catch(error => {
+            console.warn('symlink(%s, %s)', target, path, error)
+          })
+        }
+      }
+    )
+  )
+}

@xen-orchestra/backups-cli/index.js

@@ -0,0 +1,19 @@
+#!/usr/bin/env node
+
+require('./_composeCommands')({
+  'clean-vms': {
+    get main() {
+      return require('./commands/clean-vms')
+    },
+    usage: '[--force] xo-vm-backups/*',
+  },
+  'create-symlink-index': {
+    get main() {
+      return require('./commands/create-symlink-index')
+    },
+    usage: 'xo-vm-backups <field path>',
+  },
+})(process.argv.slice(2), 'xo-backups').catch(error => {
+  console.error('main', error)
+  process.exitCode = 1
+})

@xen-orchestra/backups-cli/package.json

@@ -0,0 +1,35 @@
+{
+  "private": false,
+  "bin": {
+    "xo-backups": "index.js"
+  },
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "dependencies": {
+    "@xen-orchestra/backups": "^0.1.1",
+    "@xen-orchestra/fs": "^0.10.3",
+    "filenamify": "^4.1.0",
+    "getopts": "^2.2.5",
+    "lodash": "^4.17.15",
+    "promise-toolbox": "^0.15.0",
+    "proper-lockfile": "^4.1.1",
+    "vhd-lib": "^0.7.2"
+  },
+  "engines": {
+    "node": ">=7.10.1"
+  },
+  "files": [
+    "commands",
+    "*.js"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/backups-cli",
+  "name": "@xen-orchestra/backups-cli",
+  "repository": {
+    "directory": "@xen-orchestra/backups-cli",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  },
+  "version": "0.0.0"
+}

@xen-orchestra/backups/extractIdsFromSimplePattern.js

@@ -0,0 +1,30 @@
+function extractIdsFromSimplePattern(pattern) {
+  if (pattern === undefined) {
+    return []
+  }
+
+  if (pattern !== null && typeof pattern === 'object') {
+    let keys = Object.keys(pattern)
+
+    if (keys.length === 1 && keys[0] === 'id') {
+      pattern = pattern.id
+      if (typeof pattern === 'string') {
+        return [pattern]
+      }
+      if (pattern !== null && typeof pattern === 'object') {
+        keys = Object.keys(pattern)
+        if (
+          keys.length === 1 &&
+          keys[0] === '__or' &&
+          Array.isArray((pattern = pattern.__or)) &&
+          pattern.every(_ => typeof _ === 'string')
+        ) {
+          return pattern
+        }
+      }
+    }
+  }
+
+  throw new Error('invalid pattern')
+}
+exports.extractIdsFromSimplePattern = extractIdsFromSimplePattern

@xen-orchestra/backups/filenameDate.js

@@ -0,0 +1,6 @@
+const { utcFormat, utcParse } = require('d3-time-format')
+
+// Format a date in ISO 8601 in a safe way to be used in filenames
+// (even on Windows).
+exports.formatFilenameDate = utcFormat('%Y%m%dT%H%M%SZ')
+exports.parseFilenameDate = utcParse('%Y%m%dT%H%M%SZ')

@xen-orchestra/backups/getOldEntries.js

@@ -0,0 +1,7 @@
+// returns all entries but the last retention-th
+exports.getOldEntries = (retention, entries) =>
+  entries === undefined
+    ? []
+    : retention > 0
+    ? entries.slice(0, -retention)
+    : entries

@xen-orchestra/backups/isValidXva.js

@@ -0,0 +1,65 @@
+const assert = require('assert')
+const fs = require('fs-extra')
+
+const isGzipFile = async fd => {
+  // https://tools.ietf.org/html/rfc1952.html#page-5
+  const magicNumber = Buffer.allocUnsafe(2)
+  assert.strictEqual(
+    (await fs.read(fd, magicNumber, 0, magicNumber.length, 0)).bytesRead,
+    magicNumber.length
+  )
+  return magicNumber[0] === 31 && magicNumber[1] === 139
+}
+
+// TODO: better check?
+//
+// our heuristic is not good enough, there has been some false positives
+// (detected as invalid by us but valid by `tar` and imported with success),
+// either THOUGH THEY MAY HAVE BEEN COMPRESSED FILES:
+// - these files were normal but the check is incorrect
+// - these files were invalid but without data loss
+// - these files were invalid but with silent data loss
+//
+// maybe reading the end of the file looking for a file named
+// /^Ref:\d+/\d+\.checksum$/ and then validating the tar structure from it
+//
+// https://github.com/npm/node-tar/issues/234#issuecomment-538190295
+const isValidTar = async (size, fd) => {
+  if (size <= 1024 || size % 512 !== 0) {
+    return false
+  }
+
+  const buf = Buffer.allocUnsafe(1024)
+  assert.strictEqual(
+    (await fs.read(fd, buf, 0, buf.length, size - buf.length)).bytesRead,
+    buf.length
+  )
+  return buf.every(_ => _ === 0)
+}
+
+// TODO: find an heuristic for compressed files
+const isValidXva = async path => {
+  try {
+    const fd = await fs.open(path, 'r')
+    try {
+      const { size } = await fs.fstat(fd)
+      if (size < 20) {
+        // neither a valid gzip not tar
+        return false
+      }
+
+      return (await isGzipFile(fd))
+        ? true // gzip files cannot be validated at this time
+        : await isValidTar(size, fd)
+    } finally {
+      fs.close(fd).catch(noop)
+    }
+  } catch (error) {
+    // never throw, log and report as valid to avoid side effects
+    console.error('isValidXva', path, error)
+    return true
+  }
+}
+exports.isValidXva = isValidXva
+
+const noop = Function.prototype

@xen-orchestra/backups/package.json

@@ -0,0 +1,22 @@
+{
+  "private": false,
+  "name": "@xen-orchestra/backups",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/backups",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@xen-orchestra/backups",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "version": "0.1.1",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  },
+  "dependencies": {
+    "d3-time-format": "^2.2.3",
+    "fs-extra": "^8.1.0"
+  }
+}

@xen-orchestra/backups/watchStreamSize.js

@@ -0,0 +1,11 @@
+exports.watchStreamSize = stream => {
+  const container = { size: 0 }
+  const isPaused = stream.isPaused()
+  stream.on('data', data => {
+    container.size += data.length
+  })
+  if (isPaused) {
+    stream.pause()
+  }
+  return container
+}

@xen-orchestra/cr-seed-cli/package.json

@@ -1,4 +1,5 @@
 {
+  "private": false,
   "name": "@xen-orchestra/cr-seed-cli",
   "version": "0.2.0",
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/cr-seed-cli",
@@ -16,6 +17,9 @@
   },
   "dependencies": {
     "golike-defer": "^0.4.1",
-    "xen-api": "^0.24.5"
+    "xen-api": "^0.28.4"
+  },
+  "scripts": {
+    "postversion": "npm publish"
   }
 }

@xen-orchestra/cron/README.md

@@ -16,7 +16,6 @@ Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/cron)
 <minute> <hour> <day of month> <month> <day of week>
 ```
 
-
 Each entry can be:
 
 - a single value
@@ -29,12 +28,12 @@ A wildcard (`*`) can be used as a shortcut for the whole range
 Step values can be used in conjunctions with ranges. For instance,
 `1-7/2` is the same as `1,3,5,7`.
 
-| Field            | Allowed values |
-|------------------|----------------|
-| minute           | 0-59           |
-| hour             | 0-23           |
-| day of the month | 1-31 or 3-letter names (`jan`, `feb`, …) |
-| month            | 0-11           |
+| Field            | Allowed values                                                     |
+| ---------------- | ------------------------------------------------------------------ |
+| minute           | 0-59                                                               |
+| hour             | 0-23                                                               |
+| day of the month | 1-31 or 3-letter names (`jan`, `feb`, …)                           |
+| month            | 0-11                                                               |
 | day of week      | 0-7 (0 and 7 both mean Sunday) or 3-letter names (`mon`, `tue`, …) |
 
 > Note: the month range is 0-11 to be compatible with
@@ -131,7 +130,7 @@ job.stop()
 
 ## Contributions
 
-Contributions are *very* welcomed, either on the documentation or on
+Contributions are _very_ welcomed, either on the documentation or on
 the code.
 
 You may:

@xen-orchestra/cron/package.json

@@ -1,6 +1,7 @@
 {
+  "private": false,
   "name": "@xen-orchestra/cron",
-  "version": "1.0.3",
+  "version": "1.0.6",
   "license": "ISC",
   "description": "Focused, well maintained, cron parser/scheduler",
   "keywords": [
@@ -46,8 +47,8 @@
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
     "@babel/preset-flow": "^7.0.0",
-    "cross-env": "^5.1.3",
-    "rimraf": "^2.6.2"
+    "cross-env": "^6.0.3",
+    "rimraf": "^3.0.0"
   },
   "scripts": {
     "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
@@ -55,6 +56,7 @@
     "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
     "prebuild": "yarn run clean",
     "predev": "yarn run clean",
-    "prepublishOnly": "yarn run build"
+    "prepublishOnly": "yarn run build",
+    "postversion": "npm publish"
   }
 }

@xen-orchestra/cron/src/index.js

@@ -7,7 +7,21 @@ const MAX_DELAY = 2 ** 31 - 1
 
 class Job {
   constructor(schedule, fn) {
+    let scheduledDate
     const wrapper = () => {
+      const now = Date.now()
+      if (scheduledDate > now) {
+        // we're early, delay
+        //
+        // no need to check _isEnabled, we're just delaying the existing timeout
+        //
+        // see https://github.com/vatesfr/xen-orchestra/issues/4625
+        this._timeout = setTimeout(wrapper, scheduledDate - now)
+        return
+      }
+
+      this._isRunning = true
+
       let result
       try {
         result = fn()
@@ -22,23 +36,36 @@ class Job {
       }
     }
     const scheduleNext = () => {
-      const delay = schedule._nextDelay()
-      this._timeout =
-        delay < MAX_DELAY
-          ? setTimeout(wrapper, delay)
-          : setTimeout(scheduleNext, MAX_DELAY)
+      this._isRunning = false
+
+      if (this._isEnabled) {
+        const now = schedule._createDate()
+        scheduledDate = +next(schedule._schedule, now)
+        const delay = scheduledDate - now
+        this._timeout =
+          delay < MAX_DELAY
+            ? setTimeout(wrapper, delay)
+            : setTimeout(scheduleNext, MAX_DELAY)
+      }
     }
 
+    this._isEnabled = false
+    this._isRunning = false
     this._scheduleNext = scheduleNext
     this._timeout = undefined
   }
 
   start() {
     this.stop()
-    this._scheduleNext()
+
+    this._isEnabled = true
+    if (!this._isRunning) {
+      this._scheduleNext()
+    }
   }
 
   stop() {
+    this._isEnabled = false
     clearTimeout(this._timeout)
   }
 }
@@ -68,11 +95,6 @@ class Schedule {
     return dates
   }
 
-  _nextDelay() {
-    const now = this._createDate()
-    return next(this._schedule, now) - now
-  }
-
   startJob(fn) {
     const job = this.createJob(fn)
     job.start()

@xen-orchestra/cron/src/index.spec.js

@@ -0,0 +1,78 @@
+/* eslint-env jest */
+
+import { createSchedule } from './'
+
+const wrap = value => () => value
+
+describe('issues', () => {
+  let originalDateNow
+  beforeAll(() => {
+    originalDateNow = Date.now
+  })
+  afterAll(() => {
+    Date.now = originalDateNow
+    originalDateNow = undefined
+  })
+
+  test('stop during async execution', async () => {
+    let nCalls = 0
+    let resolve, promise
+
+    const schedule = createSchedule('* * * * *')
+    const job = schedule.createJob(() => {
+      ++nCalls
+
+      // eslint-disable-next-line promise/param-names
+      promise = new Promise(r => {
+        resolve = r
+      })
+      return promise
+    })
+
+    job.start()
+    Date.now = wrap(+schedule.next(1)[0])
+    jest.runAllTimers()
+
+    expect(nCalls).toBe(1)
+
+    job.stop()
+
+    resolve()
+    await promise
+
+    jest.runAllTimers()
+    expect(nCalls).toBe(1)
+  })
+
+  test('stop then start during async job execution', async () => {
+    let nCalls = 0
+    let resolve, promise
+
+    const schedule = createSchedule('* * * * *')
+    const job = schedule.createJob(() => {
+      ++nCalls
+
+      // eslint-disable-next-line promise/param-names
+      promise = new Promise(r => {
+        resolve = r
+      })
+      return promise
+    })
+
+    job.start()
+    Date.now = wrap(+schedule.next(1)[0])
+    jest.runAllTimers()
+
+    expect(nCalls).toBe(1)
+
+    job.stop()
+    job.start()
+
+    resolve()
+    await promise
+
+    Date.now = wrap(+schedule.next(1)[0])
+    jest.runAllTimers()
+    expect(nCalls).toBe(2)
+  })
+})

@xen-orchestra/defined/README.md

@@ -1,13 +1,13 @@
-# ${pkg.name} [![Build Status](https://travis-ci.org/${pkg.shortGitHubPath}.png?branch=master)](https://travis-ci.org/${pkg.shortGitHubPath})
+# @xen-orchestra/defined [![Build Status](https://travis-ci.org/${pkg.shortGitHubPath}.png?branch=master)](https://travis-ci.org/${pkg.shortGitHubPath})
 
 > ${pkg.description}
 
 ## Install
 
-Installation of the [npm package](https://npmjs.org/package/${pkg.name}):
+Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/defined):
 
 ```
-> npm install --save ${pkg.name}
+> npm install --save @xen-orchestra/defined
 ```
 
 ## Usage
@@ -35,15 +35,15 @@ Installation of the [npm package](https://npmjs.org/package/${pkg.name}):
 
 ## Contributions
 
-Contributions are *very* welcomed, either on the documentation or on
+Contributions are _very_ welcomed, either on the documentation or on
 the code.
 
 You may:
 
-- report any [issue](${pkg.bugs})
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
   you've encountered;
 - fork and create a pull request.
 
 ## License
 
-${pkg.license} © [${pkg.author.name}](${pkg.author.url})
+ISC © [Vates SAS](https://vates.fr)

@xen-orchestra/defined/package.json

@@ -1,4 +1,5 @@
 {
+  "private": false,
   "name": "@xen-orchestra/defined",
   "version": "0.0.0",
   "license": "ISC",
@@ -34,8 +35,8 @@
     "@babel/preset-env": "^7.0.0",
     "@babel/preset-flow": "^7.0.0",
     "babel-plugin-lodash": "^3.3.2",
-    "cross-env": "^5.1.3",
-    "rimraf": "^2.6.2"
+    "cross-env": "^6.0.3",
+    "rimraf": "^3.0.0"
   },
   "scripts": {
     "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
@@ -43,6 +44,7 @@
     "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
     "prebuild": "yarn run clean",
     "predev": "yarn run prebuild",
-    "prepublishOnly": "yarn run build"
+    "prepublishOnly": "yarn run build",
+    "postversion": "npm publish"
   }
 }

@xen-orchestra/emit-async/README.md

@@ -19,7 +19,7 @@ import emitAsync from '@xen-orchestra/emit-async'
 const ee = new EE()
 ee.emitAsync = emitAsync
 
-ee.on('start', async function () {
+ee.on('start', async function() {
   // whatever
 })
 
@@ -29,11 +29,14 @@ await ee.emitAsync('start')
 
 // by default, it will rejects as soon as one listener reject, you can customise
 // error handling though:
-await ee.emitAsync({
-  onError (error) {
-    console.warn(error)
-  }
-}, 'start')
+await ee.emitAsync(
+  {
+    onError(error) {
+      console.warn(error)
+    },
+  },
+  'start'
+)
 ```
 
 ## Development
@@ -57,15 +60,15 @@ await ee.emitAsync({
 
 ## Contributions
 
-Contributions are *very* welcomed, either on the documentation or on
+Contributions are _very_ welcomed, either on the documentation or on
 the code.
 
 You may:
 
-- report any [issue](${pkg.bugs})
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
   you've encountered;
 - fork and create a pull request.
 
 ## License
 
-${pkg.license} © [${pkg.author.name}](${pkg.author.url})
+ISC © [Vates SAS](https://vates.fr)

@xen-orchestra/emit-async/package.json

@@ -1,4 +1,5 @@
 {
+  "private": false,
   "name": "@xen-orchestra/emit-async",
   "version": "0.0.0",
   "license": "ISC",
@@ -33,8 +34,8 @@
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
     "babel-plugin-lodash": "^3.3.2",
-    "cross-env": "^5.1.3",
-    "rimraf": "^2.6.2"
+    "cross-env": "^6.0.3",
+    "rimraf": "^3.0.0"
   },
   "scripts": {
     "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
@@ -42,6 +43,7 @@
     "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
     "prebuild": "yarn run clean",
     "predev": "yarn run prebuild",
-    "prepublishOnly": "yarn run build"
+    "prepublishOnly": "yarn run build",
+    "postversion": "npm publish"
   }
 }

@xen-orchestra/fs/package.json

@@ -1,6 +1,7 @@
 {
+  "private": false,
   "name": "@xen-orchestra/fs",
-  "version": "0.7.1",
+  "version": "0.10.3",
   "license": "AGPL-3.0",
   "description": "The File System for Xen Orchestra backups.",
   "keywords": [],
@@ -18,21 +19,22 @@
     "dist/"
   ],
   "engines": {
-    "node": ">=6"
+    "node": ">=8.10"
   },
   "dependencies": {
-    "@marsaud/smb2": "^0.13.0",
-    "@sindresorhus/df": "^2.1.0",
+    "@marsaud/smb2": "^0.15.0",
+    "@sindresorhus/df": "^3.1.1",
     "@xen-orchestra/async-map": "^0.0.0",
-    "decorator-synchronized": "^0.3.0",
-    "execa": "^1.0.0",
-    "fs-extra": "^7.0.0",
-    "get-stream": "^4.0.0",
+    "decorator-synchronized": "^0.5.0",
+    "execa": "^3.2.0",
+    "fs-extra": "^8.0.1",
+    "get-stream": "^5.1.0",
+    "limit-concurrency-decorator": "^0.4.0",
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.11.0",
+    "promise-toolbox": "^0.15.0",
     "readable-stream": "^3.0.6",
     "through2": "^3.0.0",
-    "tmp": "^0.0.33",
+    "tmp": "^0.1.0",
     "xo-remote-parser": "^0.5.0"
   },
   "devDependencies": {
@@ -40,14 +42,15 @@
     "@babel/core": "^7.0.0",
     "@babel/plugin-proposal-decorators": "^7.1.6",
     "@babel/plugin-proposal-function-bind": "^7.0.0",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.4.4",
     "@babel/preset-env": "^7.0.0",
     "@babel/preset-flow": "^7.0.0",
     "async-iterator-to-stream": "^1.1.0",
     "babel-plugin-lodash": "^3.3.2",
-    "cross-env": "^5.1.3",
-    "dotenv": "^6.1.0",
+    "cross-env": "^6.0.3",
+    "dotenv": "^8.0.0",
     "index-modules": "^0.3.0",
-    "rimraf": "^2.6.2"
+    "rimraf": "^3.0.0"
   },
   "scripts": {
     "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
@@ -55,6 +58,7 @@
     "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
     "prebuild": "yarn run clean",
     "predev": "yarn run clean",
-    "prepare": "yarn run build"
+    "prepare": "yarn run build",
+    "postversion": "npm publish"
   }
 }

@xen-orchestra/fs/src/abstract.js

@@ -4,6 +4,7 @@
 import getStream from 'get-stream'
 
 import asyncMap from '@xen-orchestra/async-map'
+import limit from 'limit-concurrency-decorator'
 import path from 'path'
 import synchronized from 'decorator-synchronized'
 import { fromCallback, fromEvent, ignoreErrors, timeout } from 'promise-toolbox'
@@ -25,8 +26,13 @@ type RemoteInfo = { used?: number, size?: number }
 type File = FileDescriptor | string
 
 const checksumFile = file => file + '.checksum'
+const computeRate = (hrtime: number[], size: number) => {
+  const seconds = hrtime[0] + hrtime[1] / 1e9
+  return size / seconds
+}
 
 const DEFAULT_TIMEOUT = 6e5 // 10 min
+const DEFAULT_MAX_PARALLEL_OPERATIONS = 10
 
 const ignoreEnoent = error => {
   if (error == null || error.code !== 'ENOENT') {
@@ -79,6 +85,25 @@ export default class RemoteHandlerAbstract {
       }
     }
     ;({ timeout: this._timeout = DEFAULT_TIMEOUT } = options)
+
+    const sharedLimit = limit(
+      options.maxParallelOperations ?? DEFAULT_MAX_PARALLEL_OPERATIONS
+    )
+    this.closeFile = sharedLimit(this.closeFile)
+    this.getInfo = sharedLimit(this.getInfo)
+    this.getSize = sharedLimit(this.getSize)
+    this.list = sharedLimit(this.list)
+    this.mkdir = sharedLimit(this.mkdir)
+    this.openFile = sharedLimit(this.openFile)
+    this.outputFile = sharedLimit(this.outputFile)
+    this.read = sharedLimit(this.read)
+    this.readFile = sharedLimit(this.readFile)
+    this.rename = sharedLimit(this.rename)
+    this.rmdir = sharedLimit(this.rmdir)
+    this.truncate = sharedLimit(this.truncate)
+    this.unlink = sharedLimit(this.unlink)
+    this.write = sharedLimit(this.write)
+    this.writeFile = sharedLimit(this.writeFile)
   }
 
   // Public members
@@ -93,7 +118,7 @@ export default class RemoteHandlerAbstract {
   }
 
   async closeFile(fd: FileDescriptor): Promise<void> {
-    await timeout.call(this._closeFile(fd.fd), this._timeout)
+    await this.__closeFile(fd)
   }
 
   async createOutputStream(
@@ -258,30 +283,15 @@ export default class RemoteHandlerAbstract {
   }
 
   async mkdir(dir: string): Promise<void> {
-    dir = normalizePath(dir)
-    try {
-      await this._mkdir(dir)
-    } catch (error) {
-      if (error == null || error.code !== 'EEXIST') {
-        throw error
-      }
-
-      // this operation will throw if it's not already a directory
-      await this._list(dir)
-    }
+    await this.__mkdir(normalizePath(dir))
   }
 
   async mktree(dir: string): Promise<void> {
     await this._mktree(normalizePath(dir))
   }
 
-  async openFile(path: string, flags: string): Promise<FileDescriptor> {
-    path = normalizePath(path)
-
-    return {
-      fd: await timeout.call(this._openFile(path, flags), this._timeout),
-      path,
-    }
+  openFile(path: string, flags: string): Promise<FileDescriptor> {
+    return this.__openFile(path, flags)
   }
 
   async outputFile(
@@ -362,18 +372,27 @@ export default class RemoteHandlerAbstract {
   }
 
   async test(): Promise<Object> {
+    const SIZE = 1024 * 1024 * 10
     const testFileName = normalizePath(`${Date.now()}.test`)
-    const data = await fromCallback(cb => randomBytes(1024 * 1024, cb))
+    const data = await fromCallback(randomBytes, SIZE)
     let step = 'write'
     try {
+      const writeStart = process.hrtime()
       await this._outputFile(testFileName, data, { flags: 'wx' })
+      const writeDuration = process.hrtime(writeStart)
+
       step = 'read'
+      const readStart = process.hrtime()
       const read = await this._readFile(testFileName, { flags: 'r' })
+      const readDuration = process.hrtime(readStart)
+
       if (!data.equals(read)) {
         throw new Error('output and input did not match')
       }
       return {
         success: true,
+        writeRate: computeRate(writeDuration, SIZE),
+        readRate: computeRate(readDuration, SIZE),
       }
     } catch (error) {
       return {
@@ -387,6 +406,10 @@ export default class RemoteHandlerAbstract {
     }
   }
 
+  async truncate(file: string, len: number): Promise<void> {
+    await this._truncate(file, len)
+  }
+
   async unlink(file: string, { checksum = true }: Object = {}): Promise<void> {
     file = normalizePath(file)
 
@@ -397,6 +420,18 @@ export default class RemoteHandlerAbstract {
     await this._unlink(file).catch(ignoreEnoent)
   }
 
+  async write(
+    file: File,
+    buffer: Buffer,
+    position: number
+  ): Promise<{| bytesWritten: number, buffer: Buffer |}> {
+    await this._write(
+      typeof file === 'string' ? normalizePath(file) : file,
+      buffer,
+      position
+    )
+  }
+
   async writeFile(
     file: string,
     data: Data,
@@ -405,6 +440,34 @@ export default class RemoteHandlerAbstract {
     await this._writeFile(normalizePath(file), data, { flags })
   }
 
+  // Methods that can be called by private methods to avoid parallel limit on public methods
+
+  async __closeFile(fd: FileDescriptor): Promise<void> {
+    await timeout.call(this._closeFile(fd.fd), this._timeout)
+  }
+
+  async __mkdir(dir: string): Promise<void> {
+    try {
+      await this._mkdir(dir)
+    } catch (error) {
+      if (error == null || error.code !== 'EEXIST') {
+        throw error
+      }
+
+      // this operation will throw if it's not already a directory
+      await this._list(dir)
+    }
+  }
+
+  async __openFile(path: string, flags: string): Promise<FileDescriptor> {
+    path = normalizePath(path)
+
+    return {
+      fd: await timeout.call(this._openFile(path, flags), this._timeout),
+      path,
+    }
+  }
+
   // Methods that can be implemented by inheriting classes
 
   async _closeFile(fd: mixed): Promise<void> {
@@ -453,7 +516,7 @@ export default class RemoteHandlerAbstract {
 
   async _mktree(dir: string): Promise<void> {
     try {
-      return await this.mkdir(dir)
+      return await this.__mkdir(dir)
     } catch (error) {
       if (error.code !== 'ENOENT') {
         throw error
@@ -533,6 +596,28 @@ export default class RemoteHandlerAbstract {
     throw new Error('Not implemented')
   }
 
+  async _write(file: File, buffer: Buffer, position: number): Promise<void> {
+    const isPath = typeof file === 'string'
+    if (isPath) {
+      file = await this.__openFile(file, 'r+')
+    }
+    try {
+      return await this._writeFd(file, buffer, position)
+    } finally {
+      if (isPath) {
+        await this.__closeFile(file)
+      }
+    }
+  }
+
+  async _writeFd(
+    fd: FileDescriptor,
+    buffer: Buffer,
+    position: number
+  ): Promise<void> {
+    throw new Error('Not implemented')
+  }
+
   async _writeFile(
     file: string,
     data: Data,

@xen-orchestra/fs/src/fs.spec.js

@@ -3,9 +3,9 @@
 import 'dotenv/config'
 import asyncIteratorToStream from 'async-iterator-to-stream'
 import getStream from 'get-stream'
+import { forOwn, random } from 'lodash'
 import { fromCallback } from 'promise-toolbox'
 import { pipeline } from 'readable-stream'
-import { random } from 'lodash'
 import { tmpdir } from 'os'
 
 import { getHandler } from '.'
@@ -86,7 +86,7 @@ handlers.forEach(url => {
     describe('#createOutputStream()', () => {
       it('creates parent dir if missing', async () => {
         const stream = await handler.createOutputStream('dir/file')
-        await fromCallback(cb => pipeline(createTestDataStream(), stream, cb))
+        await fromCallback(pipeline, createTestDataStream(), stream)
         await expect(await handler.readFile('dir/file')).toEqual(TEST_DATA)
       })
     })
@@ -106,7 +106,7 @@ handlers.forEach(url => {
     describe('#createWriteStream()', () => {
       testWithFileDescriptor('file', 'wx', async ({ file, flags }) => {
         const stream = await handler.createWriteStream(file, { flags })
-        await fromCallback(cb => pipeline(createTestDataStream(), stream, cb))
+        await fromCallback(pipeline, createTestDataStream(), stream)
         await expect(await handler.readFile('file')).toEqual(TEST_DATA)
       })
 
@@ -219,6 +219,12 @@ handlers.forEach(url => {
         const error = await rejectionOf(handler.outputFile('file', ''))
         expect(error.code).toBe('EEXIST')
       })
+
+      it("shouldn't timeout in case of the respect of the parallel execution restriction", async () => {
+        const handler = getHandler({ url }, { maxParallelOperations: 1 })
+        await handler.sync()
+        await handler.outputFile(`xo-fs-tests-${Date.now()}/test`, '')
+      }, 40)
     })
 
     describe('#read()', () => {
@@ -290,9 +296,11 @@ handlers.forEach(url => {
 
     describe('#test()', () => {
       it('tests the remote appears to be working', async () => {
-        expect(await handler.test()).toEqual({
-          success: true,
-        })
+        const answer = await handler.test()
+
+        expect(answer.success).toBe(true)
+        expect(typeof answer.writeRate).toBe('number')
+        expect(typeof answer.readRate).toBe('number')
       })
     })
 
@@ -308,5 +316,70 @@ handlers.forEach(url => {
         await handler.unlink('file')
       })
     })
+
+    describe('#write()', () => {
+      beforeEach(() => handler.outputFile('file', TEST_DATA))
+
+      const PATCH_DATA_LEN = Math.ceil(TEST_DATA_LEN / 2)
+      const PATCH_DATA = unsecureRandomBytes(PATCH_DATA_LEN)
+
+      forOwn(
+        {
+          'dont increase file size': (() => {
+            const offset = random(0, TEST_DATA_LEN - PATCH_DATA_LEN)
+
+            const expected = Buffer.from(TEST_DATA)
+            PATCH_DATA.copy(expected, offset)
+
+            return { offset, expected }
+          })(),
+          'increase file size': (() => {
+            const offset = random(
+              TEST_DATA_LEN - PATCH_DATA_LEN + 1,
+              TEST_DATA_LEN
+            )
+
+            const expected = Buffer.alloc(offset + PATCH_DATA_LEN)
+            TEST_DATA.copy(expected)
+            PATCH_DATA.copy(expected, offset)
+
+            return { offset, expected }
+          })(),
+        },
+        ({ offset, expected }, title) => {
+          describe(title, () => {
+            testWithFileDescriptor('file', 'r+', async ({ file }) => {
+              await handler.write(file, PATCH_DATA, offset)
+              await expect(await handler.readFile('file')).toEqual(expected)
+            })
+          })
+        }
+      )
+    })
+
+    describe('#truncate()', () => {
+      forOwn(
+        {
+          'shrinks file': (() => {
+            const length = random(0, TEST_DATA_LEN)
+            const expected = TEST_DATA.slice(0, length)
+            return { length, expected }
+          })(),
+          'grows file': (() => {
+            const length = random(TEST_DATA_LEN, TEST_DATA_LEN * 2)
+            const expected = Buffer.alloc(length)
+            TEST_DATA.copy(expected)
+            return { length, expected }
+          })(),
+        },
+        ({ length, expected }, title) => {
+          it(title, async () => {
+            await handler.outputFile('file', TEST_DATA)
+            await handler.truncate('file', length)
+            await expect(await handler.readFile('file')).toEqual(expected)
+          })
+        }
+      )
+    })
   })
 })

@xen-orchestra/fs/src/local.js

@@ -47,8 +47,19 @@ export default class LocalHandler extends RemoteHandlerAbstract {
     })
   }
 
-  _getInfo() {
-    return df.file(this._getFilePath('/'))
+  async _getInfo() {
+    // df.file() resolves with an object with the following properties:
+    // filesystem, type, size, used, available, capacity and mountpoint.
+    // size, used, available and capacity may be `NaN` so we remove any `NaN`
+    // value from the object.
+    const info = await df.file(this._getFilePath('/'))
+    Object.keys(info).forEach(key => {
+      if (Number.isNaN(info[key])) {
+        delete info[key]
+      }
+    })
+
+    return info
   }
 
   async _getSize(file) {
@@ -106,10 +117,18 @@ export default class LocalHandler extends RemoteHandlerAbstract {
     await fs.access(path, fs.R_OK | fs.W_OK)
   }
 
+  _truncate(file, len) {
+    return fs.truncate(this._getFilePath(file), len)
+  }
+
   async _unlink(file) {
     return fs.unlink(this._getFilePath(file))
   }
 
+  _writeFd(file, buffer, position) {
+    return fs.write(file.fd, buffer, 0, buffer.length, position)
+  }
+
   _writeFile(file, data, { flags }) {
     return fs.writeFile(this._getFilePath(file), data, { flag: flags })
   }

@xen-orchestra/fs/src/smb.js

@@ -155,10 +155,20 @@ export default class SmbHandler extends RemoteHandlerAbstract {
     return this.list('.')
   }
 
+  _truncate(file, len) {
+    return this._client
+      .truncate(this._getFilePath(file), len)
+      .catch(normalizeError)
+  }
+
   _unlink(file) {
     return this._client.unlink(this._getFilePath(file)).catch(normalizeError)
   }
 
+  _writeFd(file, buffer, position) {
+    return this._client.write(file.fd, buffer, 0, buffer.length, position)
+  }
+
   _writeFile(file, data, options) {
     return this._client
       .writeFile(this._getFilePath(file), data, options)

@xen-orchestra/log/README.md

@@ -15,7 +15,7 @@ Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/log):
 Everywhere something should be logged:
 
 ```js
-import createLogger from '@xen-orchestra/log'
+import { createLogger } from '@xen-orchestra/log'
 
 const log = createLogger('my-module')
 
@@ -24,11 +24,25 @@ log.info('this information is relevant to the user')
 log.warn('something went wrong but did not prevent current action')
 log.error('something went wrong')
 log.fatal('service/app is going down')
+
+// you can add contextual info
+log.debug('new API request', {
+  method: 'foo',
+  params: [ 'bar', 'baz' ]
+  user: 'qux'
+})
+
+// by convention, errors go into the `error` field
+log.error('could not join server', {
+  error,
+  server: 'example.org',
+})
 ```
 
 Then, at application level, configure the logs are handled:
 
 ```js
+import { createLogger } from '@xen-orchestra/log'
 import { configure, catchGlobalErrors } from '@xen-orchestra/log/configure'
 import transportConsole from '@xen-orchestra/log/transports/console'
 import transportEmail from '@xen-orchestra/log/transports/email'
@@ -37,13 +51,10 @@ const transport = transportEmail({
   service: 'gmail',
   auth: {
     user: 'jane.smith@gmail.com',
-    pass: 'H&NbECcpXF|pyXe#%ZEb'
+    pass: 'H&NbECcpXF|pyXe#%ZEb',
   },
   from: 'jane.smith@gmail.com',
-  to: [
-    'jane.smith@gmail.com',
-    'sam.doe@yahoo.com'
-  ]
+  to: ['jane.smith@gmail.com', 'sam.doe@yahoo.com'],
 })
 
 configure([
@@ -53,19 +64,19 @@ configure([
     // matched against the namespace of the logs
     filter: process.env.DEBUG,
 
-    transport: transportConsole()
+    transport: transportConsole(),
   },
   {
     // only levels >= warn
     level: 'warn',
 
-    transport
-  }
+    transport,
+  },
 ])
 
 // send all global errors (uncaught exceptions, warnings, unhandled rejections)
-// to this transport
-catchGlobalErrors(transport)
+// to this logger
+catchGlobalErrors(createLogger('app'))
 ```
 
 ### Transports
@@ -91,18 +102,17 @@ Configuration:
 ```js
 import transportEmail from '@xen-orchestra/log/transports/email'
 
-configure(transportEmail({
-  service: 'gmail',
-  auth: {
-    user: 'jane.smith@gmail.com',
-    pass: 'H&NbECcpXF|pyXe#%ZEb'
-  },
-  from: 'jane.smith@gmail.com',
-  to: [
-    'jane.smith@gmail.com',
-    'sam.doe@yahoo.com'
-  ]
-}))
+configure(
+  transportEmail({
+    service: 'gmail',
+    auth: {
+      user: 'jane.smith@gmail.com',
+      pass: 'H&NbECcpXF|pyXe#%ZEb',
+    },
+    from: 'jane.smith@gmail.com',
+    to: ['jane.smith@gmail.com', 'sam.doe@yahoo.com'],
+  })
+)
 ```
 
 #### Syslog
@@ -146,7 +156,7 @@ configure(transportSyslog('tcp://syslog.company.lan'))
 
 ## Contributions
 
-Contributions are *very* welcomed, either on the documentation or on
+Contributions are _very_ welcomed, either on the documentation or on
 the code.
 
 You may:

@xen-orchestra/log/package.json

@@ -1,6 +1,7 @@
 {
+  "private": false,
   "name": "@xen-orchestra/log",
-  "version": "0.1.4",
+  "version": "0.2.0",
   "license": "ISC",
   "description": "",
   "keywords": [],
@@ -27,20 +28,20 @@
     ">2%"
   ],
   "engines": {
-    "node": ">=4"
+    "node": ">=6"
   },
   "dependencies": {
     "lodash": "^4.17.4",
-    "promise-toolbox": "^0.11.0"
+    "promise-toolbox": "^0.15.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.0.0",
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
     "babel-plugin-lodash": "^3.3.2",
-    "cross-env": "^5.1.3",
+    "cross-env": "^6.0.3",
     "index-modules": "^0.3.0",
-    "rimraf": "^2.6.2"
+    "rimraf": "^3.0.0"
   },
   "scripts": {
     "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
@@ -48,6 +49,7 @@
     "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
     "prebuild": "yarn run clean",
     "predev": "yarn run prebuild",
-    "prepublishOnly": "yarn run build"
+    "prepare": "yarn run build",
+    "postversion": "npm publish"
   }
 }

@xen-orchestra/log/src/configure.js

@@ -19,7 +19,8 @@ const createTransport = config => {
     }
   }
 
-  let { filter, transport } = config
+  let { filter } = config
+  let transport = createTransport(config.transport)
   const level = resolve(config.level)
 
   if (filter !== undefined) {
@@ -51,11 +52,12 @@ const symbol =
     ? Symbol.for('@xen-orchestra/log')
     : '@@@xen-orchestra/log'
 
+const { env } = process
 global[symbol] = createTransport({
   // display warnings or above, and all that are enabled via DEBUG or
   // NODE_DEBUG env
-  filter: process.env.DEBUG || process.env.NODE_DEBUG,
-  level: LEVELS.INFO,
+  filter: [env.DEBUG, env.NODE_DEBUG].filter(Boolean).join(','),
+  level: resolve(env.LOG_LEVEL, LEVELS.INFO),
 
   transport: createConsoleTransport(),
 })

@xen-orchestra/log/src/index.js

@@ -1,5 +1,5 @@
 import createTransport from './transports/console'
-import LEVELS from './levels'
+import LEVELS, { resolve } from './levels'
 
 const symbol =
   typeof Symbol !== 'undefined'
@@ -9,7 +9,8 @@ if (!(symbol in global)) {
   // the default behavior, without requiring `configure` is to avoid
   // logging anything unless it's a real error
   const transport = createTransport()
-  global[symbol] = log => log.level > LEVELS.WARN && transport(log)
+  const level = resolve(process.env.LOG_LEVEL, LEVELS.WARN)
+  global[symbol] = log => log.level >= level && transport(log)
 }
 
 // -------------------------------------------------------------------
@@ -72,5 +73,5 @@ prototype.wrap = function(message, fn) {
   }
 }
 
-const createLogger = namespace => new Logger(namespace)
+export const createLogger = namespace => new Logger(namespace)
 export { createLogger as default }

@xen-orchestra/log/src/levels.js

@@ -13,11 +13,22 @@ for (const name in LEVELS) {
   NAMES[LEVELS[name]] = name
 }
 
-export const resolve = level => {
-  if (typeof level === 'string') {
-    level = LEVELS[level.toUpperCase()]
+// resolves to the number representation of a level
+//
+// returns `defaultLevel` if invalid
+export const resolve = (level, defaultLevel) => {
+  const type = typeof level
+  if (type === 'number') {
+    if (level in NAMES) {
+      return level
+    }
+  } else if (type === 'string') {
+    const nLevel = LEVELS[level.toUpperCase()]
+    if (nLevel !== undefined) {
+      return nLevel
+    }
   }
-  return level
+  return defaultLevel
 }
 
 Object.freeze(LEVELS)

@xen-orchestra/log/src/transports/console.js

@@ -1,24 +1,107 @@
 import LEVELS, { NAMES } from '../levels'
 
-// Bind console methods (necessary for browsers)
-const debugConsole = console.log.bind(console)
-const infoConsole = console.info.bind(console)
-const warnConsole = console.warn.bind(console)
-const errorConsole = console.error.bind(console)
+const { DEBUG, ERROR, FATAL, INFO, WARN } = LEVELS
 
-const { ERROR, INFO, WARN } = LEVELS
+let formatLevel, formatNamespace
+if (
+  process.stdout !== undefined &&
+  process.stdout.isTTY &&
+  process.stderr !== undefined &&
+  process.stderr.isTTY
+) {
+  const ansi = (style, str) => `\x1b[${style}m${str}\x1b[0m`
+
+  const LEVEL_STYLES = {
+    [DEBUG]: '2',
+    [ERROR]: '1;31',
+    [FATAL]: '1;31',
+    [INFO]: '1',
+    [WARN]: '1;33',
+  }
+  formatLevel = level => {
+    const style = LEVEL_STYLES[level]
+    const name = NAMES[level]
+    return style === undefined ? name : ansi(style, name)
+  }
+
+  const NAMESPACE_COLORS = [
+    196,
+    202,
+    208,
+    214,
+    220,
+    226,
+    190,
+    154,
+    118,
+    82,
+    46,
+    47,
+    48,
+    49,
+    50,
+    51,
+    45,
+    39,
+    33,
+    27,
+    21,
+    57,
+    93,
+    129,
+    165,
+    201,
+    200,
+    199,
+    198,
+    197,
+  ]
+  formatNamespace = namespace => {
+    // https://werxltd.com/wp/2010/05/13/javascript-implementation-of-javas-string-hashcode-method/
+    let hash = 0
+    for (let i = 0, n = namespace.length; i < n; ++i) {
+      hash = ((hash << 5) - hash + namespace.charCodeAt(i)) | 0
+    }
+    // // select a hue (HSV)
+    // const h = (Math.abs(hash) % 20) * 18
+    // // convert to RGB
+    // const f = (n, k = (n + h / 60) % 6) =>
+    //   Math.round(255 * (1 - Math.max(Math.min(k, 4 - k, 1), 0)))
+    // const r = f(5)
+    // const g = f(3)
+    // const b = f(1)
+    // return ansi(`38;2;${r};${g};${b}`, namespace)
+    return ansi(
+      `1;38;5;${NAMESPACE_COLORS[Math.abs(hash) % NAMESPACE_COLORS.length]}`,
+      namespace
+    )
+  }
+} else {
+  formatLevel = str => NAMES[str]
+  formatNamespace = str => str
+}
 
 const consoleTransport = ({ data, level, namespace, message, time }) => {
   const fn =
+    /* eslint-disable no-console */
     level < INFO
-      ? debugConsole
+      ? console.log
       : level < WARN
-      ? infoConsole
+      ? console.info
       : level < ERROR
-      ? warnConsole
-      : errorConsole
+      ? console.warn
+      : console.error
+  /* eslint-enable no-console */
 
-  fn('%s - %s - [%s] %s', time.toISOString(), namespace, NAMES[level], message)
-  data != null && fn(data)
+  const args = [
+    time.toISOString(),
+    formatNamespace(namespace),
+    formatLevel(level),
+    message,
+  ]
+  if (data != null) {
+    args.push(data)
+  }
+  fn.apply(console, args)
 }
 export default () => consoleTransport

@xen-orchestra/log/src/transports/syslog.js

@@ -1,7 +1,6 @@
 import fromCallback from 'promise-toolbox/fromCallback'
-import splitHost from 'split-host' // eslint-disable-line node/no-extraneous-import node/no-missing-import
-import startsWith from 'lodash/startsWith'
-import { createClient, Facility, Severity, Transport } from 'syslog-client' // eslint-disable-line node/no-extraneous-import node/no-missing-import
+import splitHost from 'split-host'
+import { createClient, Facility, Severity, Transport } from 'syslog-client'
 
 import LEVELS from '../levels'
 
@@ -19,10 +18,10 @@ const facility = Facility.User
 export default target => {
   const opts = {}
   if (target !== undefined) {
-    if (startsWith(target, 'tcp://')) {
+    if (target.startsWith('tcp://')) {
       target = target.slice(6)
       opts.transport = Transport.Tcp
-    } else if (startsWith(target, 'udp://')) {
+    } else if (target.startsWith('udp://')) {
       target = target.slice(6)
       opts.transport = Transport.Udp
     }

@xen-orchestra/mixin/README.md

@@ -1,13 +1,13 @@
-# ${pkg.name} [![Build Status](https://travis-ci.org/${pkg.shortGitHubPath}.png?branch=master)](https://travis-ci.org/${pkg.shortGitHubPath})
+# @xen-orchestra/mixin [![Build Status](https://travis-ci.org/${pkg.shortGitHubPath}.png?branch=master)](https://travis-ci.org/${pkg.shortGitHubPath})
 
 > ${pkg.description}
 
 ## Install
 
-Installation of the [npm package](https://npmjs.org/package/${pkg.name}):
+Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/mixin):
 
 ```
-> npm install --save ${pkg.name}
+> npm install --save @xen-orchestra/mixin
 ```
 
 ## Usage
@@ -35,15 +35,15 @@ Installation of the [npm package](https://npmjs.org/package/${pkg.name}):
 
 ## Contributions
 
-Contributions are *very* welcomed, either on the documentation or on
+Contributions are _very_ welcomed, either on the documentation or on
 the code.
 
 You may:
 
-- report any [issue](${pkg.bugs})
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
   you've encountered;
 - fork and create a pull request.
 
 ## License
 
-${pkg.license} © [${pkg.author.name}](${pkg.author.url})
+ISC © [Vates SAS](https://vates.fr)

@xen-orchestra/mixin/package.json

@@ -1,4 +1,5 @@
 {
+  "private": false,
   "name": "@xen-orchestra/mixin",
   "version": "0.0.0",
   "license": "ISC",
@@ -36,8 +37,8 @@
     "@babel/preset-env": "^7.0.0",
     "babel-plugin-dev": "^1.0.0",
     "babel-plugin-lodash": "^3.3.2",
-    "cross-env": "^5.1.3",
-    "rimraf": "^2.6.2"
+    "cross-env": "^6.0.3",
+    "rimraf": "^3.0.0"
   },
   "scripts": {
     "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
@@ -45,6 +46,7 @@
     "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
     "prebuild": "yarn run clean",
     "predev": "yarn run prebuild",
-    "prepublishOnly": "yarn run build"
+    "prepublishOnly": "yarn run build",
+    "postversion": "npm publish"
   }
 }

@xen-orchestra/self-signed/README.md

@@ -0,0 +1,42 @@
+# @xen-orchestra/self-signed
+
+> Minimalist wrapper around openssl to generate a self signed certificate
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/self-signed):
+
+```
+> npm install --save @xen-orchestra/self-signed
+```
+
+## Usage
+
+```js
+import { genSelfSigned } from '@xen-orchestra/self-signed'
+
+console.log(await genSelfSigned())
+// {
+//   cert: '-----BEGIN CERTIFICATE-----\n' +
+//     // content…
+//     '-----END CERTIFICATE-----\n',
+//   key: '-----BEGIN RSA PRIVATE KEY-----\n' +
+//     // content…
+//     '-----END RSA PRIVATE KEY-----\n'
+// }
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+ISC © [Vates SAS](https://vates.fr)

@xen-orchestra/self-signed/index.js

@@ -0,0 +1,25 @@
+const { execFile } = require('child_process')
+
+const openssl = (cmd, args, { input, ...opts } = {}) =>
+  new Promise((resolve, reject) => {
+    const child = execFile('openssl', [cmd, ...args], opts, (error, stdout) =>
+      error != null ? reject(error) : resolve(stdout)
+    )
+    if (input !== undefined) {
+      child.stdin.end(input)
+    }
+  })
+
+exports.genSelfSignedCert = async () => {
+  const key = await openssl('genrsa', ['2048'])
+  return {
+    cert: await openssl(
+      'req',
+      ['-batch', '-new', '-key', '-', '-x509', '-days', '360', '-nodes'],
+      {
+        input: key,
+      }
+    ),
+    key,
+  }
+}

@xen-orchestra/self-signed/package.json

@@ -0,0 +1,19 @@
+{
+  "private": false,
+  "name": "@xen-orchestra/self-signed",
+  "description": "Minimalist wrapper around openssl to generate a self signed certificate",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/self-signed",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@xen-orchestra/self-signed",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "version": "0.1.0",
+  "engines": {
+    "node": ">=8.10"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@xen-orchestra/template/.babelrc.js

@@ -0,0 +1,3 @@
+module.exports = require('../../@xen-orchestra/babel-config')(
+  require('./package.json')
+)

@xen-orchestra/template/README.md

@@ -0,0 +1,65 @@
+# @xen-orchestra/template [![Build Status](https://travis-ci.org/vatesfr/xen-orchestra.png?branch=master)](https://travis-ci.org/vatesfr/xen-orchestra)
+
+## Install
+
+Installation of the [npm package](https://npmjs.org/package/@xen-orchestra/template):
+
+```
+> npm install --save @xen-orchestra/template
+```
+
+## Usage
+
+Create a string replacer based on a pattern and a list of rules.
+
+```js
+const myReplacer = compileTemplate('{name}_COPY_{name}_{id}_%%', {
+  '{name}': vm => vm.name_label,
+  '{id}': vm => vm.id,
+  '%': (_, i) => i,
+})
+
+const newString = myReplacer(
+  {
+    name_label: 'foo',
+    id: 42,
+  },
+  32
+)
+
+newString === 'foo_COPY_{name}_42_32%' // true
+```
+
+## Development
+
+```
+# Install dependencies
+> yarn
+
+# Run the tests
+> yarn test
+
+# Continuously compile
+> yarn dev
+
+# Continuously run the tests
+> yarn dev-test
+
+# Build for production (automatically called by npm install)
+> yarn build
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+ISC © [Vates SAS](https://vates.fr)

@xen-orchestra/template/package.json

@@ -1,46 +1,36 @@
 {
-  "name": "xo-server-cloud",
-  "version": "0.2.4",
+  "private": false,
+  "name": "@xen-orchestra/template",
+  "version": "0.1.0",
   "license": "ISC",
-  "description": "",
-  "keywords": [
-    "cloud",
-    "orchestra",
-    "plugin",
-    "xen",
-    "xen-orchestra",
-    "xo-server"
-  ],
-  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-cloud",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/template",
   "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
   "repository": {
-    "directory": "packages/xo-server-cloud",
+    "directory": "@xen-orchestra/template",
     "type": "git",
     "url": "https://github.com/vatesfr/xen-orchestra.git"
   },
   "author": {
-    "name": "Pierre Donias",
-    "email": "pierre.donias@gmail.com"
+    "name": "Julien Fontanet",
+    "email": "julien.fontanet@vates.fr"
   },
   "preferGlobal": false,
   "main": "dist/",
-  "bin": {},
   "files": [
     "dist/"
   ],
+  "browserslist": [
+    ">2%"
+  ],
   "engines": {
     "node": ">=6"
   },
-  "dependencies": {
-    "http-request-plus": "^0.7.2",
-    "jsonrpc-websocket-client": "^0.4.1"
-  },
   "devDependencies": {
     "@babel/cli": "^7.0.0",
     "@babel/core": "^7.0.0",
     "@babel/preset-env": "^7.0.0",
-    "cross-env": "^5.1.3",
-    "rimraf": "^2.5.4"
+    "cross-env": "^6.0.3",
+    "rimraf": "^3.0.0"
   },
   "scripts": {
     "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
@@ -48,6 +38,10 @@
     "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
     "prebuild": "yarn run clean",
     "predev": "yarn run prebuild",
-    "prepublishOnly": "yarn run build"
+    "prepublishOnly": "yarn run build",
+    "postversion": "npm publish --access public"
+  },
+  "dependencies": {
+    "lodash": "^4.17.15"
   }
 }

@xen-orchestra/template/src/index.js

@@ -0,0 +1,19 @@
+import escapeRegExp from 'lodash/escapeRegExp'
+
+const compareLengthDesc = (a, b) => b.length - a.length
+
+export function compileTemplate(pattern, rules) {
+  const matches = Object.keys(rules)
+    .sort(compareLengthDesc)
+    .map(escapeRegExp)
+    .join('|')
+  const regExp = new RegExp(`\\\\(?:\\\\|${matches})|${matches}`, 'g')
+  return (...params) =>
+    pattern.replace(regExp, match => {
+      if (match[0] === '\\') {
+        return match.slice(1)
+      }
+      const rule = rules[match]
+      return typeof rule === 'function' ? rule(...params) : rule
+    })
+}

@xen-orchestra/template/src/index.spec.js

@@ -0,0 +1,14 @@
+/* eslint-env jest */
+import { compileTemplate } from '.'
+
+it("correctly replaces the template's variables", () => {
+  const replacer = compileTemplate(
+    '{property}_\\{property}_\\\\{property}_{constant}_%_FOO',
+    {
+      '{property}': obj => obj.name,
+      '{constant}': 1235,
+      '%': (_, i) => i,
+    }
+  )
+  expect(replacer({ name: 'bar' }, 5)).toBe('bar_{property}_\\bar_1235_5_FOO')
+})

@xen-orchestra/upload-ova/.babelrc.js

@@ -0,0 +1,3 @@
+module.exports = require('../../@xen-orchestra/babel-config')(
+  require('./package.json')
+)

@xen-orchestra/upload-ova/.npmignore

@@ -0,0 +1,24 @@
+/benchmark/
+/benchmarks/
+*.bench.js
+*.bench.js.map
+
+/examples/
+example.js
+example.js.map
+*.example.js
+*.example.js.map
+
+/fixture/
+/fixtures/
+*.fixture.js
+*.fixture.js.map
+*.fixtures.js
+*.fixtures.js.map
+
+/test/
+/tests/
+*.spec.js
+*.spec.js.map
+
+__snapshots__/

@xen-orchestra/upload-ova/README.md

@@ -0,0 +1,62 @@
+# XO-UPLOAD-OVA
+
+> Basic CLI to upload ova files to Xen-Orchestra
+
+## Usage
+
+```
+Usage:
+
+  xo-upload-ova --register [--expiresIn duration] <XO-Server URL> <username> [<password>]
+    Registers the XO instance to use.
+
+    --expiresIn duration
+      Can be used to change the validity duration of the
+      authorization token (default: one month).
+
+  xo-upload-ova --unregister
+    Remove stored credentials.
+
+  xo-upload-ova --inspect <file>
+    Displays the data that would be imported from the ova.
+
+  xo-upload-ova --upload <file> <sr> [--override <key>=<value> [<key>=<value>]+]
+    Actually imports the VM contained in <file> to the Storage Repository <sr>.
+    Some parameters can be overridden from the file, consult --inspect to get the list.
+    Note: --override has to come last. By default arguments are string, prefix them with <json:> to type
+    them, ex. " --override nameLabel='new VM'  memory=json:67108864 disks.vmdisk1.capacity=json:134217728"
+
+xo-upload-ova v0.1.0
+
+```
+
+#### Register your XO instance
+
+```
+> xo-upload-ova --register http://xo.my-company.net admin@admin.net admin
+Successfully logged with admin@admin.net
+```
+
+Note: only a token will be saved in the configuration file.
+
+#### Import your .ova file
+
+```
+> xo-upload-ova --upload dsl.ova a7c630bf-b38c-489e-d3c3-e62507948980 --override 'nameLabel=dsl ' descriptionLabel='short desc' memory=json:671088640 disks.vmdisk1.descriptionLabel='disk description' disks.vmdisk1.capacity=json:1342177280
+```
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+XO-UPLOAD-OVA is released under the [AGPL
+v3](http://www.gnu.org/licenses/agpl-3.0-standalone.html).

@xen-orchestra/upload-ova/package.json

@@ -0,0 +1,72 @@
+{
+  "name": "@xen-orchestra/upload-ova",
+  "version": "0.1.3",
+  "license": "AGPL-3.0",
+  "description": "Basic CLI to upload ova files to Xen-Orchestra",
+  "keywords": [
+    "import",
+    "orchestra",
+    "ova",
+    "xcp-ng",
+    "xcp",
+    "xen-orchestra",
+    "xen-server",
+    "xen",
+    "xo"
+  ],
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/upload-ova",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@xen-orchestra/upload-ova",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "preferGlobal": true,
+  "main": "dist/",
+  "bin": {
+    "xo-upload-ova": "dist/index.js"
+  },
+  "files": [
+    "dist/"
+  ],
+  "engines": {
+    "node": ">=8.10"
+  },
+  "dependencies": {
+    "@babel/polyfill": "^7.0.0",
+    "chalk": "^2.2.0",
+    "exec-promise": "^0.7.0",
+    "fs-extra": "^7.0.0",
+    "fs-promise": "^2.0.3",
+    "get-stream": "^4.1.0",
+    "http-request-plus": "^0.8.0",
+    "human-format": "^0.10.0",
+    "l33teral": "^3.0.3",
+    "lodash": "^4.17.4",
+    "nice-pipe": "0.0.0",
+    "pretty-ms": "^4.0.0",
+    "progress-stream": "^2.0.0",
+    "pw": "^0.0.4",
+    "strip-indent": "^2.0.0",
+    "xdg-basedir": "^3.0.0",
+    "xo-lib": "^0.9.0",
+    "xo-vmdk-to-vhd": "^1.1.2"
+  },
+  "devDependencies": {
+    "@babel/cli": "^7.0.0",
+    "@babel/core": "^7.0.0",
+    "@babel/preset-env": "^7.0.0",
+    "@babel/preset-flow": "^7.0.0",
+    "babel-plugin-lodash": "^3.3.2",
+    "cross-env": "^5.1.3",
+    "rimraf": "^2.6.2"
+  },
+  "scripts": {
+    "build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
+    "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
+    "prebuild": "rimraf dist/",
+    "predev": "yarn run prebuild",
+    "prepublishOnly": "yarn run build"
+  },
+  "private": true
+}

@xen-orchestra/upload-ova/src/config.js

@@ -0,0 +1,44 @@
+'use strict'
+
+// ===================================================================
+
+import assign from 'lodash/assign'
+import l33t from 'l33teral'
+import xdgBasedir from 'xdg-basedir'
+
+import { mkdirp, readFile, writeFile } from 'fs-extra'
+
+const configPath = xdgBasedir.config + '/xo-upload-ova'
+const configFile = configPath + '/config.json'
+
+export async function load() {
+  try {
+    return JSON.parse(await readFile(configFile))
+  } catch (e) {
+    return {}
+  }
+}
+
+export async function get(path) {
+  const config = await load()
+  return l33t(config).tap(path)
+}
+
+export async function save(config) {
+  await mkdirp(configPath)
+  await writeFile(configFile, JSON.stringify(config))
+}
+
+export async function set(data) {
+  const config = await load()
+  await save(assign(config, data))
+}
+
+export async function unset(paths) {
+  const config = await load()
+  const l33tConfig = l33t(config)
+  ;[].concat(paths).forEach(function(path) {
+    l33tConfig.purge(path, true)
+  })
+  return save(config)
+}

@xen-orchestra/upload-ova/src/index.js

@@ -0,0 +1,308 @@
+#!/usr/bin/env node
+
+/* eslint no-console: "off" */
+
+import chalk from 'chalk'
+import execPromise from 'exec-promise'
+import { createReadStream } from 'fs'
+import { stat } from 'fs-promise'
+import getStream from 'get-stream'
+import hrp from 'http-request-plus'
+import humanFormat from 'human-format'
+import l33t from 'l33teral'
+import isObject from 'lodash/isObject'
+import getKeys from 'lodash/keys'
+import startsWith from 'lodash/startsWith'
+import nicePipe from 'nice-pipe'
+import prettyMs from 'pretty-ms'
+import progressStream from 'progress-stream'
+import pw from 'pw'
+import stripIndent from 'strip-indent'
+import { URL } from 'url'
+import Xo from 'xo-lib'
+import { parseOVAFile } from 'xo-vmdk-to-vhd'
+
+import pkg from '../package'
+import {
+  load as loadConfig,
+  set as setConfig,
+  unset as unsetConfig,
+} from './config'
+
+function help() {
+  return stripIndent(
+    `
+    Usage:
+
+      $name --register [--expiresIn duration] <XO-Server URL> <username> [<password>]
+        Registers the XO instance to use.
+
+        --expiresIn duration
+          Can be used to change the validity duration of the
+          authorization token (default: one month).
+
+      $name --unregister
+        Remove stored credentials.
+
+      $name --inspect <file>
+        Displays the data that would be imported from the ova.
+
+      $name --upload <file> <sr> [--override <key>=<value> [<key>=<value>]+]
+        Actually imports the VM contained in <file> to the Storage Repository <sr>.
+        Some parameters can be overridden from the file, consult --inspect to get the list.
+        Note: --override has to come last. By default arguments are string, prefix them with <json:> to type
+        them, ex. " --override nameLabel='new VM'  memory=json:67108864 disks.vmdisk1.capacity=json:134217728"
+
+    $name v$version
+  `
+  ).replace(/<([^>]+)>|\$(\w+)/g, function(_, arg, key) {
+    if (arg) {
+      return '<' + chalk.yellow(arg) + '>'
+    }
+
+    if (key === 'name') {
+      return chalk.bold(pkg[key])
+    }
+
+    return pkg[key]
+  })
+}
+
+async function connect() {
+  const { server, token } = await loadConfig()
+  if (server === undefined) {
+    throw new Error('no server to connect to!')
+  }
+
+  if (token === undefined) {
+    throw new Error('no token available')
+  }
+
+  const xo = new Xo({ url: server })
+  await xo.open()
+  await xo.signIn({ token })
+  return xo
+}
+
+export function unregister() {
+  return unsetConfig(['server', 'token'])
+}
+
+export async function register(args) {
+  let expiresIn
+  if (args[0] === '--expiresIn') {
+    expiresIn = args[1]
+    args = args.slice(2)
+  }
+
+  const [
+    url,
+    email,
+    password = await new Promise(resolve => {
+      process.stdout.write('Password: ')
+      pw(resolve)
+    }),
+  ] = args
+
+  const xo = new Xo({ url })
+  await xo.open()
+  await xo.signIn({ email, password })
+  console.log('Successfully logged with', xo.user.email)
+
+  await setConfig({
+    server: url,
+    token: await xo.call('token.create', { expiresIn }),
+  })
+}
+
+function nodeStringDecoder(buffer, encoder) {
+  return Buffer.from(buffer).toString(encoder)
+}
+
+export async function inspect(args) {
+  const file = args[0]
+  const data = await parseOVAFile(
+    new NodeParsableFile(file, (await stat(file)).size),
+    nodeStringDecoder,
+    true
+  )
+  console.log('file metadata:', data)
+}
+
+function parseOverride(args) {
+  const flag = args.shift()
+  if (flag !== '--override') {
+    throw new Error('Third argument has to be --override')
+  }
+
+  if (args.length === 0) {
+    throw new Error('Missing actual override')
+  }
+  const overrides = {}
+  for (const definition of args) {
+    const index = definition.indexOf('=')
+    const key = definition.slice(0, index)
+    let value = definition.slice(index + 1)
+    if (startsWith(value, 'json:')) {
+      value = JSON.parse(value.slice(5))
+    }
+    overrides[key] = value
+  }
+  return overrides
+}
+
+export async function upload(args) {
+  const file = args.shift()
+  const srId = args.shift()
+  let overrides = {}
+  if (args.length > 1) {
+    overrides = parseOverride(args)
+  }
+
+  const data = await parseOVAFile(
+    new NodeParsableFile(file, (await stat(file)).size),
+    nodeStringDecoder
+  )
+  const params = { sr: srId }
+  const xo = await connect()
+  const getXoObject = async filter =>
+    Object.values(await xo.call('xo.getAllObjects', { filter }))[0]
+  const sr = await getXoObject({ id: srId })
+  const pool = await getXoObject({ id: sr.$poolId })
+  const master = await getXoObject({ id: pool.master })
+  const pif = await getXoObject({
+    type: 'PIF',
+    management: true,
+    $host: master.id,
+  })
+  data.networks = data.networks.map(() => pif.$network)
+  console.log('data', data)
+  const l33tData = l33t(data)
+  const overridesKeys = Object.keys(overrides)
+  const missingKeys = overridesKeys.filter(k => !l33tData.probe(k))
+  if (missingKeys.length) {
+    // eslint-disable-next-line no-throw-literal
+    throw `those override keys don't exist in the metadata: ${missingKeys}`
+  }
+  for (const key of overridesKeys) {
+    l33tData.plant(key, overrides[key])
+  }
+  data.disks = Object.values(data.disks)
+  params.data = l33tData.obj
+  params.type = 'ova'
+  const method = 'vm.import'
+
+  // FIXME: do not use private properties.
+  const baseUrl = xo._url.replace(/^ws/, 'http')
+
+  const result = await xo.call(method, params)
+  let keys, key, url
+  if (isObject(result) && (keys = getKeys(result)).length === 1) {
+    key = keys[0]
+
+    if (key === '$sendTo') {
+      if (typeof file !== 'string') {
+        // eslint-disable-next-line no-throw-literal
+        throw 'file parameter should be a path'
+      }
+      url = new URL(result[key], baseUrl)
+
+      const { size: length } = await stat(file)
+      const input = nicePipe([
+        createReadStream(file),
+        progressStream(
+          {
+            length,
+            time: 1e3,
+          },
+          printProgress
+        ),
+      ])
+
+      try {
+        return await hrp
+          .post(url.toString(), {
+            body: input,
+            headers: {
+              'content-length': length,
+            },
+          })
+          .readAll('utf-8')
+      } catch (e) {
+        console.log('ERROR', e)
+        console.log('ERROR content', await e.response.readAll('utf-8'))
+        throw e
+      }
+    }
+  }
+}
+
+export class NodeParsableFile {
+  constructor(fileName, fileLength = Infinity) {
+    this._fileName = fileName
+    this._start = 0
+    this._end = fileLength
+  }
+
+  slice(start, end) {
+    const newFile = new NodeParsableFile(this._fileName)
+    newFile._start = start < 0 ? this._end + start : this._start + start
+    newFile._end = end < 0 ? this._end + end : this._start + end
+    return newFile
+  }
+
+  async read() {
+    const result = await getStream.buffer(
+      createReadStream(this._fileName, {
+        start: this._start,
+        end: this._end - 1,
+      })
+    )
+    // crazy stuff to get a browser-compatible ArrayBuffer from a node buffer
+    // https://stackoverflow.com/a/31394257/72637
+    return result.buffer.slice(
+      result.byteOffset,
+      result.byteOffset + result.byteLength
+    )
+  }
+}
+
+const humanFormatOpts = {
+  unit: 'B',
+  scale: 'binary',
+}
+
+function printProgress(progress) {
+  if (progress.length) {
+    console.warn(
+      '%s% of %s @ %s/s - ETA %s',
+      Math.round(progress.percentage),
+      humanFormat(progress.length, humanFormatOpts),
+      humanFormat(progress.speed, humanFormatOpts),
+      prettyMs(progress.eta * 1e3)
+    )
+  } else {
+    console.warn(
+      '%s @ %s/s',
+      humanFormat(progress.transferred, humanFormatOpts),
+      humanFormat(progress.speed, humanFormatOpts)
+    )
+  }
+}
+
+export default async function main(args) {
+  if (!args || !args.length || args[0] === '-h' || args[0] === '--help') {
+    return help()
+  }
+  const fnName = args[0].replace(/^--|-\w/g, match =>
+    match === '--' ? '' : match[1].toUpperCase()
+  )
+  if (fnName in exports) {
+    return exports[fnName](args.slice(1))
+  }
+  return help()
+}
+
+if (!module.parent) {
+  execPromise(main)
+}

CHANGELOG.unreleased.md

@@ -1,28 +1,31 @@
 > This file contains all changes that have not been released yet.
+>
+> Keep in mind the changelog is addressed to **users** and should be
+> understandable by them.
 
 ### Enhancements
 
-- [SR/Disk] Disable actions on unmanaged VDIs [#3988](https://github.com/vatesfr/xen-orchestra/issues/3988) (PR [#4000](https://github.com/vatesfr/xen-orchestra/pull/4000))
-- [Pool] Specify automatic networks on a Pool [#3916](https://github.com/vatesfr/xen-orchestra/issues/3916) (PR [#3958](https://github.com/vatesfr/xen-orchestra/pull/3958))
-- [VM/advanced] Manage start delay for VM [#3909](https://github.com/vatesfr/xen-orchestra/issues/3909) (PR [#4002](https://github.com/vatesfr/xen-orchestra/pull/4002))
-- [New/Vm] SR section: Display warning message when the selected SRs aren't in the same host [#3911](https://github.com/vatesfr/xen-orchestra/issues/3911) (PR [#3967](https://github.com/vatesfr/xen-orchestra/pull/3967))
-- Enable compression for HTTP requests (and initial objects fetch)
-- [VDI migration] Display same-pool SRs first in the selector [#3945](https://github.com/vatesfr/xen-orchestra/issues/3945) (PR [#3996](https://github.com/vatesfr/xen-orchestra/pull/3996))
-- [Home] Save the current page in url [#3993](https://github.com/vatesfr/xen-orchestra/issues/3993) (PR [#3999](https://github.com/vatesfr/xen-orchestra/pull/3999))
-- [VDI] Ensure suspend VDI is destroyed when destroying a VM  [#4027](https://github.com/vatesfr/xen-orchestra/issues/4027) (PR [#4038](https://github.com/vatesfr/xen-orchestra/pull/4038))
-- [VM/disk]: Warning when 2 VDIs are on 2 different hosts' local SRs [#3911](https://github.com/vatesfr/xen-orchestra/issues/3911) (PR [#3969](https://github.com/vatesfr/xen-orchestra/pull/3969))
+> Users must be able to say: “Nice enhancement, I'm eager to test it”
 
 ### Bug fixes
 
-- [New network] PIF was wrongly required which prevented from creating a private network (PR [#4010](https://github.com/vatesfr/xen-orchestra/pull/4010))
-- [Google authentication] Migrate to new endpoint
-- [Backup NG] Better handling of huge logs [#4025](https://github.com/vatesfr/xen-orchestra/issues/4025) (PR [#4026](https://github.com/vatesfr/xen-orchestra/pull/4026))
-- [Home/VM] Bulk migration: fixed VM VDIs not migrated to the selected SR [#3986](https://github.com/vatesfr/xen-orchestra/issues/3986) (PR [#3987](https://github.com/vatesfr/xen-orchestra/pull/3987))
-- [Stats] Fix cache usage with simultaneous requests [#4017](https://github.com/vatesfr/xen-orchestra/issues/4017) (PR [#4028](https://github.com/vatesfr/xen-orchestra/pull/4028))
-- [Backup NG] Fix compression displayed for the wrong backup mode (PR [#4021](https://github.com/vatesfr/xen-orchestra/pull/4021))
+> Users must be able to say: “I had this issue, happy to know it's fixed”
 
 ### Released packages
 
-- xo-server-auth-google v0.2.1
-- xo-server v5.37.0
-- xo-web v5.37.0
+> Packages will be released in the order they are here, therefore, they should
+> be listed by inverse order of dependency.
+>
+> Rule of thumb: add packages on top.
+>
+> The format is the following: - `$packageName` `$version`
+>
+> Where `$version` is
+>
+> - patch: if the change is a bug fix or a simple code improvement
+> - minor: if the change is a new feature
+> - major: if the change breaks compatibility
+>
+> In case of conflict, the highest (lowest in previous list) `$version` wins.
+
+- xo-web patch

CODE_OF_CONDUCT.md

@@ -8,19 +8,19 @@ In the interest of fostering an open and welcoming environment, we as contributo
 
 Examples of behavior that contributes to creating a positive environment include:
 
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
+- Using welcoming and inclusive language
+- Being respectful of differing viewpoints and experiences
+- Gracefully accepting constructive criticism
+- Focusing on what is best for the community
+- Showing empathy towards other community members
 
 Examples of unacceptable behavior by participants include:
 
-* The use of sexualized language or imagery and unwelcome sexual attention or advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a professional setting
+- The use of sexualized language or imagery and unwelcome sexual attention or advances
+- Trolling, insulting/derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others' private information, such as a physical or electronic address, without explicit permission
+- Other conduct which could reasonably be considered inappropriate in a professional setting
 
 ## Our Responsibilities
 

ISSUE_TEMPLATE.md

@@ -14,9 +14,9 @@ the issues :)
 
 - **XO origin**: the sources / XO Appliance
 - **Versions**:
-   - Node: **FILL HERE**
-   - xo-web: **FILL HERE**
-   - xo-server: **FILL HERE**
+  - Node: **FILL HERE**
+  - xo-web: **FILL HERE**
+  - xo-server: **FILL HERE**
 
 ### Expected behavior
 

PULL_REQUEST_TEMPLATE.md

@@ -1,18 +1,28 @@
 ### Check list
 
-> Check items when done or if not relevant
+> Check if done, if not relevant leave unchecked.
 
-- [ ] PR reference the relevant issue (e.g. `Fixes #007`)
+- [ ] PR reference the relevant issue (e.g. `Fixes #007` or `See xoa-support#42`)
 - [ ] if UI changes, a screenshot has been added to the PR
-- [ ] `CHANGELOG.unreleased.md`:
-   - enhancement/bug fix entry added
-   - list of packages to release updated (`${name} v${new version}`)
 - [ ] documentation updated
-- [ ] **I have tested added/updated features** (and impacted code)
+- `CHANGELOG.unreleased.md`:
+  - [ ] enhancement/bug fix entry added
+  - [ ] list of packages to release updated (`${name} v${new version}`)
+- **I have tested added/updated features** (and impacted code)
+  - [ ] unit tests (e.g. [`cron/parse.spec.js`](https://github.com/vatesfr/xen-orchestra/blob/b24400b21de1ebafa1099c56bac1de5c988d9202/%40xen-orchestra/cron/src/parse.spec.js))
+  - [ ] if `xo-server` API changes, the corresponding test has been added to/updated on [`xo-server-test`](https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-test)
+  - [ ] at least manual testing
 
 ### Process
 
 1. create a PR as soon as possible
 1. mark it as `WiP:` (Work in Progress) if not ready to be merged
-1. when you want a review, add a reviewer
+1. when you want a review, add a reviewer (and only one)
 1. if necessary, update your PR, and re- add a reviewer
+
+From [_the Four Agreements_](https://en.wikipedia.org/wiki/Don_Miguel_Ruiz#The_Four_Agreements):
+
+1. Be impeccable with your word.
+1. Don't take anything personally.
+1. Don't make assumptions.
+1. Always do your best.

README.md

@@ -1,4 +1,4 @@
-# Xen Orchestra [![Chat with us](https://storage.crisp.im/plugins/images/936925df-f37b-4ba8-bab0-70cd2edcb0be/badge.svg)](https://go.crisp.im/chat/embed/?website_id=-JzqzzwddSV7bKGtEyAQ) [![Build Status](https://travis-ci.org/vatesfr/xen-orchestra.png?branch=master)](https://travis-ci.org/vatesfr/xen-orchestra)
+# Xen Orchestra [![Build Status](https://travis-ci.org/vatesfr/xen-orchestra.png?branch=master)](https://travis-ci.org/vatesfr/xen-orchestra)
 
 ![](http://i.imgur.com/tRffA5y.png)
 

docs/README.md

@@ -1,4 +1,3 @@
-
 # Xen Orchestra
 
 ## Introduction
@@ -9,23 +8,23 @@ XO is a web interface to visualize and administer your XenServer (or XAPI enable
 
 It aims to be easy to use on any device supporting modern web technologies (HTML 5, CSS 3, JavaScript), such as your desktop computer or your smartphone.
 
-![](https://pbs.twimg.com/profile_images/601775622675898368/xWbbafyO_400x400.png)
+![Xen Orchestra logo](./assets/logo.png)
 
 ## XOA quick deploy
 
-SSH to your XenServer, and execute the following:
+Log in to your account and use the deploy form available on [this page](https://xen-orchestra.com/#!/xoa)
 
-```
-bash -c "$(curl -s http://xoa.io/deploy)"
-```
+> **Note:** no data will be sent to our servers, it's running only between your browser and your host!
+
+[![](./assets/deploy_form.png)](https://xen-orchestra.com/#!/xoa)
 
 ### XOA credentials
 
-* Web UI: `admin@admin.net` / `admin`
-* Console/SSH: `xoa` / `xoa` (first login)
+- Web UI: `admin@admin.net` / `admin`
+- Console/SSH: `xoa` / `xoa` (first login)
 
 ## Must read
 
-* [XOA installation](xoa.md)
-* [Main features](features.md)
-* [Pro Support](support.md)
+- [XOA installation](xoa.md)
+- [Main features](features.md)
+- [Pro Support](support.md)

docs/SUMMARY.md

@@ -1,79 +1,81 @@
 # Summary
 
-* [Introduction](README.md)
-* [Architecture](architecture.md)
-   * [xo-server](xo-server.md)
-   * [xo-web](xo-web.md)
-   * [xo-cli](xo-cli.md)
-   * [others](others.md)
-* [Installation](installation.md)
-   * [XOA](xoa.md)
-       * [Updater](updater.md)
-       * [Trial activation](trial_activation.md)
-       * [Plugins](plugins.md)
-       * [Logs](logs.md)
-       * [Compatibility](supported-version.md)
-       * [Troubleshooting](troubleshooting.md)
-   * [From the sources](from_the_sources.md)
-* [Configuration](configuration.md)
-* [Features](features.md)
-   * [Administration](administration.md)
-       * [Home view](user_interface.md)
-       * [Search and filters](search.md)
-       * [VM management](vm_management.md)
-       * [VM creation](vm_creation.md)
-       * [VM import and export](vm_import_export.md)
-       * [XenServer Patching](patching.md)
-   * [Docker support](docker_support.md)
-   * [Backup and DR](backups.md)
-       * [Full backups](full_backups.md)
-       * [Rolling snapshots](rolling_snapshots.md)
-       * [Continuous Delta backups](delta_backups.md)
-       * [Continuous Replication](continuous_replication.md)
-       * [Disaster recovery](disaster_recovery.md)
-       * [Smart Backup](smart_backup.md)
-       * [File level Restore](file_level_restore.md)
-       * [Metadata Backup](metadata_backup.md)
-       * [Backup Concurrency](concurrency.md)
-       * [Configure backup reports](backup_reports.md)
-       * [Backup troubleshooting](backup_troubleshooting.md)
-   * [User authentication](authentication.md)
-       * [Built-in](built-in.md)
-       * [LDAP](ldap.md)
-       * [SAML](saml.md)
-       * [GitHub](github.md)
-       * [Google](google.md)
-   * [Resources delegation](resources_delegation.md)
-       * [ACLs](acls.md)
-   * [CloudInit](cloudinit.md)
-   * [Self Service](self_service.md)
-   * [Visualizations](visualizations.md)
-   * [Health](health.md)
-   * [Job manager](scheduler.md)
-   * [Alerts](alerts.md)
-   * [Load balancing](load_balancing.md)
-   * [Emergency Shutdown](emergency_shutdown.md)
-   * [Auto scalability](auto_scalability.md)
-   * [Forecaster](forecaster.md)
-* [Recipes](recipes.md)
-   * [Reverse proxy](reverse_proxy.md)
-* [How to contribute?](contributing.md)
-* [Support](support.md)
-* [Roadmap](roadmap.md)
-* [Purchase](purchase.md)
-   * [Direct purchase](directpurchase.md)
-   * [Through purchase department](through_purchase_department.md)
-   * [Reseller](reseller.md)
-   * [Editions](editions.md)
-   * [Trial](trial.md)
-   * [Invoices](invoices.md)
-   * [Upgrade](upgrade.md)
-* [XOSAN](xosan.md)
-  * [Requirements](xosan_requirements.md)
-  * [Types](xosan_types.md)
-    * [Replicated](xosan_replicated.md)
-    * [Disperse](xosan_disperse.md)
-  * [Creation](xosan_create.md)
-  * [Trial](xosan_trial.md)
-* [General Troubleshooting](general-troubleshooting.md)
-* [Glossary](glossary.md)
+- [Introduction](README.md)
+- [Architecture](architecture.md)
+  - [xo-server](xo-server.md)
+  - [xo-web](xo-web.md)
+  - [xo-cli](xo-cli.md)
+  - [others](others.md)
+- [Installation](installation.md)
+  - [XOA](xoa.md)
+    - [Updater](updater.md)
+    - [Trial activation](trial_activation.md)
+    - [Plugins](plugins.md)
+    - [Logs](logs.md)
+    - [Compatibility](supported-version.md)
+    - [Troubleshooting](troubleshooting.md)
+  - [From the sources](from_the_sources.md)
+- [Configuration](configuration.md)
+- [Features](features.md)
+  - [Administration](administration.md)
+    - [Home view](user_interface.md)
+    - [Search and filters](search.md)
+    - [VM management](vm_management.md)
+    - [VM creation](vm_creation.md)
+    - [VM import and export](vm_import_export.md)
+    - [XenServer Patching](patching.md)
+  - [Docker support](docker_support.md)
+  - [Backup and DR](backups.md)
+    - [Full backups](full_backups.md)
+    - [Rolling snapshots](rolling_snapshots.md)
+    - [Continuous Delta backups](delta_backups.md)
+    - [Continuous Replication](continuous_replication.md)
+    - [Disaster recovery](disaster_recovery.md)
+    - [Smart Backup](smart_backup.md)
+    - [File level Restore](file_level_restore.md)
+    - [Metadata Backup](metadata_backup.md)
+    - [Backup Concurrency](concurrency.md)
+    - [Configure backup reports](backup_reports.md)
+    - [Backup troubleshooting](backup_troubleshooting.md)
+  - [User authentication](authentication.md)
+    - [Built-in](built-in.md)
+    - [LDAP](ldap.md)
+    - [SAML](saml.md)
+    - [GitHub](github.md)
+    - [Google](google.md)
+  - [Resources delegation](resources_delegation.md)
+    - [ACLs](acls.md)
+  - [CloudInit](cloudinit.md)
+  - [Self Service](self_service.md)
+  - [Visualizations](visualizations.md)
+  - [Health](health.md)
+  - [Job manager](scheduler.md)
+  - [Alerts](alerts.md)
+  - [Web hooks](web-hooks.md)
+  - [Load balancing](load_balancing.md)
+  - [Emergency Shutdown](emergency_shutdown.md)
+  - [Auto scalability](auto_scalability.md)
+  - [Forecaster](forecaster.md)
+  - [SDN Controller](sdn_controller.md)
+- [Recipes](recipes.md)
+  - [Reverse proxy](reverse_proxy.md)
+- [How to contribute?](contributing.md)
+- [Support](support.md)
+- [Roadmap](roadmap.md)
+- [Purchase](purchase.md)
+  - [Direct purchase](directpurchase.md)
+  - [Through purchase department](through_purchase_department.md)
+  - [Reseller](reseller.md)
+  - [Editions](editions.md)
+  - [Trial](trial.md)
+  - [Invoices](invoices.md)
+  - [Upgrade](upgrade.md)
+- [XOSAN](xosan.md)
+  - [Requirements](xosan_requirements.md)
+  - [Types](xosan_types.md)
+    - [Replicated](xosan_replicated.md)
+    - [Disperse](xosan_disperse.md)
+  - [Creation](xosan_create.md)
+  - [Trial](xosan_trial.md)
+- [General Troubleshooting](general-troubleshooting.md)
+- [Glossary](glossary.md)

docs/acls.md

@@ -23,32 +23,32 @@ You can edit/remove existing ACLs here.
 
 There are 3 different roles for your users:
 
-* Admin
-* Operator
-* Viewer
+- Admin
+- Operator
+- Viewer
 
 ### Admin
 
 An object admin can do everything on it, even destroy it. E.g with its admin VM:
 
-* remove it
-* migrate it (to a host with admin permission on it)
-* modify the VM resources, name and description
-* clone it
-* copy it
-* convert it into a template
-* snapshot it (even revert from a snapshot)
-* export it
-* attach/add visible disks
-* same for network cards
+- remove it
+- migrate it (to a host with admin permission on it)
+- modify the VM resources, name and description
+- clone it
+- copy it
+- convert it into a template
+- snapshot it (even revert from a snapshot)
+- export it
+- attach/add visible disks
+- same for network cards
 
 ### Operator
 
 An operator can make everyday operations on assigned objects. E.g on a VM:
 
-* eject a CD
-* insert a CD (if he can view the ISO storage repository)
-* start, restart, shutdown, suspend/resume it
+- eject a CD
+- insert a CD (if he can view the ISO storage repository)
+- start, restart, shutdown, suspend/resume it
 
 All other operations are forbidden.
 
@@ -60,7 +60,7 @@ A viewer can only see the VM state and its metrics. That's all!
 
 Objects have a hierarchy: a Pool contains all its hosts, containing itself all its VMs.
 
-If you give a *view* permission to a user (or a group) on a pool, he will automatically see all the objects inside this pool (SRs, hosts, VMs).
+If you give a _view_ permission to a user (or a group) on a pool, he will automatically see all the objects inside this pool (SRs, hosts, VMs).
 
 ## Examples
 
@@ -68,5 +68,5 @@ If you give a *view* permission to a user (or a group) on a pool, he will automa
 
 If the OS install needs an ISO, you need to give this user 2 permissions:
 
-* *Operate* on the VM (e.g to start it)
-* *View* on the ISO Storage containing the needed ISO.
+- _Operate_ on the VM (e.g to start it)
+- _View_ on the ISO Storage containing the needed ISO.

docs/administration.md

@@ -2,11 +2,11 @@
 
 This section contains everyday XenServer administration tasks.
 
-* [Home view](user_interface.md)
-* [Search and filters](search.md)
-* [VM management](vm_management.md)
-* [VM creation](vm_creation.md)
-* [VM import and export](vm_import_export.md)
-* [XenServer Patching](patching.md)
+- [Home view](user_interface.md)
+- [Search and filters](search.md)
+- [VM management](vm_management.md)
+- [VM creation](vm_creation.md)
+- [VM import and export](vm_import_export.md)
+- [XenServer Patching](patching.md)
 
 ![](./assets/xo5homevms.png)

docs/alerts.md

@@ -8,12 +8,12 @@ The administrator will configure alerts based on performance thresholds.
 
 The configurable metrics are:
 
-* CPU usage (VM, host)
-* RAM usage (VM, host)
-* network bandwidth (VM, host)
-* load average (host)
-* disk IO (VM)
-* total IO (SR, only for XenServer Dundee and higher)
+- CPU usage (VM, host)
+- RAM usage (VM, host)
+- network bandwidth (VM, host)
+- load average (host)
+- disk IO (VM)
+- total IO (SR, only for XenServer Dundee and higher)
 
 If any configured values exceed the threshold during a selected period of time, an alert will be sent.
 
@@ -31,5 +31,5 @@ You can choose to be notified only if it fails or even after each backup job.
 
 Current supported alerts system:
 
-* Email
-* XMPP
+- Email
+- XMPP

docs/architecture.md

@@ -1,4 +1,3 @@
-
 # Architecture
 
 Xen Orchestra (XO) is software built with a server and clients, such as the web client `xo-web`, but also a CLI capable client, called `xo-cli`.
@@ -7,7 +6,7 @@ Xen Orchestra (XO) is software built with a server and clients, such as the web
 
 ## XOA
 
-*Xen Orchestra Virtual Appliance* (XOA) is a virtual machine with Xen Orchestra already installed, thus working out-of-the-box.
+_Xen Orchestra Virtual Appliance_ (XOA) is a virtual machine with Xen Orchestra already installed, thus working out-of-the-box.
 
 This is the easiest way to try Xen Orchestra quickly.
 
@@ -20,9 +19,9 @@ Your XOA is connected to all your hosts, or the pool master only if you are usin
 ![](./assets/xo-arch.jpg)
 
 Xen Orchestra itself is built as a modular solution. Each part has its role:
+
 - The core is "[xo-server](https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server/)" - a daemon dealing directly with XenServer or XAPI capable hosts. This is where users are stored, and it's the center point for talking to your whole Xen infrastructure.
-- The web interface is "[xo-web](https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-web)" - it runs directly from your browser. The connection with ```xo-server``` is done via *WebSockets*.
+- The web interface is "[xo-web](https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-web)" - it runs directly from your browser. The connection with `xo-server` is done via _WebSockets_.
 - "[xo-cli](https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-cli)" is a module allowing you to send commands directly from the command line.
 
-
 We have other modules as well (like the LDAP plugin for example). It allows us to use this modular architecture to add further parts as we need them. It's completely flexible, allowing us to adapt Xen Orchestra to every existing workflow.

docs/authentication.md

@@ -1,16 +1,14 @@
 # User authentication
 
-
 Xen Orchestra supports various types of user authentication, internal or even external thanks to the usage of the [Passport library](http://passportjs.org/).
 
 There are 2 types of XO users:
 
-* admins, with all rights on all connected resources
-* users, with no rights by default
+- admins, with all rights on all connected resources
+- users, with no rights by default
 
 All users will land on the "flat" view, which displays no hierarchy, only all their visible objects (or no object if they are not configured).
 
-
 ACLs will thus apply only to "users".
 
 > Any account created by an external authentication process (LDAP, SSO...) will be a **user** without any permission.

docs/backup_reports.md

@@ -6,15 +6,15 @@ At the end of a backup job, you can configure Xen Orchestra to send backup repor
 
 ### Step-by-step
 
-1. On the "settings/plugins" view you have to activate and configure the "Backup-reports" plugin. 
-![](./assets/backup-reports-plugin.png)
+1. On the "settings/plugins" view you have to activate and configure the "Backup-reports" plugin.
+   ![](./assets/backup-reports-plugin.png)
 
 2. Still in the plugins view, you also have to configure the "transport-email" plugin.
-![](./assets/transport-email-plugin.png)
+   ![](./assets/transport-email-plugin.png)
 
-3. Once it's done, you can now create your backup job. In the "report" selection you can choose the situation in wish you want to receive an email (always, never or failure). 
-![](./assets/backup-report-config.png)
-> Note: You can also modify existing backup jobs and change the behaviour of the report system.
+3. Once it's done, you can now create your backup job. In the "report" selection you can choose the situation in wish you want to receive an email (always, never or failure).
+   ![](./assets/backup-report-config.png)
+   > Note: You can also modify existing backup jobs and change the behaviour of the report system.
 
 ## XMPP nofications
 
@@ -67,9 +67,9 @@ Like all other xo-server plugins, it can be configured directly via the web inte
 
 You need to be an admin:
 
-* Go into the MatterMost menu, then Integration
-* Click on "Add Incoming webhook"
-* "Add Incoming Webhook"
+- Go into the MatterMost menu, then Integration
+- Click on "Add Incoming webhook"
+- "Add Incoming Webhook"
 
 ### Testing the plugin
 

docs/backup_troubleshooting.md

@@ -12,33 +12,37 @@ Another good way to check if there is activity is the XOA VM stats view (on the
 
 ### VDI chain protection
 
-This means your previous VM disks and snapshots should be "merged" (*coalesced* in the XenServer world) before we can take a new snapshot. This mechanism is handled by XenServer itself, not Xen Orchestra. But we can check your existing VDI chain and avoid creating more snapshots than your storage can merge. Otherwise, this will lead to catastrophic consequences. Xen Orchestra is the **only** XenServer/XCP backup product dealing with this.
+Backup jobs regularly delete snapshots. When a snapshot is deleted, either manually or via a backup job, it triggers the need for Xenserver to coalesce the VDI chain - to merge the remaining VDIs and base copies in the chain. This means generally we cannot take too many new snapshots on said VM until Xenserver has finished running a coalesce job on the VDI chain.
+
+This mechanism and scheduling is handled by XenServer itself, not Xen Orchestra. But we can check your existing VDI chain and avoid creating more snapshots than your storage can merge. If we don't, this will lead to catastrophic consequences. Xen Orchestra is the **only** XenServer/XCP backup product that takes this into account and offers protection.
 
 Without this detection, you could have 2 potential issues:
 
-* `The Snapshot Chain is too Long`
-* `SR_BACKEND_FAILURE_44 (insufficient space)`
+- `The Snapshot Chain is too Long`
+- `SR_BACKEND_FAILURE_44 (insufficient space)`
 
 The first issue is a chain that contains more than 30 elements (fixed XenServer limit), and the other one means it's full because the "coalesce" process couldn't keep up the pace and the storage filled up.
 
-In the end, this message is a **protection mechanism against damaging your SR**. The backup job will fail, but XenServer itself should eventually automatically coalesce the snapshot chain, and the the next time the backup job should complete.
+In the end, this message is a **protection mechanism preventing damage to your SR**. The backup job will fail, but XenServer itself should eventually automatically coalesce the snapshot chain, and the the next time the backup job should complete.
 
-Just remember this: **coalesce will happen every time a snapshot is removed**.
+Just remember this: **a coalesce should happen every time a snapshot is removed**.
 
 > You can read more on this on our dedicated blog post regarding [XenServer coalesce detection](https://xen-orchestra.com/blog/xenserver-coalesce-detection-in-xen-orchestra/).
 
 ### Troubleshooting a constant VDI Chain Protection message (XenServer failure to coalesce)
 
- As previously mentioned, this message can be normal and it just means XenServer needs to perform a coalesce to merge old snapshots. However if you repeatedly get this message and it seems XenServer is not coalescing, You can take a few steps to determine why.  
+As previously mentioned, this message can be normal and it just means XenServer needs to perform a coalesce to merge old snapshots. However if you repeatedly get this message and it seems XenServer is not coalescing, You can take a few steps to determine why.
 
-First check SMlog on the XenServer host for messages relating to VDI corruption or coalesce job failure. For example, by running `cat /var/log/SMlog | grep -i exception` or `cat /var/log/SMlog | grep -i error` on the XenServer host with the affected storage.  
+First check SMlog on the XenServer host for messages relating to VDI corruption or coalesce job failure. For example, by running `cat /var/log/SMlog | grep -i exception` or `cat /var/log/SMlog | grep -i error` on the XenServer host with the affected storage.
 
 Coalesce jobs can also fail to run if the SR does not have enough free space. Check the problematic SR and make sure it has enough free space, generally 30% or more free is recommended depending on VM size. You can check if this is the issue by searching `SMlog` with `grep -i coales /var/log/SMlog` (you may have to look at previous logs such as `SMlog.1`).
 
-You can check if a coalesce job is currently active by running `ps axf | grep vhd` on the XenServer host and looking for a VHD process in the results (one of the resulting processes will be the grep command you just ran, ignore that one).  
+You can check if a coalesce job is currently active by running `ps axf | grep vhd` on the XenServer host and looking for a VHD process in the results (one of the resulting processes will be the grep command you just ran, ignore that one).
 
 If you don't see any running coalesce jobs, and can't find any other reason that XenServer has not started one, you can attempt to make it start a coalesce job by rescanning the SR. This is harmless to try, but will not always result in a coalesce. Visit the problematic SR in the XOA UI, then click the "Rescan All Disks" button towards the top right: it looks like a refresh circle icon. This should begin the coalesce process - if you click the Advanced tab in the SR view, the "disks needing to be coalesced" list should become smaller and smaller.
 
+As a last resort, migrating the VM (more specifically, its disks) to a new storage repository will also force a coalesce and solve this issue. That means migrating a VM to another host (with its own storage) and back will force the VDI chain for that VM to be coalesced, and get rid of the `VDI Chain Protection` message.
+
 ### Parse Error
 
 This is most likely due to running a backup job that uses Delta functionality (eg: delta backups, or continuous replication) on a version of XenServer older than 6.5. To use delta functionality you must run [XenServer 6.5 or later](https://xen-orchestra.com/docs/supported-version.html).
@@ -51,10 +55,10 @@ The Storage Repository (where your VM disks are currently stored) is full. Note
 
 Workarounds:
 
-* use a thin provisioned SR (local ext, NFS, XOSAN)
-* wait for Citrix to release thin provisioning on LVM
-* wait for Citrix to allow another mechanism besides snapshot to be able to export disks
-* use less than 50% of SR space or don't backup all VMs
+- use a thin provisioned SR (local ext, NFS, XOSAN)
+- wait for Citrix to release thin provisioning on LVM
+- wait for Citrix to allow another mechanism besides snapshot to be able to export disks
+- use less than 50% of SR space or don't backup all VMs
 
 ### Could not find the base VM
 
@@ -68,12 +72,12 @@ To solve it, you have to change a parameter in your VM. `xe vm-param-set has-ven
 
 ### ENOSPC: no space left on device
 
-This message appears when you do not have enough free space on the target remote when running a backup to it.  
+This message appears when you do not have enough free space on the target remote when running a backup to it.
 
 To check your free space, enter your XOA and run `xoa check` to check free system space and `df -h` to check free space on your chosen remote storage.
 
 ### Error: no VMs match this pattern
 
-This is happening when you have a *smart backup job* that doesn't match any VMs. For example: you created a job to backup all running VMs. If no VMs are running on backup schedule, you'll have this message. This could also happen if you lost connection with your pool master (the VMs aren't visible anymore from Xen Orchestra).
+This is happening when you have a _smart backup job_ that doesn't match any VMs. For example: you created a job to backup all running VMs. If no VMs are running on backup schedule, you'll have this message. This could also happen if you lost connection with your pool master (the VMs aren't visible anymore from Xen Orchestra).
 
 Edit your job and try to see matching VMs or check if your pool is connected to XOA.

docs/backups.md

@@ -6,13 +6,13 @@ This section is dedicated to all existing methods of rolling back or backing up
 
 There are several ways to protect your VMs:
 
-* [Full Backups](full_backups.md) [*Starter Edition*]
-* [Rolling Snapshots](rolling_snapshots.md) [*Starter Edition*]
-* [Delta Backups](delta_backups.md) (best of both previous ones) [*Enterprise Edition*]
-* [Disaster Recovery](disaster_recovery.md) [*Enterprise Edition*]
-* [Metadata Backups](metadata_backup.md) [*Enterprise Edition*]
-* [Continuous Replication](continuous_replication.md) [*Premium Edition*]
-* [File Level Restore](file_level_restore.md) [*Premium Edition*]
+- [Full Backups](full_backups.md) [*Starter Edition*]
+- [Rolling Snapshots](rolling_snapshots.md) [*Starter Edition*]
+- [Delta Backups](delta_backups.md) (best of both previous ones) [*Enterprise Edition*]
+- [Disaster Recovery](disaster_recovery.md) [*Enterprise Edition*]
+- [Metadata Backups](metadata_backup.md) [*Enterprise Edition*]
+- [Continuous Replication](continuous_replication.md) [*Premium Edition*]
+- [File Level Restore](file_level_restore.md) [*Premium Edition*]
 
 > Don't forget to take a look at the [backup troubleshooting](backup_troubleshooting.md) section. You can also take a look at the [backup reports](backup_reports.md) section for configuring notifications.
 
@@ -54,16 +54,15 @@ The tooltip confirms this:
 
 ## Remotes
 
-> Remotes are places where your *backup* and *delta backup* files will be stored.
+> Remotes are places where your _backup_ and _delta backup_ files will be stored.
 
-To add a *remote*, go to the **Settings/Remotes** menu.
+To add a _remote_, go to the **Settings/Remotes** menu.
 
 Supported remote types:
 
-* Local (any folder in XOA filesystem)
-* NFS
-* SMB (CIFS)
-
+- Local (any folder in XOA filesystem)
+- NFS
+- SMB (CIFS)
 
 > **WARNING**: the initial "/" or "\\" is automatically added.
 
@@ -73,15 +72,15 @@ On your NFS server, authorize XOA's IP address and permissions for subfolders. T
 
 ### SMB
 
-We support SMB storage on *Windows Server 2012 R2*.
+We support SMB storage on _Windows Server 2012 R2_.
 
 > WARNING: For continuous delta backup, SMB is **NOT** recommended (or only for small VMs, eg < 50GB)
 
 Also, read the UI twice when you add an SMB store. If you have:
 
-* `192.168.1.99` as SMB host
-* `Backups` as folder
-* no subfolder
+- `192.168.1.99` as SMB host
+- `Backups` as folder
+- no subfolder
 
 You'll have to fill it like this:
 
@@ -112,13 +111,13 @@ All your scheduled backups are acccessible in the "Restore" view in the backup s
 
 ## About backup compression
 
-By default, *Backups* are compressed (using GZIP, done on XenServer side). There is no absolute rule but in general uncompressed backups are faster (but sometimes much larger).
+By default, _Backups_ are compressed (using GZIP, done on XenServer side). There is no absolute rule but in general uncompressed backups are faster (but sometimes much larger).
 
 XenServer uses Gzip compression, which is:
 
-* slow (single threaded)
-* space efficient
-* consumes less bandwidth (helpful if your NFS share is far away)
+- slow (single threaded)
+- space efficient
+- consumes less bandwidth (helpful if your NFS share is far away)
 
 If you have compression on your NFS share (or destination filesystem like ZFS), you can disable compression in Xen Orchestra.
 
@@ -152,4 +151,4 @@ Replicated VMs HA are taken into account by XS/XCP-ng. To avoid the resultant tr
 ![](./assets/disabled-dr-ha-tag.png)
 ![](./assets/disabled-cr-ha-tag.png)
 
- > The tag won't be automatically removed by XO on the replicated VMs, even if HA is re-enabled.
+> The tag won't be automatically removed by XO on the replicated VMs, even if HA is re-enabled.

docs/backups_and_disaster_recovery.md

@@ -22,16 +22,15 @@ All your scheduled backup are acccessible in the "Restore" view in backup sectio
 
 ![](https://xen-orchestra.com/blog/content/images/2015/11/restore.png)
 
-
 ### Backup compression
 
 By default, Backup are compressed (using GZIP, done in XenServer side). There is no absolute rule about using compression or not, but there is some rules.
 
 Gzip compression is:
 
-* slow
-* space efficient
-* consume less bandwidth (if your NFS share is far)
+- slow
+- space efficient
+- consume less bandwidth (if your NFS share is far)
 
 If you have compression on your NFS share (or destination file-system like ZFS), you can disable compression in Xen Orchestra.
 
@@ -45,4 +44,4 @@ This feature is close to Backups, but it creates a snapshot when planned to do s
 
 **Warning**: snapshots are not backups. They help to rollback to a previous state, but all snapshots are on the same Storage than their original disk. If you lose the original VDI (or the SR), you'll **lose all your snapshots**.
 
-[Read more about it](https://xen-orchestra.com/blog/xen-orchestra-4-2/#schedulerollingsnapshots).
+[Read more about it](https://xen-orchestra.com/blog/xen-orchestra-4-2/#schedulerollingsnapshots).

docs/built-in.md

@@ -3,8 +3,8 @@
 This is the default method. Creating a user is very simple:
 
 1. Go into the Settings view, select "Users"
-2. You can create a *user* or an *admin*, with their password (or generate one)
+2. You can create a _user_ or an _admin_, with their password (or generate one)
 
 ![](./assets/usercreation.png)
 
-By default, a *user* won't have any permissions. At the opposite, an *admin* will have all rights.
+By default, a _user_ won't have any permissions. At the opposite, an _admin_ will have all rights.

docs/cloudinit.md

@@ -2,11 +2,11 @@
 
 Cloud-init is a program "that handles the early initialization of a cloud instance"[^n]. In other words, you can, on a "cloud-init"-ready template VM, pass a lot of data at first boot:
 
-* setting the hostname
-* add ssh keys
-* automatically grow the file system
-* create users
-* and a lot more!
+- setting the hostname
+- add ssh keys
+- automatically grow the file system
+- create users
+- and a lot more!
 
 This tool is pretty standard and used everywhere. A lot of existing cloud templates are using it.
 
@@ -40,19 +40,19 @@ Finally, create the VM:
 
 Now start the VM and SSH to its IP:
 
-* **the system has the right VM hostname** (from VM name) 
-* you don't need to use a password to access it (thanks to your SSH key):
+- **the system has the right VM hostname** (from VM name)
+- you don't need to use a password to access it (thanks to your SSH key):
 
 ```
 $ ssh centos@192.168.100.226
-[centos@tmp-app1 ~]$ 
+[centos@tmp-app1 ~]$
 ```
 
 The default `cloud-init` configuration can allow you to be to be a sudoer directly:
 
 ```
 [centos@tmp-app1 ~]$ sudo -s
-[root@tmp-app1 centos]# 
+[root@tmp-app1 centos]#
 ```
 
 Check the root file system size: indeed, **it was automatically increased** to what you need:

docs/concurrency.md

@@ -1,6 +1,5 @@
 # Backup Concurrency
 
-
 Xen Orchestra 5.20 introduces new tools to manage backup concurrency. Below is an overview of the backup process and ways you can control concurrency in your own environment.
 
 ## Backup process
@@ -15,14 +14,14 @@ Xen Orchestra will fetch the content of the snapshot made in step 1. This operat
 
 ### 3. Snapshot removal
 
-When it's done exporting, we'll remove the snapshot. Note: this operation will trigger a coalesce on your storage in the near future.
+When it's done exporting, we'll remove the snapshot. Note: this operation will trigger a coalesce on your storage in the near future (a coalesce is required every time a snapshot is removed).
 
 ## Concurrency
 
 Let's say you want to backup 50 VMs (each with 1x disk) at 3:00 AM. There are **2 different strategies**:
 
-1. backup VM #1 (snapshot, export, delete snapshots) **then** backup VM #2 -> *fully sequential strategy*
-2. snapshot all VMs, **then** export all snapshots, **then** delete all snapshots for finished exports -> *fully parallel strategy*
+1. backup VM #1 (snapshot, export, delete snapshots) **then** backup VM #2 -> _fully sequential strategy_
+2. snapshot all VMs, **then** export all snapshots, **then** delete all snapshots for finished exports -> _fully parallel strategy_
 
 The first purely sequential strategy will lead to a big problem: **you can't predict when a snapshot of your data will occur**. Because you can't predict the first VM export time (let's say 3 hours), then your second VM will have its snapshot taken 3 hours later, at 6 AM. We assume that's not what you meant when you specified "backup everything at 3 AM". You would end up with data from 6 AM (and later) for other VMs.
 
@@ -32,19 +31,19 @@ So what's the best choice? Continue below to learn how to best configure concurr
 
 ### Best choice
 
-By default the *parallel strategy* is, on paper, the most logical one. But we need to give it some limits on concurrency.
+By default the _parallel strategy_ is, on paper, the most logical one. But we need to give it some limits on concurrency.
 
 > Note: Xen Orchestra can be connected to multiple pools at once. So the concurrency number applies **per pool**.
 
 Each step has its own concurrency to fit its requirements:
 
-* **snapshot process** needs to be performed with the lowest concurrency possible. 2 is a good compromise: one snapshot is fast, but a stuck snapshot won't block the whole job. That's why a concurrency of 2 is not too bad on your storage. Basically, at 3 AM, we'll do all the VM snapshots needed, 2 at a time.
-* **disk export process** is bottlenecked by XCP-ng/XenServer - so to get the most of it, you can use up to 12 in parallel. As soon a snapshot is done, the export process will start, until reaching 12 at once. Then as soon as one in those 12 is finished, another one will appear until there is nothing more to export.
-* **snapshot deletion** can't happen all at once because the previous step durations are random - no need to implement concurrency on this one.
-
-This is how it currently works in Xen Orchestra. But sometimes, you also want to have *sequential* backups combined with the *parallel strategy*. That's why we introduced a sequential option in the advanced section of backup-ng:
+- **snapshot process** needs to be performed with the lowest concurrency possible. 2 is a good compromise: one snapshot is fast, but a stuck snapshot won't block the whole job. That's why a concurrency of 2 is not too bad on your storage. Basically, at 3 AM, we'll do all the VM snapshots needed, 2 at a time.
+- **disk export process** is bottlenecked by XCP-ng/XenServer - so to get the most of it, you can use up to 12 in parallel. As soon a snapshot is done, the export process will start, until reaching 12 at once. Then as soon as one in those 12 is finished, another one will appear until there is nothing more to export.
+- **VM export process:** the 12 disk export limit mentioned above applies to VDI exports, which happen during delta exports. For full VM exports (for example, for full backup job types), there is a built in limit of 2. This means if you have a full backup job of 6 VMs, only 2 will be exported at once.
+- **snapshot deletion** can't happen all at once because the previous step durations are random - no need to implement concurrency on this one.
 
+This is how it currently works in Xen Orchestra. But sometimes, you also want to have _sequential_ backups combined with the _parallel strategy_. That's why we introduced a sequential option in the advanced section of backup-ng:
 
 > Note: 0 means it will be fully **parallel** for all VMs.
 
-If you job contains 50 VMs for example, you could specify a sequential backup with a limit of "25 at once" (enter 25 in the concurrency field). This means at 3 AM, we'll do 25 snapshots (2 at a time), then exports. As soon as the first VM backup is completely finished (snapshot removed), then we'll start the 26th and so on, to always keep a max of 25x VM backups going in parallel.
+If you job contains 50 VMs for example, you could specify a sequential backup with a limit of "25 at once" (enter 25 in the concurrency field). This means at 3 AM, we'll do 25 snapshots (2 at a time), then exports. As soon as the first VM backup is completely finished (snapshot removed), then we'll start the 26th and so on, to always keep a max of 25x VM backups going in parallel.

docs/configuration.md

@@ -15,14 +15,14 @@ user = 'nobody'
 group = 'nogroup'
 ```
 
-**Warning!** A non-privileged user requires the use of ``sudo`` to mount NFS shares. See [installation from the sources](from_the_sources.md).
+**Warning!** A non-privileged user requires the use of `sudo` to mount NFS shares. See [installation from the sources](from_the_sources.md).
 
 ### HTTP listen address and port
 
 By default, XO-server listens on all addresses (0.0.0.0) and runs on port 80. If you need to, you can change this in the `# Basic HTTP` section:
 
 ```toml
-host = '0.0.0.0'
+hostname = '0.0.0.0'
 port = 80
 ```
 
@@ -31,7 +31,7 @@ port = 80
 XO-server can also run in HTTPS (you can run HTTP and HTTPS at the same time) - just modify what's needed in the `# Basic HTTPS` section, this time with the certificates/keys you need and their path:
 
 ```toml
-host = '0.0.0.0'
+hostname = '0.0.0.0'
 port = 443
 certificate = './certificate.pem'
 key = './key.pem'
@@ -43,10 +43,10 @@ key = './key.pem'
 
 If you want to redirect everything to HTTPS, you can modify the configuration like this:
 
-```
+```toml
 # If set to true, all HTTP traffic will be redirected to the first HTTPs configuration.
 
-  redirectToHttps: true
+redirectToHttps = true
 ```
 
 This should be written just before the `mount` option, inside the `http:` block.
@@ -77,6 +77,8 @@ Don't forget to reload `systemd` conf and restart `xo-server`:
 # systemctl restart xo-server.service
 ```
 
+**Note:** The `--use-openssl-ca` option is ignored by Node if Xen-Orchestra is run with Linux capabilities. Capabilities are commonly used to bind applications to privileged ports (<1024) (i.e. `CAP_NET_BIND_SERVICE`). Local NAT rules (`iptables`) or a reverse proxy would be required to use privileged ports and a custom certficate authority.
+
 ### Redis server
 
 By default, XO-server will try to contact Redis server on `localhost`, with the port `6379`. But you can define whatever you want:

docs/continuous_replication.md

@@ -2,17 +2,17 @@
 
 > WARNING: it works only with XenServer 6.5 or later
 
-This feature is a continuous replication system for your XenServer VMs **without any storage vendor lock-in**. You can replicate a VM every *X* minutes/hours to any storage repository. It could be to a distant XenServer host or just another local storage target.
+This feature is a continuous replication system for your XenServer VMs **without any storage vendor lock-in**. You can replicate a VM every _X_ minutes/hours to any storage repository. It could be to a distant XenServer host or just another local storage target.
 
 This feature covers multiple objectives:
 
-* no storage vendor lock-in
-* no configuration (agent-less)
-* low Recovery Point Objective, from 10 minutes to 24 hours (or more)
-* flexibility
-* no intermediate storage needed
-* atomic replication
-* efficient DR (disaster recovery) process
+- no storage vendor lock-in
+- no configuration (agent-less)
+- low Recovery Point Objective, from 10 minutes to 24 hours (or more)
+- flexibility
+- no intermediate storage needed
+- atomic replication
+- efficient DR (disaster recovery) process
 
 If you lose your main pool, you can start the copy on the other side, with very recent data.
 
@@ -36,18 +36,25 @@ To protect the replication, we removed the possibility to boot your copied VM di
 
 ## Manual initial seed
 
-**If you can't transfer the first backup through your network because it's too large**, you can make a seed locally. In order to do this, follow this procedure (until we make it accessible directly in XO).  
+**If you can't transfer the first backup through your network because it's too large**, you can make a seed locally. In order to do this, follow this procedure (until we make it accessible directly in XO).
 
 > This is **only** if you need to make the initial copy without making the whole transfer through your network. Otherwise, **you don't need this**. These instructions are for Backup-NG jobs, and will not work to seed a legacy backup job. Please migrate any legacy jobs to Backup-NG!
 
-
 ### Job creation
 
-Create the Continuous Replication backup job, and leave it disabled for now. On the main Backup-NG page, note its identifiers, the main `backupJobId` and the ID of one on the schedules for the job, `backupScheduleId`.
+Create the Continuous Replication backup job, and leave it disabled for now. On the main Backup-NG page, copy the job's `backupJobId` by hovering to the left of the shortened ID and clicking the copy to clipboard button:
+
+![](./assets/cr-seed-1.png)
+
+Copy it somewhere temporarily. Now we need to also copy the ID of the job schedule, `backupScheduleId`. Do this by hovering over the schedule name in the same panel as before, and clicking the copy to clipboard button. Keep it with the `backupJobId` you copied previously as we will need them all later:
+
+![](./assets/cr-seed-2.png)
 
 ### Seed creation
 
-Manually create a snapshot on the VM to backup, and note its UUID as `snapshotUuid` from the snapshot panel for the VM.
+Manually create a snapshot on the VM being backed up, then copy this snapshot UUID, `snapshotUuid` from the snapshot panel of the VM:
+
+![](./assets/cr-seed-3.png)
 
 > DO NOT ever delete or alter this snapshot, feel free to rename it to make that clear.
 
@@ -55,7 +62,9 @@ Manually create a snapshot on the VM to backup, and note its UUID as `snapshotUu
 
 Export this snapshot to a file, then import it on the target SR.
 
-Note the UUID of this newly created VM as `targetVmUuid`.
+We need to copy the UUID of this newly created VM as well, `targetVmUuid`:
+
+![](./assets/cr-seed-4.png)
 
 > DO not start this VM or it will break the Continuous Replication job! You can rename this VM to more easily remember this.
 
@@ -77,7 +86,9 @@ Usage: xo-cr-seed <source XAPI URL> <source snapshot UUID> <target XAPI URL> <ta
 
 xo-cr-seed v0.2.0
 ```
+
 Putting it altogether and putting our values and UUID's into the command, it will look like this (it is a long command):
+
 ```
 xo-cr-seed https://root:password@xen1.company.tld 4a21c1cd-e8bd-4466-910a-f7524ecc07b1 https://root:password@xen2.company.tld 5aaf86ca-ae06-4a4e-b6e1-d04f0609e64d 90d11a94-a88f-4a84-b7c1-ed207d3de2f9 369a26f0-da77-41ab-a998-fa6b02c69b9a
 ```
@@ -85,3 +96,11 @@ xo-cr-seed https://root:password@xen1.company.tld 4a21c1cd-e8bd-4466-910a-f7524e
 ### Finished
 
 Your backup job should now be working correctly! Manually run the job the first time to check if everything is OK. Then, enable the job. **Now, only the deltas are sent, your initial seed saved you a LOT of time if you have a slow network.**
+
+### Failover process
+
+In the situation where you need to failover to your destination host, you simply need to start all your VMs on the destination host.
+
+> Note: If you want to start a VM on your destination host without breaking the CR jobs on the other side, you will need to make a copy of the VM and start the copy. Otherwise, you will be asked if you would like to force start the VMs.
+
+![](./assets/force-start.jpg)