feat(Backups NG): first iteration (#2705)

packages/xo-server/src/api/backup-ng.js

@@ -0,0 +1,155 @@
+export function createJob ({ schedules, ...job }) {
+  job.userId = this.user.id
+  return this.createBackupNgJob(job, schedules)
+}
+
+createJob.permission = 'admin'
+createJob.params = {
+  compression: {
+    enum: ['', 'native'],
+    optional: true,
+  },
+  mode: {
+    enum: ['full', 'delta'],
+  },
+  name: {
+    type: 'string',
+    optional: true,
+  },
+  remotes: {
+    type: 'object',
+    optional: true,
+  },
+  schedules: {
+    type: 'object',
+    optional: true,
+  },
+  settings: {
+    type: 'object',
+  },
+  vms: {
+    type: 'object',
+  },
+}
+
+export function deleteJob ({ id }) {
+  return this.deleteBackupNgJob(id)
+}
+deleteJob.permission = 'admin'
+deleteJob.params = {
+  id: {
+    type: 'string',
+  },
+}
+
+export function editJob (props) {
+  return this.updateJob(props)
+}
+
+editJob.permission = 'admin'
+editJob.params = {
+  compression: {
+    enum: ['', 'native'],
+    optional: true,
+  },
+  id: {
+    type: 'string',
+  },
+  mode: {
+    enum: ['full', 'delta'],
+    optional: true,
+  },
+  name: {
+    type: 'string',
+    optional: true,
+  },
+  remotes: {
+    type: 'object',
+    optional: true,
+  },
+  settings: {
+    type: 'object',
+    optional: true,
+  },
+  vms: {
+    type: 'object',
+    optional: true,
+  },
+}
+
+export function getAllJobs () {
+  return this.getAllBackupNgJobs()
+}
+
+getAllJobs.permission = 'admin'
+
+export function getJob ({ id }) {
+  return this.getBackupNgJob(id)
+}
+
+getJob.permission = 'admin'
+
+getJob.params = {
+  id: {
+    type: 'string',
+  },
+}
+
+export async function runJob ({ id, scheduleId }) {
+  return this.runJobSequence([id], await this.getSchedule(scheduleId))
+}
+
+runJob.permission = 'admin'
+
+runJob.params = {
+  id: {
+    type: 'string',
+  },
+  scheduleId: {
+    type: 'string',
+  },
+}
+
+// -----------------------------------------------------------------------------
+
+export function deleteVmBackup ({ id }) {
+  return this.deleteVmBackupNg(id)
+}
+
+deleteVmBackup.permission = 'admin'
+
+deleteVmBackup.params = {
+  id: {
+    type: 'string',
+  },
+}
+
+export function listVmBackups ({ remotes }) {
+  return this.listVmBackupsNg(remotes)
+}
+
+listVmBackups.permission = 'admin'
+
+listVmBackups.params = {
+  remotes: {
+    type: 'array',
+    items: {
+      type: 'string',
+    },
+  },
+}
+
+export function importVmBackupNg ({ id, sr }) {
+  return this.importVmBackupNg(id, sr)
+}
+
+importVmBackupNg.permission = 'admin'
+
+importVmBackupNg.params = {
+  id: {
+    type: 'string',
+  },
+  sr: {
+    type: 'string',
+  },
+}

packages/xo-server/src/xo-mixins/backups-ng/index.js

@@ -0,0 +1,772 @@
+// @flow
+
+// $FlowFixMe
+import defer from 'golike-defer'
+import { dirname, resolve } from 'path'
+// $FlowFixMe
+import { fromEvent, timeout as pTimeout } from 'promise-toolbox'
+// $FlowFixMe
+import { isEmpty, last, mapValues, values } from 'lodash'
+import { type Pattern, createPredicate } from 'value-matcher'
+import { PassThrough } from 'stream'
+
+import { type Executor, type Job } from '../jobs'
+import { type Schedule } from '../scheduling'
+
+import createSizeStream from '../../size-stream'
+import { asyncMap, safeDateFormat, serializeError } from '../../utils'
+// import { parseDateTime } from '../../xapi/utils'
+import { type RemoteHandlerAbstract } from '../../remote-handlers/abstract'
+import { type Xapi } from '../../xapi'
+
+type Dict<T, K = string> = { [K]: T }
+
+type Mode = 'full' | 'delta'
+
+type Settings = {|
+  deleteFirst?: boolean,
+  exportRetention?: number,
+  snapshotRetention?: number,
+  vmTimeout?: number
+|}
+
+type SimpleIdPattern = {|
+  id: string | {| __or: string[] |}
+|}
+
+export type BackupJob = {|
+  ...$Exact<Job>,
+  compression?: 'native',
+  mode: Mode,
+  remotes?: SimpleIdPattern,
+  settings: Dict<Settings>,
+  srs?: SimpleIdPattern,
+  type: 'backup',
+  vms: Pattern
+|}
+
+type BackupResult = {|
+  mergeDuration: number,
+  mergeSize: number,
+  transferDuration: number,
+  transferSize: number
+|}
+
+type MetadataBase = {|
+  jobId: string,
+  mode: Mode,
+  scheduleId: string,
+  timestamp: number,
+  version: '2.0.0',
+  vm: Object,
+  vmSnapshot: Object
+|}
+type MetadataDelta = {| ...MetadataBase, mode: 'delta' |}
+type MetadataFull = {|
+  ...MetadataBase,
+  data: string, // relative path to the XVA
+  mode: 'full'
+|}
+type Metadata = MetadataDelta | MetadataFull
+
+const compareSnapshotTime = (
+  { snapshot_time: time1 },
+  { snapshot_time: time2 }
+) => (time1 < time2 ? -1 : 1)
+
+const compareTimestamp = ({ timestamp: time1 }, { timestamp: time2 }) =>
+  time1 - time2
+
+// returns all entries but the last (retention - 1)-th
+//
+// the “-1” is because this code is usually run with entries computed before the
+// new entry is created
+//
+// FIXME: check whether it take the new one into account
+const getOldEntries = <T>(retention: number, entries?: T[]): T[] =>
+  entries === undefined
+    ? []
+    : --retention > 0 ? entries.slice(0, -retention) : entries
+
+const defaultSettings: Settings = {
+  deleteFirst: false,
+  exportRetention: 0,
+  snapshotRetention: 0,
+  vmTimeout: 0,
+}
+const getSetting = (
+  settings: Dict<Settings>,
+  name: $Keys<Settings>,
+  ...keys: string[]
+): any => {
+  for (let i = 0, n = keys.length; i < n; ++i) {
+    const objectSettings = settings[keys[i]]
+    if (objectSettings !== undefined) {
+      const setting = objectSettings[name]
+      if (setting !== undefined) {
+        return setting
+      }
+    }
+  }
+  return defaultSettings[name]
+}
+
+const BACKUP_DIR = 'xo-vm-backups'
+const getVmBackupDir = (uuid: string) => `${BACKUP_DIR}/${uuid}`
+
+const isMetadataFile = (filename: string) => filename.endsWith('.json')
+
+const listReplicatedVms = (xapi: Xapi, scheduleId: string, srId) => {
+  const { all } = xapi.objects
+  const vms = {}
+  for (const key in all) {
+    const object = all[key]
+    if (
+      object.$type === 'vm' &&
+      object.other_config['xo:backup:schedule'] === scheduleId &&
+      object.other_config['xo:backup:sr'] === srId
+    ) {
+      vms[object.$id] = object
+    }
+  }
+
+  // the replicated VMs have been created from a snapshot, therefore we can use
+  // `snapshot_time` as the creation time
+  return values(vms).sort(compareSnapshotTime)
+}
+
+const parseVmBackupId = id => {
+  const i = id.indexOf('/')
+  return {
+    metadataFilename: id.slice(i + 1),
+    remoteId: id.slice(0, i),
+  }
+}
+
+// used to resolve the data field from the metadata
+const resolveRelativeFromFile = (file, path) =>
+  resolve('/', dirname(file), path).slice(1)
+
+const unboxIds = (pattern?: SimpleIdPattern): string[] => {
+  if (pattern === undefined) {
+    return []
+  }
+  const { id } = pattern
+  return typeof id === 'string' ? [id] : id.__or
+}
+
+// File structure on remotes:
+//
+// <remote>
+// └─ xo-vm-backups
+//   ├─ index.json // TODO
+//   └─ <VM UUID>
+//      ├─ index.json // TODO
+//      ├─ vdis
+//      │  └─ <VDI UUID>
+//      │     ├─ index.json // TODO
+//      │     └─ <YYYYMMDD>T<HHmmss>.vhd
+//      ├─ <YYYYMMDD>T<HHmmss>.json // backup metadata
+//      └─ <YYYYMMDD>T<HHmmss>.xva
+export default class BackupNg {
+  _app: any
+
+  constructor (app: any) {
+    this._app = app
+
+    app.on('start', () => {
+      const executor: Executor = async ({
+        cancelToken,
+        job: job_,
+        logger,
+        runJobId,
+        schedule = {},
+      }) => {
+        const job: BackupJob = (job_: any)
+        const vms = app.getObjects({
+          filter: createPredicate({
+            type: 'VM',
+            ...job.vms,
+          }),
+        })
+        if (isEmpty(vms)) {
+          throw new Error('no VMs match this pattern')
+        }
+        const jobId = job.id
+        const scheduleId = schedule.id
+        const status: Object = {
+          calls: {},
+          runJobId,
+          start: Date.now(),
+          timezone: schedule.timezone,
+        }
+        const { calls } = status
+        await asyncMap(vms, async vm => {
+          const { uuid } = vm
+          const method = 'backup-ng'
+          const params = {
+            id: uuid,
+            tag: job.name,
+          }
+
+          const name = vm.name_label
+          const runCallId = logger.notice(
+            `Starting backup of ${name}. (${jobId})`,
+            {
+              event: 'jobCall.start',
+              method,
+              params,
+              runJobId,
+            }
+          )
+          const call: Object = (calls[runCallId] = {
+            method,
+            params,
+            start: Date.now(),
+          })
+          const vmCancel = cancelToken.fork()
+          try {
+            // $FlowFixMe injected $defer param
+            let p = this._backupVm(vmCancel.token, uuid, job, schedule)
+            const vmTimeout: number = getSetting(
+              job.settings,
+              'vmTimeout',
+              uuid,
+              scheduleId
+            )
+            if (vmTimeout !== 0) {
+              p = pTimeout.call(p, vmTimeout)
+            }
+            const returnedValue = await p
+            logger.notice(
+              `Backuping ${name} (${runCallId}) is a success. (${jobId})`,
+              {
+                event: 'jobCall.end',
+                runJobId,
+                runCallId,
+                returnedValue,
+              }
+            )
+
+            call.returnedValue = returnedValue
+            call.end = Date.now()
+          } catch (error) {
+            vmCancel.cancel()
+            logger.notice(
+              `Backuping ${name} (${runCallId}) has failed. (${jobId})`,
+              {
+                event: 'jobCall.end',
+                runJobId,
+                runCallId,
+                error: serializeError(error),
+              }
+            )
+
+            call.error = error
+            call.end = Date.now()
+
+            console.warn(error.stack) // TODO: remove
+          }
+        })
+        status.end = Date.now()
+        return status
+      }
+      app.registerJobExecutor('backup', executor)
+    })
+  }
+
+  async createBackupNgJob (
+    props: $Diff<BackupJob, {| id: string |}>,
+    schedules?: Dict<$Diff<Schedule, {| id: string |}>>
+  ): Promise<BackupJob> {
+    const app = this._app
+    props.type = 'backup'
+    const job: BackupJob = await app.createJob(props)
+
+    if (schedules !== undefined) {
+      const { id, settings } = job
+      const tmpIds = Object.keys(schedules)
+      await asyncMap(tmpIds, async (tmpId: string) => {
+        // $FlowFixMe don't know what is the problem (JFT)
+        const schedule = schedules[tmpId]
+        schedule.jobId = id
+        settings[(await app.createSchedule(schedule)).id] = settings[tmpId]
+        delete settings[tmpId]
+      })
+      await app.updateJob({ id, settings })
+    }
+
+    return job
+  }
+
+  async deleteBackupNgJob (id: string): Promise<void> {
+    const app = this._app
+    const [schedules] = await Promise.all([
+      app.getAllSchedules(),
+      app.getJob(id, 'backup'),
+    ])
+    await Promise.all([
+      app.removeJob(id),
+      asyncMap(schedules, schedule => {
+        if (schedule.id === id) {
+          app.removeSchedule(schedule.id)
+        }
+      }),
+    ])
+  }
+
+  async deleteVmBackupNg (id: string): Promise<void> {
+    const app = this._app
+    const { metadataFilename, remoteId } = parseVmBackupId(id)
+    const handler = await app.getRemoteHandler(remoteId)
+    const metadata: Metadata = JSON.parse(
+      await handler.readFile(metadataFilename)
+    )
+
+    if (metadata.mode === 'delta') {
+      throw new Error('not implemented')
+    }
+
+    metadata._filename = metadataFilename
+    await this._deleteFullVmBackups(handler, [metadata])
+  }
+
+  getAllBackupNgJobs (): Promise<BackupJob[]> {
+    return this._app.getAllJobs('backup')
+  }
+
+  getBackupNgJob (id: string): Promise<BackupJob> {
+    return this._app.getJob(id, 'backup')
+  }
+
+  async importVmBackupNg (id: string, srId: string): Promise<void> {
+    const app = this._app
+    const { metadataFilename, remoteId } = parseVmBackupId(id)
+    const handler = await app.getRemoteHandler(remoteId)
+    const metadata: Metadata = JSON.parse(
+      await handler.readFile(metadataFilename)
+    )
+
+    if (metadata.mode === 'delta') {
+      throw new Error('not implemented')
+    }
+
+    const xapi = app.getXapi(srId)
+    const sr = xapi.getObject(srId)
+    const xva = await handler.createReadStream(
+      resolveRelativeFromFile(metadataFilename, metadata.data)
+    )
+    const vm = await xapi.importVm(xva, { srId: sr.$id })
+    await Promise.all([
+      xapi.addTag(vm.$id, 'restored from backup'),
+      xapi.editVm(vm.$id, {
+        name_label: `${metadata.vm.name_label} (${safeDateFormat(
+          metadata.timestamp
+        )})`,
+      }),
+    ])
+    return vm.$id
+  }
+
+  async listVmBackupsNg (remotes: string[]) {
+    const backupsByVmByRemote: Dict<Dict<Metadata[]>> = {}
+
+    const app = this._app
+    await Promise.all(
+      remotes.map(async remoteId => {
+        const handler = await app.getRemoteHandler(remoteId)
+
+        const entries = (await handler.list(BACKUP_DIR).catch(error => {
+          if (error == null || error.code !== 'ENOENT') {
+            throw error
+          }
+          return []
+        })).filter(name => name !== 'index.json')
+
+        const backupsByVm = (backupsByVmByRemote[remoteId] = {})
+        await Promise.all(
+          entries.map(async vmId => {
+            // $FlowFixMe don't know what is the problem (JFT)
+            const backups = await this._listVmBackups(handler, vmId)
+
+            if (backups.length === 0) {
+              return
+            }
+
+            // inject an id usable by importVmBackupNg()
+            backups.forEach(backup => {
+              backup.id = `${remoteId}/${backup._filename}`
+            })
+
+            backupsByVm[vmId] = backups
+          })
+        )
+      })
+    )
+
+    return backupsByVmByRemote
+  }
+
+  // - [x] files (.tmp) should be renamed at the end of job
+  // - [ ] validate VHDs after exports and before imports
+  // - [ ] protect against concurrent backup against a single VM (JFT: why?)
+  // - [x] detect full remote
+  // - [x] can the snapshot and export retention be different? → Yes
+  // - [ ] snapshots and files of an old job should be detected and removed
+  // - [ ] adding and removing VDIs should behave
+  // - [ ] key export?
+  // - [x] deleteFirst per target
+  // - [ ] possibility to (re-)run a single VM in a backup?
+  // - [x] timeout per VM
+  // - [ ] display queued VMs
+  // - [ ] jobs should be cancelable
+  // - [ ] logs
+  // - [x] backups should be deletable from the API
+  // - [ ] check merge/transfert duration/size are what we want for delta
+  @defer
+  async _backupVm (
+    $defer: any,
+    $cancelToken: any,
+    vmId: string,
+    job: BackupJob,
+    schedule: Schedule
+  ): Promise<BackupResult> {
+    const app = this._app
+    const xapi = app.getXapi(vmId)
+    const vm = xapi.getObject(vmId)
+
+    const { id: jobId, settings } = job
+    const { id: scheduleId } = schedule
+
+    const exportRetention: number = getSetting(
+      settings,
+      'exportRetention',
+      scheduleId
+    )
+    const snapshotRetention: number = getSetting(
+      settings,
+      'snapshotRetention',
+      scheduleId
+    )
+
+    let remotes, srs
+    if (exportRetention === 0) {
+      if (snapshotRetention === 0) {
+        throw new Error('export and snapshots retentions cannot both be 0')
+      }
+    } else {
+      remotes = unboxIds(job.remotes)
+      srs = unboxIds(job.srs)
+      if (remotes.length === 0 && srs.length === 0) {
+        throw new Error('export retention must be 0 without remotes and SRs')
+      }
+    }
+
+    const snapshots = vm.$snapshots
+      .filter(_ => _.other_config['xo:backup:schedule'] === scheduleId)
+      .sort(compareSnapshotTime)
+    $defer(() =>
+      asyncMap(getOldEntries(snapshotRetention, snapshots), _ =>
+        xapi.deleteVm(_)
+      )
+    )
+
+    let snapshot = await xapi._snapshotVm(
+      $cancelToken,
+      vm,
+      `[XO Backup] ${vm.name_label}`
+    )
+    $defer.onFailure.call(xapi, '_deleteVm', snapshot)
+    await xapi._updateObjectMapProperty(snapshot, 'other_config', {
+      'xo:backup:job': jobId,
+      'xo:backup:schedule': scheduleId,
+    })
+    snapshot = await xapi.barrier(snapshot.$ref)
+
+    if (exportRetention === 0) {
+      return {
+        mergeDuration: 0,
+        mergeSize: 0,
+        transferDuration: 0,
+        transferSize: 0,
+      }
+    }
+
+    const now = Date.now()
+    const { mode } = job
+
+    const metadata: Metadata = {
+      jobId,
+      mode,
+      scheduleId,
+      timestamp: now,
+      version: '2.0.0',
+      vm,
+      vmSnapshot: snapshot,
+    }
+
+    if (mode === 'full') {
+      // TODO: do not create the snapshot if there are no snapshotRetention and
+      // the VM is not running
+      if (snapshotRetention === 0) {
+        $defer.call(xapi, 'deleteVm', snapshot)
+      }
+
+      let xva = await xapi.exportVm($cancelToken, snapshot)
+      const exportTask = xva.task
+      xva = xva.pipe(createSizeStream())
+
+      const dirname = getVmBackupDir(vm.uuid)
+      const basename = safeDateFormat(now)
+
+      const dataBasename = `${basename}.xva`
+      const metadataFilename = `${dirname}/${basename}.json`
+
+      metadata.data = `./${dataBasename}`
+      const dataFilename = `${dirname}/${dataBasename}`
+      const tmpFilename = `${dirname}/.${dataBasename}`
+
+      const jsonMetadata = JSON.stringify(metadata)
+
+      await Promise.all([
+        asyncMap(
+          remotes,
+          defer(async ($defer, remoteId) => {
+            const fork = xva.pipe(new PassThrough())
+
+            const handler = await app.getRemoteHandler(remoteId)
+
+            const oldBackups = getOldEntries(
+              exportRetention,
+              await this._listVmBackups(
+                handler,
+                vm,
+                _ => _.mode === 'full' && _.scheduleId === scheduleId
+              )
+            )
+
+            const deleteFirst = getSetting(settings, 'deleteFirst', remoteId)
+            if (deleteFirst) {
+              await this._deleteFullVmBackups(handler, oldBackups)
+            }
+
+            const output = await handler.createOutputStream(tmpFilename, {
+              checksum: true,
+            })
+            $defer.onFailure.call(handler, 'unlink', tmpFilename)
+            $defer.onSuccess.call(
+              handler,
+              'rename',
+              tmpFilename,
+              dataFilename,
+              { checksum: true }
+            )
+
+            const promise = fromEvent(output, 'finish')
+            fork.pipe(output)
+            await Promise.all([exportTask, promise])
+
+            await handler.outputFile(metadataFilename, jsonMetadata)
+
+            if (!deleteFirst) {
+              await this._deleteFullVmBackups(handler, oldBackups)
+            }
+          })
+        ),
+        asyncMap(
+          srs,
+          defer(async ($defer, srId) => {
+            const fork = xva.pipe(new PassThrough())
+            fork.task = exportTask
+
+            const xapi = app.getXapi(srId)
+            const sr = xapi.getObject(srId)
+
+            const oldVms = getOldEntries(
+              exportRetention,
+              listReplicatedVms(xapi, scheduleId, srId)
+            )
+
+            const deleteFirst = getSetting(settings, 'deleteFirst', srId)
+            if (deleteFirst) {
+              await this._deleteVms(xapi, oldVms)
+            }
+
+            const vm = await xapi.barrier(
+              await xapi._importVm($cancelToken, fork, sr, vm =>
+                xapi._setObjectProperties(vm, {
+                  nameLabel: `${metadata.vm.name_label} (${safeDateFormat(
+                    metadata.timestamp
+                  )})`,
+                })
+              )
+            )
+
+            await Promise.all([
+              xapi.addTag(vm.$ref, 'Disaster Recovery'),
+              xapi._updateObjectMapProperty(vm, 'blocked_operations', {
+                start:
+                  'Start operation for this vm is blocked, clone it if you want to use it.',
+              }),
+              xapi._updateObjectMapProperty(vm, 'other_config', {
+                'xo:backup:sr': srId,
+              }),
+            ])
+
+            if (!deleteFirst) {
+              await this._deleteVms(xapi, oldVms)
+            }
+          })
+        ),
+      ])
+
+      return {
+        mergeDuration: 0,
+        mergeSize: 0,
+        transferDuration: Date.now() - now,
+        transferSize: xva.size,
+      }
+    }
+
+    const baseSnapshot = last(snapshots)
+    if (baseSnapshot !== undefined) {
+      console.log(baseSnapshot.$id) // TODO: remove
+      // check current state
+      // await Promise.all([asyncMap(remotes, remoteId => {})])
+    }
+
+    const deltaExport = await xapi.exportDeltaVm(
+      $cancelToken,
+      snapshot,
+      baseSnapshot
+    )
+
+    // forks of the lazy streams
+    deltaExport.streams = mapValues(deltaExport.streams, lazyStream => {
+      let stream
+      return () => {
+        if (stream === undefined) {
+          stream = lazyStream()
+        }
+        return Promise.resolve(stream).then(stream => {
+          const fork = stream.pipe(new PassThrough())
+          fork.task = stream.task
+          return fork
+        })
+      }
+    })
+
+    const mergeStart = 0
+    const mergeEnd = 0
+    let transferStart = 0
+    let transferEnd = 0
+    await Promise.all([
+      asyncMap(remotes, defer(async ($defer, remote) => {})),
+      asyncMap(
+        srs,
+        defer(async ($defer, srId) => {
+          const xapi = app.getXapi(srId)
+          const sr = xapi.getObject(srId)
+
+          const oldVms = getOldEntries(
+            exportRetention,
+            listReplicatedVms(xapi, scheduleId, srId)
+          )
+
+          const deleteFirst = getSetting(settings, 'deleteFirst', srId)
+          if (deleteFirst) {
+            await this._deleteVms(xapi, oldVms)
+          }
+
+          transferStart =
+            transferStart === 0
+              ? Date.now()
+              : Math.min(transferStart, Date.now())
+
+          const { vm } = await xapi.importDeltaVm(deltaExport, {
+            disableStartAfterImport: false, // we'll take care of that
+            name_label: `${metadata.vm.name_label} (${safeDateFormat(
+              metadata.timestamp
+            )})`,
+            srId: sr.$id,
+          })
+
+          transferEnd = Math.max(transferEnd, Date.now())
+
+          await Promise.all([
+            xapi.addTag(vm.$ref, 'Continuous Replication'),
+            xapi._updateObjectMapProperty(vm, 'blocked_operations', {
+              start:
+                'Start operation for this vm is blocked, clone it if you want to use it.',
+            }),
+            xapi._updateObjectMapProperty(vm, 'other_config', {
+              'xo:backup:sr': srId,
+            }),
+          ])
+
+          if (!deleteFirst) {
+            await this._deleteVms(xapi, oldVms)
+          }
+        })
+      ),
+    ])
+
+    return {
+      mergeDuration: mergeEnd - mergeStart,
+      mergeSize: 0,
+      transferDuration: transferEnd - transferStart,
+      transferSize: 0,
+    }
+  }
+
+  async _deleteFullVmBackups (
+    handler: RemoteHandlerAbstract,
+    backups: Metadata[]
+  ): Promise<void> {
+    await asyncMap(backups, ({ _filename, data }) =>
+      Promise.all([
+        handler.unlink(_filename),
+        handler.unlink(resolveRelativeFromFile(_filename, data)),
+      ])
+    )
+  }
+
+  async _deleteVms (xapi: Xapi, vms: Object[]): Promise<void> {
+    return asyncMap(vms, vm => xapi.deleteVm(vm))
+  }
+
+  async _listVmBackups (
+    handler: RemoteHandlerAbstract,
+    vm: Object | string,
+    predicate?: Metadata => boolean
+  ): Promise<Metadata[]> {
+    const backups = []
+
+    const dir = getVmBackupDir(typeof vm === 'string' ? vm : vm.uuid)
+    try {
+      const files = await handler.list(dir)
+      await Promise.all(
+        files.filter(isMetadataFile).map(async file => {
+          const path = `${dir}/${file}`
+          try {
+            const metadata = JSON.parse(await handler.readFile(path))
+            if (predicate === undefined || predicate(metadata)) {
+              Object.defineProperty(metadata, '_filename', {
+                value: path,
+              })
+              backups.push(metadata)
+            }
+          } catch (error) {
+            console.warn('_listVmBackups', path, error)
+          }
+        })
+      )
+    } catch (error) {
+      if (error == null || error.code !== 'ENOENT') {
+        throw error
+      }
+    }
+
+    return backups.sort(compareTimestamp)
+  }
+}

packages/xo-server/src/xo-mixins/backups-ng/migration.js

@@ -0,0 +1,153 @@
+// @flow
+
+import assert from 'assert'
+
+import { type BackupJob } from '../backups-ng'
+import { type CallJob } from '../jobs'
+import { type Schedule } from '../scheduling'
+
+const createOr = (children: Array<any>): any =>
+  children.length === 1 ? children[0] : { __or: children }
+
+const methods = {
+  'vm.deltaCopy': (
+    job: CallJob,
+    { retention = 1, sr, vms },
+    schedule: Schedule
+  ) => ({
+    mode: 'delta',
+    settings: {
+      [schedule.id]: {
+        exportRetention: retention,
+        vmTimeout: job.timeout,
+      },
+    },
+    srs: { id: sr },
+    userId: job.userId,
+    vms,
+  }),
+  'vm.rollingDeltaBackup': (
+    job: CallJob,
+    { depth = 1, retention = depth, remote, vms },
+    schedule: Schedule
+  ) => ({
+    mode: 'delta',
+    remotes: { id: remote },
+    settings: {
+      [schedule.id]: {
+        exportRetention: retention,
+        vmTimeout: job.timeout,
+      },
+    },
+    vms,
+  }),
+  'vm.rollingDrCopy': (
+    job: CallJob,
+    { deleteOldBackupsFirst, depth = 1, retention = depth, sr, vms },
+    schedule: Schedule
+  ) => ({
+    mode: 'full',
+    settings: {
+      [schedule.id]: {
+        deleteFirst: deleteOldBackupsFirst,
+        exportRetention: retention,
+        vmTimeout: job.timeout,
+      },
+    },
+    srs: { id: sr },
+    vms,
+  }),
+  'vm.rollingBackup': (
+    job: CallJob,
+    { compress, depth = 1, retention = depth, remoteId, vms },
+    schedule: Schedule
+  ) => ({
+    compression: compress ? 'native' : undefined,
+    mode: 'full',
+    remotes: { id: remoteId },
+    settings: {
+      [schedule.id]: {
+        exportRetention: retention,
+        vmTimeout: job.timeout,
+      },
+    },
+    vms,
+  }),
+  'vm.rollingSnapshot': (
+    job: CallJob,
+    { depth = 1, retention = depth, vms },
+    schedule: Schedule
+  ) => ({
+    mode: 'full',
+    settings: {
+      [schedule.id]: {
+        snapshotRetention: retention,
+        vmTimeout: job.timeout,
+      },
+    },
+    vms,
+  }),
+}
+
+const parseParamsVector = vector => {
+  assert.strictEqual(vector.type, 'crossProduct')
+  const { items } = vector
+  assert.strictEqual(items.length, 2)
+
+  let vms, params
+  if (items[1].type === 'map') {
+    ;[params, vms] = items
+
+    vms = vms.collection
+    assert.strictEqual(vms.type, 'fetchObjects')
+    vms = vms.pattern
+  } else {
+    ;[vms, params] = items
+
+    assert.strictEqual(vms.type, 'set')
+    vms = vms.values
+    if (vms.length !== 0) {
+      assert.deepStrictEqual(Object.keys(vms[0]), ['id'])
+      vms = { id: createOr(vms.map(_ => _.id)) }
+    }
+  }
+
+  assert.strictEqual(params.type, 'set')
+  params = params.values
+  assert.strictEqual(params.length, 1)
+  params = params[0]
+
+  return { ...params, vms }
+}
+
+export const translateOldJobs = async (app: any): Promise<Array<BackupJob>> => {
+  const backupJobs: Array<BackupJob> = []
+  const [jobs, schedules] = await Promise.all([
+    app.getAllJobs('call'),
+    app.getAllSchedules(),
+  ])
+  jobs.forEach(job => {
+    try {
+      const { id } = job
+      let method, schedule
+      if (
+        job.type === 'call' &&
+        (method = methods[job.method]) !== undefined &&
+        (schedule = schedules.find(_ => _.jobId === id)) !== undefined
+      ) {
+        const params = parseParamsVector(job.paramsVector)
+        backupJobs.push({
+          id,
+          name: params.tag || job.name,
+          type: 'backup',
+          userId: job.userId,
+          // $FlowFixMe `method` is initialized but Flow fails to see this
+          ...method(job, params, schedule),
+        })
+      }
+    } catch (error) {
+      console.warn('translateOldJobs', job, error)
+    }
+  })
+  return backupJobs
+}

packages/xo-server/src/xo-mixins/jobs/index.js

@@ -142,6 +142,7 @@ export default class Jobs {
   }
 
   async getAllJobs (type: string = 'call'): Promise<Array<Job>> {
+    // $FlowFixMe don't know what is the problem (JFT)
     const jobs = await this._jobs.get()
     const runningJobs = this._runningJobs
     const result = []

packages/xo-web/package.json

@@ -30,6 +30,7 @@
     "node": ">=6"
   },
   "devDependencies": {
+    "@julien-f/freactal": "0.1.0",
     "@nraynaud/novnc": "0.6.1",
     "@xen-orchestra/cron": "^1.0.2",
     "ansi_up": "^2.0.2",

packages/xo-web/src/common/intl/messages.js

@@ -60,6 +60,8 @@ const messages = {
   selfServicePage: 'Self service',
   backupPage: 'Backup',
   jobsPage: 'Jobs',
+  backupNG: 'Backups NG',
+  backupNGName: 'Name',
   xoaPage: 'XOA',
   updatePage: 'Updates',
   licensesPage: 'Licenses',
@@ -354,6 +356,13 @@ const messages = {
   remoteTestSuccessMessage: 'The remote appears to work correctly',
   remoteConnectionFailed: 'Connection failed',
 
+  // ------ Backup job -----
+
+  confirmDeleteBackupJobsTitle:
+    'Delete backup job{nJobs, plural, one {} other {s}}',
+  confirmDeleteBackupJobsBody:
+    'Are you sure you want to delete {nJobs, number} backup job{nJobs, plural, one {} other {s}}?',
+
   // ------ Remote -----
   remoteName: 'Name',
   remotePath: 'Path',

packages/xo-web/src/common/xo/index.js

@@ -1602,6 +1602,52 @@ export const enableSchedule = id => editSchedule({ id, enabled: true })
 
 export const getSchedule = id => _call('schedule.get', { id })
 
+// Backup NG ---------------------------------------------------------
+
+export const subscribeBackupNgJobs = createSubscription(() =>
+  _call('backupNg.getAllJobs')
+)
+
+export const createBackupNgJob = props =>
+  _call('backupNg.createJob', props)::tap(subscribeBackupNgJobs.forceRefresh)
+
+export const deleteBackupNgJobs = async ids => {
+  const { length } = ids
+  if (length === 0) {
+    return
+  }
+  const vars = { nJobs: length }
+  try {
+    await confirm({
+      title: _('confirmDeleteBackupJobsTitle', vars),
+      body: <p>{_('confirmDeleteBackupJobsBody', vars)}</p>,
+    })
+  } catch (_) {
+    return
+  }
+
+  return Promise.all(
+    ids.map(id => _call('backupNg.deleteJob', { id: resolveId(id) }))
+  )::tap(subscribeBackupNgJobs.forceRefresh)
+}
+
+export const editBackupNgJob = props =>
+  _call('backupNg.editJob', props)::tap(subscribeBackupNgJobs.forceRefresh)
+
+export const getBackupNgJob = id => _call('backupNg.getJob', { id })
+
+export const runBackupNgJob = ({ id, scheduleId }) =>
+  _call('backupNg.runJob', { id, scheduleId })
+
+export const listVmBackups = remotes =>
+  _call('backupNg.listVmBackups', { remotes: resolveIds(remotes) })
+
+export const restoreBackup = (backup, sr) =>
+  _call('backupNg.importVmBackupNg', {
+    id: resolveId(backup),
+    sr: resolveId(sr),
+  })
+
 // Plugins -----------------------------------------------------------
 
 export const loadPlugin = async id =>

packages/xo-web/src/xo-app/backup-ng/edit.js

@@ -0,0 +1,21 @@
+import addSubscriptions from 'add-subscriptions'
+import React from 'react'
+import { injectState, provideState } from '@julien-f/freactal'
+import { Debug } from 'utils'
+import { subscribeBackupNgJobs, subscribeSchedules } from 'xo'
+
+import New from './new'
+
+export default [
+  addSubscriptions({
+    jobs: subscribeBackupNgJobs,
+    schedules: subscribeSchedules,
+  }),
+  provideState({
+    computed: {
+      value: ({ jobs, schedules }) => {},
+    },
+  }),
+  injectState,
+  props => ({ state }) => <New value={state.value} />,
+].reduceRight((value, decorator) => decorator(value))

packages/xo-web/src/xo-app/backup-ng/file-restore/index.js

@@ -0,0 +1,8 @@
+import Component from 'base-component'
+import React from 'react'
+
+export default class FileRestore extends Component {
+  render () {
+    return <p className='text-danger'>Available soon</p>
+  }
+}

packages/xo-web/src/xo-app/backup-ng/index.js

@@ -0,0 +1,210 @@
+import _ from 'intl'
+import ActionButton from 'action-button'
+import addSubscriptions from 'add-subscriptions'
+import Icon from 'icon'
+import React from 'react'
+import SortedTable from 'sorted-table'
+import { map, groupBy } from 'lodash'
+import { Card, CardHeader, CardBlock } from 'card'
+import { constructQueryString } from 'smart-backup-pattern'
+import { Container, Row, Col } from 'grid'
+import { NavLink, NavTabs } from 'nav'
+import { routes } from 'utils'
+import {
+  deleteBackupNgJobs,
+  subscribeBackupNgJobs,
+  subscribeSchedules,
+  runBackupNgJob,
+} from 'xo'
+
+import LogsTable from '../logs'
+import Page from '../page'
+
+import Edit from './edit'
+import New from './new'
+import FileRestore from './file-restore'
+import Restore from './restore'
+
+const Ul = ({ children, ...props }) => (
+  <ul {...props} style={{ display: 'inline', padding: '0 0.5em' }}>
+    {children}
+  </ul>
+)
+
+const Li = ({ children, ...props }) => (
+  <li {...props} style={{ listStyleType: 'none', display: 'inline-block' }}>
+    {children}
+  </li>
+)
+
+const Td = ({ children, ...props }) => (
+  <td {...props} style={{ borderRight: '1px solid black' }}>
+    {children}
+  </td>
+)
+
+const SchedulePreviewBody = ({ job, schedules }) => (
+  <table>
+    {map(schedules, schedule => (
+      <tr key={schedule.id}>
+        <Td>{schedule.cron}</Td>
+        <Td>{schedule.timezone}</Td>
+        <Td>{job.settings[schedule.id].exportRetention}</Td>
+        <Td>{job.settings[schedule.id].snapshotRetention}</Td>
+        <td>
+          <ActionButton
+            handler={runBackupNgJob}
+            icon='run-schedule'
+            size='small'
+            data-id={job.id}
+            data-scheduleId={schedule.id}
+            btnStyle='warning'
+          />
+        </td>
+      </tr>
+    ))}
+  </table>
+)
+
+const SchedulePreviewHeader = ({ _ }) => (
+  <Ul>
+    <Li>Schedule ID</Li> | <Li>Cron</Li> | <Li>Timezone</Li> |{' '}
+    <Li>Export retention</Li> | <Li>Snapshot retention</Li> |{' '}
+  </Ul>
+)
+
+@addSubscriptions({
+  jobs: subscribeBackupNgJobs,
+  schedulesByJob: cb =>
+    subscribeSchedules(schedules => {
+      cb(groupBy(schedules, 'jobId'))
+    }),
+})
+class JobsTable extends React.Component {
+  static contextTypes = {
+    router: React.PropTypes.object,
+  }
+
+  static tableProps = {
+    actions: [
+      {
+        handler: deleteBackupNgJobs,
+        label: _('deleteBackupSchedule'),
+        icon: 'delete',
+        level: 'danger',
+      },
+    ],
+    columns: [
+      {
+        itemRenderer: _ => _.id.slice(0, 5),
+        sortCriteria: _ => _.id,
+        name: _('jobId'),
+      },
+      {
+        itemRenderer: _ => _.name,
+        sortCriteria: _ => _.name,
+        name: _('jobName'),
+      },
+      {
+        itemRenderer: _ => _.mode,
+        sortCriteria: _ => _.mode,
+        name: 'mode',
+      },
+      {
+        component: _ => (
+          <SchedulePreviewBody
+            job={_.item}
+            schedules={_.userData.schedulesByJob[_.item.id]}
+          />
+        ),
+        name: <SchedulePreviewHeader />,
+      },
+    ],
+    individualActions: [
+      {
+        handler: (job, { goTo }) =>
+          goTo({
+            pathname: '/home',
+            query: { t: 'VM', s: constructQueryString(job.vms) },
+          }),
+        label: _('redirectToMatchingVms'),
+        icon: 'preview',
+      },
+      {
+        handler: (job, { goTo }) => goTo(`/backup-ng/${job.id}/edit`),
+        label: '',
+        icon: 'edit',
+        level: 'primary',
+      },
+    ],
+  }
+
+  _goTo = path => {
+    this.context.router.push(path)
+  }
+
+  render () {
+    return (
+      <SortedTable
+        {...JobsTable.tableProps}
+        collection={this.props.jobs}
+        data-goTo={this._goTo}
+        data-schedulesByJob={this.props.schedulesByJob}
+      />
+    )
+  }
+}
+
+const Overview = () => (
+  <div>
+    <Card>
+      <CardHeader>
+        <Icon icon='schedule' /> {_('backupSchedules')}
+      </CardHeader>
+      <CardBlock>
+        <JobsTable />
+      </CardBlock>
+    </Card>
+    <LogsTable />
+  </div>
+)
+
+const HEADER = (
+  <Container>
+    <Row>
+      <Col mediumSize={3}>
+        <h2>
+          <Icon icon='backup' /> {_('backupPage')}
+        </h2>
+      </Col>
+      <Col mediumSize={9}>
+        <NavTabs className='pull-right'>
+          <NavLink exact to='/backup-ng'>
+            <Icon icon='menu-backup-overview' /> {_('backupOverviewPage')}
+          </NavLink>
+          <NavLink to='/backup-ng/new'>
+            <Icon icon='menu-backup-new' /> {_('backupNewPage')}
+          </NavLink>
+          <NavLink to='/backup-ng/restore'>
+            <Icon icon='menu-backup-restore' /> {_('backupRestorePage')}
+          </NavLink>
+          <NavLink to='/backup-ng/file-restore'>
+            <Icon icon='menu-backup-file-restore' />{' '}
+            {_('backupFileRestorePage')}
+          </NavLink>
+        </NavTabs>
+      </Col>
+    </Row>
+  </Container>
+)
+
+export default routes(Overview, {
+  ':id/edit': Edit,
+  new: New,
+  restore: Restore,
+  'file-restore': FileRestore,
+})(({ children }) => (
+  <Page header={HEADER} title='backupPage' formatTitle>
+    {children}
+  </Page>
+))

packages/xo-web/src/xo-app/backup-ng/new.js

@@ -0,0 +1,269 @@
+import _, { messages } from 'intl'
+import ActionButton from 'action-button'
+import moment from 'moment-timezone'
+import React from 'react'
+import Scheduler, { SchedulePreview } from 'scheduling'
+import Upgrade from 'xoa-upgrade'
+import { injectState, provideState } from '@julien-f/freactal'
+import { cloneDeep, orderBy, size, isEmpty, map } from 'lodash'
+import { SelectRemote, SelectSr, SelectVm } from 'select-objects'
+import { resolveIds } from 'utils'
+import { createBackupNgJob, editBackupNgJob, editSchedule } from 'xo'
+
+const FormGroup = props => <div {...props} className='form-group' />
+const Input = props => <input {...props} className='form-control' />
+
+const DEFAULT_CRON_PATTERN = '0 0 * * *'
+const DEFAULT_TIMEZONE = moment.tz.guess()
+
+const constructPattern = values => ({
+  id: {
+    __or: resolveIds(values),
+  },
+})
+
+const removeScheduleFromSettings = tmpSchedules => {
+  const newTmpSchedules = cloneDeep(tmpSchedules)
+
+  for (let schedule in newTmpSchedules) {
+    delete newTmpSchedules[schedule].cron
+    delete newTmpSchedules[schedule].timezone
+  }
+
+  return newTmpSchedules
+}
+
+const getRandomId = () =>
+  Math.random()
+    .toString(36)
+    .slice(2)
+
+export default [
+  New => props => (
+    <Upgrade place='newBackup' required={2}>
+      <New {...props} />
+    </Upgrade>
+  ),
+  provideState({
+    initialState: () => ({
+      delta: false,
+      formId: getRandomId(),
+      tmpSchedule: {
+        cron: DEFAULT_CRON_PATTERN,
+        timezone: DEFAULT_TIMEZONE,
+      },
+      exportRetention: 0,
+      snapshotRetention: 0,
+      name: '',
+      remotes: [],
+      schedules: {},
+      srs: [],
+      vms: [],
+      tmpSchedules: {},
+    }),
+    effects: {
+      addSchedule: () => state => {
+        const id = getRandomId()
+
+        return {
+          ...state,
+          tmpSchedules: {
+            ...state.tmpSchedules,
+            [id]: {
+              ...state.tmpSchedule,
+              exportRetention: state.exportRetention,
+              snapshotRetention: state.snapshotRetention,
+            },
+          },
+        }
+      },
+      createJob: () => async state => {
+        await createBackupNgJob({
+          name: state.name,
+          mode: state.delta ? 'delta' : 'full',
+          remotes: constructPattern(state.remotes),
+          schedules: state.tmpSchedules,
+          settings: {
+            ...removeScheduleFromSettings(state.tmpSchedules),
+            ...state.schedules,
+          },
+          srs: constructPattern(state.srs),
+          vms: constructPattern(state.vms),
+        })
+      },
+      setTmpSchedule: (_, schedule) => state => ({
+        ...state,
+        tmpSchedule: {
+          cron: schedule.cronPattern,
+          timezone: schedule.timezone,
+        },
+      }),
+      setExportRetention: (_, { target: { value } }) => state => ({
+        ...state,
+        exportRetention: value,
+      }),
+      setSnapshotRetention: (_, { target: { value } }) => state => ({
+        ...state,
+        snapshotRetention: value,
+      }),
+      editSchedule: (
+        _,
+        { target: { dataset: { scheduleId } } }
+      ) => state => ({}),
+      editTmpSchedule: (_, { scheduleId }) => state => ({
+        ...state,
+        tmpSchedules: {
+          ...state.tmpSchedules,
+          [scheduleId]: {
+            ...state.tmpSchedule,
+            exportRetention: state.exportRetention,
+            snapshotRetention: state.snapshotRetention,
+          },
+        },
+      }),
+      setDelta: (_, { target: { value } }) => state => ({
+        ...state,
+        delta: value,
+      }),
+      setName: (_, { target: { value } }) => state => ({
+        ...state,
+        name: value,
+      }),
+      setRemotes: (_, remotes) => state => ({ ...state, remotes }),
+      setSrs: (_, srs) => state => ({ ...state, srs }),
+      setVms: (_, vms) => state => ({ ...state, vms }),
+    },
+    computed: {
+      isInvalid: state =>
+        state.name.trim() === '' ||
+        (isEmpty(state.schedules) && isEmpty(state.tmpSchedules)),
+      sortedSchedules: ({ schedules }) => orderBy(schedules, 'name'),
+      // TO DO: use sortedTmpSchedules
+      sortedTmpSchedules: ({ tmpSchedules }) => orderBy(tmpSchedules, 'id'),
+    },
+  }),
+  injectState,
+  ({ effects, state }) => (
+    <form id={state.formId}>
+      <FormGroup>
+        <h1>BackupNG</h1>
+        <label>
+          <strong>Name</strong>
+        </label>
+        <Input onChange={effects.setName} value={state.name} />
+      </FormGroup>
+      <FormGroup>
+        <label>Target remotes (for Export)</label>
+        <SelectRemote
+          multi
+          onChange={effects.setRemotes}
+          value={state.remotes}
+        />
+      </FormGroup>
+      <FormGroup>
+        <label>
+          Target SRs (for Replication)
+          <SelectSr multi onChange={effects.setSrs} value={state.srs} />
+        </label>
+      </FormGroup>
+      <FormGroup>
+        <label>
+          Vms to Backup
+          <SelectVm multi onChange={effects.setVms} value={state.vms} />
+        </label>
+      </FormGroup>
+      {false /* TODO: remove when implemented */ && (!isEmpty(state.srs) || !isEmpty(state.remotes)) && (
+        <Upgrade place='newBackup' required={4}>
+          <FormGroup>
+            <label>
+              <input
+                type='checkbox'
+                onChange={effects.setDelta}
+                value={state.delta}
+              />{' '}
+              Use delta
+            </label>
+          </FormGroup>
+        </Upgrade>
+      )}
+      {!isEmpty(state.schedules) && (
+        <FormGroup>
+          <h3>Saved schedules</h3>
+          <ul>
+            {state.sortedSchedules.map(schedule => (
+              <li key={schedule.id}>
+                {schedule.name} {schedule.cron} {schedule.timezone}
+                <ActionButton
+                  data-scheduleId={schedule.id}
+                  handler={effects.editSchedule}
+                  icon='edit'
+                  size='small'
+                />
+              </li>
+            ))}
+          </ul>
+        </FormGroup>
+      )}
+      {!isEmpty(state.tmpSchedules) && (
+        <FormGroup>
+          <h3>New schedules</h3>
+          <ul>
+            {map(state.tmpSchedules, (schedule, key) => (
+              <li key={key}>
+                {schedule.cron} {schedule.timezone} {schedule.exportRetention}{' '}
+                {schedule.snapshotRetention}
+                <ActionButton
+                  data-scheduleId={key}
+                  handler={effects.editTmpSchedule}
+                  icon='edit'
+                  size='small'
+                />
+              </li>
+            ))}
+          </ul>
+        </FormGroup>
+      )}
+      <FormGroup>
+        <h1>BackupNG</h1>
+        <label>
+          <strong>Export retention</strong>
+        </label>
+        <Input
+          type='number'
+          onChange={effects.setExportRetention}
+          value={state.exportRetention}
+        />
+        <label>
+          <strong>Snapshot retention</strong>
+        </label>
+        <Input
+          type='number'
+          onChange={effects.setSnapshotRetention}
+          value={state.snapshotRetention}
+        />
+        <Scheduler
+          cronPattern={state.tmpSchedule.cron}
+          onChange={effects.setTmpSchedule}
+          timezone={state.tmpSchedule.timezone}
+        />
+        <SchedulePreview
+          cronPattern={state.tmpSchedule.cron}
+          timezone={state.tmpSchedule.timezone}
+        />
+        <br />
+        <ActionButton handler={effects.addSchedule} icon='add'>
+          Add a schedule
+        </ActionButton>
+      </FormGroup>
+      <ActionButton
+        disabled={state.isInvalid}
+        form={state.formId}
+        handler={effects.createJob}
+        redirectOnSuccess='/backup-ng'
+        icon='save'
+      >
+        Create
+      </ActionButton>
+    </form>
+  ),
+].reduceRight((value, decorator) => decorator(value))

packages/xo-web/src/xo-app/backup-ng/restore/import-modal-body.js

@@ -0,0 +1,42 @@
+import _ from 'intl'
+import React from 'react'
+import Component from 'base-component'
+import { Select } from 'form'
+import { SelectSr } from 'select-objects'
+import { FormattedDate } from 'react-intl'
+
+export default class ImportModalBody extends Component {
+  get value () {
+    return this.state
+  }
+  render () {
+    return (
+      <div>
+        <div className='mb-1'>
+          <Select
+            optionRenderer={backup => (
+              <span>
+                {`[${backup.mode}] `}
+                <FormattedDate
+                  value={new Date(backup.timestamp)}
+                  month='long'
+                  day='numeric'
+                  year='numeric'
+                  hour='2-digit'
+                  minute='2-digit'
+                  second='2-digit'
+                />
+              </span>
+            )}
+            options={this.props.data.backups}
+            onChange={this.linkState('backup')}
+            placeholder={_('importBackupModalSelectBackup')}
+          />
+        </div>
+        <div>
+          <SelectSr onChange={this.linkState('sr')} />
+        </div>
+      </div>
+    )
+  }
+}

packages/xo-web/src/xo-app/backup-ng/restore/index.js

@@ -0,0 +1,134 @@
+import _ from 'intl'
+import ActionButton from 'action-button'
+import Component from 'base-component'
+import Icon from 'icon'
+import React from 'react'
+import SortedTable from 'sorted-table'
+import Upgrade from 'xoa-upgrade'
+import { addSubscriptions, noop } from 'utils'
+import { subscribeRemotes, listVmBackups, restoreBackup } from 'xo'
+import { assign, filter, forEach, map } from 'lodash'
+import { confirm } from 'modal'
+import { error } from 'notification'
+import { FormattedDate } from 'react-intl'
+
+import ImportModalBody from './import-modal-body'
+
+const BACKUPS_COLUMNS = [
+  {
+    name: 'VM',
+    itemRenderer: ({ last }) => last.vm.name_label,
+  },
+  {
+    name: 'VM description',
+    itemRenderer: ({ last }) => last.vm.name_description,
+  },
+  {
+    name: 'Last backup',
+    itemRenderer: ({ last }) => (
+      <FormattedDate
+        value={new Date(last.timestamp)}
+        month='long'
+        day='numeric'
+        year='numeric'
+        hour='2-digit'
+        minute='2-digit'
+        second='2-digit'
+      />
+    ),
+  },
+  {
+    name: 'Available backups',
+    itemRenderer: ({ count }) =>
+      map(count, (n, mode) => `${mode}: ${n}`).join(', '),
+  },
+]
+
+@addSubscriptions({
+  remotes: subscribeRemotes,
+})
+export default class Restore extends Component {
+  state = {
+    backupsByVm: {},
+  }
+
+  componentWillReceiveProps (props) {
+    if (props.remotes !== this.props.remotes) {
+      this._refreshBackupList(props.remotes)
+    }
+  }
+
+  _refreshBackupList = async remotes => {
+    const backupsByRemote = await listVmBackups(
+      filter(remotes, { enabled: true })
+    )
+    const backupsByVm = {}
+    forEach(backupsByRemote, backups => {
+      forEach(backups, (vmBackups, vmId) => {
+        if (backupsByVm[vmId] === undefined) {
+          backupsByVm[vmId] = { backups: [] }
+        }
+
+        backupsByVm[vmId].backups.push(...vmBackups)
+      })
+    })
+    // TODO: perf
+    let last
+    forEach(backupsByVm, (vmBackups, vmId) => {
+      last = { timestamp: 0 }
+      const count = {}
+      forEach(vmBackups.backups, backup => {
+        if (backup.timestamp > last.timestamp) {
+          last = backup
+        }
+        count[backup.mode] = (count[backup.mode] || 0) + 1
+      })
+
+      assign(vmBackups, { last, count })
+    })
+    this.setState({ backupsByVm })
+  }
+
+  _restore = data =>
+    confirm({
+      title: `Restore ${data.last.vm.name_label}`,
+      body: <ImportModalBody data={data} />,
+    }).then(({ backup, sr }) => {
+      if (backup == null || sr == null) {
+        error(_('backupRestoreErrorTitle'), _('backupRestoreErrorMessage'))
+        return
+      }
+
+      return restoreBackup(backup, sr)
+    }, noop)
+
+  render () {
+    return (
+      <Upgrade place='restoreBackup' available={2}>
+        <div>
+          <h2>{_('restoreBackups')}</h2>
+          <div className='mb-1'>
+            <em>
+              <Icon icon='info' /> {_('restoreBackupsInfo')}
+            </em>
+          </div>
+          <div className='mb-1'>
+            <ActionButton
+              btnStyle='primary'
+              handler={this._refreshBackupList}
+              handlerParam={this.props.remotes}
+              icon='refresh'
+            >
+              Refresh backup list
+            </ActionButton>
+          </div>
+          <SortedTable
+            collection={this.state.backupsByVm}
+            columns={BACKUPS_COLUMNS}
+            rowAction={this._restore}
+          />
+        </div>
+      </Upgrade>
+    )
+  }
+}

packages/xo-web/src/xo-app/index.js

@@ -22,6 +22,7 @@ import { Container, Row, Col } from 'grid'
 
 import About from './about'
 import Backup from './backup'
+import BackupNg from './backup-ng'
 import Dashboard from './dashboard'
 import Home from './home'
 import Host from './host'
@@ -74,6 +75,7 @@ const BODY_STYLE = {
 @routes('home', {
   about: About,
   backup: Backup,
+  'backup-ng': BackupNg,
   dashboard: Dashboard,
   home: Home,
   'hosts/:id': Host,

packages/xo-web/src/xo-app/logs/index.js

@@ -14,10 +14,15 @@ import { alert } from 'modal'
 import { Card, CardHeader, CardBlock } from 'card'
 import { connectStore, formatSize, formatSpeed } from 'utils'
 import { createFilter, createGetObject, createSelector } from 'selectors'
-import { deleteJobsLogs, subscribeJobs, subscribeJobsLogs } from 'xo'
 import { forEach, includes, keyBy, map, orderBy } from 'lodash'
 import { FormattedDate } from 'react-intl'
 import { get } from 'xo-defined'
+import {
+  deleteJobsLogs,
+  subscribeJobs,
+  subscribeJobsLogs,
+  subscribeBackupNgJobs,
+} from 'xo'
 
 // ===================================================================
 
@@ -83,7 +88,7 @@ const JobDataInfos = ({
   mergeSize,
 }) => (
   <div>
-    {transferSize !== undefined && (
+    {transferSize && transferDuration ? (
       <div>
         <strong>{_('jobTransferredDataSize')}</strong>{' '}
         {formatSize(transferSize)}
@@ -91,15 +96,15 @@ const JobDataInfos = ({
         <strong>{_('jobTransferredDataSpeed')}</strong>{' '}
         {formatSpeed(transferSize, transferDuration)}
       </div>
-    )}
+    ) : null}
-    {mergeSize !== undefined && (
+    {mergeSize && mergeDuration ? (
       <div>
         <strong>{_('jobMergedDataSize')}</strong> {formatSize(mergeSize)}
         <br />
         <strong>{_('jobMergedDataSpeed')}</strong>{' '}
         {formatSpeed(mergeSize, mergeDuration)}
       </div>
-    )}
+    ) : null}
   </div>
 )
 
@@ -382,7 +387,7 @@ const LOG_FILTERS = {
 
 export default [
   propTypes({
-    jobKeys: propTypes.array.isRequired,
+    jobKeys: propTypes.array,
   }),
   addSubscriptions(({ jobKeys }) => ({
     logs: cb =>
@@ -391,7 +396,10 @@ export default [
         forEach(rawLogs, (log, id) => {
           const data = log.data
           const { time } = log
-          if (data.event === 'job.start' && includes(jobKeys, data.key)) {
+          if (
+            data.event === 'job.start' &&
+            (jobKeys === undefined || includes(jobKeys, data.key))
+          ) {
             logs[id] = {
               id,
               jobId: data.jobId,
@@ -450,8 +458,9 @@ export default [
         cb(orderBy(logs, ['time'], ['desc']))
       }),
     jobs: cb => subscribeJobs(jobs => cb(keyBy(jobs, 'id'))),
+    ngJobs: cb => subscribeBackupNgJobs(jobs => cb(keyBy(jobs, 'id'))),
   })),
-  ({ logs, jobs }) => (
+  ({ logs, jobs, ngJobs }) => (
     <Card>
       <CardHeader>
         <Icon icon='log' /> Logs
@@ -462,7 +471,7 @@ export default [
           collection={logs}
           columns={LOG_COLUMNS}
           component={SortedTable}
-          data-jobs={jobs}
+          data-jobs={{ ...jobs, ...ngJobs }}
           emptyMessage={_('noLogs')}
           filters={LOG_FILTERS}
           individualActions={LOG_ACTIONS_INDIVIDUAL}

packages/xo-web/src/xo-app/menu/index.js

@@ -202,6 +202,11 @@ export default class Menu extends Component {
           },
         ],
       },
+      isAdmin && {
+        to: '/backup-ng',
+        icon: 'menu-backup',
+        label: ['Backup NG'],
+      },
       isAdmin && {
         to: 'xoa/update',
         icon: 'menu-xoa',
@@ -460,7 +465,10 @@ const MenuLinkItem = props => {
           size='lg'
           fixedWidth
         />
-        <span className={styles.hiddenCollapsed}> {_(label)}&nbsp;</span>
+        <span className={styles.hiddenCollapsed}>
+          {' '}
+          {typeof label === 'string' ? _(label) : label}&nbsp;
+        </span>
         {pill > 0 && <span className='tag tag-pill tag-primary'>{pill}</span>}
         {extra}
       </Link>

yarn.lock

@@ -567,6 +567,10 @@
     normalize-path "^2.0.1"
     through2 "^2.0.3"
 
+"@julien-f/freactal@0.1.0":
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/@julien-f/freactal/-/freactal-0.1.0.tgz#c3c97c1574ed82de6989f7f3c6110f34b0da3866"
+
 "@marsaud/smb2-promise@^0.2.0", "@marsaud/smb2-promise@^0.2.1":
   version "0.2.1"
   resolved "https://registry.yarnpkg.com/@marsaud/smb2-promise/-/smb2-promise-0.2.1.tgz#fee95f4baba6e4d930e8460d3377aa12560e0f0e"