diff --git a/packages/xo-server/src/api/backup-ng.js b/packages/xo-server/src/api/backup-ng.js new file mode 100644 index 000000000..430ffb826 --- /dev/null +++ b/packages/xo-server/src/api/backup-ng.js @@ -0,0 +1,155 @@ +export function createJob ({ schedules, ...job }) { + job.userId = this.user.id + return this.createBackupNgJob(job, schedules) +} + +createJob.permission = 'admin' +createJob.params = { + compression: { + enum: ['', 'native'], + optional: true, + }, + mode: { + enum: ['full', 'delta'], + }, + name: { + type: 'string', + optional: true, + }, + remotes: { + type: 'object', + optional: true, + }, + schedules: { + type: 'object', + optional: true, + }, + settings: { + type: 'object', + }, + vms: { + type: 'object', + }, +} + +export function deleteJob ({ id }) { + return this.deleteBackupNgJob(id) +} +deleteJob.permission = 'admin' +deleteJob.params = { + id: { + type: 'string', + }, +} + +export function editJob (props) { + return this.updateJob(props) +} + +editJob.permission = 'admin' +editJob.params = { + compression: { + enum: ['', 'native'], + optional: true, + }, + id: { + type: 'string', + }, + mode: { + enum: ['full', 'delta'], + optional: true, + }, + name: { + type: 'string', + optional: true, + }, + remotes: { + type: 'object', + optional: true, + }, + settings: { + type: 'object', + optional: true, + }, + vms: { + type: 'object', + optional: true, + }, +} + +export function getAllJobs () { + return this.getAllBackupNgJobs() +} + +getAllJobs.permission = 'admin' + +export function getJob ({ id }) { + return this.getBackupNgJob(id) +} + +getJob.permission = 'admin' + +getJob.params = { + id: { + type: 'string', + }, +} + +export async function runJob ({ id, scheduleId }) { + return this.runJobSequence([id], await this.getSchedule(scheduleId)) +} + +runJob.permission = 'admin' + +runJob.params = { + id: { + type: 'string', + }, + scheduleId: { + type: 'string', + }, +} + +// ----------------------------------------------------------------------------- + +export function deleteVmBackup ({ id }) { + return this.deleteVmBackupNg(id) +} + +deleteVmBackup.permission = 'admin' + +deleteVmBackup.params = { + id: { + type: 'string', + }, +} + +export function listVmBackups ({ remotes }) { + return this.listVmBackupsNg(remotes) +} + +listVmBackups.permission = 'admin' + +listVmBackups.params = { + remotes: { + type: 'array', + items: { + type: 'string', + }, + }, +} + +export function importVmBackupNg ({ id, sr }) { + return this.importVmBackupNg(id, sr) +} + +importVmBackupNg.permission = 'admin' + +importVmBackupNg.params = { + id: { + type: 'string', + }, + sr: { + type: 'string', + }, +} diff --git a/packages/xo-server/src/xo-mixins/backups-ng/index.js b/packages/xo-server/src/xo-mixins/backups-ng/index.js new file mode 100644 index 000000000..7aa5875d7 --- /dev/null +++ b/packages/xo-server/src/xo-mixins/backups-ng/index.js @@ -0,0 +1,772 @@ +// @flow + +// $FlowFixMe +import defer from 'golike-defer' +import { dirname, resolve } from 'path' +// $FlowFixMe +import { fromEvent, timeout as pTimeout } from 'promise-toolbox' +// $FlowFixMe +import { isEmpty, last, mapValues, values } from 'lodash' +import { type Pattern, createPredicate } from 'value-matcher' +import { PassThrough } from 'stream' + +import { type Executor, type Job } from '../jobs' +import { type Schedule } from '../scheduling' + +import createSizeStream from '../../size-stream' +import { asyncMap, safeDateFormat, serializeError } from '../../utils' +// import { parseDateTime } from '../../xapi/utils' +import { type RemoteHandlerAbstract } from '../../remote-handlers/abstract' +import { type Xapi } from '../../xapi' + +type Dict = { [K]: T } + +type Mode = 'full' | 'delta' + +type Settings = {| + deleteFirst?: boolean, + exportRetention?: number, + snapshotRetention?: number, + vmTimeout?: number +|} + +type SimpleIdPattern = {| + id: string | {| __or: string[] |} +|} + +export type BackupJob = {| + ...$Exact, + compression?: 'native', + mode: Mode, + remotes?: SimpleIdPattern, + settings: Dict, + srs?: SimpleIdPattern, + type: 'backup', + vms: Pattern +|} + +type BackupResult = {| + mergeDuration: number, + mergeSize: number, + transferDuration: number, + transferSize: number +|} + +type MetadataBase = {| + jobId: string, + mode: Mode, + scheduleId: string, + timestamp: number, + version: '2.0.0', + vm: Object, + vmSnapshot: Object +|} +type MetadataDelta = {| ...MetadataBase, mode: 'delta' |} +type MetadataFull = {| + ...MetadataBase, + data: string, // relative path to the XVA + mode: 'full' +|} +type Metadata = MetadataDelta | MetadataFull + +const compareSnapshotTime = ( + { snapshot_time: time1 }, + { snapshot_time: time2 } +) => (time1 < time2 ? -1 : 1) + +const compareTimestamp = ({ timestamp: time1 }, { timestamp: time2 }) => + time1 - time2 + +// returns all entries but the last (retention - 1)-th +// +// the “-1” is because this code is usually run with entries computed before the +// new entry is created +// +// FIXME: check whether it take the new one into account +const getOldEntries = (retention: number, entries?: T[]): T[] => + entries === undefined + ? [] + : --retention > 0 ? entries.slice(0, -retention) : entries + +const defaultSettings: Settings = { + deleteFirst: false, + exportRetention: 0, + snapshotRetention: 0, + vmTimeout: 0, +} +const getSetting = ( + settings: Dict, + name: $Keys, + ...keys: string[] +): any => { + for (let i = 0, n = keys.length; i < n; ++i) { + const objectSettings = settings[keys[i]] + if (objectSettings !== undefined) { + const setting = objectSettings[name] + if (setting !== undefined) { + return setting + } + } + } + return defaultSettings[name] +} + +const BACKUP_DIR = 'xo-vm-backups' +const getVmBackupDir = (uuid: string) => `${BACKUP_DIR}/${uuid}` + +const isMetadataFile = (filename: string) => filename.endsWith('.json') + +const listReplicatedVms = (xapi: Xapi, scheduleId: string, srId) => { + const { all } = xapi.objects + const vms = {} + for (const key in all) { + const object = all[key] + if ( + object.$type === 'vm' && + object.other_config['xo:backup:schedule'] === scheduleId && + object.other_config['xo:backup:sr'] === srId + ) { + vms[object.$id] = object + } + } + + // the replicated VMs have been created from a snapshot, therefore we can use + // `snapshot_time` as the creation time + return values(vms).sort(compareSnapshotTime) +} + +const parseVmBackupId = id => { + const i = id.indexOf('/') + return { + metadataFilename: id.slice(i + 1), + remoteId: id.slice(0, i), + } +} + +// used to resolve the data field from the metadata +const resolveRelativeFromFile = (file, path) => + resolve('/', dirname(file), path).slice(1) + +const unboxIds = (pattern?: SimpleIdPattern): string[] => { + if (pattern === undefined) { + return [] + } + const { id } = pattern + return typeof id === 'string' ? [id] : id.__or +} + +// File structure on remotes: +// +// +// └─ xo-vm-backups +// ├─ index.json // TODO +// └─ +// ├─ index.json // TODO +// ├─ vdis +// │ └─ +// │ ├─ index.json // TODO +// │ └─ T.vhd +// ├─ T.json // backup metadata +// └─ T.xva +export default class BackupNg { + _app: any + + constructor (app: any) { + this._app = app + + app.on('start', () => { + const executor: Executor = async ({ + cancelToken, + job: job_, + logger, + runJobId, + schedule = {}, + }) => { + const job: BackupJob = (job_: any) + const vms = app.getObjects({ + filter: createPredicate({ + type: 'VM', + ...job.vms, + }), + }) + if (isEmpty(vms)) { + throw new Error('no VMs match this pattern') + } + const jobId = job.id + const scheduleId = schedule.id + const status: Object = { + calls: {}, + runJobId, + start: Date.now(), + timezone: schedule.timezone, + } + const { calls } = status + await asyncMap(vms, async vm => { + const { uuid } = vm + const method = 'backup-ng' + const params = { + id: uuid, + tag: job.name, + } + + const name = vm.name_label + const runCallId = logger.notice( + `Starting backup of ${name}. (${jobId})`, + { + event: 'jobCall.start', + method, + params, + runJobId, + } + ) + const call: Object = (calls[runCallId] = { + method, + params, + start: Date.now(), + }) + const vmCancel = cancelToken.fork() + try { + // $FlowFixMe injected $defer param + let p = this._backupVm(vmCancel.token, uuid, job, schedule) + const vmTimeout: number = getSetting( + job.settings, + 'vmTimeout', + uuid, + scheduleId + ) + if (vmTimeout !== 0) { + p = pTimeout.call(p, vmTimeout) + } + const returnedValue = await p + logger.notice( + `Backuping ${name} (${runCallId}) is a success. (${jobId})`, + { + event: 'jobCall.end', + runJobId, + runCallId, + returnedValue, + } + ) + + call.returnedValue = returnedValue + call.end = Date.now() + } catch (error) { + vmCancel.cancel() + logger.notice( + `Backuping ${name} (${runCallId}) has failed. (${jobId})`, + { + event: 'jobCall.end', + runJobId, + runCallId, + error: serializeError(error), + } + ) + + call.error = error + call.end = Date.now() + + console.warn(error.stack) // TODO: remove + } + }) + status.end = Date.now() + return status + } + app.registerJobExecutor('backup', executor) + }) + } + + async createBackupNgJob ( + props: $Diff, + schedules?: Dict<$Diff> + ): Promise { + const app = this._app + props.type = 'backup' + const job: BackupJob = await app.createJob(props) + + if (schedules !== undefined) { + const { id, settings } = job + const tmpIds = Object.keys(schedules) + await asyncMap(tmpIds, async (tmpId: string) => { + // $FlowFixMe don't know what is the problem (JFT) + const schedule = schedules[tmpId] + schedule.jobId = id + settings[(await app.createSchedule(schedule)).id] = settings[tmpId] + delete settings[tmpId] + }) + await app.updateJob({ id, settings }) + } + + return job + } + + async deleteBackupNgJob (id: string): Promise { + const app = this._app + const [schedules] = await Promise.all([ + app.getAllSchedules(), + app.getJob(id, 'backup'), + ]) + await Promise.all([ + app.removeJob(id), + asyncMap(schedules, schedule => { + if (schedule.id === id) { + app.removeSchedule(schedule.id) + } + }), + ]) + } + + async deleteVmBackupNg (id: string): Promise { + const app = this._app + const { metadataFilename, remoteId } = parseVmBackupId(id) + const handler = await app.getRemoteHandler(remoteId) + const metadata: Metadata = JSON.parse( + await handler.readFile(metadataFilename) + ) + + if (metadata.mode === 'delta') { + throw new Error('not implemented') + } + + metadata._filename = metadataFilename + await this._deleteFullVmBackups(handler, [metadata]) + } + + getAllBackupNgJobs (): Promise { + return this._app.getAllJobs('backup') + } + + getBackupNgJob (id: string): Promise { + return this._app.getJob(id, 'backup') + } + + async importVmBackupNg (id: string, srId: string): Promise { + const app = this._app + const { metadataFilename, remoteId } = parseVmBackupId(id) + const handler = await app.getRemoteHandler(remoteId) + const metadata: Metadata = JSON.parse( + await handler.readFile(metadataFilename) + ) + + if (metadata.mode === 'delta') { + throw new Error('not implemented') + } + + const xapi = app.getXapi(srId) + const sr = xapi.getObject(srId) + const xva = await handler.createReadStream( + resolveRelativeFromFile(metadataFilename, metadata.data) + ) + const vm = await xapi.importVm(xva, { srId: sr.$id }) + await Promise.all([ + xapi.addTag(vm.$id, 'restored from backup'), + xapi.editVm(vm.$id, { + name_label: `${metadata.vm.name_label} (${safeDateFormat( + metadata.timestamp + )})`, + }), + ]) + return vm.$id + } + + async listVmBackupsNg (remotes: string[]) { + const backupsByVmByRemote: Dict> = {} + + const app = this._app + await Promise.all( + remotes.map(async remoteId => { + const handler = await app.getRemoteHandler(remoteId) + + const entries = (await handler.list(BACKUP_DIR).catch(error => { + if (error == null || error.code !== 'ENOENT') { + throw error + } + return [] + })).filter(name => name !== 'index.json') + + const backupsByVm = (backupsByVmByRemote[remoteId] = {}) + await Promise.all( + entries.map(async vmId => { + // $FlowFixMe don't know what is the problem (JFT) + const backups = await this._listVmBackups(handler, vmId) + + if (backups.length === 0) { + return + } + + // inject an id usable by importVmBackupNg() + backups.forEach(backup => { + backup.id = `${remoteId}/${backup._filename}` + }) + + backupsByVm[vmId] = backups + }) + ) + }) + ) + + return backupsByVmByRemote + } + + // - [x] files (.tmp) should be renamed at the end of job + // - [ ] validate VHDs after exports and before imports + // - [ ] protect against concurrent backup against a single VM (JFT: why?) + // - [x] detect full remote + // - [x] can the snapshot and export retention be different? → Yes + // - [ ] snapshots and files of an old job should be detected and removed + // - [ ] adding and removing VDIs should behave + // - [ ] key export? + // - [x] deleteFirst per target + // - [ ] possibility to (re-)run a single VM in a backup? + // - [x] timeout per VM + // - [ ] display queued VMs + // - [ ] jobs should be cancelable + // - [ ] logs + // - [x] backups should be deletable from the API + // - [ ] check merge/transfert duration/size are what we want for delta + @defer + async _backupVm ( + $defer: any, + $cancelToken: any, + vmId: string, + job: BackupJob, + schedule: Schedule + ): Promise { + const app = this._app + const xapi = app.getXapi(vmId) + const vm = xapi.getObject(vmId) + + const { id: jobId, settings } = job + const { id: scheduleId } = schedule + + const exportRetention: number = getSetting( + settings, + 'exportRetention', + scheduleId + ) + const snapshotRetention: number = getSetting( + settings, + 'snapshotRetention', + scheduleId + ) + + let remotes, srs + if (exportRetention === 0) { + if (snapshotRetention === 0) { + throw new Error('export and snapshots retentions cannot both be 0') + } + } else { + remotes = unboxIds(job.remotes) + srs = unboxIds(job.srs) + if (remotes.length === 0 && srs.length === 0) { + throw new Error('export retention must be 0 without remotes and SRs') + } + } + + const snapshots = vm.$snapshots + .filter(_ => _.other_config['xo:backup:schedule'] === scheduleId) + .sort(compareSnapshotTime) + $defer(() => + asyncMap(getOldEntries(snapshotRetention, snapshots), _ => + xapi.deleteVm(_) + ) + ) + + let snapshot = await xapi._snapshotVm( + $cancelToken, + vm, + `[XO Backup] ${vm.name_label}` + ) + $defer.onFailure.call(xapi, '_deleteVm', snapshot) + await xapi._updateObjectMapProperty(snapshot, 'other_config', { + 'xo:backup:job': jobId, + 'xo:backup:schedule': scheduleId, + }) + snapshot = await xapi.barrier(snapshot.$ref) + + if (exportRetention === 0) { + return { + mergeDuration: 0, + mergeSize: 0, + transferDuration: 0, + transferSize: 0, + } + } + + const now = Date.now() + const { mode } = job + + const metadata: Metadata = { + jobId, + mode, + scheduleId, + timestamp: now, + version: '2.0.0', + vm, + vmSnapshot: snapshot, + } + + if (mode === 'full') { + // TODO: do not create the snapshot if there are no snapshotRetention and + // the VM is not running + if (snapshotRetention === 0) { + $defer.call(xapi, 'deleteVm', snapshot) + } + + let xva = await xapi.exportVm($cancelToken, snapshot) + const exportTask = xva.task + xva = xva.pipe(createSizeStream()) + + const dirname = getVmBackupDir(vm.uuid) + const basename = safeDateFormat(now) + + const dataBasename = `${basename}.xva` + const metadataFilename = `${dirname}/${basename}.json` + + metadata.data = `./${dataBasename}` + const dataFilename = `${dirname}/${dataBasename}` + const tmpFilename = `${dirname}/.${dataBasename}` + + const jsonMetadata = JSON.stringify(metadata) + + await Promise.all([ + asyncMap( + remotes, + defer(async ($defer, remoteId) => { + const fork = xva.pipe(new PassThrough()) + + const handler = await app.getRemoteHandler(remoteId) + + const oldBackups = getOldEntries( + exportRetention, + await this._listVmBackups( + handler, + vm, + _ => _.mode === 'full' && _.scheduleId === scheduleId + ) + ) + + const deleteFirst = getSetting(settings, 'deleteFirst', remoteId) + if (deleteFirst) { + await this._deleteFullVmBackups(handler, oldBackups) + } + + const output = await handler.createOutputStream(tmpFilename, { + checksum: true, + }) + $defer.onFailure.call(handler, 'unlink', tmpFilename) + $defer.onSuccess.call( + handler, + 'rename', + tmpFilename, + dataFilename, + { checksum: true } + ) + + const promise = fromEvent(output, 'finish') + fork.pipe(output) + await Promise.all([exportTask, promise]) + + await handler.outputFile(metadataFilename, jsonMetadata) + + if (!deleteFirst) { + await this._deleteFullVmBackups(handler, oldBackups) + } + }) + ), + asyncMap( + srs, + defer(async ($defer, srId) => { + const fork = xva.pipe(new PassThrough()) + fork.task = exportTask + + const xapi = app.getXapi(srId) + const sr = xapi.getObject(srId) + + const oldVms = getOldEntries( + exportRetention, + listReplicatedVms(xapi, scheduleId, srId) + ) + + const deleteFirst = getSetting(settings, 'deleteFirst', srId) + if (deleteFirst) { + await this._deleteVms(xapi, oldVms) + } + + const vm = await xapi.barrier( + await xapi._importVm($cancelToken, fork, sr, vm => + xapi._setObjectProperties(vm, { + nameLabel: `${metadata.vm.name_label} (${safeDateFormat( + metadata.timestamp + )})`, + }) + ) + ) + + await Promise.all([ + xapi.addTag(vm.$ref, 'Disaster Recovery'), + xapi._updateObjectMapProperty(vm, 'blocked_operations', { + start: + 'Start operation for this vm is blocked, clone it if you want to use it.', + }), + xapi._updateObjectMapProperty(vm, 'other_config', { + 'xo:backup:sr': srId, + }), + ]) + + if (!deleteFirst) { + await this._deleteVms(xapi, oldVms) + } + }) + ), + ]) + + return { + mergeDuration: 0, + mergeSize: 0, + transferDuration: Date.now() - now, + transferSize: xva.size, + } + } + + const baseSnapshot = last(snapshots) + if (baseSnapshot !== undefined) { + console.log(baseSnapshot.$id) // TODO: remove + // check current state + // await Promise.all([asyncMap(remotes, remoteId => {})]) + } + + const deltaExport = await xapi.exportDeltaVm( + $cancelToken, + snapshot, + baseSnapshot + ) + + // forks of the lazy streams + deltaExport.streams = mapValues(deltaExport.streams, lazyStream => { + let stream + return () => { + if (stream === undefined) { + stream = lazyStream() + } + return Promise.resolve(stream).then(stream => { + const fork = stream.pipe(new PassThrough()) + fork.task = stream.task + return fork + }) + } + }) + + const mergeStart = 0 + const mergeEnd = 0 + let transferStart = 0 + let transferEnd = 0 + await Promise.all([ + asyncMap(remotes, defer(async ($defer, remote) => {})), + asyncMap( + srs, + defer(async ($defer, srId) => { + const xapi = app.getXapi(srId) + const sr = xapi.getObject(srId) + + const oldVms = getOldEntries( + exportRetention, + listReplicatedVms(xapi, scheduleId, srId) + ) + + const deleteFirst = getSetting(settings, 'deleteFirst', srId) + if (deleteFirst) { + await this._deleteVms(xapi, oldVms) + } + + transferStart = + transferStart === 0 + ? Date.now() + : Math.min(transferStart, Date.now()) + + const { vm } = await xapi.importDeltaVm(deltaExport, { + disableStartAfterImport: false, // we'll take care of that + name_label: `${metadata.vm.name_label} (${safeDateFormat( + metadata.timestamp + )})`, + srId: sr.$id, + }) + + transferEnd = Math.max(transferEnd, Date.now()) + + await Promise.all([ + xapi.addTag(vm.$ref, 'Continuous Replication'), + xapi._updateObjectMapProperty(vm, 'blocked_operations', { + start: + 'Start operation for this vm is blocked, clone it if you want to use it.', + }), + xapi._updateObjectMapProperty(vm, 'other_config', { + 'xo:backup:sr': srId, + }), + ]) + + if (!deleteFirst) { + await this._deleteVms(xapi, oldVms) + } + }) + ), + ]) + + return { + mergeDuration: mergeEnd - mergeStart, + mergeSize: 0, + transferDuration: transferEnd - transferStart, + transferSize: 0, + } + } + + async _deleteFullVmBackups ( + handler: RemoteHandlerAbstract, + backups: Metadata[] + ): Promise { + await asyncMap(backups, ({ _filename, data }) => + Promise.all([ + handler.unlink(_filename), + handler.unlink(resolveRelativeFromFile(_filename, data)), + ]) + ) + } + + async _deleteVms (xapi: Xapi, vms: Object[]): Promise { + return asyncMap(vms, vm => xapi.deleteVm(vm)) + } + + async _listVmBackups ( + handler: RemoteHandlerAbstract, + vm: Object | string, + predicate?: Metadata => boolean + ): Promise { + const backups = [] + + const dir = getVmBackupDir(typeof vm === 'string' ? vm : vm.uuid) + try { + const files = await handler.list(dir) + await Promise.all( + files.filter(isMetadataFile).map(async file => { + const path = `${dir}/${file}` + try { + const metadata = JSON.parse(await handler.readFile(path)) + if (predicate === undefined || predicate(metadata)) { + Object.defineProperty(metadata, '_filename', { + value: path, + }) + backups.push(metadata) + } + } catch (error) { + console.warn('_listVmBackups', path, error) + } + }) + ) + } catch (error) { + if (error == null || error.code !== 'ENOENT') { + throw error + } + } + + return backups.sort(compareTimestamp) + } +} diff --git a/packages/xo-server/src/xo-mixins/backups-ng/migration.js b/packages/xo-server/src/xo-mixins/backups-ng/migration.js new file mode 100644 index 000000000..5014f48c5 --- /dev/null +++ b/packages/xo-server/src/xo-mixins/backups-ng/migration.js @@ -0,0 +1,153 @@ +// @flow + +import assert from 'assert' + +import { type BackupJob } from '../backups-ng' +import { type CallJob } from '../jobs' +import { type Schedule } from '../scheduling' + +const createOr = (children: Array): any => + children.length === 1 ? children[0] : { __or: children } + +const methods = { + 'vm.deltaCopy': ( + job: CallJob, + { retention = 1, sr, vms }, + schedule: Schedule + ) => ({ + mode: 'delta', + settings: { + [schedule.id]: { + exportRetention: retention, + vmTimeout: job.timeout, + }, + }, + srs: { id: sr }, + userId: job.userId, + vms, + }), + 'vm.rollingDeltaBackup': ( + job: CallJob, + { depth = 1, retention = depth, remote, vms }, + schedule: Schedule + ) => ({ + mode: 'delta', + remotes: { id: remote }, + settings: { + [schedule.id]: { + exportRetention: retention, + vmTimeout: job.timeout, + }, + }, + vms, + }), + 'vm.rollingDrCopy': ( + job: CallJob, + { deleteOldBackupsFirst, depth = 1, retention = depth, sr, vms }, + schedule: Schedule + ) => ({ + mode: 'full', + settings: { + [schedule.id]: { + deleteFirst: deleteOldBackupsFirst, + exportRetention: retention, + vmTimeout: job.timeout, + }, + }, + srs: { id: sr }, + vms, + }), + 'vm.rollingBackup': ( + job: CallJob, + { compress, depth = 1, retention = depth, remoteId, vms }, + schedule: Schedule + ) => ({ + compression: compress ? 'native' : undefined, + mode: 'full', + remotes: { id: remoteId }, + settings: { + [schedule.id]: { + exportRetention: retention, + vmTimeout: job.timeout, + }, + }, + vms, + }), + 'vm.rollingSnapshot': ( + job: CallJob, + { depth = 1, retention = depth, vms }, + schedule: Schedule + ) => ({ + mode: 'full', + settings: { + [schedule.id]: { + snapshotRetention: retention, + vmTimeout: job.timeout, + }, + }, + vms, + }), +} + +const parseParamsVector = vector => { + assert.strictEqual(vector.type, 'crossProduct') + const { items } = vector + assert.strictEqual(items.length, 2) + + let vms, params + if (items[1].type === 'map') { + ;[params, vms] = items + + vms = vms.collection + assert.strictEqual(vms.type, 'fetchObjects') + vms = vms.pattern + } else { + ;[vms, params] = items + + assert.strictEqual(vms.type, 'set') + vms = vms.values + if (vms.length !== 0) { + assert.deepStrictEqual(Object.keys(vms[0]), ['id']) + vms = { id: createOr(vms.map(_ => _.id)) } + } + } + + assert.strictEqual(params.type, 'set') + params = params.values + assert.strictEqual(params.length, 1) + params = params[0] + + return { ...params, vms } +} + +export const translateOldJobs = async (app: any): Promise> => { + const backupJobs: Array = [] + const [jobs, schedules] = await Promise.all([ + app.getAllJobs('call'), + app.getAllSchedules(), + ]) + jobs.forEach(job => { + try { + const { id } = job + let method, schedule + if ( + job.type === 'call' && + (method = methods[job.method]) !== undefined && + (schedule = schedules.find(_ => _.jobId === id)) !== undefined + ) { + const params = parseParamsVector(job.paramsVector) + backupJobs.push({ + id, + name: params.tag || job.name, + type: 'backup', + userId: job.userId, + // $FlowFixMe `method` is initialized but Flow fails to see this + ...method(job, params, schedule), + }) + } + } catch (error) { + console.warn('translateOldJobs', job, error) + } + }) + return backupJobs +} diff --git a/packages/xo-server/src/xo-mixins/jobs/index.js b/packages/xo-server/src/xo-mixins/jobs/index.js index 07c8ffc78..adb292f7c 100644 --- a/packages/xo-server/src/xo-mixins/jobs/index.js +++ b/packages/xo-server/src/xo-mixins/jobs/index.js @@ -142,6 +142,7 @@ export default class Jobs { } async getAllJobs (type: string = 'call'): Promise> { + // $FlowFixMe don't know what is the problem (JFT) const jobs = await this._jobs.get() const runningJobs = this._runningJobs const result = [] diff --git a/packages/xo-web/package.json b/packages/xo-web/package.json index ca9cb03a7..a86dbc6fa 100644 --- a/packages/xo-web/package.json +++ b/packages/xo-web/package.json @@ -30,6 +30,7 @@ "node": ">=6" }, "devDependencies": { + "@julien-f/freactal": "0.1.0", "@nraynaud/novnc": "0.6.1", "@xen-orchestra/cron": "^1.0.2", "ansi_up": "^2.0.2", diff --git a/packages/xo-web/src/common/intl/messages.js b/packages/xo-web/src/common/intl/messages.js index 794e64869..a2d21a630 100644 --- a/packages/xo-web/src/common/intl/messages.js +++ b/packages/xo-web/src/common/intl/messages.js @@ -60,6 +60,8 @@ const messages = { selfServicePage: 'Self service', backupPage: 'Backup', jobsPage: 'Jobs', + backupNG: 'Backups NG', + backupNGName: 'Name', xoaPage: 'XOA', updatePage: 'Updates', licensesPage: 'Licenses', @@ -354,6 +356,13 @@ const messages = { remoteTestSuccessMessage: 'The remote appears to work correctly', remoteConnectionFailed: 'Connection failed', + // ------ Backup job ----- + + confirmDeleteBackupJobsTitle: + 'Delete backup job{nJobs, plural, one {} other {s}}', + confirmDeleteBackupJobsBody: + 'Are you sure you want to delete {nJobs, number} backup job{nJobs, plural, one {} other {s}}?', + // ------ Remote ----- remoteName: 'Name', remotePath: 'Path', diff --git a/packages/xo-web/src/common/xo/index.js b/packages/xo-web/src/common/xo/index.js index fa8ad4649..c3d76a0c3 100644 --- a/packages/xo-web/src/common/xo/index.js +++ b/packages/xo-web/src/common/xo/index.js @@ -1602,6 +1602,52 @@ export const enableSchedule = id => editSchedule({ id, enabled: true }) export const getSchedule = id => _call('schedule.get', { id }) +// Backup NG --------------------------------------------------------- + +export const subscribeBackupNgJobs = createSubscription(() => + _call('backupNg.getAllJobs') +) + +export const createBackupNgJob = props => + _call('backupNg.createJob', props)::tap(subscribeBackupNgJobs.forceRefresh) + +export const deleteBackupNgJobs = async ids => { + const { length } = ids + if (length === 0) { + return + } + const vars = { nJobs: length } + try { + await confirm({ + title: _('confirmDeleteBackupJobsTitle', vars), + body:

{_('confirmDeleteBackupJobsBody', vars)}

, + }) + } catch (_) { + return + } + + return Promise.all( + ids.map(id => _call('backupNg.deleteJob', { id: resolveId(id) })) + )::tap(subscribeBackupNgJobs.forceRefresh) +} + +export const editBackupNgJob = props => + _call('backupNg.editJob', props)::tap(subscribeBackupNgJobs.forceRefresh) + +export const getBackupNgJob = id => _call('backupNg.getJob', { id }) + +export const runBackupNgJob = ({ id, scheduleId }) => + _call('backupNg.runJob', { id, scheduleId }) + +export const listVmBackups = remotes => + _call('backupNg.listVmBackups', { remotes: resolveIds(remotes) }) + +export const restoreBackup = (backup, sr) => + _call('backupNg.importVmBackupNg', { + id: resolveId(backup), + sr: resolveId(sr), + }) + // Plugins ----------------------------------------------------------- export const loadPlugin = async id => diff --git a/packages/xo-web/src/xo-app/backup-ng/edit.js b/packages/xo-web/src/xo-app/backup-ng/edit.js new file mode 100644 index 000000000..07dbd421f --- /dev/null +++ b/packages/xo-web/src/xo-app/backup-ng/edit.js @@ -0,0 +1,21 @@ +import addSubscriptions from 'add-subscriptions' +import React from 'react' +import { injectState, provideState } from '@julien-f/freactal' +import { Debug } from 'utils' +import { subscribeBackupNgJobs, subscribeSchedules } from 'xo' + +import New from './new' + +export default [ + addSubscriptions({ + jobs: subscribeBackupNgJobs, + schedules: subscribeSchedules, + }), + provideState({ + computed: { + value: ({ jobs, schedules }) => {}, + }, + }), + injectState, + props => ({ state }) => , +].reduceRight((value, decorator) => decorator(value)) diff --git a/packages/xo-web/src/xo-app/backup-ng/file-restore/index.js b/packages/xo-web/src/xo-app/backup-ng/file-restore/index.js new file mode 100644 index 000000000..2bbae4507 --- /dev/null +++ b/packages/xo-web/src/xo-app/backup-ng/file-restore/index.js @@ -0,0 +1,8 @@ +import Component from 'base-component' +import React from 'react' + +export default class FileRestore extends Component { + render () { + return

Available soon

+ } +} diff --git a/packages/xo-web/src/xo-app/backup-ng/index.js b/packages/xo-web/src/xo-app/backup-ng/index.js new file mode 100644 index 000000000..2f8d3a8fb --- /dev/null +++ b/packages/xo-web/src/xo-app/backup-ng/index.js @@ -0,0 +1,210 @@ +import _ from 'intl' +import ActionButton from 'action-button' +import addSubscriptions from 'add-subscriptions' +import Icon from 'icon' +import React from 'react' +import SortedTable from 'sorted-table' +import { map, groupBy } from 'lodash' +import { Card, CardHeader, CardBlock } from 'card' +import { constructQueryString } from 'smart-backup-pattern' +import { Container, Row, Col } from 'grid' +import { NavLink, NavTabs } from 'nav' +import { routes } from 'utils' +import { + deleteBackupNgJobs, + subscribeBackupNgJobs, + subscribeSchedules, + runBackupNgJob, +} from 'xo' + +import LogsTable from '../logs' +import Page from '../page' + +import Edit from './edit' +import New from './new' +import FileRestore from './file-restore' +import Restore from './restore' + +const Ul = ({ children, ...props }) => ( +
    + {children} +
+) + +const Li = ({ children, ...props }) => ( +
  • + {children} +
    • +) + +const Td = ({ children, ...props }) => ( + + {children} + +) + +const SchedulePreviewBody = ({ job, schedules }) => ( + + {map(schedules, schedule => ( + + + + + + + + ))} +
    {schedule.cron}{schedule.timezone}{job.settings[schedule.id].exportRetention}{job.settings[schedule.id].snapshotRetention} + +
    +) + +const SchedulePreviewHeader = ({ _ }) => ( +
      +
    • Schedule ID
      • |
    • Cron
      • |
    • Timezone
      • |{' '} +
    • Export retention
      • |
    • Snapshot retention
      • |{' '} +
    +) + +@addSubscriptions({ + jobs: subscribeBackupNgJobs, + schedulesByJob: cb => + subscribeSchedules(schedules => { + cb(groupBy(schedules, 'jobId')) + }), +}) +class JobsTable extends React.Component { + static contextTypes = { + router: React.PropTypes.object, + } + + static tableProps = { + actions: [ + { + handler: deleteBackupNgJobs, + label: _('deleteBackupSchedule'), + icon: 'delete', + level: 'danger', + }, + ], + columns: [ + { + itemRenderer: _ => _.id.slice(0, 5), + sortCriteria: _ => _.id, + name: _('jobId'), + }, + { + itemRenderer: _ => _.name, + sortCriteria: _ => _.name, + name: _('jobName'), + }, + { + itemRenderer: _ => _.mode, + sortCriteria: _ => _.mode, + name: 'mode', + }, + { + component: _ => ( + + ), + name: , + }, + ], + individualActions: [ + { + handler: (job, { goTo }) => + goTo({ + pathname: '/home', + query: { t: 'VM', s: constructQueryString(job.vms) }, + }), + label: _('redirectToMatchingVms'), + icon: 'preview', + }, + { + handler: (job, { goTo }) => goTo(`/backup-ng/${job.id}/edit`), + label: '', + icon: 'edit', + level: 'primary', + }, + ], + } + + _goTo = path => { + this.context.router.push(path) + } + + render () { + return ( + + ) + } +} + +const Overview = () => ( +
    + + + {_('backupSchedules')} + + + + + + +
    +) + +const HEADER = ( + + + +

    + {_('backupPage')} +

    + + + + + {_('backupOverviewPage')} + + + {_('backupNewPage')} + + + {_('backupRestorePage')} + + + {' '} + {_('backupFileRestorePage')} + + + +     +     +) + +export default routes(Overview, { + ':id/edit': Edit, + new: New, + restore: Restore, + 'file-restore': FileRestore, +})(({ children }) => ( + + {children} + +)) diff --git a/packages/xo-web/src/xo-app/backup-ng/new.js b/packages/xo-web/src/xo-app/backup-ng/new.js new file mode 100644 index 000000000..6d9dabccf --- /dev/null +++ b/packages/xo-web/src/xo-app/backup-ng/new.js @@ -0,0 +1,269 @@ +import _, { messages } from 'intl' +import ActionButton from 'action-button' +import moment from 'moment-timezone' +import React from 'react' +import Scheduler, { SchedulePreview } from 'scheduling' +import Upgrade from 'xoa-upgrade' +import { injectState, provideState } from '@julien-f/freactal' +import { cloneDeep, orderBy, size, isEmpty, map } from 'lodash' +import { SelectRemote, SelectSr, SelectVm } from 'select-objects' +import { resolveIds } from 'utils' +import { createBackupNgJob, editBackupNgJob, editSchedule } from 'xo' + +const FormGroup = props =>
    +const Input = props => + +const DEFAULT_CRON_PATTERN = '0 0 * * *' +const DEFAULT_TIMEZONE = moment.tz.guess() + +const constructPattern = values => ({ + id: { + __or: resolveIds(values), + }, +}) + +const removeScheduleFromSettings = tmpSchedules => { + const newTmpSchedules = cloneDeep(tmpSchedules) + + for (let schedule in newTmpSchedules) { + delete newTmpSchedules[schedule].cron + delete newTmpSchedules[schedule].timezone + } + + return newTmpSchedules +} + +const getRandomId = () => + Math.random() + .toString(36) + .slice(2) + +export default [ + New => props => ( + + + + ), + provideState({ + initialState: () => ({ + delta: false, + formId: getRandomId(), + tmpSchedule: { + cron: DEFAULT_CRON_PATTERN, + timezone: DEFAULT_TIMEZONE, + }, + exportRetention: 0, + snapshotRetention: 0, + name: '', + remotes: [], + schedules: {}, + srs: [], + vms: [], + tmpSchedules: {}, + }), + effects: { + addSchedule: () => state => { + const id = getRandomId() + + return { + ...state, + tmpSchedules: { + ...state.tmpSchedules, + [id]: { + ...state.tmpSchedule, + exportRetention: state.exportRetention, + snapshotRetention: state.snapshotRetention, + }, + }, + } + }, + createJob: () => async state => { + await createBackupNgJob({ + name: state.name, + mode: state.delta ? 'delta' : 'full', + remotes: constructPattern(state.remotes), + schedules: state.tmpSchedules, + settings: { + ...removeScheduleFromSettings(state.tmpSchedules), + ...state.schedules, + }, + srs: constructPattern(state.srs), + vms: constructPattern(state.vms), + }) + }, + setTmpSchedule: (_, schedule) => state => ({ + ...state, + tmpSchedule: { + cron: schedule.cronPattern, + timezone: schedule.timezone, + }, + }), + setExportRetention: (_, { target: { value } }) => state => ({ + ...state, + exportRetention: value, + }), + setSnapshotRetention: (_, { target: { value } }) => state => ({ + ...state, + snapshotRetention: value, + }), + editSchedule: ( + _, + { target: { dataset: { scheduleId } } } + ) => state => ({}), + editTmpSchedule: (_, { scheduleId }) => state => ({ + ...state, + tmpSchedules: { + ...state.tmpSchedules, + [scheduleId]: { + ...state.tmpSchedule, + exportRetention: state.exportRetention, + snapshotRetention: state.snapshotRetention, + }, + }, + }), + setDelta: (_, { target: { value } }) => state => ({ + ...state, + delta: value, + }), + setName: (_, { target: { value } }) => state => ({ + ...state, + name: value, + }), + setRemotes: (_, remotes) => state => ({ ...state, remotes }), + setSrs: (_, srs) => state => ({ ...state, srs }), + setVms: (_, vms) => state => ({ ...state, vms }), + }, + computed: { + isInvalid: state => + state.name.trim() === '' || + (isEmpty(state.schedules) && isEmpty(state.tmpSchedules)), + sortedSchedules: ({ schedules }) => orderBy(schedules, 'name'), + // TO DO: use sortedTmpSchedules + sortedTmpSchedules: ({ tmpSchedules }) => orderBy(tmpSchedules, 'id'), + }, + }), + injectState, + ({ effects, state }) => ( +
    + +

    BackupNG

    + + +     + + + + + + + + + + + {false /* TODO: remove when implemented */ && (!isEmpty(state.srs) || !isEmpty(state.remotes)) && ( + + + + + + )} + {!isEmpty(state.schedules) && ( + +

    Saved schedules

    +
      + {state.sortedSchedules.map(schedule => ( +
    • + {schedule.name} {schedule.cron} {schedule.timezone} + +
      • + ))} +
    +     + )} + {!isEmpty(state.tmpSchedules) && ( + +

    New schedules

    +
      + {map(state.tmpSchedules, (schedule, key) => ( +
    • + {schedule.cron} {schedule.timezone} {schedule.exportRetention}{' '} + {schedule.snapshotRetention} + +
      • + ))} +
    +     + )} + +

    BackupNG

    + + + + + + +
    + + Add a schedule + +     + + Create + +
    + ), +].reduceRight((value, decorator) => decorator(value)) diff --git a/packages/xo-web/src/xo-app/backup-ng/restore/import-modal-body.js b/packages/xo-web/src/xo-app/backup-ng/restore/import-modal-body.js new file mode 100644 index 000000000..8f9649c78 --- /dev/null +++ b/packages/xo-web/src/xo-app/backup-ng/restore/import-modal-body.js @@ -0,0 +1,42 @@ +import _ from 'intl' +import React from 'react' +import Component from 'base-component' +import { Select } from 'form' +import { SelectSr } from 'select-objects' +import { FormattedDate } from 'react-intl' + +export default class ImportModalBody extends Component { + get value () { + return this.state + } + render () { + return ( +
    +
    +