From 80c1e39b53055b7af042b2bb32484fe66c310d11 Mon Sep 17 00:00:00 2001 From: Julien Fontanet Date: Mon, 12 Mar 2018 17:26:20 +0100 Subject: [PATCH] feat(Backups NG): third iteration (#2729) --- .eslintrc.js | 1 + flow-typed/lodash.js | 4 + flow-typed/promise-toolbox.js | 1 + package.json | 2 +- .../xo-server/src/remote-handlers/abstract.js | 26 +- .../xo-server/src/remote-handlers/checksum.js | 17 +- packages/xo-server/src/size-stream.js | 8 +- packages/xo-server/src/utils.js.flow | 22 + packages/xo-server/src/vhd-merge.js | 21 +- packages/xo-server/src/xapi/index.js | 8 +- packages/xo-server/src/xapi/index.js.flow | 72 +- .../src/xo-mixins/backups-ng/file-restore.js | 0 .../src/xo-mixins/backups-ng/index.js | 657 ++++++++++------ .../xo-server/src/xo-mixins/jobs/index.js | 11 +- packages/xo-web/src/common/intl/messages.js | 5 +- packages/xo-web/src/common/scheduler-tmp.js | 505 ------------ packages/xo-web/src/xo-app/backup-ng/new.js | 741 ------------------ .../xo-web/src/xo-app/backup-ng/new/index.js | 728 +++++++++++++++++ .../src/xo-app/backup-ng/new/new-schedule.js | 111 +++ .../src/xo-app/backup-ng/new/schedules.js | 168 ++++ .../xo-web/src/xo-app/backup-ng/new/utils.js | 11 + .../restore/delete-backups-modal-body.js | 20 +- .../src/xo-app/backup-ng/restore/index.js | 17 +- yarn.lock | 6 +- 24 files changed, 1599 insertions(+), 1563 deletions(-) create mode 100644 packages/xo-server/src/utils.js.flow create mode 100644 packages/xo-server/src/xo-mixins/backups-ng/file-restore.js delete mode 100644 packages/xo-web/src/common/scheduler-tmp.js delete mode 100644 packages/xo-web/src/xo-app/backup-ng/new.js create mode 100644 packages/xo-web/src/xo-app/backup-ng/new/index.js create mode 100644 packages/xo-web/src/xo-app/backup-ng/new/new-schedule.js create mode 100644 packages/xo-web/src/xo-app/backup-ng/new/schedules.js create mode 100644 packages/xo-web/src/xo-app/backup-ng/new/utils.js diff --git a/.eslintrc.js b/.eslintrc.js index a2dcaf139..1f9c30e9a 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -6,6 +6,7 @@ module.exports = { $Diff: true, $Exact: true, $Keys: true, + $PropertyType: true, $Shape: true, }, parser: 'babel-eslint', diff --git a/flow-typed/lodash.js b/flow-typed/lodash.js index 5128e932e..b990fb648 100644 --- a/flow-typed/lodash.js +++ b/flow-typed/lodash.js @@ -3,6 +3,10 @@ declare module 'lodash' { declare export function isEmpty(mixed): boolean declare export function keyBy(array: T[], iteratee: string): boolean declare export function last(array?: T[]): T | void + declare export function map( + collection: T1[], + iteratee: (T1) => T2 + ): T2[] declare export function mapValues( object: { [K]: V1 }, iteratee: (V1, K) => V2 diff --git a/flow-typed/promise-toolbox.js b/flow-typed/promise-toolbox.js index aca0b5088..62e02ab5b 100644 --- a/flow-typed/promise-toolbox.js +++ b/flow-typed/promise-toolbox.js @@ -1,4 +1,5 @@ declare module 'promise-toolbox' { + declare export function cancelable(Function): Function declare export function defer(): {| promise: Promise, reject: T => void, diff --git a/package.json b/package.json index a48b72834..d7a211f5f 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "eslint-plugin-react": "^7.6.1", "eslint-plugin-standard": "^3.0.1", "exec-promise": "^0.7.0", - "flow-bin": "^0.66.0", + "flow-bin": "^0.67.1", "globby": "^8.0.0", "husky": "^0.14.3", "jest": "^22.0.4", diff --git a/packages/xo-server/src/remote-handlers/abstract.js b/packages/xo-server/src/remote-handlers/abstract.js index 83ec43ba2..f128eb37d 100644 --- a/packages/xo-server/src/remote-handlers/abstract.js +++ b/packages/xo-server/src/remote-handlers/abstract.js @@ -92,11 +92,11 @@ export default class RemoteHandlerAbstract { await promise } - async readFile (file: string, options?: Object): Promise { + async readFile (file: string, options?: Object): Promise { return this._readFile(file, options) } - _readFile (file: string, options?: Object): Promise { + _readFile (file: string, options?: Object): Promise { return this.createReadStream(file, options).then(streamToBuffer) } @@ -119,11 +119,25 @@ export default class RemoteHandlerAbstract { throw new Error('Not implemented') } - async list (dir: string = '.') { - return this._list(dir) + async list ( + dir: string = '.', + { + filter, + prependDir = false, + }: { filter?: (name: string) => boolean, prependDir?: boolean } = {} + ): Promise { + const entries = await this._list(dir) + + if (prependDir) { + entries.forEach((entry, i) => { + entries[i] = dir + '/' + entry + }) + } + + return filter === undefined ? entries : entries.filter(filter) } - async _list (dir: string) { + async _list (dir: string): Promise { throw new Error('Not implemented') } @@ -207,7 +221,7 @@ export default class RemoteHandlerAbstract { } async refreshChecksum (path: string): Promise { - const stream: any = (await this.createReadStream(path)).pipe( + const stream = (await this.createReadStream(path)).pipe( createChecksumStream() ) stream.resume() // start reading the whole file diff --git a/packages/xo-server/src/remote-handlers/checksum.js b/packages/xo-server/src/remote-handlers/checksum.js index 77e2c39ed..7855fbe5b 100644 --- a/packages/xo-server/src/remote-handlers/checksum.js +++ b/packages/xo-server/src/remote-handlers/checksum.js @@ -1,7 +1,11 @@ -import invert from 'lodash/invert' +// @flow + +// $FlowFixMe import through2 from 'through2' import { createHash } from 'crypto' import { defer, fromEvent } from 'promise-toolbox' +import { invert } from 'lodash' +import { type Readable, type Transform } from 'stream' const ALGORITHM_TO_ID = { md5: '1', @@ -21,7 +25,9 @@ const ID_TO_ALGORITHM = invert(ALGORITHM_TO_ID) // const checksumStream = source.pipe(createChecksumStream()) // checksumStream.resume() // make the data flow without an output // console.log(await checksumStream.checksum) -export const createChecksumStream = (algorithm = 'md5') => { +export const createChecksumStream = ( + algorithm: string = 'md5' +): Transform & { checksum: Promise } => { const algorithmId = ALGORITHM_TO_ID[algorithm] if (!algorithmId) { @@ -48,7 +54,10 @@ export const createChecksumStream = (algorithm = 'md5') => { // Check if the checksum of a readable stream is equals to an expected checksum. // The given stream is wrapped in a stream which emits an error event // if the computed checksum is not equals to the expected checksum. -export const validChecksumOfReadStream = (stream, expectedChecksum) => { +export const validChecksumOfReadStream = ( + stream: Readable, + expectedChecksum: string +): Readable & { checksumVerified: Promise } => { const algorithmId = expectedChecksum.slice( 1, expectedChecksum.indexOf('$', 1) @@ -60,7 +69,7 @@ export const validChecksumOfReadStream = (stream, expectedChecksum) => { const hash = createHash(ID_TO_ALGORITHM[algorithmId]) - const wrapper = stream.pipe( + const wrapper: any = stream.pipe( through2( { highWaterMark: 0 }, (chunk, enc, callback) => { diff --git a/packages/xo-server/src/size-stream.js b/packages/xo-server/src/size-stream.js index aa8693376..bdca30619 100644 --- a/packages/xo-server/src/size-stream.js +++ b/packages/xo-server/src/size-stream.js @@ -1,6 +1,10 @@ -import through2 from 'through2' +// @flow -const createSizeStream = () => { +// $FlowFixMe +import through2 from 'through2' +import { type Readable } from 'stream' + +const createSizeStream = (): Readable & { size: number } => { const wrapper = through2((chunk, enc, cb) => { wrapper.size += chunk.length cb(null, chunk) diff --git a/packages/xo-server/src/utils.js.flow b/packages/xo-server/src/utils.js.flow new file mode 100644 index 000000000..50a7b3e2d --- /dev/null +++ b/packages/xo-server/src/utils.js.flow @@ -0,0 +1,22 @@ +// @flow + +import { type Readable } from 'stream' + +type MaybePromise = Promise | T + +declare export function asyncMap( + collection: MaybePromise, + (T1, number) => MaybePromise +): Promise +declare export function asyncMap( + collection: MaybePromise<{ [K]: V1 }>, + (V1, K) => MaybePromise +): Promise + +declare export function getPseudoRandomBytes(n: number): Buffer + +declare export function safeDateFormat(timestamp: number): string + +declare export function serializeError(error: Error): Object + +declare export function streamToBuffer(stream: Readable): Promise diff --git a/packages/xo-server/src/vhd-merge.js b/packages/xo-server/src/vhd-merge.js index 006257f83..14c77f771 100644 --- a/packages/xo-server/src/vhd-merge.js +++ b/packages/xo-server/src/vhd-merge.js @@ -6,6 +6,7 @@ import fu from '@nraynaud/struct-fu' import isEqual from 'lodash/isEqual' import { fromEvent } from 'promise-toolbox' +import type RemoteHandler from './remote-handlers/abstract' import constantStream from './constant-stream' import { noop, streamToBuffer } from './utils' @@ -34,8 +35,8 @@ const VHD_PARENT_LOCATOR_ENTRIES = 8 const VHD_PLATFORM_CODE_NONE = 0 // Types of backup treated. Others are not supported. -const HARD_DISK_TYPE_DYNAMIC = 3 // Full backup. -const HARD_DISK_TYPE_DIFFERENCING = 4 // Delta backup. +export const HARD_DISK_TYPE_DYNAMIC = 3 // Full backup. +export const HARD_DISK_TYPE_DIFFERENCING = 4 // Delta backup. // Other. const BLOCK_UNUSED = 0xffffffff @@ -666,11 +667,6 @@ export default concurrency(2)(async function vhdMerge ( throw new Error('Unable to merge, child is not a delta backup.') } - // Merging in differencing disk is prohibited in our case. - if (parentVhd.footer.diskType !== HARD_DISK_TYPE_DYNAMIC) { - throw new Error('Unable to merge, parent is not a full backup.') - } - // Allocation table map is not yet implemented. if ( parentVhd.hasBlockAllocationTableMap() || @@ -694,6 +690,7 @@ export default concurrency(2)(async function vhdMerge ( mergedDataSize += await parentVhd.coalesceBlock(childVhd, blockId) } } + const cFooter = childVhd.footer const pFooter = parentVhd.footer @@ -701,6 +698,7 @@ export default concurrency(2)(async function vhdMerge ( pFooter.diskGeometry = { ...cFooter.diskGeometry } pFooter.originalSize = { ...cFooter.originalSize } pFooter.timestamp = cFooter.timestamp + pFooter.uuid = cFooter.uuid // necessary to update values and to recreate the footer after block // creation @@ -759,3 +757,12 @@ export async function chainVhd ( return false } + +export async function readVhdMetadata (handler: RemoteHandler, path: string) { + const vhd = new Vhd(handler, path) + await vhd.readHeaderAndFooter() + return { + footer: vhd.footer, + header: vhd.header, + } +} diff --git a/packages/xo-server/src/xapi/index.js b/packages/xo-server/src/xapi/index.js index 4021df903..ceabf6c72 100644 --- a/packages/xo-server/src/xapi/index.js +++ b/packages/xo-server/src/xapi/index.js @@ -788,8 +788,8 @@ export default class Xapi extends XapiBase { async exportDeltaVm ( $defer, $cancelToken, - vmId, - baseVmId = undefined, + vmId: string, + baseVmId?: string, { bypassVdiChainsCheck = false, @@ -799,7 +799,7 @@ export default class Xapi extends XapiBase { disableBaseTags = false, snapshotNameLabel = undefined, } = {} - ) { + ): Promise { let vm = this.getObject(vmId) if (!bypassVdiChainsCheck) { this._assertHealthyVdiChains(vm) @@ -915,7 +915,7 @@ export default class Xapi extends XapiBase { @deferrable async importDeltaVm ( $defer, - delta, + delta: DeltaVmExport, { deleteBase = false, disableStartAfterImport = true, diff --git a/packages/xo-server/src/xapi/index.js.flow b/packages/xo-server/src/xapi/index.js.flow index 5a3ec3d00..81e3a9714 100644 --- a/packages/xo-server/src/xapi/index.js.flow +++ b/packages/xo-server/src/xapi/index.js.flow @@ -1,23 +1,35 @@ // @flow -import {type Readable} from 'stream' +import { type Readable } from 'stream' -export type DeltaVmExport = { - streams: $Dict<() => Promise>, - vbds: { [ref: string]: {} }, - vdis: { [ref: string]: { $SR$uuid: string } }, - vifs: { [ref: string]: {} } +type AugmentedReadable = Readable & { + size?: number, + task?: Promise } +type MaybeArray = Array | T + +export type DeltaVmExport = {| + streams: $Dict<() => Promise>, + vbds: { [ref: string]: Object }, + vdis: { [ref: string]: { + $SR$uuid: string, + snapshot_of: string, + } }, + vifs: { [ref: string]: Object }, + vm: Vm, +|} + +export type DeltaVmImport = {| + ...DeltaVmExport, + streams: $Dict Promise>>, +|} + declare class XapiObject { $id: string; $ref: string; $type: string; } -type AugmentedReadable = Readable & { - size?: number, - task?: Promise -} type Id = string | XapiObject declare export class Vm extends XapiObject { @@ -31,19 +43,39 @@ declare export class Vm extends XapiObject { declare export class Xapi { objects: { all: $Dict }; - _importVm (cancelToken: mixed, stream: AugmentedReadable, sr?: XapiObject, onVmCreation?: XapiObject => any): Promise; - _updateObjectMapProperty(object: XapiObject, property: string, entries: $Dict): Promise; - _setObjectProperties(object: XapiObject, properties: $Dict): Promise; - _snapshotVm (cancelToken: mixed, vm: Vm, nameLabel?: string): Promise; + _importVm( + cancelToken: mixed, + stream: AugmentedReadable, + sr?: XapiObject, + onVmCreation?: (XapiObject) => any + ): Promise; + _updateObjectMapProperty( + object: XapiObject, + property: string, + entries: $Dict + ): Promise; + _setObjectProperties( + object: XapiObject, + properties: $Dict + ): Promise; + _snapshotVm(cancelToken: mixed, vm: Vm, nameLabel?: string): Promise; addTag(object: Id, tag: string): Promise; barrier(): void; barrier(ref: string): XapiObject; - deleteVm (vm: Id): Promise; + deleteVm(vm: Id): Promise; editVm(vm: Id, $Dict): Promise; - exportDeltaVm (cancelToken: mixed, snapshot: Id, baseSnapshot?: Id): Promise; - exportVm(cancelToken: mixed, vm: Vm, options?: Object): Promise; - getObject (object: Id): XapiObject; - importDeltaVm (data: DeltaVmExport, options: Object): Promise<{ vm: Vm }>; - importVm (stream: AugmentedReadable, options: Object): Promise; + exportDeltaVm( + cancelToken: mixed, + snapshot: Id, + baseSnapshot?: Id + ): Promise; + exportVm( + cancelToken: mixed, + vm: Vm, + options?: Object + ): Promise; + getObject(object: Id): XapiObject; + importDeltaVm(data: DeltaVmImport, options: Object): Promise<{ vm: Vm }>; + importVm(stream: AugmentedReadable, options: Object): Promise; } diff --git a/packages/xo-server/src/xo-mixins/backups-ng/file-restore.js b/packages/xo-server/src/xo-mixins/backups-ng/file-restore.js new file mode 100644 index 000000000..e69de29bb diff --git a/packages/xo-server/src/xo-mixins/backups-ng/index.js b/packages/xo-server/src/xo-mixins/backups-ng/index.js index a9965ceaa..7733e82bb 100644 --- a/packages/xo-server/src/xo-mixins/backups-ng/index.js +++ b/packages/xo-server/src/xo-mixins/backups-ng/index.js @@ -2,19 +2,29 @@ // $FlowFixMe import defer from 'golike-defer' -import { basename, dirname, resolve } from 'path' -import { timeout as pTimeout } from 'promise-toolbox' -import { isEmpty, last, mapValues, noop, values } from 'lodash' import { type Pattern, createPredicate } from 'value-matcher' import { type Readable, PassThrough } from 'stream' +import { basename, dirname, resolve } from 'path' +import { isEmpty, last, mapValues, noop, values } from 'lodash' +import { timeout as pTimeout } from 'promise-toolbox' import { type Executor, type Job } from '../jobs' import { type Schedule } from '../scheduling' -import createSizeStream from '../../size-stream' import type RemoteHandler from '../../remote-handlers/abstract' +import createSizeStream from '../../size-stream' +import { + type DeltaVmExport, + type DeltaVmImport, + type Vm, + type Xapi, +} from '../../xapi' import { asyncMap, safeDateFormat, serializeError } from '../../utils' -import type { Vm, Xapi } from '../../xapi' +import mergeVhd, { + HARD_DISK_TYPE_DIFFERENCING, + chainVhd, + readVhdMetadata, +} from '../../vhd-merge' type Mode = 'full' | 'delta' @@ -22,11 +32,11 @@ type Settings = {| deleteFirst?: boolean, exportRetention?: number, snapshotRetention?: number, - vmTimeout?: number + vmTimeout?: number, |} type SimpleIdPattern = {| - id: string | {| __or: string[] |} + id: string | {| __or: string[] |}, |} export type BackupJob = {| @@ -37,34 +47,37 @@ export type BackupJob = {| settings: $Dict, srs?: SimpleIdPattern, type: 'backup', - vms: Pattern + vms: Pattern, |} type BackupResult = {| mergeDuration: number, mergeSize: number, transferDuration: number, - transferSize: number + transferSize: number, |} type MetadataBase = {| + _filename?: string, jobId: string, - mode: Mode, scheduleId: string, timestamp: number, version: '2.0.0', vm: Object, - vmSnapshot: Object + vmSnapshot: Object, |} type MetadataDelta = {| ...MetadataBase, mode: 'delta', - vdis: $Dict<{}> + vdis: $PropertyType, + vbds: $PropertyType, + vhds: { [vdiId: string]: string }, + vifs: $PropertyType, |} type MetadataFull = {| ...MetadataBase, - data: string, // relative path to the XVA - mode: 'full' + mode: 'full', + xva: string, |} type Metadata = MetadataDelta | MetadataFull @@ -110,6 +123,7 @@ const BACKUP_DIR = 'xo-vm-backups' const getVmBackupDir = (uuid: string) => `${BACKUP_DIR}/${uuid}` const isMetadataFile = (filename: string) => filename.endsWith('.json') +const isVhd = (filename: string) => filename.endsWith('.vhd') const listReplicatedVms = ( xapi: Xapi, @@ -134,6 +148,100 @@ const listReplicatedVms = ( return values(vms).sort(compareSnapshotTime) } +// returns the chain of parents of this VHD +// +// TODO: move to vhd-merge module +const getVhdChain = async ( + handler: RemoteHandler, + path: string +): Promise => { + const chain = [] + + while (true) { + const vhd = await readVhdMetadata(handler, path) + vhd.path = path + chain.push(vhd) + if (vhd.header.type !== HARD_DISK_TYPE_DIFFERENCING) { + break + } + path = resolveRelativeFromFile(path, vhd.header.parentUnicodeName) + } + + return chain +} + +const importers: $Dict< + ( + handler: RemoteHandler, + metadataFilename: string, + metadata: Metadata, + xapi: Xapi, + sr: { $id: string } + ) => Promise, + Mode +> = { + async delta (handler, metadataFilename, metadata, xapi, sr) { + metadata = ((metadata: any): MetadataDelta) + const { vdis, vhds, vm } = metadata + + const streams = {} + await asyncMap(vdis, async (vdi, id) => { + const chain = await getVhdChain( + handler, + resolveRelativeFromFile(metadataFilename, vhds[id]) + ) + streams[`${id}.vhd`] = await asyncMap(chain, ({ path }) => + handler.createReadStream(path, { + checksum: true, + ignoreMissingChecksum: true, + }) + ) + }) + + const delta: DeltaVmImport = { + streams, + vbds: metadata.vbds, + vdis, + vifs: metadata.vifs, + vm: { + ...vm, + name_label: `${vm.name_label} ({${safeDateFormat( + metadata.timestamp + )}})`, + tags: [...vm.tags, 'restored from backup'], + }, + } + + const { vm: newVm } = await xapi.importDeltaVm(delta, { + disableStartAfterImport: false, + srId: sr, + // TODO: support mapVdisSrs + }) + return newVm.$id + }, + async full (handler, metadataFilename, metadata, xapi, sr) { + metadata = ((metadata: any): MetadataFull) + + const xva = await handler.createReadStream( + resolveRelativeFromFile(metadataFilename, metadata.xva), + { + checksum: true, + ignoreMissingChecksum: true, // provide an easy way to opt-out + } + ) + const vm = await xapi.importVm(xva, { srId: sr.$id }) + await Promise.all([ + xapi.addTag(vm.$id, 'restored from backup'), + xapi.editVm(vm.$id, { + name_label: `${metadata.vm.name_label} (${safeDateFormat( + metadata.timestamp + )})`, + }), + ]) + return vm.$id + }, +} + const parseVmBackupId = (id: string) => { const i = id.indexOf('/') return { @@ -142,7 +250,7 @@ const parseVmBackupId = (id: string) => { } } -// used to resolve the data field from the metadata +// used to resolve the xva field from the metadata const resolveRelativeFromFile = (file: string, path: string): string => resolve('/', dirname(file), path).slice(1) @@ -226,7 +334,7 @@ export default class BackupNg { getXapi: (id: string) => Xapi, getJob: (id: string, 'backup') => Promise, updateJob: ($Shape) => Promise, - removeJob: (id: string) => Promise + removeJob: (id: string) => Promise, } constructor (app: any) { @@ -245,7 +353,7 @@ export default class BackupNg { } const job: BackupJob = (job_: any) - const vms = app.getObjects({ + const vms: $Dict = app.getObjects({ filter: createPredicate({ type: 'VM', ...job.vms, @@ -384,13 +492,15 @@ export default class BackupNg { const metadata: Metadata = JSON.parse( String(await handler.readFile(metadataFilename)) ) + metadata._filename = metadataFilename if (metadata.mode === 'delta') { - throw new Error('not implemented') + await this._deleteDeltaVmBackups(handler, [metadata]) + } else if (metadata.mode === 'full') { + await this._deleteFullVmBackups(handler, [metadata]) + } else { + throw new Error(`no deleter for backup mode ${metadata.mode}`) } - - metadata._filename = metadataFilename - await this._deleteFullVmBackups(handler, [metadata]) } async importVmBackupNg (id: string, srId: string): Promise { @@ -401,29 +511,20 @@ export default class BackupNg { String(await handler.readFile(metadataFilename)) ) - if (metadata.mode === 'delta') { - throw new Error('not implemented') + const importer = importers[metadata.mode] + if (importer === undefined) { + throw new Error(`no importer for backup mode ${metadata.mode}`) } const xapi = app.getXapi(srId) - const sr = xapi.getObject(srId) - const xva = await handler.createReadStream( - resolveRelativeFromFile(metadataFilename, metadata.data), - { - checksum: true, - ignoreMissingChecksum: true, // provide an easy way to opt-out - } + + return importer( + handler, + metadataFilename, + metadata, + xapi, + xapi.getObject(srId) ) - const vm = await xapi.importVm(xva, { srId: sr.$id }) - await Promise.all([ - xapi.addTag(vm.$id, 'restored from backup'), - xapi.editVm(vm.$id, { - name_label: `${metadata.vm.name_label} (${safeDateFormat( - metadata.timestamp - )})`, - }), - ]) - return vm.$id } async listVmBackupsNg (remotes: string[]) { @@ -469,9 +570,12 @@ export default class BackupNg { // - [ ] clones of replicated VMs should not be garbage collected // - if storing uuids in source VM, how to detect them if the source is // lost? - // - [ ] adding and removing VDIs should behave - // - [ ] validate VHDs after exports and before imports - // - [ ] isolate VHD chains by job + // - [ ] validate VHDs after exports and before imports, how? + // - [ ] in case of merge failure + // 1. delete (or isolate) the tainted VHD + // 2. next run should be a full + // - [ ] add a lock on the job/VDI during merge which should prevent other merges and restoration + // - [ ] import for delta // // Low: // - [ ] check merge/transfert duration/size are what we want for delta @@ -479,10 +583,10 @@ export default class BackupNg { // - [ ] possibility to (re-)run a single VM in a backup? // - [ ] display queued VMs // - [ ] snapshots and files of an old job should be detected and removed + // - [ ] delta import should support mapVdisSrs + // - [ ] size of the path? (base64url(Buffer.from(uuid.split('-').join(''), 'hex'))) // // Triage: - // - [ ] protect against concurrent backup against a single VM (JFT: why?) - // - shouldn't be necessary now that VHD chains are separated by job // - [ ] logs // // Done: @@ -493,6 +597,8 @@ export default class BackupNg { // - [x] deleteFirst per target // - [x] timeout per VM // - [x] backups should be deletable from the API + // - [x] adding and removing VDIs should behave + // - [x] isolate VHD chains by job @defer async _backupVm ( $defer: any, @@ -540,17 +646,17 @@ export default class BackupNg { ) ) - let snapshot = await xapi._snapshotVm( + let snapshot: Vm = (await xapi._snapshotVm( $cancelToken, vm, - `[XO Backup] ${vm.name_label}` - ) + `[XO Backup ${job.name}] ${vm.name_label}` + ): any) $defer.onFailure.call(xapi, '_deleteVm', snapshot) await xapi._updateObjectMapProperty(snapshot, 'other_config', { 'xo:backup:job': jobId, 'xo:backup:schedule': scheduleId, }) - snapshot = await xapi.barrier(snapshot.$ref) + snapshot = ((await xapi.barrier(snapshot.$ref): any): Vm) if (exportRetention === 0) { return { @@ -569,30 +675,19 @@ export default class BackupNg { const now = Date.now() const vmDir = getVmBackupDir(vm.uuid) - const { mode } = job const basename = safeDateFormat(now) const metadataFilename = `${vmDir}/${basename}.json` - const metadata: Metadata = { - jobId, - mode, - scheduleId, - timestamp: now, - version: '2.0.0', - vm, - vmSnapshot: snapshot, - } - - if (mode === 'full') { + if (job.mode === 'full') { // TODO: do not create the snapshot if there are no snapshotRetention and // the VM is not running if (snapshotRetention === 0) { $defer.call(xapi, 'deleteVm', snapshot) } - let xva = await xapi.exportVm($cancelToken, snapshot, { + let xva: any = await xapi.exportVm($cancelToken, snapshot, { compress: job.compression === 'native', }) const exportTask = xva.task @@ -600,7 +695,16 @@ export default class BackupNg { const dataBasename = `${basename}.xva` - metadata.data = `./${dataBasename}` + const metadata: MetadataFull = { + jobId, + mode: 'full', + scheduleId, + timestamp: now, + version: '2.0.0', + vm, + vmSnapshot: snapshot, + xva: `./${dataBasename}`, + } const dataFilename = `${vmDir}/${dataBasename}` const jsonMetadata = JSON.stringify(metadata) @@ -614,14 +718,14 @@ export default class BackupNg { const handler = await app.getRemoteHandler(remoteId) - const oldBackups = getOldEntries( + const oldBackups: MetadataFull[] = (getOldEntries( exportRetention, await this._listVmBackups( handler, vm, _ => _.mode === 'full' && _.scheduleId === scheduleId ) - ) + ): any) const deleteFirst = getSetting(settings, 'deleteFirst', remoteId) if (deleteFirst) { @@ -697,190 +801,277 @@ export default class BackupNg { transferDuration: Date.now() - now, transferSize: xva.size, } - } - - // const vdiDir = `${vmDir}/${jobId}/vdis` - - const baseSnapshot = last(snapshots) - if (baseSnapshot !== undefined) { - console.log(baseSnapshot.$id) // TODO: remove - // check current state - // await Promise.all([asyncMap(remotes, remoteId => {})]) - } - - const deltaExport = await xapi.exportDeltaVm( - $cancelToken, - snapshot, - baseSnapshot - ) - - metadata.vbds = deltaExport.vbds - metadata.vdis = deltaExport.vdis - metadata.vifs = deltaExport.vifs - // const jsonMetadata = JSON.stringify(metadata) - - // create a fork of the delta export - const forkExport = (() => { - // replace the stream factories by fork factories - const streams = mapValues(deltaExport.streams, lazyStream => { - let forks = [] - return () => { - if (forks === undefined) { - throw new Error('cannot fork the stream after it has been created') - } - if (forks.length === 0) { - lazyStream().then( - stream => { - // $FlowFixMe - forks.forEach(({ resolve }) => { - const fork = stream.pipe(new PassThrough()) - fork.task = stream.task - resolve(fork) - }) - forks = undefined - }, - error => { - // $FlowFixMe - forks.forEach(({ reject }) => { - reject(error) - }) - forks = undefined - } - ) - } - return new Promise((resolve, reject) => { - // $FlowFixMe - forks.push({ reject, resolve }) - }) - } - }) - return () => { - return { - __proto__: deltaExport, - streams, - } + } else if (job.mode === 'delta') { + const baseSnapshot = last(snapshots) + if (baseSnapshot !== undefined) { + console.log(baseSnapshot.$id) // TODO: remove + // check current state + // await Promise.all([asyncMap(remotes, remoteId => {})]) } - })() - const mergeStart = 0 - const mergeEnd = 0 - let transferStart = 0 - let transferEnd = 0 - const errors = [] - await waitAll( - [ - // ...remotes.map( - // defer(async ($defer, remoteId) => { - // const fork = forkExport() - // - // const handler = await app.getRemoteHandler(remoteId) - // - // // const oldBackups = getOldEntries( - // // exportRetention, - // // await this._listVmBackups( - // // handler, - // // vm, - // // _ => _.mode === 'delta' && _.scheduleId === scheduleId - // // ) - // // ) - // - // const deleteFirst = getSetting(settings, 'deleteFirst', remoteId) - // if (deleteFirst) { - // // TODO - // } - // - // await asyncMap(fork.vdis, (vdi, id) => - // writeStream( - // fork.streams[`${id}.vhd`](), - // handler, - // `${vdiDir}/${vdi.uuid}/${basename}.vhd` - // ) - // ) - // - // await handler.outputFile(metadataFilename, jsonMetadata) - // - // if (!deleteFirst) { - // // TODO - // } - // }) - // ), - ...srs.map( - defer(async ($defer, srId) => { - const fork = forkExport() + const deltaExport = await xapi.exportDeltaVm( + $cancelToken, + snapshot, + baseSnapshot + ) - const xapi = app.getXapi(srId) - const sr = xapi.getObject(srId) - - const oldVms = getOldEntries( - exportRetention, - listReplicatedVms(xapi, scheduleId, srId) - ) - - const deleteFirst = getSetting(settings, 'deleteFirst', srId) - if (deleteFirst) { - await this._deleteVms(xapi, oldVms) - } - - transferStart = Math.min(transferStart, Date.now()) - - const { vm } = await xapi.importDeltaVm(fork, { - disableStartAfterImport: false, // we'll take care of that - name_label: `${metadata.vm.name_label} (${safeDateFormat( - metadata.timestamp - )})`, - srId: sr.$id, - }) - - transferEnd = Math.max(transferEnd, Date.now()) - - await Promise.all([ - xapi.addTag(vm.$ref, 'Continuous Replication'), - xapi._updateObjectMapProperty(vm, 'blocked_operations', { - start: - 'Start operation for this vm is blocked, clone it if you want to use it.', - }), - xapi._updateObjectMapProperty(vm, 'other_config', { - 'xo:backup:sr': srId, - }), - ]) - - if (!deleteFirst) { - await this._deleteVms(xapi, oldVms) - } - }) + const metadata: MetadataDelta = { + jobId, + mode: 'delta', + scheduleId, + timestamp: now, + vbds: deltaExport.vbds, + vdis: deltaExport.vdis, + version: '2.0.0', + vifs: deltaExport.vifs, + vhds: mapValues( + deltaExport.vdis, + vdi => + `vdis/${jobId}/${ + (xapi.getObject(vdi.snapshot_of): Object).uuid + }/${basename}.vhd` ), - ], - error => { - console.warn(error) - errors.push(error) + vm, + vmSnapshot: snapshot, } - ) - if (errors.length !== 0) { - throw errors - } - return { - mergeDuration: mergeEnd - mergeStart, - mergeSize: 0, - transferDuration: transferEnd - transferStart, - transferSize: 0, + const jsonMetadata = JSON.stringify(metadata) + + // create a fork of the delta export + const forkExport = (() => { + // replace the stream factories by fork factories + const streams: any = mapValues(deltaExport.streams, lazyStream => { + let forks = [] + return () => { + if (forks === undefined) { + throw new Error( + 'cannot fork the stream after it has been created' + ) + } + if (forks.length === 0) { + lazyStream().then( + stream => { + // $FlowFixMe + forks.forEach(({ resolve }) => { + const fork: any = stream.pipe(new PassThrough()) + fork.task = stream.task + resolve(fork) + }) + forks = undefined + }, + error => { + // $FlowFixMe + forks.forEach(({ reject }) => { + reject(error) + }) + forks = undefined + } + ) + } + return new Promise((resolve, reject) => { + // $FlowFixMe + forks.push({ reject, resolve }) + }) + } + }) + return () => { + return { + __proto__: deltaExport, + streams, + } + } + })() + + const mergeStart = 0 + const mergeEnd = 0 + let transferStart = 0 + let transferEnd = 0 + const errors = [] + await waitAll( + [ + ...remotes.map( + defer(async ($defer, remoteId) => { + const fork = forkExport() + + const handler = await app.getRemoteHandler(remoteId) + + const oldBackups: MetadataDelta[] = (getOldEntries( + exportRetention, + await this._listVmBackups( + handler, + vm, + _ => _.mode === 'delta' && _.scheduleId === scheduleId + ) + ): any) + + const deleteFirst = getSetting(settings, 'deleteFirst', remoteId) + if (deleteFirst) { + this._deleteDeltaVmBackups(handler, oldBackups) + } + + await asyncMap( + fork.vdis, + defer(async ($defer, vdi, id) => { + const path = `${vmDir}/${metadata.vhds[id]}` + + const isDelta = 'xo:base_delta' in vdi.other_config + let parentPath + if (isDelta) { + const vdiDir = dirname(path) + const parent = (await handler.list(vdiDir)) + .filter(isVhd) + .sort() + .pop() + parentPath = `${vdiDir}/${parent}` + } + + await writeStream(fork.streams[`${id}.vhd`](), handler, path) + $defer.onFailure.call(handler, 'unlink', path) + + if (isDelta) { + await chainVhd(handler, parentPath, handler, path) + } + }) + ) + + await handler.outputFile(metadataFilename, jsonMetadata) + + if (!deleteFirst) { + this._deleteDeltaVmBackups(handler, oldBackups) + } + }) + ), + ...srs.map( + defer(async ($defer, srId) => { + const fork = forkExport() + + const xapi = app.getXapi(srId) + const sr = xapi.getObject(srId) + + const oldVms = getOldEntries( + exportRetention, + listReplicatedVms(xapi, scheduleId, srId) + ) + + const deleteFirst = getSetting(settings, 'deleteFirst', srId) + if (deleteFirst) { + await this._deleteVms(xapi, oldVms) + } + + transferStart = Math.min(transferStart, Date.now()) + + const { vm } = await xapi.importDeltaVm(fork, { + disableStartAfterImport: false, // we'll take care of that + name_label: `${metadata.vm.name_label} (${safeDateFormat( + metadata.timestamp + )})`, + srId: sr.$id, + }) + + transferEnd = Math.max(transferEnd, Date.now()) + + await Promise.all([ + xapi.addTag(vm.$ref, 'Continuous Replication'), + xapi._updateObjectMapProperty(vm, 'blocked_operations', { + start: + 'Start operation for this vm is blocked, clone it if you want to use it.', + }), + xapi._updateObjectMapProperty(vm, 'other_config', { + 'xo:backup:sr': srId, + }), + ]) + + if (!deleteFirst) { + await this._deleteVms(xapi, oldVms) + } + }) + ), + ], + error => { + console.warn(error) + errors.push(error) + } + ) + if (errors.length !== 0) { + throw errors + } + + return { + mergeDuration: mergeEnd - mergeStart, + mergeSize: 0, + transferDuration: transferEnd - transferStart, + transferSize: 0, + } + } else { + throw new Error(`no exporter for backup mode ${job.mode}`) } } + async _deleteDeltaVmBackups ( + handler: RemoteHandler, + backups: MetadataDelta[] + ): Promise { + // TODO: remove VHD as well + await asyncMap(backups, async backup => { + const filename = ((backup._filename: any): string) + + return Promise.all([ + handler.unlink(filename), + asyncMap(backup.vhds, _ => + // $FlowFixMe injected $defer param + this._deleteVhd(handler, resolveRelativeFromFile(filename, _)) + ), + ]) + }) + } + async _deleteFullVmBackups ( handler: RemoteHandler, - backups: Metadata[] + backups: MetadataFull[] ): Promise { - await asyncMap(backups, ({ _filename, data }) => - Promise.all([ + await asyncMap(backups, ({ _filename, xva }) => { + _filename = ((_filename: any): string) + return Promise.all([ handler.unlink(_filename), - handler.unlink(resolveRelativeFromFile(_filename, data)), + handler.unlink(resolveRelativeFromFile(_filename, xva)), ]) - ) + }) } - async _deleteVms (xapi: Xapi, vms: Object[]): Promise { - return asyncMap(vms, vm => xapi.deleteVm(vm)) + // FIXME: synchronize by job/VDI, otherwise it can cause issues with the merge + @defer + async _deleteVhd ($defer: any, handler: RemoteHandler, path: string) { + const vhds = await asyncMap( + await handler.list(dirname(path), { filter: isVhd, prependDir: true }), + async path => { + const metadata = await readVhdMetadata(handler, path) + metadata.path = path + return metadata + } + ) + const base = basename(path) + const child = vhds.find(_ => _.header.parentUnicodeName === base) + if (child === undefined) { + return handler.unlink(path) + } + + $defer.onFailure.call(handler, 'unlink', path) + + const childPath = child.path + + await Promise.all([ + mergeVhd(handler, path, handler, childPath), + handler.unlink(path + '.checksum'), + ]) + + await Promise.all([ + handler.rename(path, childPath), + handler.unlink(childPath + '.checksum'), + ]) + } + + async _deleteVms (xapi: Xapi, vms: Vm[]): Promise { + await asyncMap(vms, vm => xapi.deleteVm(vm)) } async _listVmBackups ( diff --git a/packages/xo-server/src/xo-mixins/jobs/index.js b/packages/xo-server/src/xo-mixins/jobs/index.js index 8be061f13..ef4326304 100644 --- a/packages/xo-server/src/xo-mixins/jobs/index.js +++ b/packages/xo-server/src/xo-mixins/jobs/index.js @@ -2,13 +2,13 @@ import type { Pattern } from 'value-matcher' -// $FlowFixMe import { cancelable } from 'promise-toolbox' +import { map as mapToArray } from 'lodash' import { noSuchObject } from 'xo-common/api-errors' import Collection from '../../collection/redis' import patch from '../../patch' -import { mapToArray, serializeError } from '../../utils' +import { serializeError } from '../../utils' import type Logger from '../logs/loggers/abstract' import { type Schedule } from '../scheduling' @@ -165,12 +165,15 @@ export default class Jobs { } async getJob (id: string, type?: string): Promise { - const job = await this._jobs.first(id) + let job = await this._jobs.first(id) if (job === null || (type !== undefined && job.properties.type !== type)) { throw noSuchObject(id, 'job') } - return job.properties + job = job.properties + job.runId = this._runningJobs[id] + + return job } createJob (job: $Diff): Promise { diff --git a/packages/xo-web/src/common/intl/messages.js b/packages/xo-web/src/common/intl/messages.js index 1a0406f82..9bd94bef2 100644 --- a/packages/xo-web/src/common/intl/messages.js +++ b/packages/xo-web/src/common/intl/messages.js @@ -290,6 +290,8 @@ const messages = { jobEditMessage: 'You are editing job {name} ({id}). Saving will override previous job state.', scheduleEdit: 'Edit', + scheduleSave: 'Save', + cancelScheduleEdition: 'Cancel', scheduleAdd: 'Add a schedule', scheduleDelete: 'Delete', deleteSelectedSchedules: 'Delete selected schedules', @@ -316,6 +318,8 @@ const messages = { smartBackupModeSelection: 'Select backup mode:', normalBackup: 'Normal backup', smartBackup: 'Smart backup', + exportRetention: 'Export retention', + snapshotRetention: 'Snapshot retention', backupName: 'Name', useDelta: 'Use delta', useCompression: 'Use compression', @@ -1192,7 +1196,6 @@ const messages = { deleteVmBackupsTitle: 'Delete {vm} backups', deleteVmBackupsSelect: 'Select backups to delete:', deleteVmBackupsSelectAll: 'All', - deleteVmBackupsDeltaInfo: 'Delta backup deletion will be available soon', deleteVmBackupsBulkTitle: 'Delete backups', deleteVmBackupsBulkMessage: 'Are you sure you want to delete all the backups from {nVms, number} VM{nVms, plural, one {} other {s}}?', diff --git a/packages/xo-web/src/common/scheduler-tmp.js b/packages/xo-web/src/common/scheduler-tmp.js deleted file mode 100644 index abcccab9e..000000000 --- a/packages/xo-web/src/common/scheduler-tmp.js +++ /dev/null @@ -1,505 +0,0 @@ -import classNames from 'classnames' -import React from 'react' -import { forEach, includes, isArray, map, sortedIndex } from 'lodash' -import { FormattedDate } from 'react-intl' - -import _ from './intl' -import Button from './button' -import Component from './base-component' -import propTypes from './prop-types-decorator' -import TimezonePicker from './timezone-picker' -import Icon from './icon' -import Tooltip from './tooltip' -import { Card, CardHeader, CardBlock } from './card' -import { Col, Row } from './grid' -import { Range, Toggle } from './form' - -// =================================================================== - -const CLICKABLE = { cursor: 'pointer' } - -// =================================================================== - -const UNITS = ['minute', 'hour', 'monthDay', 'month', 'weekDay'] - -const MINUTES_RANGE = [2, 30] -const HOURS_RANGE = [2, 12] -const MONTH_DAYS_RANGE = [2, 15] -const MONTHS_RANGE = [2, 6] - -const MONTHS = [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]] - -const DAYS = (() => { - const days = [] - - for (let i = 0; i < 4; i++) { - days[i] = [] - - for (let j = 1; j < 8; j++) { - days[i].push(7 * i + j) - } - } - - days.push([29, 30, 31]) - - return days -})() - -const WEEK_DAYS = [[0, 1, 2], [3, 4, 5], [6]] - -const HOURS = (() => { - const hours = [] - - for (let i = 0; i < 4; i++) { - hours[i] = [] - - for (let j = 0; j < 6; j++) { - hours[i].push(6 * i + j) - } - } - - return hours -})() - -const MINS = (() => { - const minutes = [] - - for (let i = 0; i < 6; i++) { - minutes[i] = [] - - for (let j = 0; j < 10; j++) { - minutes[i].push(10 * i + j) - } - } - - return minutes -})() - -const PICKTIME_TO_ID = { - minute: 0, - hour: 1, - monthDay: 2, - month: 3, - weekDay: 4, -} - -// =================================================================== - -// monthNum: [ 0 : 11 ] -const getMonthName = monthNum => ( - -) - -// dayNum: [ 0 : 6 ] -const getDayName = dayNum => ( - // January, 1970, 5th => Monday - -) - -// =================================================================== - -@propTypes({ - children: propTypes.any.isRequired, - onChange: propTypes.func.isRequired, - tdId: propTypes.number.isRequired, - value: propTypes.bool.isRequired, -}) -class ToggleTd extends Component { - _onClick = () => { - const { props } = this - props.onChange(props.tdId, !props.value) - } - - render () { - const { props } = this - return ( - - {props.children} - - ) - } -} - -// =================================================================== - -@propTypes({ - labelId: propTypes.string.isRequired, - options: propTypes.array.isRequired, - optionRenderer: propTypes.func, - onChange: propTypes.func.isRequired, - value: propTypes.array.isRequired, -}) -class TableSelect extends Component { - static defaultProps = { - optionRenderer: value => value, - } - - _reset = () => { - this.props.onChange([]) - } - - _handleChange = (tdId, tdValue) => { - const { props } = this - - const newValue = props.value.slice() - const index = sortedIndex(newValue, tdId) - - if (tdValue) { - // Add - if (newValue[index] !== tdId) { - newValue.splice(index, 0, tdId) - } - } else { - // Remove - if (newValue[index] === tdId) { - newValue.splice(index, 1) - } - } - - props.onChange(newValue) - } - - render () { - const { labelId, options, optionRenderer, value } = this.props - - return ( -
- - - {map(options, (line, i) => ( - - {map(line, tdOption => ( - - ))} - - ))} - -
- -
- ) - } -} - -// =================================================================== - -// "2,7" => [2,7] "*/2" => 2 "*" => [] -const cronToValue = (cron, range) => { - if (cron.indexOf('/') === 1) { - return +cron.split('/')[1] - } - - if (cron === '*') { - return [] - } - - return map(cron.split(','), Number) -} - -// [2,7] => "2,7" 2 => "*/2" [] => "*" -const valueToCron = value => { - if (!isArray(value)) { - return `*/${value}` - } - - if (!value.length) { - return '*' - } - - return value.join(',') -} - -@propTypes({ - headerAddon: propTypes.node, - optionRenderer: propTypes.func, - onChange: propTypes.func.isRequired, - range: propTypes.array, - labelId: propTypes.string.isRequired, - value: propTypes.any.isRequired, -}) -class TimePicker extends Component { - _update = cron => { - const { tableValue, rangeValue } = this.state - - const newValue = cronToValue(cron) - const periodic = !isArray(newValue) - - this.setState({ - periodic, - tableValue: periodic ? tableValue : newValue, - rangeValue: periodic ? newValue : rangeValue, - }) - } - - componentWillReceiveProps (props) { - if (props.value !== this.props.value) { - this._update(props.value) - } - } - - componentDidMount () { - this._update(this.props.value) - } - - _onChange = value => { - this.props.onChange(valueToCron(value)) - } - - _tableTab = () => this._onChange(this.state.tableValue || []) - _periodicTab = () => - this._onChange(this.state.rangeValue || this.props.range[0]) - - render () { - const { headerAddon, labelId, options, optionRenderer, range } = this.props - - const { periodic, tableValue, rangeValue } = this.state - - return ( - - - {_(`scheduling${labelId}`)} - {headerAddon} - - - {range && ( - - )} - {periodic ? ( - - ) : ( - - )} - - - ) - } -} - -const isWeekDayMode = ({ monthDayPattern, weekDayPattern }) => { - if (monthDayPattern === '*' && weekDayPattern === '*') { - return - } - - return weekDayPattern !== '*' -} - -@propTypes({ - monthDayPattern: propTypes.string.isRequired, - weekDayPattern: propTypes.string.isRequired, -}) -class DayPicker extends Component { - state = { - weekDayMode: isWeekDayMode(this.props), - } - - componentWillReceiveProps (props) { - const weekDayMode = isWeekDayMode(props) - - if (weekDayMode !== undefined) { - this.setState({ weekDayMode }) - } - } - - _setWeekDayMode = weekDayMode => { - this.props.onChange(['*', '*']) - this.setState({ weekDayMode }) - } - - _onChange = cron => { - const isMonthDayPattern = !this.state.weekDayMode || includes(cron, '/') - - this.props.onChange([ - isMonthDayPattern ? cron : '*', - isMonthDayPattern ? '*' : cron, - ]) - } - - render () { - const { monthDayPattern, weekDayPattern } = this.props - const { weekDayMode } = this.state - - const dayModeToggle = ( - - - - - - ) - - return ( - - ) - } -} - -// =================================================================== - -@propTypes({ - cronPattern: propTypes.string, - onChange: propTypes.func, - timezone: propTypes.string, - value: propTypes.shape({ - cronPattern: propTypes.string.isRequired, - timezone: propTypes.string, - }), -}) -export default class Scheduler extends Component { - constructor (props) { - super(props) - - this._onCronChange = newCrons => { - const cronPattern = this._getCronPattern().split(' ') - forEach(newCrons, (cron, unit) => { - cronPattern[PICKTIME_TO_ID[unit]] = cron - }) - - this.props.onChange({ - cronPattern: cronPattern.join(' '), - timezone: this._getTimezone(), - }) - } - - forEach(UNITS, unit => { - this[`_${unit}Change`] = cron => this._onCronChange({ [unit]: cron }) - }) - this._dayChange = ([monthDay, weekDay]) => - this._onCronChange({ monthDay, weekDay }) - } - - _onTimezoneChange = timezone => { - this.props.onChange({ - cronPattern: this._getCronPattern(), - timezone, - }) - } - - _getCronPattern = () => { - const { value, cronPattern = value.cronPattern } = this.props - return cronPattern - } - - _getTimezone = () => { - const { value, timezone = value && value.timezone } = this.props - return timezone - } - - render () { - const cronPatternArr = this._getCronPattern().split(' ') - const timezone = this._getTimezone() - - return ( -
- - - - - - - - - - - - - - -
- - - -
- ) - } -} diff --git a/packages/xo-web/src/xo-app/backup-ng/new.js b/packages/xo-web/src/xo-app/backup-ng/new.js deleted file mode 100644 index 3dff6f026..000000000 --- a/packages/xo-web/src/xo-app/backup-ng/new.js +++ /dev/null @@ -1,741 +0,0 @@ -import _ from 'intl' -import ActionButton from 'action-button' -import Component from 'base-component' -import moment from 'moment-timezone' -import React from 'react' -import Scheduler from 'scheduler-tmp' -import SmartBackupPreview from 'smart-backup-preview' -import SortedTable from 'sorted-table' -import Upgrade from 'xoa-upgrade' -import { Card, CardBlock, CardHeader } from 'card' -import { connectStore, resolveIds } from 'utils' -import { confirm } from 'modal' -import { SchedulePreview } from 'scheduling' -import { - constructSmartPattern, - destructSmartPattern, -} from 'smart-backup-pattern' -import { createGetObjectsOfType } from 'selectors' -import { injectState, provideState } from '@julien-f/freactal' -import { Select, Toggle } from 'form' -import { findKey, flatten, get, isEmpty, map, size, some } from 'lodash' -import { - SelectPool, - SelectRemote, - SelectSr, - SelectTag, - SelectVm, -} from 'select-objects' -import { - createBackupNgJob, - createSchedule, - deleteSchedule, - editBackupNgJob, - editSchedule, -} from 'xo' - -// =================================================================== - -const SMART_MODE_INITIAL_STATE = { - powerState: 'All', - $pool: {}, - tags: {}, -} - -const SMART_MODE_FUNCTIONS = { - setPowerState: (_, powerState) => state => ({ - ...state, - powerState, - }), - setPoolValues: (_, values) => state => ({ - ...state, - $pool: { - ...state.$pool, - values, - }, - }), - setPoolNotValues: (_, notValues) => state => ({ - ...state, - $pool: { - ...state.$pool, - notValues, - }, - }), - setTagValues: (_, values) => state => ({ - ...state, - tags: { - ...state.tags, - values, - }, - }), - setTagNotValues: (_, notValues) => state => ({ - ...state, - tags: { - ...state.tags, - notValues, - }, - }), -} - -const normaliseTagValues = values => resolveIds(values).map(value => [value]) - -const SMART_MODE_COMPUTED = { - vmsSmartPattern: ({ $pool, powerState, tags }) => ({ - $pool: constructSmartPattern($pool, resolveIds), - power_state: powerState === 'All' ? undefined : powerState, - tags: constructSmartPattern(tags, normaliseTagValues), - type: 'VM', - }), - allVms: (state, { allVms }) => allVms, -} - -const VMS_STATUSES_OPTIONS = [ - { value: 'All', label: _('vmStateAll') }, - { value: 'Running', label: _('vmStateRunning') }, - { value: 'Halted', label: _('vmStateHalted') }, -] - -const SmartBackup = injectState(({ state, effects }) => ( - - {_('smartBackupModeTitle')} - - - - - - - - - - - - - ) - } -} - -// =================================================================== - -const constructPattern = values => ({ - id: { - __or: resolveIds(values), - }, -}) - -const destructPattern = pattern => pattern.id.__or - -const destructVmsPattern = pattern => - pattern.id === undefined - ? { - powerState: pattern.power_state || 'All', - $pool: destructSmartPattern(pattern.$pool), - tags: destructSmartPattern(pattern.tags, flatten), - } - : { - vms: destructPattern(pattern), - } - -const FormGroup = props =>
-const Input = props => - -const getNewSettings = schedules => { - const newSettings = {} - - for (const schedule in schedules) { - newSettings[schedule] = { - exportRetention: +schedules[schedule].exportRetention, - snapshotRetention: +schedules[schedule].snapshotRetention, - } - } - - return newSettings -} - -const getRandomId = () => - Math.random() - .toString(36) - .slice(2) - -export default [ - New => props => ( - - - - ), - connectStore({ - allVms: createGetObjectsOfType('VM'), - }), - provideState({ - initialState: () => ({ - compression: true, - delta: false, - formId: getRandomId(), - name: '', - paramsUpdated: false, - remotes: [], - smartMode: false, - srs: [], - vms: [], - ...SMART_MODE_INITIAL_STATE, - ...SCHEDULES_INITIAL_STATE, - }), - effects: { - addSchedule: () => async state => { - const { schedule, snapshotRetention, exportRetention } = await confirm({ - title: 'New schedule', - body: , - }) - - if (isScheduleValid(snapshotRetention, exportRetention)) { - const id = getRandomId() - return { - ...state, - tmpSchedules: { - ...state.tmpSchedules, - [id]: { - ...schedule, - exportRetention: exportRetention, - snapshotRetention: snapshotRetention, - }, - }, - } - } - }, - createJob: () => async state => { - await createBackupNgJob({ - name: state.name, - mode: state.delta ? 'delta' : 'full', - compression: state.compression ? 'native' : '', - schedules: state.tmpSchedules, - settings: { - ...getNewSettings(state.tmpSchedules), - }, - remotes: constructPattern(state.remotes), - srs: constructPattern(state.srs), - vms: state.smartMode - ? state.vmsSmartPattern - : constructPattern(state.vms), - }) - }, - editJob: () => async (state, props) => { - const newSettings = {} - if (!isEmpty(state.tmpSchedules)) { - await Promise.all( - map(state.tmpSchedules, async schedule => { - const scheduleId = (await createSchedule(props.job.id, { - cron: schedule.cron, - timezone: schedule.timezone, - })).id - newSettings[scheduleId] = { - exportRetention: +schedule.exportRetention, - snapshotRetention: +schedule.snapshotRetention, - } - }) - ) - } - - await editBackupNgJob({ - id: props.job.id, - name: state.name, - mode: state.delta ? 'delta' : 'full', - compression: state.compression ? 'native' : '', - remotes: constructPattern(state.remotes), - srs: constructPattern(state.srs), - vms: state.smartMode - ? state.vmsSmartPattern - : constructPattern(state.vms), - settings: { - ...newSettings, - ...props.job.settings, - }, - }) - }, - setDelta: (_, { target: { checked } }) => state => ({ - ...state, - delta: checked, - }), - setCompression: (_, { target: { checked } }) => state => ({ - ...state, - compression: checked, - }), - setSmartMode: (_, smartMode) => state => ({ - ...state, - smartMode, - }), - setName: (_, { target: { value } }) => state => ({ - ...state, - name: value, - }), - setRemotes: (_, remotes) => state => ({ ...state, remotes }), - setSrs: (_, srs) => state => ({ ...state, srs }), - setVms: (_, vms) => state => ({ ...state, vms }), - updateParams: () => (state, { job }) => ({ - ...state, - compression: job.compression === 'native', - delta: job.mode === 'delta', - name: job.name, - paramsUpdated: true, - smartMode: job.vms.id === undefined, - remotes: destructPattern(job.remotes), - srs: destructPattern(job.srs), - ...destructVmsPattern(job.vms), - }), - ...SMART_MODE_FUNCTIONS, - ...SCHEDULES_FUNCTIONS, - }, - computed: { - needUpdateParams: (state, { job }) => - job !== undefined && !state.paramsUpdated, - isInvalid: state => - state.name.trim() === '' || - (isEmpty(state.schedules) && isEmpty(state.tmpSchedules)) || - (isEmpty(state.vms) && !state.smartMode), - showCompression: (state, { job }) => - !state.delta && - (some( - state.tmpSchedules, - schedule => +schedule.exportRetention !== 0 - ) || - (job && - some(job.settings, schedule => schedule.exportRetention !== 0))), - ...SMART_MODE_COMPUTED, - ...SCHEDULES_COMPUTED, - }, - }), - injectState, - ({ effects, state }) => { - if (state.needUpdateParams) { - effects.updateParams() - } - - return ( -
- - - - - - - - - - - - - - -
- - - {state.smartMode ? ( - - - - ) : ( - - - - - )} - {(!isEmpty(state.srs) || !isEmpty(state.remotes)) && ( - - - - - - )} - {state.showCompression && ( - - )} -
- - {_('scheduleAdd')} - -
-
- -
- {state.paramsUpdated ? ( - - {_('scheduleEdit')} - - ) : ( - - {_('createBackupJob')} - - )} - - ) - }, -].reduceRight((value, decorator) => decorator(value)) diff --git a/packages/xo-web/src/xo-app/backup-ng/new/index.js b/packages/xo-web/src/xo-app/backup-ng/new/index.js new file mode 100644 index 000000000..2e60507d3 --- /dev/null +++ b/packages/xo-web/src/xo-app/backup-ng/new/index.js @@ -0,0 +1,728 @@ +import _ from 'intl' +import ActionButton from 'action-button' +import React from 'react' +import renderXoItem, { renderXoItemFromId } from 'render-xo-item' +import SmartBackupPreview from 'smart-backup-preview' +import Tooltip from 'tooltip' +import Upgrade from 'xoa-upgrade' +import { addSubscriptions, connectStore, resolveId, resolveIds } from 'utils' +import { Card, CardBlock, CardHeader } from 'card' +import { Container, Col, Row } from 'grid' +import { createGetObjectsOfType } from 'selectors' +import { flatten, get, keyBy, isEmpty, map, some } from 'lodash' +import { injectState, provideState } from '@julien-f/freactal' +import { Select, Toggle } from 'form' +import { + constructSmartPattern, + destructSmartPattern, +} from 'smart-backup-pattern' +import { + SelectPool, + SelectRemote, + SelectSr, + SelectTag, + SelectVm, +} from 'select-objects' +import { + createBackupNgJob, + createSchedule, + deleteSchedule, + editBackupNgJob, + editSchedule, + subscribeRemotes, +} from 'xo' + +import Schedules from './schedules' +import { FormGroup, getRandomId, Input, Ul, Li } from './utils' + +// =================================================================== + +const SMART_MODE_INITIAL_STATE = { + powerState: 'All', + $pool: {}, + tags: {}, +} + +const SMART_MODE_FUNCTIONS = { + setPowerState: (_, powerState) => state => ({ + ...state, + powerState, + }), + setPoolValues: (_, values) => state => ({ + ...state, + $pool: { + ...state.$pool, + values, + }, + }), + setPoolNotValues: (_, notValues) => state => ({ + ...state, + $pool: { + ...state.$pool, + notValues, + }, + }), + setTagValues: (_, values) => state => ({ + ...state, + tags: { + ...state.tags, + values, + }, + }), + setTagNotValues: (_, notValues) => state => ({ + ...state, + tags: { + ...state.tags, + notValues, + }, + }), +} + +const normaliseTagValues = values => resolveIds(values).map(value => [value]) + +const SMART_MODE_COMPUTED = { + vmsSmartPattern: ({ $pool, powerState, tags }) => ({ + $pool: constructSmartPattern($pool, resolveIds), + power_state: powerState === 'All' ? undefined : powerState, + tags: constructSmartPattern(tags, normaliseTagValues), + type: 'VM', + }), + allVms: (state, { allVms }) => allVms, +} + +const VMS_STATUSES_OPTIONS = [ + { value: 'All', label: _('vmStateAll') }, + { value: 'Running', label: _('vmStateRunning') }, + { value: 'Halted', label: _('vmStateHalted') }, +] + +const SmartBackup = injectState(({ state, effects }) => ( +
+ + + + + {state.smartMode ? ( + + + + ) : ( + + + + + )} + {state.showCompression && ( + + )} + + + + +
+ + {_('rollingSnapshot')} + + + {_('backup')} + + + {_('deltaBackup')} + + + {_('disasterRecovery')} + + + {_('continuousReplication')} + +
+ + + {(state.backupMode || state.deltaMode) && ( + + + {_(state.backupMode ? 'backup' : 'deltaBackup')} + + + + + +
+
    + {map(state.remotes, (id, key) => ( +
  • + {state.allRemotes && + renderXoItem({ + type: 'remote', + value: state.allRemotes[id], + })} + +
    • + ))} +
+ + + + )} + {(state.drMode || state.crMode) && ( + + + {_( + state.drMode + ? 'disasterRecovery' + : 'continuousReplication' + )} + + + + + +
+
    + {map(state.srs, (id, key) => ( +
  • + {renderXoItemFromId(id)} + +
    • + ))} +
+ + + + )} + + + + + + + {state.paramsUpdated ? ( + + {_('scheduleEdit')} + + ) : ( + + {_('createBackupJob')} + + )} + + + + ) + }, +].reduceRight((value, decorator) => decorator(value)) diff --git a/packages/xo-web/src/xo-app/backup-ng/new/new-schedule.js b/packages/xo-web/src/xo-app/backup-ng/new/new-schedule.js new file mode 100644 index 000000000..cb18ff7c2 --- /dev/null +++ b/packages/xo-web/src/xo-app/backup-ng/new/new-schedule.js @@ -0,0 +1,111 @@ +import _ from 'intl' +import ActionButton from 'action-button' +import moment from 'moment-timezone' +import React from 'react' +import Scheduler, { SchedulePreview } from 'scheduling' +import { Card, CardBlock } from 'card' +import { injectState, provideState } from '@julien-f/freactal' + +import { FormGroup, getRandomId, Input } from './utils' + +export default [ + injectState, + provideState({ + initialState: ({ + schedule: { + cron = '0 0 * * *', + exportRetention = 0, + snapshotRetention = 0, + timezone = moment.tz.guess(), + }, + }) => ({ + cron, + exportRetention, + formId: getRandomId(), + snapshotRetention, + timezone, + }), + effects: { + setExportRetention: (_, { target: { value } }) => state => ({ + ...state, + exportRetention: value, + }), + setSnapshotRetention: (_, { target: { value } }) => state => ({ + ...state, + snapshotRetention: value, + }), + setSchedule: (_, { cronPattern, timezone }) => state => ({ + ...state, + cron: cronPattern, + timezone, + }), + }, + computed: { + isScheduleInvalid: ({ snapshotRetention, exportRetention }) => + (+snapshotRetention === 0 || snapshotRetention === '') && + (+exportRetention === 0 || exportRetention === ''), + }, + }), + injectState, + ({ effects, state }) => ( +
+ + + {state.exportMode && ( + + + + + )} + {state.snapshotMode && ( + + + + + )} + + +
+ + {_('scheduleSave')} + + + {_('cancelScheduleEdition')} + + + +
+ ), +].reduceRight((value, decorator) => decorator(value)) diff --git a/packages/xo-web/src/xo-app/backup-ng/new/schedules.js b/packages/xo-web/src/xo-app/backup-ng/new/schedules.js new file mode 100644 index 000000000..9cda2cfb5 --- /dev/null +++ b/packages/xo-web/src/xo-app/backup-ng/new/schedules.js @@ -0,0 +1,168 @@ +import _ from 'intl' +import ActionButton from 'action-button' +import React from 'react' +import SortedTable from 'sorted-table' +import { Card, CardBlock, CardHeader } from 'card' +import { injectState, provideState } from '@julien-f/freactal' +import { isEmpty, findKey } from 'lodash' + +import NewSchedule from './new-schedule' +import { FormGroup } from './utils' + +// =================================================================== + +const SCHEDULES_COLUMNS = [ + { + itemRenderer: _ => _.cron, + sortCriteria: 'cron', + name: _('scheduleCron'), + }, + { + itemRenderer: _ => _.timezone, + sortCriteria: 'timezone', + name: _('scheduleTimezone'), + }, + { + itemRenderer: _ => _.exportRetention, + sortCriteria: _ => _.exportRetention, + name: _('scheduleExportRetention'), + }, + { + itemRenderer: _ => _.snapshotRetention, + sortCriteria: _ => _.snapshotRetention, + name: _('scheduleSnapshotRetention'), + }, +] + +const SAVED_SCHEDULES_COLUMNS = [ + { + itemRenderer: _ => _.name, + sortCriteria: 'name', + name: _('scheduleName'), + default: true, + }, + ...SCHEDULES_COLUMNS, +] + +const rowTransform = (schedule, { settings }) => { + const { exportRetention, snapshotRetention } = settings[schedule.id] || {} + + return { + ...schedule, + exportRetention, + snapshotRetention, + } +} + +const SAVED_SCHEDULES_INDIVIDUAL_ACTIONS = [ + { + handler: (schedule, { editSchedule }) => editSchedule(schedule), + label: _('scheduleEdit'), + icon: 'edit', + disabled: (_, { disabledEdition }) => disabledEdition, + level: 'primary', + }, + { + handler: (schedule, { deleteSchedule }) => deleteSchedule(schedule.id), + label: _('scheduleDelete'), + disabled: (_, { disabledDeletion }) => disabledDeletion, + icon: 'delete', + level: 'danger', + }, +] + +const NEW_SCHEDULES_INDIVIDUAL_ACTIONS = [ + { + handler: (schedule, { editNewSchedule, newSchedules }) => + editNewSchedule({ + id: findKey(newSchedules, schedule), + ...schedule, + }), + label: _('scheduleEdit'), + disabled: (_, { disabledEdition }) => disabledEdition, + icon: 'edit', + level: 'primary', + }, + { + handler: (schedule, { deleteNewSchedule, newSchedules }) => + deleteNewSchedule(findKey(newSchedules, schedule)), + label: _('scheduleDelete'), + icon: 'delete', + level: 'danger', + }, +] + +// =================================================================== + +export default [ + injectState, + provideState({ + computed: { + disabledDeletion: state => state.schedules.length <= 1, + disabledEdition: state => + state.editionMode !== undefined || + (!state.exportMode && !state.snapshotMode), + }, + }), + injectState, + ({ effects, state }) => ( +
+ + + {_('backupSchedules')} + + + + {isEmpty(state.schedules) && + isEmpty(state.newSchedules) && ( +

{_('noSchedules')}

+ )} + {!isEmpty(state.schedules) && ( + + + + + )} + {!isEmpty(state.newSchedules) && ( + + + + + )} + + + {state.editionMode !== undefined && ( + + )} +
+ ), +].reduceRight((value, decorator) => decorator(value)) diff --git a/packages/xo-web/src/xo-app/backup-ng/new/utils.js b/packages/xo-web/src/xo-app/backup-ng/new/utils.js new file mode 100644 index 000000000..98d0726bd --- /dev/null +++ b/packages/xo-web/src/xo-app/backup-ng/new/utils.js @@ -0,0 +1,11 @@ +import React from 'react' + +export const FormGroup = props =>
+export const Input = props => +export const Ul = props =>
    +export const Li = props =>
  • + +export const getRandomId = () => + Math.random() + .toString(36) + .slice(2) diff --git a/packages/xo-web/src/xo-app/backup-ng/restore/delete-backups-modal-body.js b/packages/xo-web/src/xo-app/backup-ng/restore/delete-backups-modal-body.js index a58289d15..e372ecb85 100644 --- a/packages/xo-web/src/xo-app/backup-ng/restore/delete-backups-modal-body.js +++ b/packages/xo-web/src/xo-app/backup-ng/restore/delete-backups-modal-body.js @@ -1,7 +1,6 @@ import _ from 'intl' import classNames from 'classnames' import Component from 'base-component' -import Icon from 'icon' import React from 'react' import { FormattedDate } from 'react-intl' import { forEach, map, orderBy } from 'lodash' @@ -19,8 +18,7 @@ export default class DeleteBackupsModalBody extends Component { const selected = this._getSelectedBackups().length === 0 const state = {} - // TODO: [DELTA] remove filter - forEach(this.props.backups.filter(b => b.mode !== 'delta'), backup => { + forEach(this.props.backups, backup => { state[_escapeDot(backup.id)] = selected }) @@ -38,9 +36,7 @@ export default class DeleteBackupsModalBody extends Component { _getAllSelected = createSelector( () => this.props.backups, this._getSelectedBackups, - (backups, selectedBackups) => - // TODO: [DELTA] remove filter - backups.filter(b => b.mode !== 'delta').length === selectedBackups.length + (backups, selectedBackups) => backups.length === selectedBackups.length ) _getBackups = createSelector( @@ -58,15 +54,11 @@ export default class DeleteBackupsModalBody extends Component { className={classNames( 'list-group-item', 'list-group-item-action', - backup.mode === 'delta' && 'disabled', // TODO: [DELTA] remove this.state[_escapeDot(backup.id)] && 'active' )} data-id={backup.id} key={backup.id} - onClick={ - backup.mode !== 'delta' && // TODO: [DELTA] remove - this.toggleState(_escapeDot(backup.id)) - } + onClick={this.toggleState(_escapeDot(backup.id))} type='button' > {' '} {_('deleteVmBackupsSelectAll')}
- {/* TODO: [DELTA] remove div and i18n message */} -
- - {_('deleteVmBackupsDeltaInfo')} - -
) } diff --git a/packages/xo-web/src/xo-app/backup-ng/restore/index.js b/packages/xo-web/src/xo-app/backup-ng/restore/index.js index 9213f6373..ebbe2377d 100644 --- a/packages/xo-web/src/xo-app/backup-ng/restore/index.js +++ b/packages/xo-web/src/xo-app/backup-ng/restore/index.js @@ -202,24 +202,11 @@ export default class Restore extends Component { strongConfirm: { messageId: 'deleteVmBackupsBulkConfirmText', values: { - nBackups: reduce( - datas, - (sum, data) => - // TODO: [DELTA] remove filter - sum + data.backups.filter(b => b.mode !== 'delta').length, - 0 - ), + nBackups: reduce(datas, (sum, data) => sum + data.backups.length, 0), }, }, }) - .then( - () => - deleteBackups( - // TODO: [DELTA] remove filter - flatMap(datas, 'backups').filter(b => b.mode !== 'delta') - ), - noop - ) + .then(() => deleteBackups(flatMap(datas, 'backups')), noop) .then(() => this._refreshBackupList()) // --------------------------------------------------------------------------- diff --git a/yarn.lock b/yarn.lock index 880cdba85..1be65a345 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4735,9 +4735,9 @@ flat-cache@^1.2.1: graceful-fs "^4.1.2" write "^0.2.1" -flow-bin@^0.66.0: - version "0.66.0" - resolved "https://registry.yarnpkg.com/flow-bin/-/flow-bin-0.66.0.tgz#a96dde7015dc3343fd552a7b4963c02be705ca26" +flow-bin@^0.67.1: + version "0.67.1" + resolved "https://registry.yarnpkg.com/flow-bin/-/flow-bin-0.67.1.tgz#eabb7197cce870ac9442cfd04251c7ddc30377db" flush-write-stream@^1.0.2: version "1.0.2"