From aad4ebf287f2f80bc9e309cb79e438dd3beb4d2e Mon Sep 17 00:00:00 2001 From: Fabrice Marsaud Date: Mon, 7 Dec 2015 17:08:02 +0100 Subject: [PATCH 01/11] Remote handlers refactored, and adding a smb handler --- package.json | 2 + src/api/vm.coffee | 21 +++- src/remote-handler.js | 140 ------------------------- src/remote-handlers/abstract.js | 49 +++++++++ src/remote-handlers/local.js | 70 +++++++++++++ src/remote-handlers/nfs.js | 119 +++++++++++++++++++++ src/remote-handlers/smb.js | 134 ++++++++++++++++++++++++ src/xo-mixins/backups.js | 179 ++++++++++++++++---------------- src/xo-mixins/remotes.js | 45 ++++++-- 9 files changed, 517 insertions(+), 242 deletions(-) delete mode 100644 src/remote-handler.js create mode 100644 src/remote-handlers/abstract.js create mode 100644 src/remote-handlers/local.js create mode 100644 src/remote-handlers/nfs.js create mode 100644 src/remote-handlers/smb.js diff --git a/package.json b/package.json index 26cded281..f437ef516 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "node": ">=0.12 <5" }, "dependencies": { + "@marsaud/smb2-promise": "^0.1.0", "app-conf": "^0.4.0", "babel-runtime": "^5", "base64url": "^1.0.5", @@ -96,6 +97,7 @@ "lodash.sortby": "^3.1.4", "lodash.startswith": "^3.0.1", "loud-rejection": "^1.2.0", + "lodash.trim": "^3.0.1", "make-error": "^1", "micromatch": "^2.3.2", "minimist": "^1.2.0", diff --git a/src/api/vm.coffee b/src/api/vm.coffee index 9ab4e1d0e..459bd585d 100644 --- a/src/api/vm.coffee +++ b/src/api/vm.coffee @@ -493,6 +493,11 @@ exports.snapshot = snapshot #--------------------------------------------------------------------- rollingDeltaBackup = $coroutine ({vm, remote, tag, depth}) -> + _remote = yield @getRemote remote + if not _remote?.path? + throw new Error "No such Remote #{remote}" + if not _remote.enabled + throw new Error "Backup remote #{remote} is disabled" return yield @rollingDeltaVmBackup({ vm, remoteId: remote, @@ -572,12 +577,18 @@ exports.rollingSnapshot = rollingSnapshot #--------------------------------------------------------------------- -backup = $coroutine ({vm, pathToFile, compress, onlyMetadata}) -> - yield @backupVm({vm, pathToFile, compress, onlyMetadata}) +backup = $coroutine ({vm, remoteId, file, compress, onlyMetadata}) -> + remote = yield @getRemote remoteId + if not remote?.path? + throw new Error "No such Remote #{remoteId}" + if not remote.enabled + throw new Error "Backup remote #{remoteId} is disabled" + yield @backupVm({vm, remoteId, file, compress, onlyMetadata}) backup.params = { - id: { type: 'string' } - pathToFile: { type: 'string' } + id: {type: 'string'} + remoteId: { type: 'string' } + file: { type: 'string' } compress: { type: 'boolean', optional: true } onlyMetadata: { type: 'boolean', optional: true } } @@ -622,7 +633,7 @@ rollingBackup = $coroutine ({vm, remoteId, tag, depth, compress, onlyMetadata}) throw new Error "Backup remote #{remoteId} is disabled" return yield @rollingBackupVm({ vm, - path: remote.path, + remoteId, tag, depth, compress, diff --git a/src/remote-handler.js b/src/remote-handler.js deleted file mode 100644 index fe565020d..000000000 --- a/src/remote-handler.js +++ /dev/null @@ -1,140 +0,0 @@ -import filter from 'lodash.filter' -import fs from 'fs-promise' -import {exec} from 'child_process' - -import { - forEach, - noop, - promisify -} from './utils' - -const execAsync = promisify(exec) - -class NfsMounter { - async _loadRealMounts () { - let stdout - try { - [stdout] = await execAsync('findmnt -P -t nfs,nfs4 --output SOURCE,TARGET --noheadings') - } catch (exc) { - // When no mounts are found, the call pretends to fail... - } - const mounted = {} - if (stdout) { - const regex = /^SOURCE="([^:]*):(.*)" TARGET="(.*)"$/ - forEach(stdout.split('\n'), m => { - if (m) { - const match = regex.exec(m) - mounted[match[3]] = { - host: match[1], - share: match[2] - } - } - }) - } - this._realMounts = mounted - return mounted - } - - _fullPath (path) { - return path - } - - _matchesRealMount (mount) { - return this._fullPath(mount.path) in this._realMounts - } - - async _mount (mount) { - const path = this._fullPath(mount.path) - await fs.ensureDir(path) - return await execAsync(`mount -t nfs ${mount.host}:${mount.share} ${path}`) - } - - async forget (mount) { - try { - await this._umount(mount) - } catch (_) { - // We have to go on... - } - } - - async _umount (mount) { - const path = this._fullPath(mount.path) - await execAsync(`umount ${path}`) - } - - async sync (mount) { - await this._loadRealMounts() - if (this._matchesRealMount(mount) && !mount.enabled) { - try { - await this._umount(mount) - } catch (exc) { - mount.enabled = true - mount.error = exc.message - } - } else if (!this._matchesRealMount(mount) && mount.enabled) { - try { - await this._mount(mount) - } catch (exc) { - mount.enabled = false - mount.error = exc.message - } - } - return mount - } - - async disableAll (mounts) { - await this._loadRealMounts() - forEach(mounts, async mount => { - if (this._matchesRealMount(mount)) { - try { - await this._umount(mount) - } catch (_) { - // We have to go on... - } - } - }) - } -} - -class LocalHandler { - constructor () { - this.forget = noop - this.disableAll = noop - } - - async sync (local) { - if (local.enabled) { - try { - await fs.ensureDir(local.path) - await fs.access(local.path, fs.R_OK | fs.W_OK) - } catch (exc) { - local.enabled = false - local.error = exc.message - } - } - return local - } -} - -export default class RemoteHandler { - constructor () { - this.handlers = { - nfs: new NfsMounter(), - local: new LocalHandler() - } - } - - async sync (remote) { - return await this.handlers[remote.type].sync(remote) - } - - async forget (remote) { - return await this.handlers[remote.type].forget(remote) - } - - async disableAll (remotes) { - const promises = [] - forEach(['local', 'nfs'], type => promises.push(this.handlers[type].disableAll(filter(remotes, remote => remote.type === type)))) - await Promise.all(promises) - } -} diff --git a/src/remote-handlers/abstract.js b/src/remote-handlers/abstract.js new file mode 100644 index 000000000..e871bfa50 --- /dev/null +++ b/src/remote-handlers/abstract.js @@ -0,0 +1,49 @@ +export default class RemoteHandlerAbstract { + constructor (remote) { + this._remote = remote + } + + set (remote) { + this._remote = remote + } + + async sync () { + throw new Error('Not implemented') + } + + async forget () { + throw new Error('Not implemented') + } + + async outputFile (file, data, options) { + throw new Error('Not implemented') + } + + async readFile (file, options) { + throw new Error('Not implemented') + } + + async rename (oldPath, newPath) { + throw new Error('Not implemented') + } + + async list (dir = undefined) { + throw new Error('Not implemented') + } + + async createReadStream (file) { + throw new Error('Not implemented') + } + + async createOutputStream (file) { + throw new Error('Not implemented') + } + + async unlink (file) { + throw new Error('Not implemented') + } + + async getSize (file) { + throw new Error('Not implement') + } +} diff --git a/src/remote-handlers/local.js b/src/remote-handlers/local.js new file mode 100644 index 000000000..2dc08e919 --- /dev/null +++ b/src/remote-handlers/local.js @@ -0,0 +1,70 @@ +import fs from 'fs-promise' +import RemoteHandlerAbstract from './abstract' +import {dirname} from 'path' +import {noop} from '../utils' + +export default class LocalHandler extends RemoteHandlerAbstract { + constructor (remote) { + super(remote) + this.forget = noop + } + + _getFilePath (file) { + const parts = [this._remote.path] + if (file) { + parts.push(file) + } + return parts.join('/') + } + + async sync () { + if (this._remote.enabled) { + try { + await fs.ensureDir(this._remote.path) + await fs.access(this._remote.path, fs.R_OK | fs.W_OK) + } catch (exc) { + this._remote.enabled = false + this._remote.error = exc.message + } + } + return this._remote + } + + async outputFile (file, data, options) { + const path = this._getFilePath(file) + await fs.ensureDir(dirname(path)) + await fs.writeFile(path, data, options) + } + + async readFile (file, options) { + return await fs.readFile(this._getFilePath(file), options) + } + + async rename (oldPath, newPath) { + return await fs.rename(this._getFilePath(oldPath), this._getFilePath(newPath)) + } + + async list (dir = undefined) { + return await fs.readdir(this._getFilePath(dir)) + } + + async createReadStream (file) { + return fs.createReadStream(this._getFilePath(file)) + } + + async createOutputStream (file, options) { + const path = this._getFilePath(file) + await fs.ensureDir(dirname(path)) + return fs.createWriteStream(path, options) + } + + async unlink (file) { + return fs.unlink(this._getFilePath(file)) + } + + async getSize (file) { + const stats = await fs.stat(this._getFilePath(file)) + return stats.size + } + +} diff --git a/src/remote-handlers/nfs.js b/src/remote-handlers/nfs.js new file mode 100644 index 000000000..c4b8d309d --- /dev/null +++ b/src/remote-handlers/nfs.js @@ -0,0 +1,119 @@ +import fs from 'fs-promise' +import RemoteHandlerAbstract from './abstract' +import {dirname} from 'path' +import {exec} from 'child_process' +import {forEach, promisify} from '../utils' + +const execAsync = promisify(exec) + +export default class NfsHandler extends RemoteHandlerAbstract { + _getFilePath (file) { + const parts = [this._remote.path] + if (file) { + parts.push(file) + } + return parts.join('/') + } + + async _loadRealMounts () { + let stdout + try { + [stdout] = await execAsync('findmnt -P -t nfs,nfs4 --output SOURCE,TARGET --noheadings') + } catch (exc) { + // When no mounts are found, the call pretends to fail... + } + const mounted = {} + if (stdout) { + const regex = /^SOURCE="([^:]*):(.*)" TARGET="(.*)"$/ + forEach(stdout.split('\n'), m => { + if (m) { + const match = regex.exec(m) + mounted[match[3]] = { + host: match[1], + share: match[2] + } + } + }) + } + this._realMounts = mounted + return mounted + } + + _matchesRealMount (remote) { + return remote.path in this._realMounts + } + + async _mount (remote) { + await fs.ensureDir(remote.path) + return await execAsync(`mount -t nfs ${remote.host}:${remote.share} ${remote.path}`) + } + + async sync () { + await this._loadRealMounts() + if (this._matchesRealMount(this._remote) && !this._remote.enabled) { + try { + await this._umount(this._remote) + } catch (exc) { + this._remote.enabled = true + this._remote.error = exc.message + } + } else if (!this._matchesRealMount(this._remote) && this._remote.enabled) { + try { + await this._mount(this._remote) + } catch (exc) { + this._remote.enabled = false + this._remote.error = exc.message + } + } + return this._remote + } + + async forget () { + try { + await this._umount(this._remote) + } catch (_) { + // We have to go on... + } + } + + async _umount (remote) { + await execAsync(`umount ${remote.path}`) + } + + async outputFile (file, data, options) { + const path = this._getFilePath(file) + await fs.ensureDir(dirname(path)) + await fs.writeFile(path, data, options) + } + + async readFile (file, options) { + return await fs.readFile(this._getFilePath(file), options) + } + + async rename (oldPath, newPath) { + return await fs.rename(this._getFilePath(oldPath), this._getFilePath(newPath)) + } + + async list (dir = undefined) { + return await fs.readdir(this._getFilePath(dir)) + } + + async createReadStream (file) { + return fs.createReadStream(this._getFilePath(file)) + } + + async createOutputStream (file, options) { + const path = this._getFilePath(file) + await fs.ensureDir(dirname(path)) + return fs.createWriteStream(path, options) + } + + async unlink (file) { + return fs.unlink(this._getFilePath(file)) + } + + async getSize (file) { + const stats = await fs.stat(this._getFilePath(file)) + return stats.size + } +} diff --git a/src/remote-handlers/smb.js b/src/remote-handlers/smb.js new file mode 100644 index 000000000..e6f8a6918 --- /dev/null +++ b/src/remote-handlers/smb.js @@ -0,0 +1,134 @@ +import Smb2 from '@marsaud/smb2-promise' +import {noop} from '../utils' +import RemoteHandlerAbstract from './abstract' + +export default class SmbHandler extends RemoteHandlerAbstract { + constructor (remote) { + super(remote) + this.forget = noop + } + + _getClient (remote) { + return new Smb2({ + share: `\\\\${remote.host}`, + domain: remote.domain, + username: remote.username, + password: remote.password, + autoCloseTimeout: 0 + }) + } + + _getFilePath (file) { + const parts = [] + if (this._remote.path !== '') { + parts.push(this._remote.path) + } + if (file) { + parts.push(file.split('/')) + } + return parts.join('\\') + } + + _getDirname (file) { + const parts = file.split('\\') + parts.pop() + return parts.join('\\') + } + + async sync () { + if (this._remote.enabled) { + try { + // Check access (smb2 does not expose connect in public so far...) + await this.list() + } catch (error) { + this._remote.enabled = false + this._remote.error = error.message + } + } + return this._remote + } + + async outputFile (file, data, options) { + const client = this._getClient(this._remote) + const path = this._getFilePath(file) + const dir = this._getDirname(path) + try { + if (dir) { + await client.ensureDir(dir) + } + return await client.writeFile(path, data, options) + } finally { + client.close() + } + } + + async readFile (file, options) { + const client = this._getClient(this._remote) + try { + return await client.readFile(this._getFilePath(file), options) + } finally { + client.close() + } + } + + async rename (oldPath, newPath) { + const client = this._getClient(this._remote) + try { + return await client.rename(this._getFilePath(oldPath), this._getFilePath(newPath)) + } finally { + client.close() + } + } + + async list (dir = undefined) { + const client = this._getClient(this._remote) + try { + return await client.readdir(this._getFilePath(dir)) + } finally { + client.close() + } + } + + async createReadStream (file) { + const client = this._getClient(this._remote) + const stream = await client.createReadStream(this._getFilePath(file)) + stream.on('end', () => client.close()) + return stream + } + + async createOutputStream (file, options) { + const client = this._getClient(this._remote) + const path = this._getFilePath(file) + const dir = this._getDirname(path) + let stream + try { + if (dir) { + await client.ensureDir(dir) + } + stream = await client.createWriteStream(path, options/* , { flags: 'wx' }*/) // TODO ensure that wx flag is properly handled by @marsaud/smb2 + } catch (err) { + client.close() + throw err + } + stream.on('finish', () => client.close()) + return stream + } + + async unlink (file) { + const client = this._getClient(this._remote) + try { + return await client.unlink(this._getFilePath(file)) + } finally { + client.close() + } + } + + async getSize (file) { + const client = await this._getClient(this._remote) + try { + return await client.getSize(this._getFilePath(file)) + } finally { + client.close() + } + } +} diff --git a/src/xo-mixins/backups.js b/src/xo-mixins/backups.js index 6c17d4c48..f98b1f56f 100644 --- a/src/xo-mixins/backups.js +++ b/src/xo-mixins/backups.js @@ -6,17 +6,6 @@ import filter from 'lodash.filter' import findIndex from 'lodash.findindex' import sortBy from 'lodash.sortby' import startsWith from 'lodash.startswith' -import { - createReadStream, - createWriteStream, - ensureDir, - readdir, - readFile, - rename, - stat, - unlink, - writeFile -} from 'fs-promise' import { basename, dirname @@ -60,19 +49,19 @@ export default class { async listRemoteBackups (remoteId) { const remote = await this._xo.getRemote(remoteId) - const path = remote.path + const handler = this._xo.getRemoteHandler(remote) // List backups. (Except delta backups) const xvaFilter = file => endsWith(file, '.xva') - const files = await readdir(path) + const files = await handler.list() const backups = filter(files, xvaFilter) // List delta backups. const deltaDirs = filter(files, file => startsWith(file, 'vm_delta_')) for (const deltaDir of deltaDirs) { - const files = await readdir(`${path}/${deltaDir}`) + const files = await handler.list(deltaDir) const deltaBackups = filter(files, xvaFilter) backups.push(...mapToArray( @@ -84,11 +73,12 @@ export default class { return backups } - // TODO: move into utils and rename! - async _openAndwaitReadableFile (path, errorMessage) { - const stream = createReadStream(path) - + // TODO: move into utils and rename! NO, until we may pass a handler instead of a remote...? + async _openAndwaitReadableFile (remote, file, errorMessage) { + const handler = this._xo.getRemoteHandler(remote) + let stream try { + stream = await handler.createReadStream(file) await eventToPromise(stream, 'readable') } catch (error) { if (error.code === 'ENOENT') { @@ -97,16 +87,15 @@ export default class { throw error } - stream.length = (await stat(path)).size - + stream.length = await handler.getSize(file) return stream } async importVmBackup (remoteId, file, sr) { const remote = await this._xo.getRemote(remoteId) - const path = `${remote.path}/${file}` const stream = await this._openAndwaitReadableFile( - path, + remote, + file, 'VM to import not found in this remote' ) @@ -179,39 +168,50 @@ export default class { // TODO: The other backup methods must use this function ! // Prerequisite: The backups array must be ordered. (old to new backups) - async _removeOldBackups (backups, path, n) { + async _removeOldBackups (backups, remote, dir, n) { if (n <= 0) { return } + const handler = this._xo.getRemoteHandler(remote) + const getPath = (file, dir) => dir ? `${dir}/${file}` : file await Promise.all( - mapToArray(backups.slice(0, n), backup => unlink(`${path}/${backup}`)) + mapToArray(backups.slice(0, n), async backup => await handler.unlink(getPath(backup, dir))) ) } // ----------------------------------------------------------------- - async _listVdiBackups (path) { - const files = await readdir(path) + async _listVdiBackups (remote, dir) { + const handler = this._xo.getRemoteHandler(remote) + let files + try { + files = await handler.list(dir) + } catch (error) { + if (error.code === 'ENOENT') { + files = [] + } else { + throw error + } + } const backups = sortBy(filter(files, fileName => isVdiBackup(fileName))) let i // Avoid unstable state: No full vdi found to the beginning of array. (base) for (i = 0; i < backups.length && isDeltaVdiBackup(backups[i]); i++); - await this._removeOldBackups(backups, path, i) + await this._removeOldBackups(backups, remote, dir, i) return backups.slice(i) } - async _deltaVdiBackup ({vdi, path, depth}) { + async _deltaVdiBackup ({vdi, remote, dir, depth}) { const xapi = this._xo.getXapi(vdi) const backupDirectory = `vdi_${vdi.uuid}` vdi = xapi.getObject(vdi._xapiId) - path = `${path}/${backupDirectory}` - await ensureDir(path) + dir = `${dir}/${backupDirectory}` - const backups = await this._listVdiBackups(path) + const backups = await this._listVdiBackups(remote, dir) // Make snapshot. const date = safeDateFormat(new Date()) @@ -234,15 +234,16 @@ export default class { // Export full or delta backup. const vdiFilename = `${date}_${isFull ? 'full' : 'delta'}.vhd` - const backupFullPath = `${path}/${vdiFilename}` + const backupFullPath = `${dir}/${vdiFilename}` + const handler = this._xo.getRemoteHandler(remote) try { const sourceStream = await xapi.exportVdi(currentSnapshot.$id, { baseId: isFull ? undefined : base.$id, format: VDI_FORMAT_VHD }) - const targetStream = createWriteStream(backupFullPath, { flags: 'wx' }) + const targetStream = await handler.createOutputStream(backupFullPath, { flags: 'wx' }) sourceStream.on('error', error => targetStream.emit('error', error)) await Promise.all([ @@ -252,8 +253,7 @@ export default class { } catch (error) { // Remove new backup. (corrupt) and delete new vdi base. xapi.deleteVdi(currentSnapshot.$id).catch(noop) - await unlink(backupFullPath).catch(noop) - + await handler.unlink(backupFullPath).catch(noop) throw error } @@ -266,8 +266,8 @@ export default class { } } - async _mergeDeltaVdiBackups ({path, depth}) { - const backups = await this._listVdiBackups(path) + async _mergeDeltaVdiBackups ({remote, dir, depth}) { + const backups = await this._listVdiBackups(remote, dir) let i = backups.length - depth // No merge. @@ -278,37 +278,39 @@ export default class { const newFull = `${getVdiTimestamp(backups[i])}_full.vhd` const vhdUtil = `${__dirname}/../../bin/vhd-util` + const handler = this._xo.getRemoteHandler(remote) for (; i > 0 && isDeltaVdiBackup(backups[i]); i--) { - const backup = `${path}/${backups[i]}` - const parent = `${path}/${backups[i - 1]}` + const backup = `${dir}/${backups[i]}` + const parent = `${dir}/${backups[i - 1]}` try { - await execa(vhdUtil, ['modify', '-n', backup, '-p', parent]) - await execa(vhdUtil, ['coalesce', '-n', backup]) + await execa(vhdUtil, ['modify', '-n', `${remote.path}/${backup}`, '-p', `${remote.path}/${parent}`]) // FIXME not ok at least with smb remotes + await execa(vhdUtil, ['coalesce', '-n', `${remote.path}/${backup}`]) // FIXME not ok at least with smb remotes } catch (e) { console.error('Unable to use vhd-util.', e) throw e } - await unlink(backup) + await handler.unlink(backup) } // The base was removed, it exists two full backups or more ? // => Remove old backups before the most recent full. if (i > 0) { for (i--; i >= 0; i--) { - await unlink(`${path}/${backups[i]}`) + await remote.unlink(`${dir}/${backups[i]}`) } return } // Rename the first old full backup to the new full backup. - await rename(`${path}/${backups[0]}`, `${path}/${newFull}`) + await handler.rename(`${dir}/${backups[0]}`, `${dir}/${newFull}`) } - async _importVdiBackupContent (xapi, file, vdiId) { + async _importVdiBackupContent (xapi, remote, file, vdiId) { const stream = await this._openAndwaitReadableFile( + remote, file, 'VDI to import not found in this remote' ) @@ -320,10 +322,10 @@ export default class { async importDeltaVdiBackup ({vdi, remoteId, filePath}) { const remote = await this._xo.getRemote(remoteId) - const path = dirname(`${remote.path}/${filePath}`) + const dir = dirname(filePath) const filename = basename(filePath) - const backups = await this._listVdiBackups(path) + const backups = await this._listVdiBackups(remote, dir) // Search file. (delta or full backup) const i = findIndex(backups, backup => @@ -347,34 +349,33 @@ export default class { const xapi = this._xo.getXapi(vdi) for (; j <= i; j++) { - await this._importVdiBackupContent(xapi, `${path}/${backups[j]}`, vdi._xapiId) + await this._importVdiBackupContent(xapi, remote, `${dir}/${backups[j]}`, vdi._xapiId) } } // ----------------------------------------------------------------- - async _listDeltaVmBackups (path) { - const files = await readdir(path) + async _listDeltaVmBackups (remote, dir) { + const handler = this._xo.getRemoteHandler(remote) + const files = await handler.list(dir) return await sortBy(filter(files, (fileName) => /^\d+T\d+Z_.*\.(?:xva|json)$/.test(fileName))) } - async _failedRollingDeltaVmBackup (xapi, path, fulFilledVdiBackups) { + async _failedRollingDeltaVmBackup (xapi, remote, dir, fulFilledVdiBackups) { + const handler = this._xo.getRemoteHandler(remote) await Promise.all( mapToArray(fulFilledVdiBackups, async vdiBackup => { const { newBaseId, backupDirectory, vdiFilename } = vdiBackup.value() await xapi.deleteVdi(newBaseId) - await unlink(`${path}/${backupDirectory}/${vdiFilename}`).catch(noop) + await handler.unlink(`${dir}/${backupDirectory}/${vdiFilename}`).catch(noop) }) ) } async rollingDeltaVmBackup ({vm, remoteId, tag, depth}) { const remote = await this._xo.getRemote(remoteId) - const directory = `vm_delta_${tag}_${vm.uuid}` - const path = `${remote.path}/${directory}` - - await ensureDir(path) + const dir = `vm_delta_${tag}_${vm.uuid}` const info = { vbds: [], @@ -408,7 +409,7 @@ export default class { if (!info.vdis[vdiUUID]) { info.vdis[vdiUUID] = { ...vdi } promises.push( - this._deltaVdiBackup({vdi: vdiXo, path, depth}).then( + this._deltaVdiBackup({remote, vdi: vdiXo, dir, depth}).then( vdiBackup => { const { backupDirectory, vdiFilename } = vdiBackup info.vdis[vdiUUID].xoPath = `${backupDirectory}/${vdiFilename}` @@ -435,29 +436,31 @@ export default class { } if (fail) { - console.error(`Remove successful backups in ${path}`, fulFilledVdiBackups) - await this._failedRollingDeltaVmBackup(xapi, path, fulFilledVdiBackups) + console.error(`Remove successful backups in ${remote.path}/${dir}`, fulFilledVdiBackups) + await this._failedRollingDeltaVmBackup(xapi, remote, dir, fulFilledVdiBackups) throw new Error('Rolling delta vm backup failed.') } - const backups = await this._listDeltaVmBackups(path) + const backups = await this._listDeltaVmBackups(remote, dir) const date = safeDateFormat(new Date()) const backupFormat = `${date}_${vm.name_label}` - const xvaPath = `${path}/${backupFormat}.xva` - const infoPath = `${path}/${backupFormat}.json` + const xvaPath = `${dir}/${backupFormat}.xva` + const infoPath = `${dir}/${backupFormat}.json` + + const handler = this._xo.getRemoteHandler(remote) try { await Promise.all([ - this.backupVm({vm, pathToFile: xvaPath, onlyMetadata: true}), - writeFile(infoPath, JSON.stringify(info), {flag: 'wx'}) + this.backupVm({vm, remoteId, file: xvaPath, onlyMetadata: true}), + handler.outputFile(infoPath, JSON.stringify(info), {flag: 'wx'}) ]) } catch (e) { await Promise.all([ - unlink(xvaPath).catch(noop), - unlink(infoPath).catch(noop), - this._failedRollingDeltaVmBackup(xapi, path, fulFilledVdiBackups) + handler.unlink(xvaPath).catch(noop), + handler.unlink(infoPath).catch(noop), + this._failedRollingDeltaVmBackup(xapi, remote, dir, fulFilledVdiBackups) ]) throw e @@ -467,12 +470,12 @@ export default class { await Promise.all( mapToArray(vdiBackups, vdiBackup => { const { backupDirectory } = vdiBackup.value() - return this._mergeDeltaVdiBackups({path: `${path}/${backupDirectory}`, depth}) + return this._mergeDeltaVdiBackups({remote, dir: `${dir}/${backupDirectory}`, depth}) }) ) // Remove x2 files : json AND xva files. - await this._removeOldBackups(backups, path, backups.length - (depth - 1) * 2) + await this._removeOldBackups(backups, remote, dir, backups.length - (depth - 1) * 2) // Remove old vdi bases. Promise.all( @@ -486,11 +489,12 @@ export default class { ).catch(noop) // Returns relative path. - return `${directory}/${backupFormat}` + return `${dir}/${backupFormat}` } - async _importVmMetadata (xapi, file) { + async _importVmMetadata (xapi, remote, file) { const stream = await this._openAndwaitReadableFile( + remote, file, 'VM metadata to import not found in this remote' ) @@ -512,11 +516,10 @@ export default class { async importDeltaVmBackup ({sr, remoteId, filePath}) { const remote = await this._xo.getRemote(remoteId) - const fullBackupPath = `${remote.path}/${filePath}` const xapi = this._xo.getXapi(sr) // Import vm metadata. - const vm = await this._importVmMetadata(xapi, `${fullBackupPath}.xva`) + const vm = await this._importVmMetadata(xapi, remote, `${filePath}.xva`) const vmName = vm.name_label // Disable start and change the VM name label during import. @@ -529,7 +532,8 @@ export default class { // Because XenServer creates Vbds linked to the vdis of the backup vm if it exists. await xapi.destroyVbdsFromVm(vm.uuid) - const info = JSON.parse(await readFile(`${fullBackupPath}.json`)) + const handler = this._xo.getRemoteHandler(remote) + const info = JSON.parse(await handler.readFile(`${filePath}.json`)) // Import VDIs. const vdiIds = {} @@ -565,8 +569,10 @@ export default class { // ----------------------------------------------------------------- - async backupVm ({vm, pathToFile, compress, onlyMetadata}) { - const targetStream = createWriteStream(pathToFile, { flags: 'wx' }) + async backupVm ({vm, remoteId, file, compress, onlyMetadata}) { + const remote = await this._xo.getRemote(remoteId) + const handler = this._xo.getRemoteHandler(remote) + const targetStream = await handler.createOutputStream(file, { flags: 'wx' }) const promise = eventToPromise(targetStream, 'finish') const sourceStream = await this._xo.getXapi(vm).exportVm(vm._xapiId, { @@ -578,26 +584,19 @@ export default class { await promise } - async rollingBackupVm ({vm, path, tag, depth, compress, onlyMetadata}) { - await ensureDir(path) - const files = await readdir(path) + async rollingBackupVm ({vm, remoteId, tag, depth, compress, onlyMetadata}) { + const remote = await this._xo.getRemote(remoteId) + const handler = this._xo.getRemoteHandler(remote) + const files = await handler.list() const reg = new RegExp('^[^_]+_' + escapeStringRegexp(`${tag}_${vm.name_label}.xva`)) const backups = sortBy(filter(files, (fileName) => reg.test(fileName))) const date = safeDateFormat(new Date()) - const backupFullPath = `${path}/${date}_${tag}_${vm.name_label}.xva` + const file = `${date}_${tag}_${vm.name_label}.xva` - await this.backupVm({vm, pathToFile: backupFullPath, compress, onlyMetadata}) - - const promises = [] - for (let surplus = backups.length - (depth - 1); surplus > 0; surplus--) { - const oldBackup = backups.shift() - promises.push(unlink(`${path}/${oldBackup}`)) - } - await Promise.all(promises) - - return backupFullPath + await this.backupVm({vm, remoteId, file, compress, onlyMetadata}) + await this._removeOldBackups(backups, remote, undefined, backups.length - (depth - 1)) } async rollingSnapshotVm (vm, tag, depth) { diff --git a/src/xo-mixins/remotes.js b/src/xo-mixins/remotes.js index 1f7f344fa..7178ef0db 100644 --- a/src/xo-mixins/remotes.js +++ b/src/xo-mixins/remotes.js @@ -1,6 +1,8 @@ import startsWith from 'lodash.startswith' -import RemoteHandler from '../remote-handler' +import RemoteHandlerLocal from '../remote-handlers/local' +import RemoteHandlerNfs from '../remote-handlers/nfs' +import RemoteHandlerSmb from '../remote-handlers/smb' import { Remotes } from '../models/remote' import { NoSuchObject @@ -30,12 +32,26 @@ export default class { xo.on('start', async () => { // TODO: Should it be private? - this.remoteHandler = new RemoteHandler() + this._remoteHandlers = {} await this.initRemotes() await this.syncAllRemotes() }) - xo.on('stop', () => this.disableAllRemotes()) + xo.on('stop', () => this.forgetAllRemotes()) + } + + getRemoteHandler (remote) { + if (!(remote.id in this._remoteHandlers)) { + const handlers = { + 'local': RemoteHandlerLocal, + 'nfs': RemoteHandlerNfs, + 'smb': RemoteHandlerSmb + } + this._remoteHandlers[remote.id] = new handlers[remote.type](remote) + } + const handler = this._remoteHandlers[remote.id] + handler.set(remote) + return handler } _developRemote (remote) { @@ -50,6 +66,18 @@ export default class { _remote.path = '/tmp/xo-server/mounts/' + _remote.id _remote.host = host _remote.share = share + } else if (startsWith(_remote.url, 'smb://')) { + _remote.type = 'smb' + const url = _remote.url.slice(6) + const [auth, smb] = url.split('@') + const [username, password] = auth.split(':') + const [domain, sh] = smb.split('\\\\') + const [host, path] = sh.split('\0') + _remote.host = host + _remote.path = path + _remote.domain = domain + _remote.username = username + _remote.password = password } return _remote } @@ -79,7 +107,8 @@ export default class { async updateRemote (id, {name, url, enabled, error}) { const remote = await this._getRemote(id) this._updateRemote(remote, {name, url, enabled, error}) - const props = await this.remoteHandler.sync(this._developRemote(remote.properties)) + const r = this._developRemote(remote.properties) + const props = await this.getRemoteHandler(r).sync() this._updateRemote(remote, props) return await this._developRemote(this._remotes.save(remote).properties) } @@ -97,7 +126,7 @@ export default class { async removeRemote (id) { const remote = await this.getRemote(id) - await this.remoteHandler.forget(remote) + await this.getRemoteHandler(remote).forget() await this._remotes.remove(id) } @@ -110,9 +139,11 @@ export default class { } // TODO: Should it be private? - async disableAllRemotes () { + async forgetAllRemotes () { const remotes = await this.getAllRemotes() - this.remoteHandler.disableAll(remotes) + for (let remote of remotes) { + await this.getRemoteHandler(remote).forget() + } } // TODO: Should it be private? From c304d9cc62c28fee7215adfd2ca4554c68163044 Mon Sep 17 00:00:00 2001 From: Fabrice Marsaud Date: Wed, 20 Jan 2016 09:36:38 +0100 Subject: [PATCH 02/11] No vdi merge through smb --- src/api/vm.coffee | 2 ++ src/xo-mixins/backups.js | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/api/vm.coffee b/src/api/vm.coffee index 459bd585d..6a05aa3d5 100644 --- a/src/api/vm.coffee +++ b/src/api/vm.coffee @@ -498,6 +498,8 @@ rollingDeltaBackup = $coroutine ({vm, remote, tag, depth}) -> throw new Error "No such Remote #{remote}" if not _remote.enabled throw new Error "Backup remote #{remote} is disabled" + if _remote.type == 'smb' + throw new Error "Delta Backup is not supported for smb remotes" return yield @rollingDeltaVmBackup({ vm, remoteId: remote, diff --git a/src/xo-mixins/backups.js b/src/xo-mixins/backups.js index f98b1f56f..93a7a9d7f 100644 --- a/src/xo-mixins/backups.js +++ b/src/xo-mixins/backups.js @@ -267,6 +267,9 @@ export default class { } async _mergeDeltaVdiBackups ({remote, dir, depth}) { + if (remote.type === 'smb') { + throw new Error('VDI merging is not available through SMB') + } const backups = await this._listVdiBackups(remote, dir) let i = backups.length - depth From e8380b8a12dc6119aa57addaecda6cd6ce0f753b Mon Sep 17 00:00:00 2001 From: Fabrice Marsaud Date: Fri, 22 Jan 2016 15:37:23 +0100 Subject: [PATCH 03/11] PR feedback --- src/api/vm.coffee | 12 ------- src/remote-handlers/abstract.js | 62 ++++++++++++++++++++++++++++++--- src/remote-handlers/local.js | 48 ++++++++++++------------- src/remote-handlers/nfs.js | 54 +++------------------------- src/remote-handlers/smb.js | 33 ++++++++++-------- src/xo-mixins/backups.js | 19 ++++++++++ 6 files changed, 122 insertions(+), 106 deletions(-) diff --git a/src/api/vm.coffee b/src/api/vm.coffee index 6a05aa3d5..5485bda1c 100644 --- a/src/api/vm.coffee +++ b/src/api/vm.coffee @@ -493,13 +493,6 @@ exports.snapshot = snapshot #--------------------------------------------------------------------- rollingDeltaBackup = $coroutine ({vm, remote, tag, depth}) -> - _remote = yield @getRemote remote - if not _remote?.path? - throw new Error "No such Remote #{remote}" - if not _remote.enabled - throw new Error "Backup remote #{remote} is disabled" - if _remote.type == 'smb' - throw new Error "Delta Backup is not supported for smb remotes" return yield @rollingDeltaVmBackup({ vm, remoteId: remote, @@ -628,11 +621,6 @@ exports.importBackup = importBackup #--------------------------------------------------------------------- rollingBackup = $coroutine ({vm, remoteId, tag, depth, compress, onlyMetadata}) -> - remote = yield @getRemote remoteId - if not remote?.path? - throw new Error "No such Remote #{remoteId}" - if not remote.enabled - throw new Error "Backup remote #{remoteId} is disabled" return yield @rollingBackupVm({ vm, remoteId, diff --git a/src/remote-handlers/abstract.js b/src/remote-handlers/abstract.js index e871bfa50..95f292bfd 100644 --- a/src/remote-handlers/abstract.js +++ b/src/remote-handlers/abstract.js @@ -1,3 +1,5 @@ +import eventToPromise from 'event-to-promise' + export default class RemoteHandlerAbstract { constructor (remote) { this._remote = remote @@ -8,42 +10,92 @@ export default class RemoteHandlerAbstract { } async sync () { + return await this._sync() + } + + async _sync () { throw new Error('Not implemented') } async forget () { + return await this._forget() + } + + async _forget () { throw new Error('Not implemented') } async outputFile (file, data, options) { - throw new Error('Not implemented') + return await this._outputFile(file, data, options) + } + + async _outputFile (file, data, options) { + const stream = this.createOutputStream(file) + const promise = eventToPromise(stream, 'finish') + stream.end(data) + return promise } async readFile (file, options) { - throw new Error('Not implemented') + return await this._readFile(file, options) + } + + async _readFile (file, options) { + const stream = this.createReadStream(file, options) + let data = '' + stream.on('data', d => data += d) + await eventToPromise(stream, 'end') + return data } async rename (oldPath, newPath) { + return await this._rename(oldPath, newPath) + } + + async _rename (oldPath, newPath) { throw new Error('Not implemented') } - async list (dir = undefined) { + async list (dir = '.') { + return await this._list(dir) + } + + async _list (dir = '.') { throw new Error('Not implemented') } - async createReadStream (file) { + async createReadStream (file, options) { + const length = await this.getSize(file) + const stream = await this._createReadStream(file) + stream.length = length + return stream + } + + async _createReadStream (file, options) { throw new Error('Not implemented') } - async createOutputStream (file) { + async createOutputStream (file, options) { + return await this._createOutputStream(file) + } + + async _createOutputStream (file, options) { throw new Error('Not implemented') } async unlink (file) { + return await this._unlink(file) + } + + async _unlink (file) { throw new Error('Not implemented') } async getSize (file) { + return await this._getSize(file) + } + + async _getSize (file) { throw new Error('Not implement') } } diff --git a/src/remote-handlers/local.js b/src/remote-handlers/local.js index 2dc08e919..0c2c8530a 100644 --- a/src/remote-handlers/local.js +++ b/src/remote-handlers/local.js @@ -1,23 +1,23 @@ import fs from 'fs-promise' import RemoteHandlerAbstract from './abstract' -import {dirname} from 'path' +import startsWith from 'lodash.startswith' import {noop} from '../utils' +import {resolve} from 'path' export default class LocalHandler extends RemoteHandlerAbstract { - constructor (remote) { - super(remote) - this.forget = noop - } - _getFilePath (file) { const parts = [this._remote.path] if (file) { parts.push(file) } - return parts.join('/') + const path = resolve.apply(null, parts) + if (!startsWith(path, this._remote.path)) { + throw new Error('Remote path is unavailable') + } + return path } - async sync () { + async _sync () { if (this._remote.enabled) { try { await fs.ensureDir(this._remote.path) @@ -30,39 +30,39 @@ export default class LocalHandler extends RemoteHandlerAbstract { return this._remote } - async outputFile (file, data, options) { - const path = this._getFilePath(file) - await fs.ensureDir(dirname(path)) - await fs.writeFile(path, data, options) + async _forget () { + return noop() } - async readFile (file, options) { + async _outputFile (file, data, options) { + await fs.outputFile(this._getFilePath(file), data, options) + } + + async _readFile (file, options) { return await fs.readFile(this._getFilePath(file), options) } - async rename (oldPath, newPath) { + async _rename (oldPath, newPath) { return await fs.rename(this._getFilePath(oldPath), this._getFilePath(newPath)) } - async list (dir = undefined) { + async _list (dir = '.') { return await fs.readdir(this._getFilePath(dir)) } - async createReadStream (file) { - return fs.createReadStream(this._getFilePath(file)) + async _createReadStream (file, options) { + return await fs.createReadStream(this._getFilePath(file), options) } - async createOutputStream (file, options) { - const path = this._getFilePath(file) - await fs.ensureDir(dirname(path)) - return fs.createWriteStream(path, options) + async _createOutputStream (file, options) { + return await fs.createOutputStream(this._getFilePath(file), options) } - async unlink (file) { - return fs.unlink(this._getFilePath(file)) + async _unlink (file) { + return await fs.unlink(this._getFilePath(file)) } - async getSize (file) { + async _getSize (file) { const stats = await fs.stat(this._getFilePath(file)) return stats.size } diff --git a/src/remote-handlers/nfs.js b/src/remote-handlers/nfs.js index c4b8d309d..b7f882eb2 100644 --- a/src/remote-handlers/nfs.js +++ b/src/remote-handlers/nfs.js @@ -1,20 +1,11 @@ import fs from 'fs-promise' -import RemoteHandlerAbstract from './abstract' -import {dirname} from 'path' +import LocalHandler from './local' import {exec} from 'child_process' import {forEach, promisify} from '../utils' const execAsync = promisify(exec) -export default class NfsHandler extends RemoteHandlerAbstract { - _getFilePath (file) { - const parts = [this._remote.path] - if (file) { - parts.push(file) - } - return parts.join('/') - } - +export default class NfsHandler extends LocalHandler { async _loadRealMounts () { let stdout try { @@ -48,7 +39,7 @@ export default class NfsHandler extends RemoteHandlerAbstract { return await execAsync(`mount -t nfs ${remote.host}:${remote.share} ${remote.path}`) } - async sync () { + async _sync () { await this._loadRealMounts() if (this._matchesRealMount(this._remote) && !this._remote.enabled) { try { @@ -68,7 +59,7 @@ export default class NfsHandler extends RemoteHandlerAbstract { return this._remote } - async forget () { + async _forget () { try { await this._umount(this._remote) } catch (_) { @@ -79,41 +70,4 @@ export default class NfsHandler extends RemoteHandlerAbstract { async _umount (remote) { await execAsync(`umount ${remote.path}`) } - - async outputFile (file, data, options) { - const path = this._getFilePath(file) - await fs.ensureDir(dirname(path)) - await fs.writeFile(path, data, options) - } - - async readFile (file, options) { - return await fs.readFile(this._getFilePath(file), options) - } - - async rename (oldPath, newPath) { - return await fs.rename(this._getFilePath(oldPath), this._getFilePath(newPath)) - } - - async list (dir = undefined) { - return await fs.readdir(this._getFilePath(dir)) - } - - async createReadStream (file) { - return fs.createReadStream(this._getFilePath(file)) - } - - async createOutputStream (file, options) { - const path = this._getFilePath(file) - await fs.ensureDir(dirname(path)) - return fs.createWriteStream(path, options) - } - - async unlink (file) { - return fs.unlink(this._getFilePath(file)) - } - - async getSize (file) { - const stats = await fs.stat(this._getFilePath(file)) - return stats.size - } } diff --git a/src/remote-handlers/smb.js b/src/remote-handlers/smb.js index e6f8a6918..8eb74420e 100644 --- a/src/remote-handlers/smb.js +++ b/src/remote-handlers/smb.js @@ -5,7 +5,7 @@ import RemoteHandlerAbstract from './abstract' export default class SmbHandler extends RemoteHandlerAbstract { constructor (remote) { super(remote) - this.forget = noop + this._forget = noop } _getClient (remote) { @@ -19,6 +19,9 @@ export default class SmbHandler extends RemoteHandlerAbstract { } _getFilePath (file) { + if (file === '.') { + file = undefined + } const parts = [] if (this._remote.path !== '') { parts.push(this._remote.path) @@ -29,13 +32,13 @@ export default class SmbHandler extends RemoteHandlerAbstract { return parts.join('\\') } - _getDirname (file) { + _dirname (file) { const parts = file.split('\\') parts.pop() return parts.join('\\') } - async sync () { + async _sync () { if (this._remote.enabled) { try { // Check access (smb2 does not expose connect in public so far...) @@ -48,10 +51,10 @@ export default class SmbHandler extends RemoteHandlerAbstract { return this._remote } - async outputFile (file, data, options) { + async _outputFile (file, data, options) { const client = this._getClient(this._remote) const path = this._getFilePath(file) - const dir = this._getDirname(path) + const dir = this._dirname(path) try { if (dir) { await client.ensureDir(dir) @@ -62,7 +65,7 @@ export default class SmbHandler extends RemoteHandlerAbstract { } } - async readFile (file, options) { + async _readFile (file, options) { const client = this._getClient(this._remote) try { return await client.readFile(this._getFilePath(file), options) @@ -71,7 +74,7 @@ export default class SmbHandler extends RemoteHandlerAbstract { } } - async rename (oldPath, newPath) { + async _rename (oldPath, newPath) { const client = this._getClient(this._remote) try { return await client.rename(this._getFilePath(oldPath), this._getFilePath(newPath)) @@ -80,7 +83,7 @@ export default class SmbHandler extends RemoteHandlerAbstract { } } - async list (dir = undefined) { + async _list (dir = '.') { const client = this._getClient(this._remote) try { return await client.readdir(this._getFilePath(dir)) @@ -89,23 +92,23 @@ export default class SmbHandler extends RemoteHandlerAbstract { } } - async createReadStream (file) { + async _createReadStream (file, options) { const client = this._getClient(this._remote) - const stream = await client.createReadStream(this._getFilePath(file)) + const stream = await client.createReadStream(this._getFilePath(file), options) // FIXME ensure that options are properly handled by @marsaud/smb2 stream.on('end', () => client.close()) return stream } - async createOutputStream (file, options) { + async _createOutputStream (file, options) { const client = this._getClient(this._remote) const path = this._getFilePath(file) - const dir = this._getDirname(path) + const dir = this._dirname(path) let stream try { if (dir) { await client.ensureDir(dir) } - stream = await client.createWriteStream(path, options/* , { flags: 'wx' }*/) // TODO ensure that wx flag is properly handled by @marsaud/smb2 + stream = await client.createWriteStream(path, options) // FIXME ensure that options are properly handled by @marsaud/smb2 } catch (err) { client.close() throw err @@ -114,7 +117,7 @@ export default class SmbHandler extends RemoteHandlerAbstract { return stream } - async unlink (file) { + async _unlink (file) { const client = this._getClient(this._remote) try { return await client.unlink(this._getFilePath(file)) @@ -123,7 +126,7 @@ export default class SmbHandler extends RemoteHandlerAbstract { } } - async getSize (file) { + async _getSize (file) { const client = await this._getClient(this._remote) try { return await client.getSize(this._getFilePath(file)) diff --git a/src/xo-mixins/backups.js b/src/xo-mixins/backups.js index 93a7a9d7f..02def985a 100644 --- a/src/xo-mixins/backups.js +++ b/src/xo-mixins/backups.js @@ -378,6 +378,17 @@ export default class { async rollingDeltaVmBackup ({vm, remoteId, tag, depth}) { const remote = await this._xo.getRemote(remoteId) + + if (!remote) { + throw new Error(`No such Remote ${remoteId}`) + } + if (!remote.enabled) { + throw new Error(`Remote ${remoteId} is disabled`) + } + if (remote.type === 'smb') { + throw new Error('Delta Backup is not supported for smb remotes') + } + const dir = `vm_delta_${tag}_${vm.uuid}` const info = { @@ -589,6 +600,14 @@ export default class { async rollingBackupVm ({vm, remoteId, tag, depth, compress, onlyMetadata}) { const remote = await this._xo.getRemote(remoteId) + + if (!remote) { + throw new Error(`No such Remote s{remoteId}`) + } + if (!remote.enabled) { + throw new Error(`Backup remote ${remoteId} is disabled`) + } + const handler = this._xo.getRemoteHandler(remote) const files = await handler.list() From 1d5d59c4c0209f7b512e884d368b3427f444f4fa Mon Sep 17 00:00:00 2001 From: Fabrice Marsaud Date: Mon, 25 Jan 2016 17:00:48 +0100 Subject: [PATCH 04/11] Remote handler reworked --- src/remote-handlers/abstract.js | 8 ++-- src/remote-handlers/local.js | 20 ++++++++-- src/remote-handlers/nfs.js | 14 +++++++ src/remote-handlers/smb.js | 23 ++++++++++- src/xo-mixins/backups.js | 59 +++++++++++----------------- src/xo-mixins/remotes.js | 68 ++++++++++----------------------- 6 files changed, 100 insertions(+), 92 deletions(-) diff --git a/src/remote-handlers/abstract.js b/src/remote-handlers/abstract.js index 95f292bfd..bd7a662de 100644 --- a/src/remote-handlers/abstract.js +++ b/src/remote-handlers/abstract.js @@ -2,11 +2,11 @@ import eventToPromise from 'event-to-promise' export default class RemoteHandlerAbstract { constructor (remote) { - this._remote = remote + this._remote = this._getInfo({...remote}) } - set (remote) { - this._remote = remote + _getInfo (remote) { + throw new Error('Not implemented') } async sync () { @@ -76,7 +76,7 @@ export default class RemoteHandlerAbstract { } async createOutputStream (file, options) { - return await this._createOutputStream(file) + return await this._createOutputStream(file, options) } async _createOutputStream (file, options) { diff --git a/src/remote-handlers/local.js b/src/remote-handlers/local.js index 0c2c8530a..564b0d00d 100644 --- a/src/remote-handlers/local.js +++ b/src/remote-handlers/local.js @@ -2,9 +2,19 @@ import fs from 'fs-promise' import RemoteHandlerAbstract from './abstract' import startsWith from 'lodash.startswith' import {noop} from '../utils' -import {resolve} from 'path' +import {dirname, resolve} from 'path' export default class LocalHandler extends RemoteHandlerAbstract { + _getInfo (remote) { + if (!startsWith(remote.url, 'file://')) { + throw new Error('Incorrect remote type') + } + this.type = 'local' + remote.path = remote.url.split('://')[1] + remote.path = `/${remote.path}` // FIXME the heading slash has been forgotten on client side + return remote + } + _getFilePath (file) { const parts = [this._remote.path] if (file) { @@ -35,7 +45,9 @@ export default class LocalHandler extends RemoteHandlerAbstract { } async _outputFile (file, data, options) { - await fs.outputFile(this._getFilePath(file), data, options) + const path = this._getFilePath(file) + await fs.ensureDir(dirname(path)) + await fs.writeFile(this._getFilePath(file), data, options) } async _readFile (file, options) { @@ -55,7 +67,9 @@ export default class LocalHandler extends RemoteHandlerAbstract { } async _createOutputStream (file, options) { - return await fs.createOutputStream(this._getFilePath(file), options) + const path = this._getFilePath(file) + await fs.ensureDir(dirname(path)) + return await fs.createWriteStream(path, options) } async _unlink (file) { diff --git a/src/remote-handlers/nfs.js b/src/remote-handlers/nfs.js index b7f882eb2..5530b3c79 100644 --- a/src/remote-handlers/nfs.js +++ b/src/remote-handlers/nfs.js @@ -1,11 +1,25 @@ import fs from 'fs-promise' import LocalHandler from './local' +import startsWith from 'lodash.startswith' import {exec} from 'child_process' import {forEach, promisify} from '../utils' const execAsync = promisify(exec) export default class NfsHandler extends LocalHandler { + _getInfo (remote) { + if (!startsWith(remote.url, 'nfs://')) { + throw new Error('Incorrect remote type') + } + this.type = 'nfs' + const url = remote.url.split('://')[1] + const [host, share] = url.split(':') + remote.path = '/tmp/xo-server/mounts/' + remote.id + remote.host = host + remote.share = share + return remote + } + async _loadRealMounts () { let stdout try { diff --git a/src/remote-handlers/smb.js b/src/remote-handlers/smb.js index 8eb74420e..60e611395 100644 --- a/src/remote-handlers/smb.js +++ b/src/remote-handlers/smb.js @@ -1,6 +1,7 @@ -import Smb2 from '@marsaud/smb2-promise' -import {noop} from '../utils' import RemoteHandlerAbstract from './abstract' +import Smb2 from '@marsaud/smb2-promise' +import startsWith from 'lodash.startswith' +import {noop} from '../utils' export default class SmbHandler extends RemoteHandlerAbstract { constructor (remote) { @@ -8,6 +9,24 @@ export default class SmbHandler extends RemoteHandlerAbstract { this._forget = noop } + _getInfo (remote) { + if (!startsWith(remote.url, 'smb://')) { + throw new Error('Incorrect remote type') + } + this.type = 'smb' + const url = remote.url.split('://')[1] + const [auth, smb] = url.split('@') + const [username, password] = auth.split(':') + const [domain, sh] = smb.split('\\\\') + const [host, path] = sh.split('\0') + remote.host = host + remote.path = path + remote.domain = domain + remote.username = username + remote.password = password + return remote + } + _getClient (remote) { return new Smb2({ share: `\\\\${remote.host}`, diff --git a/src/xo-mixins/backups.js b/src/xo-mixins/backups.js index 02def985a..57eec27b6 100644 --- a/src/xo-mixins/backups.js +++ b/src/xo-mixins/backups.js @@ -49,19 +49,18 @@ export default class { async listRemoteBackups (remoteId) { const remote = await this._xo.getRemote(remoteId) - const handler = this._xo.getRemoteHandler(remote) // List backups. (Except delta backups) const xvaFilter = file => endsWith(file, '.xva') - const files = await handler.list() + const files = await remote.handler.list() const backups = filter(files, xvaFilter) // List delta backups. const deltaDirs = filter(files, file => startsWith(file, 'vm_delta_')) for (const deltaDir of deltaDirs) { - const files = await handler.list(deltaDir) + const files = await remote.handler.list(deltaDir) const deltaBackups = filter(files, xvaFilter) backups.push(...mapToArray( @@ -75,10 +74,9 @@ export default class { // TODO: move into utils and rename! NO, until we may pass a handler instead of a remote...? async _openAndwaitReadableFile (remote, file, errorMessage) { - const handler = this._xo.getRemoteHandler(remote) let stream try { - stream = await handler.createReadStream(file) + stream = await remote.handler.createReadStream(file) await eventToPromise(stream, 'readable') } catch (error) { if (error.code === 'ENOENT') { @@ -87,7 +85,7 @@ export default class { throw error } - stream.length = await handler.getSize(file) + stream.length = await remote.handler.getSize(file) return stream } @@ -172,21 +170,19 @@ export default class { if (n <= 0) { return } - const handler = this._xo.getRemoteHandler(remote) const getPath = (file, dir) => dir ? `${dir}/${file}` : file await Promise.all( - mapToArray(backups.slice(0, n), async backup => await handler.unlink(getPath(backup, dir))) + mapToArray(backups.slice(0, n), async backup => await remote.handler.unlink(getPath(backup, dir))) ) } // ----------------------------------------------------------------- async _listVdiBackups (remote, dir) { - const handler = this._xo.getRemoteHandler(remote) let files try { - files = await handler.list(dir) + files = await remote.handler.list(dir) } catch (error) { if (error.code === 'ENOENT') { files = [] @@ -236,14 +232,13 @@ export default class { const vdiFilename = `${date}_${isFull ? 'full' : 'delta'}.vhd` const backupFullPath = `${dir}/${vdiFilename}` - const handler = this._xo.getRemoteHandler(remote) try { const sourceStream = await xapi.exportVdi(currentSnapshot.$id, { baseId: isFull ? undefined : base.$id, format: VDI_FORMAT_VHD }) - const targetStream = await handler.createOutputStream(backupFullPath, { flags: 'wx' }) + const targetStream = await remote.handler.createOutputStream(backupFullPath, { flags: 'wx' }) sourceStream.on('error', error => targetStream.emit('error', error)) await Promise.all([ @@ -253,7 +248,7 @@ export default class { } catch (error) { // Remove new backup. (corrupt) and delete new vdi base. xapi.deleteVdi(currentSnapshot.$id).catch(noop) - await handler.unlink(backupFullPath).catch(noop) + await remote.handler.unlink(backupFullPath).catch(noop) throw error } @@ -267,7 +262,7 @@ export default class { } async _mergeDeltaVdiBackups ({remote, dir, depth}) { - if (remote.type === 'smb') { + if (remote.handler.type === 'smb') { throw new Error('VDI merging is not available through SMB') } const backups = await this._listVdiBackups(remote, dir) @@ -281,20 +276,19 @@ export default class { const newFull = `${getVdiTimestamp(backups[i])}_full.vhd` const vhdUtil = `${__dirname}/../../bin/vhd-util` - const handler = this._xo.getRemoteHandler(remote) for (; i > 0 && isDeltaVdiBackup(backups[i]); i--) { const backup = `${dir}/${backups[i]}` const parent = `${dir}/${backups[i - 1]}` try { - await execa(vhdUtil, ['modify', '-n', `${remote.path}/${backup}`, '-p', `${remote.path}/${parent}`]) // FIXME not ok at least with smb remotes - await execa(vhdUtil, ['coalesce', '-n', `${remote.path}/${backup}`]) // FIXME not ok at least with smb remotes + await execa(vhdUtil, ['modify', '-n', `${remote.handler.path}/${backup}`, '-p', `${remote.handler.path}/${parent}`]) // FIXME not ok at least with smb remotes + await execa(vhdUtil, ['coalesce', '-n', `${remote.handler.path}/${backup}`]) // FIXME not ok at least with smb remotes } catch (e) { console.error('Unable to use vhd-util.', e) throw e } - await handler.unlink(backup) + await remote.handler.unlink(backup) } // The base was removed, it exists two full backups or more ? @@ -308,7 +302,7 @@ export default class { } // Rename the first old full backup to the new full backup. - await handler.rename(`${dir}/${backups[0]}`, `${dir}/${newFull}`) + await remote.handler.rename(`${dir}/${backups[0]}`, `${dir}/${newFull}`) } async _importVdiBackupContent (xapi, remote, file, vdiId) { @@ -359,19 +353,17 @@ export default class { // ----------------------------------------------------------------- async _listDeltaVmBackups (remote, dir) { - const handler = this._xo.getRemoteHandler(remote) - const files = await handler.list(dir) + const files = await remote.handler.list(dir) return await sortBy(filter(files, (fileName) => /^\d+T\d+Z_.*\.(?:xva|json)$/.test(fileName))) } async _failedRollingDeltaVmBackup (xapi, remote, dir, fulFilledVdiBackups) { - const handler = this._xo.getRemoteHandler(remote) await Promise.all( mapToArray(fulFilledVdiBackups, async vdiBackup => { const { newBaseId, backupDirectory, vdiFilename } = vdiBackup.value() await xapi.deleteVdi(newBaseId) - await handler.unlink(`${dir}/${backupDirectory}/${vdiFilename}`).catch(noop) + await remote.handler.unlink(`${dir}/${backupDirectory}/${vdiFilename}`).catch(noop) }) ) } @@ -385,7 +377,7 @@ export default class { if (!remote.enabled) { throw new Error(`Remote ${remoteId} is disabled`) } - if (remote.type === 'smb') { + if (remote.handler.type === 'smb') { throw new Error('Delta Backup is not supported for smb remotes') } @@ -450,7 +442,7 @@ export default class { } if (fail) { - console.error(`Remove successful backups in ${remote.path}/${dir}`, fulFilledVdiBackups) + console.error(`Remove successful backups in ${remote.handler.path}/${dir}`, fulFilledVdiBackups) await this._failedRollingDeltaVmBackup(xapi, remote, dir, fulFilledVdiBackups) throw new Error('Rolling delta vm backup failed.') @@ -463,17 +455,15 @@ export default class { const xvaPath = `${dir}/${backupFormat}.xva` const infoPath = `${dir}/${backupFormat}.json` - const handler = this._xo.getRemoteHandler(remote) - try { await Promise.all([ this.backupVm({vm, remoteId, file: xvaPath, onlyMetadata: true}), - handler.outputFile(infoPath, JSON.stringify(info), {flag: 'wx'}) + remote.handler.outputFile(infoPath, JSON.stringify(info), {flag: 'wx'}) ]) } catch (e) { await Promise.all([ - handler.unlink(xvaPath).catch(noop), - handler.unlink(infoPath).catch(noop), + remote.handler.unlink(xvaPath).catch(noop), + remote.handler.unlink(infoPath).catch(noop), this._failedRollingDeltaVmBackup(xapi, remote, dir, fulFilledVdiBackups) ]) @@ -546,8 +536,7 @@ export default class { // Because XenServer creates Vbds linked to the vdis of the backup vm if it exists. await xapi.destroyVbdsFromVm(vm.uuid) - const handler = this._xo.getRemoteHandler(remote) - const info = JSON.parse(await handler.readFile(`${filePath}.json`)) + const info = JSON.parse(await remote.handler.readFile(`${filePath}.json`)) // Import VDIs. const vdiIds = {} @@ -585,8 +574,7 @@ export default class { async backupVm ({vm, remoteId, file, compress, onlyMetadata}) { const remote = await this._xo.getRemote(remoteId) - const handler = this._xo.getRemoteHandler(remote) - const targetStream = await handler.createOutputStream(file, { flags: 'wx' }) + const targetStream = await remote.handler.createOutputStream(file, { flags: 'wx' }) const promise = eventToPromise(targetStream, 'finish') const sourceStream = await this._xo.getXapi(vm).exportVm(vm._xapiId, { @@ -608,8 +596,7 @@ export default class { throw new Error(`Backup remote ${remoteId} is disabled`) } - const handler = this._xo.getRemoteHandler(remote) - const files = await handler.list() + const files = await remote.handler.list() const reg = new RegExp('^[^_]+_' + escapeStringRegexp(`${tag}_${vm.name_label}.xva`)) const backups = sortBy(filter(files, (fileName) => reg.test(fileName))) diff --git a/src/xo-mixins/remotes.js b/src/xo-mixins/remotes.js index 7178ef0db..7f3b0205f 100644 --- a/src/xo-mixins/remotes.js +++ b/src/xo-mixins/remotes.js @@ -1,5 +1,3 @@ -import startsWith from 'lodash.startswith' - import RemoteHandlerLocal from '../remote-handlers/local' import RemoteHandlerNfs from '../remote-handlers/nfs' import RemoteHandlerSmb from '../remote-handlers/smb' @@ -40,50 +38,26 @@ export default class { xo.on('stop', () => this.forgetAllRemotes()) } - getRemoteHandler (remote) { - if (!(remote.id in this._remoteHandlers)) { - const handlers = { - 'local': RemoteHandlerLocal, - 'nfs': RemoteHandlerNfs, - 'smb': RemoteHandlerSmb - } - this._remoteHandlers[remote.id] = new handlers[remote.type](remote) + _addHandler (remote) { + const Handler = { + file: RemoteHandlerLocal, + smb: RemoteHandlerSmb, + nfs: RemoteHandlerNfs } - const handler = this._remoteHandlers[remote.id] - handler.set(remote) - return handler - } - - _developRemote (remote) { - const _remote = { ...remote } - if (startsWith(_remote.url, 'file://')) { - _remote.type = 'local' - _remote.path = _remote.url.slice(6) - } else if (startsWith(_remote.url, 'nfs://')) { - _remote.type = 'nfs' - const url = _remote.url.slice(6) - const [host, share] = url.split(':') - _remote.path = '/tmp/xo-server/mounts/' + _remote.id - _remote.host = host - _remote.share = share - } else if (startsWith(_remote.url, 'smb://')) { - _remote.type = 'smb' - const url = _remote.url.slice(6) - const [auth, smb] = url.split('@') - const [username, password] = auth.split(':') - const [domain, sh] = smb.split('\\\\') - const [host, path] = sh.split('\0') - _remote.host = host - _remote.path = path - _remote.domain = domain - _remote.username = username - _remote.password = password + const type = remote.url.split('://')[0] + if (!Handler[type]) { + throw new Error('Unhandled remote type') } - return _remote + Object.defineProperty(remote, 'handler', { + value: new Handler[type](remote) + }) + remote.handler._getInfo(remote) // FIXME this has to be done by a specific code SHARED with the handler, not by the handler itself + remote.type = remote.handler.type // FIXME subsequent workaround + return remote } async getAllRemotes () { - return mapToArray(await this._remotes.get(), this._developRemote) + return mapToArray(await this._remotes.get(), this._addHandler) } async _getRemote (id) { @@ -96,7 +70,7 @@ export default class { } async getRemote (id) { - return this._developRemote((await this._getRemote(id)).properties) + return this._addHandler((await this._getRemote(id)).properties) } async createRemote ({name, url}) { @@ -107,10 +81,10 @@ export default class { async updateRemote (id, {name, url, enabled, error}) { const remote = await this._getRemote(id) this._updateRemote(remote, {name, url, enabled, error}) - const r = this._developRemote(remote.properties) - const props = await this.getRemoteHandler(r).sync() + const r = this._addHandler(remote.properties) + const props = await r.sync() this._updateRemote(remote, props) - return await this._developRemote(this._remotes.save(remote).properties) + return await this._addHandler(this._remotes.save(remote).properties) } _updateRemote (remote, {name, url, enabled, error}) { @@ -126,7 +100,7 @@ export default class { async removeRemote (id) { const remote = await this.getRemote(id) - await this.getRemoteHandler(remote).forget() + await remote.handler.forget() await this._remotes.remove(id) } @@ -142,7 +116,7 @@ export default class { async forgetAllRemotes () { const remotes = await this.getAllRemotes() for (let remote of remotes) { - await this.getRemoteHandler(remote).forget() + await remote.handler.forget() } } From 62564d747fa8763a7aa7262556260a5567757f55 Mon Sep 17 00:00:00 2001 From: Fabrice Marsaud Date: Mon, 25 Jan 2016 17:28:14 +0100 Subject: [PATCH 05/11] Errors moved from API to core --- src/api/vm.coffee | 5 ----- src/xo-mixins/backups.js | 6 ++++++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/api/vm.coffee b/src/api/vm.coffee index 5485bda1c..b9f43aa38 100644 --- a/src/api/vm.coffee +++ b/src/api/vm.coffee @@ -573,11 +573,6 @@ exports.rollingSnapshot = rollingSnapshot #--------------------------------------------------------------------- backup = $coroutine ({vm, remoteId, file, compress, onlyMetadata}) -> - remote = yield @getRemote remoteId - if not remote?.path? - throw new Error "No such Remote #{remoteId}" - if not remote.enabled - throw new Error "Backup remote #{remoteId} is disabled" yield @backupVm({vm, remoteId, file, compress, onlyMetadata}) backup.params = { diff --git a/src/xo-mixins/backups.js b/src/xo-mixins/backups.js index 57eec27b6..c216ec59f 100644 --- a/src/xo-mixins/backups.js +++ b/src/xo-mixins/backups.js @@ -574,6 +574,12 @@ export default class { async backupVm ({vm, remoteId, file, compress, onlyMetadata}) { const remote = await this._xo.getRemote(remoteId) + if (!remote) { + throw new Error(`No such Remote ${remoteId}`) + } + if (!remote.enabled) { + throw new Error(`Backup remote ${remoteId} is disabled`) + } const targetStream = await remote.handler.createOutputStream(file, { flags: 'wx' }) const promise = eventToPromise(targetStream, 'finish') From f7f13b9e07bc25c0d49a8b5c47759847bea84b6a Mon Sep 17 00:00:00 2001 From: Fabrice Marsaud Date: Tue, 26 Jan 2016 09:47:47 +0100 Subject: [PATCH 06/11] PR feedback 2 --- src/remote-handlers/abstract.js | 40 +++++++++++++++++---------------- src/remote-handlers/local.js | 21 +++++++++-------- src/remote-handlers/nfs.js | 17 +++++++------- src/remote-handlers/smb.js | 19 +++++++++------- 4 files changed, 53 insertions(+), 44 deletions(-) diff --git a/src/remote-handlers/abstract.js b/src/remote-handlers/abstract.js index bd7a662de..4f63fc0b1 100644 --- a/src/remote-handlers/abstract.js +++ b/src/remote-handlers/abstract.js @@ -1,4 +1,5 @@ import eventToPromise from 'event-to-promise' +import getStream from 'get-stream' export default class RemoteHandlerAbstract { constructor (remote) { @@ -10,7 +11,7 @@ export default class RemoteHandlerAbstract { } async sync () { - return await this._sync() + return this._sync() } async _sync () { @@ -18,7 +19,7 @@ export default class RemoteHandlerAbstract { } async forget () { - return await this._forget() + return this._forget() } async _forget () { @@ -26,30 +27,27 @@ export default class RemoteHandlerAbstract { } async outputFile (file, data, options) { - return await this._outputFile(file, data, options) + return this._outputFile(file, data, options) } async _outputFile (file, data, options) { - const stream = this.createOutputStream(file) + const stream = await this.createOutputStream(file) const promise = eventToPromise(stream, 'finish') stream.end(data) return promise } async readFile (file, options) { - return await this._readFile(file, options) + return this._readFile(file, options) } async _readFile (file, options) { - const stream = this.createReadStream(file, options) - let data = '' - stream.on('data', d => data += d) - await eventToPromise(stream, 'end') - return data + const stream = await this.createReadStream(file, options) + return getStream(stream) } async rename (oldPath, newPath) { - return await this._rename(oldPath, newPath) + return this._rename(oldPath, newPath) } async _rename (oldPath, newPath) { @@ -57,17 +55,21 @@ export default class RemoteHandlerAbstract { } async list (dir = '.') { - return await this._list(dir) + return this._list(dir) } - async _list (dir = '.') { + async _list (dir) { throw new Error('Not implemented') } async createReadStream (file, options) { - const length = await this.getSize(file) const stream = await this._createReadStream(file) - stream.length = length + if (!('length' in stream) || stream.length === null) { + try { + const length = await this.getSize(file) + stream.length = length + } catch (_) {} + } return stream } @@ -76,7 +78,7 @@ export default class RemoteHandlerAbstract { } async createOutputStream (file, options) { - return await this._createOutputStream(file, options) + return this._createOutputStream(file, options) } async _createOutputStream (file, options) { @@ -84,7 +86,7 @@ export default class RemoteHandlerAbstract { } async unlink (file) { - return await this._unlink(file) + return this._unlink(file) } async _unlink (file) { @@ -92,10 +94,10 @@ export default class RemoteHandlerAbstract { } async getSize (file) { - return await this._getSize(file) + return this._getSize(file) } async _getSize (file) { - throw new Error('Not implement') + throw new Error('Not implemented') } } diff --git a/src/remote-handlers/local.js b/src/remote-handlers/local.js index 564b0d00d..deef8ebe8 100644 --- a/src/remote-handlers/local.js +++ b/src/remote-handlers/local.js @@ -1,15 +1,18 @@ import fs from 'fs-promise' import RemoteHandlerAbstract from './abstract' import startsWith from 'lodash.startswith' -import {noop} from '../utils' -import {dirname, resolve} from 'path' +import { noop } from '../utils' +import { dirname, resolve } from 'path' export default class LocalHandler extends RemoteHandlerAbstract { + get type () { + return 'local' + } + _getInfo (remote) { if (!startsWith(remote.url, 'file://')) { throw new Error('Incorrect remote type') } - this.type = 'local' remote.path = remote.url.split('://')[1] remote.path = `/${remote.path}` // FIXME the heading slash has been forgotten on client side return remote @@ -51,29 +54,29 @@ export default class LocalHandler extends RemoteHandlerAbstract { } async _readFile (file, options) { - return await fs.readFile(this._getFilePath(file), options) + return fs.readFile(this._getFilePath(file), options) } async _rename (oldPath, newPath) { - return await fs.rename(this._getFilePath(oldPath), this._getFilePath(newPath)) + return fs.rename(this._getFilePath(oldPath), this._getFilePath(newPath)) } async _list (dir = '.') { - return await fs.readdir(this._getFilePath(dir)) + return fs.readdir(this._getFilePath(dir)) } async _createReadStream (file, options) { - return await fs.createReadStream(this._getFilePath(file), options) + return fs.createReadStream(this._getFilePath(file), options) } async _createOutputStream (file, options) { const path = this._getFilePath(file) await fs.ensureDir(dirname(path)) - return await fs.createWriteStream(path, options) + return fs.createWriteStream(path, options) } async _unlink (file) { - return await fs.unlink(this._getFilePath(file)) + return fs.unlink(this._getFilePath(file)) } async _getSize (file) { diff --git a/src/remote-handlers/nfs.js b/src/remote-handlers/nfs.js index 5530b3c79..129e6b82f 100644 --- a/src/remote-handlers/nfs.js +++ b/src/remote-handlers/nfs.js @@ -1,17 +1,18 @@ import fs from 'fs-promise' import LocalHandler from './local' import startsWith from 'lodash.startswith' -import {exec} from 'child_process' -import {forEach, promisify} from '../utils' - -const execAsync = promisify(exec) +import execa from 'execa' +import { forEach } from '../utils' export default class NfsHandler extends LocalHandler { + get type () { + return 'nfs' + } + _getInfo (remote) { if (!startsWith(remote.url, 'nfs://')) { throw new Error('Incorrect remote type') } - this.type = 'nfs' const url = remote.url.split('://')[1] const [host, share] = url.split(':') remote.path = '/tmp/xo-server/mounts/' + remote.id @@ -23,7 +24,7 @@ export default class NfsHandler extends LocalHandler { async _loadRealMounts () { let stdout try { - [stdout] = await execAsync('findmnt -P -t nfs,nfs4 --output SOURCE,TARGET --noheadings') + [stdout] = await execa('findmnt', ['-P', '-t', 'nfs,nfs4', '--output', 'SOURCE,TARGET', '--noheadings']) } catch (exc) { // When no mounts are found, the call pretends to fail... } @@ -50,7 +51,7 @@ export default class NfsHandler extends LocalHandler { async _mount (remote) { await fs.ensureDir(remote.path) - return await execAsync(`mount -t nfs ${remote.host}:${remote.share} ${remote.path}`) + return execa('mount', ['-t', 'nfs', `${remote.host}:${remote.share}`, remote.path]) } async _sync () { @@ -82,6 +83,6 @@ export default class NfsHandler extends LocalHandler { } async _umount (remote) { - await execAsync(`umount ${remote.path}`) + await execa('umount', [remote.path]) } } diff --git a/src/remote-handlers/smb.js b/src/remote-handlers/smb.js index 60e611395..8634361d2 100644 --- a/src/remote-handlers/smb.js +++ b/src/remote-handlers/smb.js @@ -1,7 +1,7 @@ import RemoteHandlerAbstract from './abstract' import Smb2 from '@marsaud/smb2-promise' import startsWith from 'lodash.startswith' -import {noop} from '../utils' +import { noop } from '../utils' export default class SmbHandler extends RemoteHandlerAbstract { constructor (remote) { @@ -9,11 +9,14 @@ export default class SmbHandler extends RemoteHandlerAbstract { this._forget = noop } + get type () { + return 'smb' + } + _getInfo (remote) { if (!startsWith(remote.url, 'smb://')) { throw new Error('Incorrect remote type') } - this.type = 'smb' const url = remote.url.split('://')[1] const [auth, smb] = url.split('@') const [username, password] = auth.split(':') @@ -78,7 +81,7 @@ export default class SmbHandler extends RemoteHandlerAbstract { if (dir) { await client.ensureDir(dir) } - return await client.writeFile(path, data, options) + return client.writeFile(path, data, options) } finally { client.close() } @@ -87,7 +90,7 @@ export default class SmbHandler extends RemoteHandlerAbstract { async _readFile (file, options) { const client = this._getClient(this._remote) try { - return await client.readFile(this._getFilePath(file), options) + return client.readFile(this._getFilePath(file), options) } finally { client.close() } @@ -96,7 +99,7 @@ export default class SmbHandler extends RemoteHandlerAbstract { async _rename (oldPath, newPath) { const client = this._getClient(this._remote) try { - return await client.rename(this._getFilePath(oldPath), this._getFilePath(newPath)) + return client.rename(this._getFilePath(oldPath), this._getFilePath(newPath)) } finally { client.close() } @@ -105,7 +108,7 @@ export default class SmbHandler extends RemoteHandlerAbstract { async _list (dir = '.') { const client = this._getClient(this._remote) try { - return await client.readdir(this._getFilePath(dir)) + return client.readdir(this._getFilePath(dir)) } finally { client.close() } @@ -139,7 +142,7 @@ export default class SmbHandler extends RemoteHandlerAbstract { async _unlink (file) { const client = this._getClient(this._remote) try { - return await client.unlink(this._getFilePath(file)) + return client.unlink(this._getFilePath(file)) } finally { client.close() } @@ -148,7 +151,7 @@ export default class SmbHandler extends RemoteHandlerAbstract { async _getSize (file) { const client = await this._getClient(this._remote) try { - return await client.getSize(this._getFilePath(file)) + return client.getSize(this._getFilePath(file)) } finally { client.close() } From d6e1c13c3938f4e12c9175f1a2c10241f59ff7c7 Mon Sep 17 00:00:00 2001 From: Fabrice Marsaud Date: Tue, 26 Jan 2016 17:20:24 +0100 Subject: [PATCH 07/11] Handler and remotes reworked --- package.json | 5 +- src/models/remote.js | 4 +- src/remote-handlers/abstract.js | 27 +++++-- src/remote-handlers/local.js | 21 +++-- src/remote-handlers/nfs.js | 31 +++----- src/remote-handlers/smb.js | 25 ++---- src/xo-mixins/backups.js | 137 +++++++++++++++++--------------- src/xo-mixins/remotes.js | 38 ++++----- 8 files changed, 143 insertions(+), 145 deletions(-) diff --git a/package.json b/package.json index f437ef516..2b2fb47e3 100644 --- a/package.json +++ b/package.json @@ -96,8 +96,8 @@ "lodash.pick": "^3.0.0", "lodash.sortby": "^3.1.4", "lodash.startswith": "^3.0.1", - "loud-rejection": "^1.2.0", "lodash.trim": "^3.0.1", + "loud-rejection": "^1.2.0", "make-error": "^1", "micromatch": "^2.3.2", "minimist": "^1.2.0", @@ -116,7 +116,8 @@ "ws": "~0.8.0", "xen-api": "^0.7.2", "xml2js": "~0.4.6", - "xo-collection": "^0.4.0" + "xo-collection": "^0.4.0", + "xo-remote-parser": "0.0.0" }, "devDependencies": { "babel-eslint": "^4.0.10", diff --git a/src/models/remote.js b/src/models/remote.js index 9df009278..e99ca7891 100644 --- a/src/models/remote.js +++ b/src/models/remote.js @@ -1,6 +1,8 @@ import Collection from '../collection/redis' import Model from '../model' -import { forEach } from '../utils' +import { + forEach +} from '../utils' // =================================================================== diff --git a/src/remote-handlers/abstract.js b/src/remote-handlers/abstract.js index 4f63fc0b1..a3c7ce92d 100644 --- a/src/remote-handlers/abstract.js +++ b/src/remote-handlers/abstract.js @@ -1,15 +1,29 @@ import eventToPromise from 'event-to-promise' import getStream from 'get-stream' +import { + parse +} from 'xo-remote-parse' + export default class RemoteHandlerAbstract { constructor (remote) { - this._remote = this._getInfo({...remote}) + this._remote = parse({...remote}) + if (!this._remote.type === this.type) { + throw new Error('Incorrect remote type') + } } _getInfo (remote) { throw new Error('Not implemented') } + get type () { + throw new Error('Not implemented') + } + + /** + * Asks the handler to sync the state of the effective remote with its' metadata + */ async sync () { return this._sync() } @@ -18,6 +32,9 @@ export default class RemoteHandlerAbstract { throw new Error('Not implemented') } + /** + * Free the resources possibly dedicated to put the remote at work, when it is no more needed + */ async forget () { return this._forget() } @@ -42,8 +59,7 @@ export default class RemoteHandlerAbstract { } async _readFile (file, options) { - const stream = await this.createReadStream(file, options) - return getStream(stream) + return getStream(await this.createReadStream(file, options)) } async rename (oldPath, newPath) { @@ -64,10 +80,9 @@ export default class RemoteHandlerAbstract { async createReadStream (file, options) { const stream = await this._createReadStream(file) - if (!('length' in stream) || stream.length === null) { + if (stream.length === undefined) { try { - const length = await this.getSize(file) - stream.length = length + stream.length = await this.getSize(file) } catch (_) {} } return stream diff --git a/src/remote-handlers/local.js b/src/remote-handlers/local.js index deef8ebe8..5c8a846a3 100644 --- a/src/remote-handlers/local.js +++ b/src/remote-handlers/local.js @@ -1,23 +1,20 @@ import fs from 'fs-promise' -import RemoteHandlerAbstract from './abstract' import startsWith from 'lodash.startswith' -import { noop } from '../utils' -import { dirname, resolve } from 'path' +import { + dirname, + resolve +} from 'path' + +import RemoteHandlerAbstract from './abstract' +import { + noop +} from '../utils' export default class LocalHandler extends RemoteHandlerAbstract { get type () { return 'local' } - _getInfo (remote) { - if (!startsWith(remote.url, 'file://')) { - throw new Error('Incorrect remote type') - } - remote.path = remote.url.split('://')[1] - remote.path = `/${remote.path}` // FIXME the heading slash has been forgotten on client side - return remote - } - _getFilePath (file) { const parts = [this._remote.path] if (file) { diff --git a/src/remote-handlers/nfs.js b/src/remote-handlers/nfs.js index 129e6b82f..12be2ab6d 100644 --- a/src/remote-handlers/nfs.js +++ b/src/remote-handlers/nfs.js @@ -1,35 +1,21 @@ -import fs from 'fs-promise' -import LocalHandler from './local' -import startsWith from 'lodash.startswith' import execa from 'execa' -import { forEach } from '../utils' +import fs from 'fs-promise' + +import LocalHandler from './local' +import { + forEach +} from '../utils' export default class NfsHandler extends LocalHandler { get type () { return 'nfs' } - _getInfo (remote) { - if (!startsWith(remote.url, 'nfs://')) { - throw new Error('Incorrect remote type') - } - const url = remote.url.split('://')[1] - const [host, share] = url.split(':') - remote.path = '/tmp/xo-server/mounts/' + remote.id - remote.host = host - remote.share = share - return remote - } - async _loadRealMounts () { let stdout + const mounted = {} try { [stdout] = await execa('findmnt', ['-P', '-t', 'nfs,nfs4', '--output', 'SOURCE,TARGET', '--noheadings']) - } catch (exc) { - // When no mounts are found, the call pretends to fail... - } - const mounted = {} - if (stdout) { const regex = /^SOURCE="([^:]*):(.*)" TARGET="(.*)"$/ forEach(stdout.split('\n'), m => { if (m) { @@ -40,7 +26,10 @@ export default class NfsHandler extends LocalHandler { } } }) + } catch (exc) { + // When no mounts are found, the call pretends to fail... } + this._realMounts = mounted return mounted } diff --git a/src/remote-handlers/smb.js b/src/remote-handlers/smb.js index 8634361d2..5bf416c61 100644 --- a/src/remote-handlers/smb.js +++ b/src/remote-handlers/smb.js @@ -1,7 +1,9 @@ -import RemoteHandlerAbstract from './abstract' import Smb2 from '@marsaud/smb2-promise' -import startsWith from 'lodash.startswith' -import { noop } from '../utils' + +import RemoteHandlerAbstract from './abstract' +import { + noop +} from '../utils' export default class SmbHandler extends RemoteHandlerAbstract { constructor (remote) { @@ -13,23 +15,6 @@ export default class SmbHandler extends RemoteHandlerAbstract { return 'smb' } - _getInfo (remote) { - if (!startsWith(remote.url, 'smb://')) { - throw new Error('Incorrect remote type') - } - const url = remote.url.split('://')[1] - const [auth, smb] = url.split('@') - const [username, password] = auth.split(':') - const [domain, sh] = smb.split('\\\\') - const [host, path] = sh.split('\0') - remote.host = host - remote.path = path - remote.domain = domain - remote.username = username - remote.password = password - return remote - } - _getClient (remote) { return new Smb2({ share: `\\\\${remote.host}`, diff --git a/src/xo-mixins/backups.js b/src/xo-mixins/backups.js index c216ec59f..e7895bd76 100644 --- a/src/xo-mixins/backups.js +++ b/src/xo-mixins/backups.js @@ -48,19 +48,19 @@ export default class { } async listRemoteBackups (remoteId) { - const remote = await this._xo.getRemote(remoteId) + const handler = await this._xo.getRemoteHandler(remoteId) // List backups. (Except delta backups) const xvaFilter = file => endsWith(file, '.xva') - const files = await remote.handler.list() + const files = await handler.list() const backups = filter(files, xvaFilter) // List delta backups. const deltaDirs = filter(files, file => startsWith(file, 'vm_delta_')) for (const deltaDir of deltaDirs) { - const files = await remote.handler.list(deltaDir) + const files = await handler.list(deltaDir) const deltaBackups = filter(files, xvaFilter) backups.push(...mapToArray( @@ -73,10 +73,10 @@ export default class { } // TODO: move into utils and rename! NO, until we may pass a handler instead of a remote...? - async _openAndwaitReadableFile (remote, file, errorMessage) { + async _openAndwaitReadableFile (handler, file, errorMessage) { let stream try { - stream = await remote.handler.createReadStream(file) + stream = await handler.createReadStream(file) await eventToPromise(stream, 'readable') } catch (error) { if (error.code === 'ENOENT') { @@ -85,14 +85,13 @@ export default class { throw error } - stream.length = await remote.handler.getSize(file) return stream } async importVmBackup (remoteId, file, sr) { - const remote = await this._xo.getRemote(remoteId) + const handler = await this._xo.getRemoteHandler(remoteId) const stream = await this._openAndwaitReadableFile( - remote, + handler, file, 'VM to import not found in this remote' ) @@ -166,23 +165,23 @@ export default class { // TODO: The other backup methods must use this function ! // Prerequisite: The backups array must be ordered. (old to new backups) - async _removeOldBackups (backups, remote, dir, n) { + async _removeOldBackups (backups, handler, dir, n) { if (n <= 0) { return } const getPath = (file, dir) => dir ? `${dir}/${file}` : file await Promise.all( - mapToArray(backups.slice(0, n), async backup => await remote.handler.unlink(getPath(backup, dir))) + mapToArray(backups.slice(0, n), async backup => await handler.unlink(getPath(backup, dir))) ) } // ----------------------------------------------------------------- - async _listVdiBackups (remote, dir) { + async _listVdiBackups (handler, dir) { let files try { - files = await remote.handler.list(dir) + files = await handler.list(dir) } catch (error) { if (error.code === 'ENOENT') { files = [] @@ -195,19 +194,19 @@ export default class { // Avoid unstable state: No full vdi found to the beginning of array. (base) for (i = 0; i < backups.length && isDeltaVdiBackup(backups[i]); i++); - await this._removeOldBackups(backups, remote, dir, i) + await this._removeOldBackups(backups, handler, dir, i) return backups.slice(i) } - async _deltaVdiBackup ({vdi, remote, dir, depth}) { + async _deltaVdiBackup ({vdi, handler, dir, depth}) { const xapi = this._xo.getXapi(vdi) const backupDirectory = `vdi_${vdi.uuid}` vdi = xapi.getObject(vdi._xapiId) dir = `${dir}/${backupDirectory}` - const backups = await this._listVdiBackups(remote, dir) + const backups = await this._listVdiBackups(handler, dir) // Make snapshot. const date = safeDateFormat(new Date()) @@ -238,7 +237,7 @@ export default class { format: VDI_FORMAT_VHD }) - const targetStream = await remote.handler.createOutputStream(backupFullPath, { flags: 'wx' }) + const targetStream = await handler.createOutputStream(backupFullPath, { flags: 'wx' }) sourceStream.on('error', error => targetStream.emit('error', error)) await Promise.all([ @@ -248,7 +247,7 @@ export default class { } catch (error) { // Remove new backup. (corrupt) and delete new vdi base. xapi.deleteVdi(currentSnapshot.$id).catch(noop) - await remote.handler.unlink(backupFullPath).catch(noop) + await handler.unlink(backupFullPath).catch(noop) throw error } @@ -261,11 +260,11 @@ export default class { } } - async _mergeDeltaVdiBackups ({remote, dir, depth}) { - if (remote.handler.type === 'smb') { + async _mergeDeltaVdiBackups ({handler, dir, depth}) { + if (handler.type === 'smb') { throw new Error('VDI merging is not available through SMB') } - const backups = await this._listVdiBackups(remote, dir) + const backups = await this._listVdiBackups(handler, dir) let i = backups.length - depth // No merge. @@ -281,33 +280,33 @@ export default class { const parent = `${dir}/${backups[i - 1]}` try { - await execa(vhdUtil, ['modify', '-n', `${remote.handler.path}/${backup}`, '-p', `${remote.handler.path}/${parent}`]) // FIXME not ok at least with smb remotes - await execa(vhdUtil, ['coalesce', '-n', `${remote.handler.path}/${backup}`]) // FIXME not ok at least with smb remotes + await execa(vhdUtil, ['modify', '-n', `${handler.path}/${backup}`, '-p', `${handler.path}/${parent}`]) // FIXME not ok at least with smb remotes + await execa(vhdUtil, ['coalesce', '-n', `${handler.path}/${backup}`]) // FIXME not ok at least with smb remotes } catch (e) { console.error('Unable to use vhd-util.', e) throw e } - await remote.handler.unlink(backup) + await handler.unlink(backup) } // The base was removed, it exists two full backups or more ? // => Remove old backups before the most recent full. if (i > 0) { for (i--; i >= 0; i--) { - await remote.unlink(`${dir}/${backups[i]}`) + await handler.unlink(`${dir}/${backups[i]}`) } return } // Rename the first old full backup to the new full backup. - await remote.handler.rename(`${dir}/${backups[0]}`, `${dir}/${newFull}`) + await handler.rename(`${dir}/${backups[0]}`, `${dir}/${newFull}`) } - async _importVdiBackupContent (xapi, remote, file, vdiId) { + async _importVdiBackupContent (xapi, handler, file, vdiId) { const stream = await this._openAndwaitReadableFile( - remote, + handler, file, 'VDI to import not found in this remote' ) @@ -318,11 +317,14 @@ export default class { } async importDeltaVdiBackup ({vdi, remoteId, filePath}) { - const remote = await this._xo.getRemote(remoteId) + const handler = await this._xo.getRemoteHandler(remoteId) + return this._importDeltaVdiBackup(vdi, handler, filePath) + } + async _importDeltaVdiBackup (vdi, handler, filePath) { const dir = dirname(filePath) const filename = basename(filePath) - const backups = await this._listVdiBackups(remote, dir) + const backups = await this._listVdiBackups(handler, dir) // Search file. (delta or full backup) const i = findIndex(backups, backup => @@ -346,24 +348,24 @@ export default class { const xapi = this._xo.getXapi(vdi) for (; j <= i; j++) { - await this._importVdiBackupContent(xapi, remote, `${dir}/${backups[j]}`, vdi._xapiId) + await this._importVdiBackupContent(xapi, handler, `${dir}/${backups[j]}`, vdi._xapiId) } } // ----------------------------------------------------------------- - async _listDeltaVmBackups (remote, dir) { - const files = await remote.handler.list(dir) + async _listDeltaVmBackups (handler, dir) { + const files = await handler.list(dir) return await sortBy(filter(files, (fileName) => /^\d+T\d+Z_.*\.(?:xva|json)$/.test(fileName))) } - async _failedRollingDeltaVmBackup (xapi, remote, dir, fulFilledVdiBackups) { + async _failedRollingDeltaVmBackup (xapi, handler, dir, fulFilledVdiBackups) { await Promise.all( mapToArray(fulFilledVdiBackups, async vdiBackup => { const { newBaseId, backupDirectory, vdiFilename } = vdiBackup.value() await xapi.deleteVdi(newBaseId) - await remote.handler.unlink(`${dir}/${backupDirectory}/${vdiFilename}`).catch(noop) + await handler.unlink(`${dir}/${backupDirectory}/${vdiFilename}`).catch(noop) }) ) } @@ -377,7 +379,9 @@ export default class { if (!remote.enabled) { throw new Error(`Remote ${remoteId} is disabled`) } - if (remote.handler.type === 'smb') { + + const handler = this._xo.getRemoteHandler(remote) + if (handler.type === 'smb') { throw new Error('Delta Backup is not supported for smb remotes') } @@ -415,7 +419,7 @@ export default class { if (!info.vdis[vdiUUID]) { info.vdis[vdiUUID] = { ...vdi } promises.push( - this._deltaVdiBackup({remote, vdi: vdiXo, dir, depth}).then( + this._deltaVdiBackup({handler, vdi: vdiXo, dir, depth}).then( vdiBackup => { const { backupDirectory, vdiFilename } = vdiBackup info.vdis[vdiUUID].xoPath = `${backupDirectory}/${vdiFilename}` @@ -442,13 +446,13 @@ export default class { } if (fail) { - console.error(`Remove successful backups in ${remote.handler.path}/${dir}`, fulFilledVdiBackups) - await this._failedRollingDeltaVmBackup(xapi, remote, dir, fulFilledVdiBackups) + console.error(`Remove successful backups in ${handler.path}/${dir}`, fulFilledVdiBackups) + await this._failedRollingDeltaVmBackup(xapi, handler, dir, fulFilledVdiBackups) throw new Error('Rolling delta vm backup failed.') } - const backups = await this._listDeltaVmBackups(remote, dir) + const backups = await this._listDeltaVmBackups(handler, dir) const date = safeDateFormat(new Date()) const backupFormat = `${date}_${vm.name_label}` @@ -457,14 +461,14 @@ export default class { try { await Promise.all([ - this.backupVm({vm, remoteId, file: xvaPath, onlyMetadata: true}), - remote.handler.outputFile(infoPath, JSON.stringify(info), {flag: 'wx'}) + this._backupVm(vm, handler, xvaPath, {onlyMetadata: true}), + handler.outputFile(infoPath, JSON.stringify(info), {flag: 'wx'}) ]) } catch (e) { await Promise.all([ - remote.handler.unlink(xvaPath).catch(noop), - remote.handler.unlink(infoPath).catch(noop), - this._failedRollingDeltaVmBackup(xapi, remote, dir, fulFilledVdiBackups) + handler.unlink(xvaPath).catch(noop), + handler.unlink(infoPath).catch(noop), + this._failedRollingDeltaVmBackup(xapi, handler, dir, fulFilledVdiBackups) ]) throw e @@ -474,12 +478,12 @@ export default class { await Promise.all( mapToArray(vdiBackups, vdiBackup => { const { backupDirectory } = vdiBackup.value() - return this._mergeDeltaVdiBackups({remote, dir: `${dir}/${backupDirectory}`, depth}) + return this._mergeDeltaVdiBackups({handler, dir: `${dir}/${backupDirectory}`, depth}) }) ) // Remove x2 files : json AND xva files. - await this._removeOldBackups(backups, remote, dir, backups.length - (depth - 1) * 2) + await this._removeOldBackups(backups, handler, dir, backups.length - (depth - 1) * 2) // Remove old vdi bases. Promise.all( @@ -496,34 +500,34 @@ export default class { return `${dir}/${backupFormat}` } - async _importVmMetadata (xapi, remote, file) { + async _importVmMetadata (xapi, handler, file) { const stream = await this._openAndwaitReadableFile( - remote, + handler, file, 'VM metadata to import not found in this remote' ) return await xapi.importVm(stream, { onlyMetadata: true }) } - async _importDeltaVdiBackupFromVm (xapi, vmId, remoteId, directory, vdiInfo) { + async _importDeltaVdiBackupFromVm (xapi, vmId, handler, directory, vdiInfo) { const vdi = await xapi.createVdi(vdiInfo.virtual_size, vdiInfo) const vdiId = vdi.$id - await this.importDeltaVdiBackup({ - vdi: this._xo.getObject(vdiId), - remoteId, - filePath: `${directory}/${vdiInfo.xoPath}` - }) + await this._importDeltaVdiBackup( + this._xo.getObject(vdiId), + handler, + `${directory}/${vdiInfo.xoPath}` + ) return vdiId } async importDeltaVmBackup ({sr, remoteId, filePath}) { - const remote = await this._xo.getRemote(remoteId) + const handler = await this._xo.getRemoteHandler(remoteId) const xapi = this._xo.getXapi(sr) // Import vm metadata. - const vm = await this._importVmMetadata(xapi, remote, `${filePath}.xva`) + const vm = await this._importVmMetadata(xapi, handler, `${filePath}.xva`) const vmName = vm.name_label // Disable start and change the VM name label during import. @@ -536,7 +540,7 @@ export default class { // Because XenServer creates Vbds linked to the vdis of the backup vm if it exists. await xapi.destroyVbdsFromVm(vm.uuid) - const info = JSON.parse(await remote.handler.readFile(`${filePath}.json`)) + const info = JSON.parse(await handler.readFile(`${filePath}.json`)) // Import VDIs. const vdiIds = {} @@ -574,13 +578,20 @@ export default class { async backupVm ({vm, remoteId, file, compress, onlyMetadata}) { const remote = await this._xo.getRemote(remoteId) + if (!remote) { throw new Error(`No such Remote ${remoteId}`) } if (!remote.enabled) { throw new Error(`Backup remote ${remoteId} is disabled`) } - const targetStream = await remote.handler.createOutputStream(file, { flags: 'wx' }) + + const handler = this._xo.getRemoteHandler(remote) + return this._backupVm(vm, handler, file, {compress, onlyMetadata}) + } + + async _backupVm (vm, handler, file, {compress, onlyMetadata}) { + const targetStream = await handler.createOutputStream(file, { flags: 'wx' }) const promise = eventToPromise(targetStream, 'finish') const sourceStream = await this._xo.getXapi(vm).exportVm(vm._xapiId, { @@ -596,13 +607,15 @@ export default class { const remote = await this._xo.getRemote(remoteId) if (!remote) { - throw new Error(`No such Remote s{remoteId}`) + throw new Error(`No such Remote ${remoteId}`) } if (!remote.enabled) { throw new Error(`Backup remote ${remoteId} is disabled`) } - const files = await remote.handler.list() + const handler = this._xo.getRemoteHandler(remote) + + const files = await handler.list() const reg = new RegExp('^[^_]+_' + escapeStringRegexp(`${tag}_${vm.name_label}.xva`)) const backups = sortBy(filter(files, (fileName) => reg.test(fileName))) @@ -610,8 +623,8 @@ export default class { const date = safeDateFormat(new Date()) const file = `${date}_${tag}_${vm.name_label}.xva` - await this.backupVm({vm, remoteId, file, compress, onlyMetadata}) - await this._removeOldBackups(backups, remote, undefined, backups.length - (depth - 1)) + await this._backupVm(vm, handler, file, {compress, onlyMetadata}) + await this._removeOldBackups(backups, handler, undefined, backups.length - (depth - 1)) } async rollingSnapshotVm (vm, tag, depth) { diff --git a/src/xo-mixins/remotes.js b/src/xo-mixins/remotes.js index 7f3b0205f..ac60821c6 100644 --- a/src/xo-mixins/remotes.js +++ b/src/xo-mixins/remotes.js @@ -1,14 +1,15 @@ import RemoteHandlerLocal from '../remote-handlers/local' import RemoteHandlerNfs from '../remote-handlers/nfs' import RemoteHandlerSmb from '../remote-handlers/smb' -import { Remotes } from '../models/remote' +import { + forEach +} from '../utils' import { NoSuchObject } from '../api-errors' import { - forEach, - mapToArray -} from '../utils' + Remotes +} from '../models/remote' // =================================================================== @@ -29,16 +30,16 @@ export default class { }) xo.on('start', async () => { - // TODO: Should it be private? - this._remoteHandlers = {} - await this.initRemotes() await this.syncAllRemotes() }) xo.on('stop', () => this.forgetAllRemotes()) } - _addHandler (remote) { + async getRemoteHandler (remote) { + if (typeof remote === 'string') { + remote = await this.getRemote(remote) + } const Handler = { file: RemoteHandlerLocal, smb: RemoteHandlerSmb, @@ -48,16 +49,11 @@ export default class { if (!Handler[type]) { throw new Error('Unhandled remote type') } - Object.defineProperty(remote, 'handler', { - value: new Handler[type](remote) - }) - remote.handler._getInfo(remote) // FIXME this has to be done by a specific code SHARED with the handler, not by the handler itself - remote.type = remote.handler.type // FIXME subsequent workaround - return remote + return new Handler[type](remote) } async getAllRemotes () { - return mapToArray(await this._remotes.get(), this._addHandler) + return this._remotes.get() } async _getRemote (id) { @@ -70,7 +66,7 @@ export default class { } async getRemote (id) { - return this._addHandler((await this._getRemote(id)).properties) + return (await this._getRemote(id)).properties } async createRemote ({name, url}) { @@ -81,10 +77,10 @@ export default class { async updateRemote (id, {name, url, enabled, error}) { const remote = await this._getRemote(id) this._updateRemote(remote, {name, url, enabled, error}) - const r = this._addHandler(remote.properties) - const props = await r.sync() + const handler = this.getRemoteHandler(remote.properties) + const props = await handler.sync() this._updateRemote(remote, props) - return await this._addHandler(this._remotes.save(remote).properties) + return (await this._remotes.save(remote)).properties } _updateRemote (remote, {name, url, enabled, error}) { @@ -99,8 +95,8 @@ export default class { } async removeRemote (id) { - const remote = await this.getRemote(id) - await remote.handler.forget() + const handler = await this.getRemoteHandler(id) + await handler.forget() await this._remotes.remove(id) } From 495b59c2e5c977dd6f46b2deafe83a51a62f3125 Mon Sep 17 00:00:00 2001 From: Fabrice Marsaud Date: Tue, 26 Jan 2016 17:34:00 +0100 Subject: [PATCH 08/11] update dependency --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2b2fb47e3..a80aefb39 100644 --- a/package.json +++ b/package.json @@ -117,7 +117,7 @@ "xen-api": "^0.7.2", "xml2js": "~0.4.6", "xo-collection": "^0.4.0", - "xo-remote-parser": "0.0.0" + "xo-remote-parser": "^0.1.0" }, "devDependencies": { "babel-eslint": "^4.0.10", From 261f0b4bf0b3e909dc74032a9dad7a7dd10ac526 Mon Sep 17 00:00:00 2001 From: Fabrice Marsaud Date: Wed, 27 Jan 2016 09:11:45 +0100 Subject: [PATCH 09/11] typo fix --- src/remote-handlers/abstract.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/remote-handlers/abstract.js b/src/remote-handlers/abstract.js index a3c7ce92d..12ca37f89 100644 --- a/src/remote-handlers/abstract.js +++ b/src/remote-handlers/abstract.js @@ -3,7 +3,7 @@ import getStream from 'get-stream' import { parse -} from 'xo-remote-parse' +} from 'xo-remote-parser' export default class RemoteHandlerAbstract { constructor (remote) { From ccdc744748271e49c454fb3822ac5bfd08915192 Mon Sep 17 00:00:00 2001 From: Fabrice Marsaud Date: Wed, 27 Jan 2016 10:08:59 +0100 Subject: [PATCH 10/11] fixes --- src/api/remote.js | 5 +++-- src/xo-mixins/backups.js | 6 +++--- src/xo-mixins/remotes.js | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/api/remote.js b/src/api/remote.js index 0af3a8d97..afd66458d 100644 --- a/src/api/remote.js +++ b/src/api/remote.js @@ -5,7 +5,7 @@ export async function getAll () { getAll.permission = 'admin' getAll.description = 'Gets all existing fs remote points' -export async function get (id) { +export async function get ({id}) { return await this.getRemote(id) } @@ -15,7 +15,8 @@ get.params = { id: {type: 'string'} } -export async function list (id) { +export async function list ({id}) { + console.log(id) return await this.listRemoteBackups(id) } diff --git a/src/xo-mixins/backups.js b/src/xo-mixins/backups.js index e7895bd76..6bd7c61cf 100644 --- a/src/xo-mixins/backups.js +++ b/src/xo-mixins/backups.js @@ -380,7 +380,7 @@ export default class { throw new Error(`Remote ${remoteId} is disabled`) } - const handler = this._xo.getRemoteHandler(remote) + const handler = await this._xo.getRemoteHandler(remote) if (handler.type === 'smb') { throw new Error('Delta Backup is not supported for smb remotes') } @@ -586,7 +586,7 @@ export default class { throw new Error(`Backup remote ${remoteId} is disabled`) } - const handler = this._xo.getRemoteHandler(remote) + const handler = await this._xo.getRemoteHandler(remote) return this._backupVm(vm, handler, file, {compress, onlyMetadata}) } @@ -613,7 +613,7 @@ export default class { throw new Error(`Backup remote ${remoteId} is disabled`) } - const handler = this._xo.getRemoteHandler(remote) + const handler = await this._xo.getRemoteHandler(remote) const files = await handler.list() diff --git a/src/xo-mixins/remotes.js b/src/xo-mixins/remotes.js index ac60821c6..8b3fbddbb 100644 --- a/src/xo-mixins/remotes.js +++ b/src/xo-mixins/remotes.js @@ -77,7 +77,7 @@ export default class { async updateRemote (id, {name, url, enabled, error}) { const remote = await this._getRemote(id) this._updateRemote(remote, {name, url, enabled, error}) - const handler = this.getRemoteHandler(remote.properties) + const handler = await this.getRemoteHandler(remote.properties) const props = await handler.sync() this._updateRemote(remote, props) return (await this._remotes.save(remote)).properties From 3b53f5ac11a3c45f7167e72138bd0e0d70be6053 Mon Sep 17 00:00:00 2001 From: Fabrice Marsaud Date: Wed, 27 Jan 2016 10:58:16 +0100 Subject: [PATCH 11/11] fixes --- src/api/remote.js | 1 - src/remote-handlers/abstract.js | 6 +----- src/xo-mixins/remotes.js | 4 +++- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/api/remote.js b/src/api/remote.js index afd66458d..fb191297f 100644 --- a/src/api/remote.js +++ b/src/api/remote.js @@ -16,7 +16,6 @@ get.params = { } export async function list ({id}) { - console.log(id) return await this.listRemoteBackups(id) } diff --git a/src/remote-handlers/abstract.js b/src/remote-handlers/abstract.js index 12ca37f89..ed8d3b87a 100644 --- a/src/remote-handlers/abstract.js +++ b/src/remote-handlers/abstract.js @@ -8,15 +8,11 @@ import { export default class RemoteHandlerAbstract { constructor (remote) { this._remote = parse({...remote}) - if (!this._remote.type === this.type) { + if (this._remote.type !== this.type) { throw new Error('Incorrect remote type') } } - _getInfo (remote) { - throw new Error('Not implemented') - } - get type () { throw new Error('Not implemented') } diff --git a/src/xo-mixins/remotes.js b/src/xo-mixins/remotes.js index 8b3fbddbb..e6ec7d479 100644 --- a/src/xo-mixins/remotes.js +++ b/src/xo-mixins/remotes.js @@ -112,7 +112,9 @@ export default class { async forgetAllRemotes () { const remotes = await this.getAllRemotes() for (let remote of remotes) { - await remote.handler.forget() + try { + (await this.getRemoteHandler(remote)).forget() + } catch (_) {} } }