diff --git a/packages/xo-server/package.json b/packages/xo-server/package.json
index 029eed0cb..50ef009ab 100644
--- a/packages/xo-server/package.json
+++ b/packages/xo-server/package.json
@@ -116,7 +116,8 @@
"xo-collection": "^0.4.1",
"xo-common": "^0.1.1",
"xo-remote-parser": "^0.3",
- "xo-vmdk-to-vhd": "0.0.12"
+ "xo-vmdk-to-vhd": "0.0.12",
+ "yazl": "^2.4.3"
},
"devDependencies": {
"@babel/cli": "7.0.0-beta.44",
diff --git a/packages/xo-server/src/api/backup-ng.js b/packages/xo-server/src/api/backup-ng.js
index 422ef80f6..9e1940fe2 100644
--- a/packages/xo-server/src/api/backup-ng.js
+++ b/packages/xo-server/src/api/backup-ng.js
@@ -1,3 +1,7 @@
+import { basename } from 'path'
+
+import { safeDateFormat } from '../utils'
+
export function createJob ({ schedules, ...job }) {
job.userId = this.user.id
return this.createBackupNgJob(job, schedules)
@@ -171,3 +175,88 @@ importVmBackup.params = {
type: 'string',
},
}
+
+// -----------------------------------------------------------------------------
+
+export function listPartitions ({ remote, disk }) {
+ return this.listBackupNgDiskPartitions(remote, disk)
+}
+
+listPartitions.permission = 'admin'
+
+listPartitions.params = {
+ disk: {
+ type: 'string',
+ },
+ remote: {
+ type: 'string',
+ },
+}
+
+export function listFiles ({ remote, disk, partition, path }) {
+ return this.listBackupNgPartitionFiles(remote, disk, partition, path)
+}
+
+listFiles.permission = 'admin'
+
+listFiles.params = {
+ disk: {
+ type: 'string',
+ },
+ partition: {
+ type: 'string',
+ optional: true,
+ },
+ path: {
+ type: 'string',
+ },
+ remote: {
+ type: 'string',
+ },
+}
+
+async function handleFetchFiles (req, res, { remote, disk, partition, paths }) {
+ const zipStream = await this.fetchBackupNgPartitionFiles(
+ remote,
+ disk,
+ partition,
+ paths
+ )
+
+ res.setHeader('content-disposition', 'attachment')
+ res.setHeader('content-type', 'application/octet-stream')
+ return zipStream
+}
+
+export async function fetchFiles (params) {
+ const { paths } = params
+ let filename = `restore_${safeDateFormat(new Date())}`
+ if (paths.length === 1) {
+ filename += `_${basename(paths[0])}`
+ }
+ filename += '.zip'
+
+ return this.registerHttpRequest(handleFetchFiles, params, {
+ suffix: encodeURI(`/${filename}`),
+ }).then(url => ({ $getFrom: url }))
+}
+
+fetchFiles.permission = 'admin'
+
+fetchFiles.params = {
+ disk: {
+ type: 'string',
+ },
+ partition: {
+ optional: true,
+ type: 'string',
+ },
+ paths: {
+ items: { type: 'string' },
+ minLength: 1,
+ type: 'array',
+ },
+ remote: {
+ type: 'string',
+ },
+}
diff --git a/packages/xo-server/src/lvm.js b/packages/xo-server/src/lvm.js
index d05a35e41..17d8b757a 100644
--- a/packages/xo-server/src/lvm.js
+++ b/packages/xo-server/src/lvm.js
@@ -1,16 +1,15 @@
import execa from 'execa'
import splitLines from 'split-lines'
import { createParser } from 'parse-pairs'
-import { isArray, map } from 'lodash'
// ===================================================================
const parse = createParser({
keyTransform: key => key.slice(5).toLowerCase(),
})
-const makeFunction = command => (fields, ...args) =>
- execa
- .stdout(command, [
+const makeFunction = command => async (fields, ...args) => {
+ return splitLines(
+ await execa.stdout(command, [
'--noheading',
'--nosuffix',
'--nameprefixes',
@@ -21,17 +20,8 @@ const makeFunction = command => (fields, ...args) =>
String(fields),
...args,
])
- .then(stdout =>
- map(
- splitLines(stdout),
- isArray(fields)
- ? parse
- : line => {
- const data = parse(line)
- return data[fields]
- }
- )
- )
+ ).map(Array.isArray(fields) ? parse : line => parse(line)[fields])
+}
export const lvs = makeFunction('lvs')
export const pvs = makeFunction('pvs')
diff --git a/packages/xo-server/src/xo-mixins/backups-ng/file-restore.js b/packages/xo-server/src/xo-mixins/backups-ng/file-restore.js
deleted file mode 100644
index e69de29bb..000000000
diff --git a/packages/xo-server/src/xo-mixins/backups-ng/index.js b/packages/xo-server/src/xo-mixins/backups-ng/index.js
index 3d4f78c78..e7b9a4c14 100644
--- a/packages/xo-server/src/xo-mixins/backups-ng/index.js
+++ b/packages/xo-server/src/xo-mixins/backups-ng/index.js
@@ -539,6 +539,16 @@ export default class BackupNg {
// inject an id usable by importVmBackupNg()
backups.forEach(backup => {
backup.id = `${remoteId}/${backup._filename}`
+
+ const { vdis, vhds } = backup
+ backup.disks = Object.keys(vhds).map(vdiId => {
+ const vdi = vdis[vdiId]
+ return {
+ id: `${dirname(backup._filename)}/${vhds[vdiId]}`,
+ name: vdi.name_label,
+ uuid: vdi.uuid,
+ }
+ })
})
backupsByVm[vmUuid] = backups
@@ -1096,7 +1106,11 @@ export default class BackupNg {
})
)
} catch (error) {
- if (error == null || error.code !== 'ENOENT') {
+ let code
+ if (
+ error == null ||
+ ((code = error.code) !== 'ENOENT' && code !== 'ENOTDIR')
+ ) {
throw error
}
}
diff --git a/packages/xo-server/src/xo-mixins/file-restore-ng.js b/packages/xo-server/src/xo-mixins/file-restore-ng.js
new file mode 100644
index 000000000..96883b81d
--- /dev/null
+++ b/packages/xo-server/src/xo-mixins/file-restore-ng.js
@@ -0,0 +1,332 @@
+import defer from 'golike-defer'
+import execa from 'execa'
+import splitLines from 'split-lines'
+import { createParser as createPairsParser } from 'parse-pairs'
+import { normalize } from 'path'
+import { readdir, rmdir, stat } from 'fs-extra'
+import { ZipFile } from 'yazl'
+
+import { lvs, pvs } from '../lvm'
+import { resolveSubpath, tmpDir } from '../utils'
+
+const IGNORED_PARTITION_TYPES = {
+ // https://github.com/jhermsmeier/node-mbr/blob/master/lib/partition.js#L38
+ 0x05: true,
+ 0x0f: true,
+ 0x15: true,
+ 0x5e: true,
+ 0x5f: true,
+ 0x85: true,
+ 0x91: true,
+ 0x9b: true,
+ 0xc5: true,
+ 0xcf: true,
+ 0xd5: true,
+
+ 0x82: true, // swap
+}
+
+const PARTITION_TYPE_NAMES = {
+ 0x07: 'NTFS',
+ 0x0c: 'FAT',
+ 0x83: 'linux',
+}
+
+const RE_VHDI = /^vhdi(\d+)$/
+
+const parsePartxLine = createPairsParser({
+ keyTransform: key => (key === 'UUID' ? 'id' : key.toLowerCase()),
+ valueTransform: (value, key) =>
+ key === 'start' || key === 'size'
+ ? +value
+ : key === 'type' ? PARTITION_TYPE_NAMES[+value] || value : value,
+})
+
+const listLvmLogicalVolumes = defer(
+ async ($defer, devicePath, partition, results = []) => {
+ const pv = await mountLvmPhysicalVolume(devicePath, partition)
+ $defer(pv.unmount)
+
+ const lvs = await pvs(['lv_name', 'lv_path', 'lv_size', 'vg_name'], pv.path)
+ const partitionId = partition !== undefined ? partition.id : ''
+ lvs.forEach((lv, i) => {
+ const name = lv.lv_name
+ if (name !== '') {
+ results.push({
+ id: `${partitionId}/${lv.vg_name}/${name}`,
+ name,
+ size: lv.lv_size,
+ })
+ }
+ })
+ return results
+ }
+)
+
+async function mountLvmPhysicalVolume (devicePath, partition) {
+ const args = []
+ if (partition !== undefined) {
+ args.push('-o', partition.start * 512)
+ }
+ args.push('--show', '-f', devicePath)
+ const path = (await execa.stdout('losetup', args)).trim()
+ await execa('pvscan', ['--cache', path])
+
+ return {
+ path,
+ unmount: async () => {
+ try {
+ const vgNames = await pvs('vg_name', path)
+ await execa('vgchange', ['-an', ...vgNames])
+ } finally {
+ await execa('losetup', ['-d', path])
+ }
+ },
+ }
+}
+
+const mountPartition = defer(async ($defer, devicePath, partition) => {
+ const options = ['loop', 'ro']
+
+ if (partition !== undefined) {
+ const { start } = partition
+ if (start !== undefined) {
+ options.push(`offset=${start * 512}`)
+ }
+ }
+
+ const path = await tmpDir()
+ $defer.onFailure(rmdir, path)
+
+ const mount = options =>
+ execa('mount', [
+ `--options=${options.join(',')}`,
+ `--source=${devicePath}`,
+ `--target=${path}`,
+ ])
+
+ // `norecovery` option is used for ext3/ext4/xfs, if it fails it might be
+ // another fs, try without
+ try {
+ await mount([...options, 'norecovery'])
+ } catch (error) {
+ await mount(options)
+ }
+ const unmount = async () => {
+ await execa('umount', ['--lazy', path])
+ return rmdir(path)
+ }
+ $defer.onFailure(unmount)
+
+ return { path, unmount }
+})
+
+// - [x] list partitions
+// - [x] list files in a partition
+// - [x] list files in a bare partition
+// - [x] list LVM partitions
+//
+// - [ ] partitions with unmount debounce
+// - [ ] handle directory restore
+// - [ ] handle multiple entries restore (both dirs and files)
+// - [ ] by default use common path as root
+// - [ ] handle LVM partitions on multiple disks
+// - [ ] find mounted disks/partitions on start (in case of interruptions)
+//
+// - [ ] manual mount/unmount (of disk) for advance file restore
+// - could it stay mounted during the backup process?
+// - [ ] mountDisk (VHD)
+// - [ ] unmountDisk (only for manual mount)
+// - [ ] getMountedDisks
+// - [ ] mountPartition (optional)
+// - [ ] getMountedPartitions
+// - [ ] unmountPartition
+export default class BackupNgFileRestore {
+ constructor (app) {
+ this._app = app
+ this._mounts = { __proto__: null }
+ }
+
+ @defer
+ async fetchBackupNgPartitionFiles (
+ $defer,
+ remoteId,
+ diskId,
+ partitionId,
+ paths
+ ) {
+ const disk = await this._mountDisk(remoteId, diskId)
+ $defer.onFailure(disk.unmount)
+
+ const partition = await this._mountPartition(disk.path, partitionId)
+ $defer.onFailure(partition.unmount)
+
+ const zip = new ZipFile()
+ paths.forEach(file => {
+ zip.addFile(resolveSubpath(partition.path, file), normalize('./' + file))
+ })
+ zip.end()
+ return zip.outputStream.on('end', () =>
+ partition.unmount().then(disk.unmount)
+ )
+ }
+
+ @defer
+ async listBackupNgDiskPartitions ($defer, remoteId, diskId) {
+ const disk = await this._mountDisk(remoteId, diskId)
+ $defer(disk.unmount)
+ return this._listPartitions(disk.path)
+ }
+
+ @defer
+ async listBackupNgPartitionFiles (
+ $defer,
+ remoteId,
+ diskId,
+ partitionId,
+ path
+ ) {
+ const disk = await this._mountDisk(remoteId, diskId)
+ $defer(disk.unmount)
+
+ const partition = await this._mountPartition(disk.path, partitionId)
+ $defer(partition.unmount)
+
+ path = resolveSubpath(partition.path, path)
+
+ const entriesMap = {}
+ await Promise.all(
+ readdir(path).map(async name => {
+ try {
+ const stats = await stat(`${path}/${name}`)
+ entriesMap[stats.isDirectory() ? `${name}/` : name] = {}
+ } catch (error) {
+ if (error == null || error.code !== 'ENOENT') {
+ throw error
+ }
+ }
+ })
+ )
+ return entriesMap
+ }
+
+ async _findPartition (devicePath, partitionId) {
+ const partitions = await this._listPartitions(devicePath, false)
+ const partition = partitions.find(_ => _.id === partitionId)
+ if (partition === undefined) {
+ throw new Error(`partition ${partitionId} not found`)
+ }
+ return partition
+ }
+
+ async _listPartitions (devicePath, inspectLvmPv = true) {
+ const stdout = await execa.stdout('partx', [
+ '--bytes',
+ '--output=NR,START,SIZE,NAME,UUID,TYPE',
+ '--pairs',
+ devicePath,
+ ])
+
+ const promises = []
+ const partitions = []
+ splitLines(stdout).forEach(line => {
+ const partition = parsePartxLine(line)
+ let { type } = partition
+ if (type == null || (type = +type) in IGNORED_PARTITION_TYPES) {
+ return
+ }
+
+ if (inspectLvmPv && type === 0x8e) {
+ promises.push(listLvmLogicalVolumes(devicePath, partition, partitions))
+ return
+ }
+
+ partitions.push(partition)
+ })
+
+ await Promise.all(promises)
+
+ return partitions
+ }
+
+ @defer
+ async _mountDisk ($defer, remoteId, diskId) {
+ const handler = await this._app.getRemoteHandler(remoteId)
+ if (handler._getFilePath === undefined) {
+ throw new Error(`this remote is not supported`)
+ }
+
+ const diskPath = handler._getFilePath(diskId)
+ const mountDir = await tmpDir()
+ $defer.onFailure(rmdir, mountDir)
+
+ await execa('vhdimount', [diskPath, mountDir])
+ const unmount = async () => {
+ await execa('fusermount', ['-uz', mountDir])
+ return rmdir(mountDir)
+ }
+ $defer.onFailure(unmount)
+
+ let max = 0
+ let maxEntry
+ const entries = await readdir(mountDir)
+ entries.forEach(entry => {
+ const matches = RE_VHDI.exec(entry)
+ if (matches !== null) {
+ const value = +matches[1]
+ if (value > max) {
+ max = value
+ maxEntry = entry
+ }
+ }
+ })
+ if (max === 0) {
+ throw new Error('no disks found')
+ }
+
+ return {
+ path: `${mountDir}/${maxEntry}`,
+ unmount,
+ }
+ }
+
+ @defer
+ async _mountPartition ($defer, devicePath, partitionId) {
+ if (partitionId === undefined) {
+ return mountPartition(devicePath)
+ }
+
+ if (partitionId.includes('/')) {
+ const [pvId, vgName, lvName] = partitionId.split('/')
+ const lvmPartition =
+ pvId !== '' ? await this._findPartition(devicePath, pvId) : undefined
+
+ const pv = await mountLvmPhysicalVolume(devicePath, lvmPartition)
+
+ const unmountQueue = [pv.unmount]
+ const unmount = async () => {
+ let fn
+ while ((fn = unmountQueue.pop()) !== undefined) {
+ await fn()
+ }
+ }
+ $defer.onFailure(unmount)
+
+ await execa('vgchange', ['-ay', vgName])
+ unmountQueue.push(() => execa('vgchange', ['-an', vgName]))
+
+ const partition = await mountPartition(
+ (await lvs(['lv_name', 'lv_path'], vgName)).find(
+ _ => _.lv_name === lvName
+ ).lv_path
+ )
+ unmountQueue.push(partition.unmount)
+ return { ...partition, unmount }
+ }
+
+ return mountPartition(
+ devicePath,
+ await this._findPartition(devicePath, partitionId)
+ )
+ }
+}
diff --git a/packages/xo-server/src/xo-mixins/jobs/index.js b/packages/xo-server/src/xo-mixins/jobs/index.js
index 381b8f933..91f0681d8 100644
--- a/packages/xo-server/src/xo-mixins/jobs/index.js
+++ b/packages/xo-server/src/xo-mixins/jobs/index.js
@@ -225,9 +225,10 @@ export default class Jobs {
runningJobs[id] = runJobId
+ let session
try {
const app = this._app
- const session = app.createUserConnection()
+ session = app.createUserConnection()
session.set('user_id', job.userId)
const status = await executor({
@@ -255,6 +256,9 @@ export default class Jobs {
throw error
} finally {
delete runningJobs[id]
+ if (session !== undefined) {
+ session.close()
+ }
}
}
diff --git a/packages/xo-server/src/xo.js b/packages/xo-server/src/xo.js
index bb6769068..bf3135d0a 100644
--- a/packages/xo-server/src/xo.js
+++ b/packages/xo-server/src/xo.js
@@ -140,7 +140,11 @@ export default class Xo extends EventEmitter {
}).then(
result => {
if (result != null) {
- res.end(JSON.stringify(result))
+ if (typeof result.pipe === 'function') {
+ result.pipe(res)
+ } else {
+ res.end(JSON.stringify(result))
+ }
}
},
error => {
diff --git a/packages/xo-web/src/common/render-xo-item.js b/packages/xo-web/src/common/render-xo-item.js
index 4ade1211d..c2c7debed 100644
--- a/packages/xo-web/src/common/render-xo-item.js
+++ b/packages/xo-web/src/common/render-xo-item.js
@@ -5,6 +5,7 @@ import { startsWith } from 'lodash'
import Icon from './icon'
import propTypes from './prop-types-decorator'
import { createGetObject } from './selectors'
+import { FormattedDate } from 'react-intl'
import { isSrWritable } from './xo'
import { connectStore, formatSize } from './utils'
@@ -203,10 +204,29 @@ const xoItemToRender = {
: group.name_label}
),
+
+ backup: backup => (
+
+
+ {backup.mode}
+ {' '}
+ {backup.remote.name}{' '}
+
{_('deleteVmBackupsBulkMessage', { nVms: datas.length })}
, + icon: 'delete', + strongConfirm: { + messageId: 'deleteVmBackupsBulkConfirmText', + values: { + nBackups: reduce(datas, (sum, data) => sum + data.backups.length, 0), + }, + }, + }) + .then(() => deleteBackups(flatMap(datas, 'backups')), noop) + .then(() => this._refreshBackupList()) + + // --------------------------------------------------------------------------- + + _actions = [ + { + handler: this._bulkDelete, + icon: 'delete', + individualHandler: this._delete, + label: _('deleteVmBackups'), + level: 'danger', + }, + ] + + _individualActions = [ + { + handler: this._restore, + icon: 'restore', + label: _('restoreVmBackups'), + level: 'primary', + }, + ] -export default class FileRestore extends Component { render () { - returnAvailable soon
+ return ( ++ {path} {scanningFiles &&+ +} + {listFilesError && } +
{file.path}+ +