NuKVM/xen-orchestra
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(xo-server#fetchBackupNgPartitionFiles): use @xen-orchestra/backups lib (#5606)

Browse Source
This commit is contained in:
badrAZ
2021-02-23 17:48:11 +01:00
committed by GitHub
parent 8a5fe86193
commit 5c9a47b6b7
4 changed files with 19 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
packages/xo-server/package.json
View File

@@ -145,8 +145,7 @@
"xo-collection": "^0.4.1",
"xo-common": "^0.5.0",
"xo-remote-parser": "^0.6.0",
"xo-vmdk-to-vhd": "^2.0.0",
"yazl": "^2.4.3"
"xo-vmdk-to-vhd": "^2.0.0"
},
"devDependencies": {
"@babel/cli": "^7.0.0",

45
packages/xo-server/src/_dedupeUnmount.js
View File

@@ -1,45 +0,0 @@
import assert from 'assert'
import { MultiKeyMap } from '@vates/multi-key-map'
import ensureArray from './_ensureArray'
function State() {
this.i = 0
this.value = undefined
}
export const dedupeUnmount = (fn, keyFn) => {
const states = new MultiKeyMap()
return function () {
const keys = ensureArray(keyFn.apply(this, arguments))
let state = states.get(keys)
if (state === undefined) {
state = new State()
states.set(keys, state)
const mount = async () => {
try {
const value = await fn.apply(this, arguments)
return {
__proto__: value,
async unmount() {
assert(state.i > 0)
if (--state.i === 0) {
states.delete(keys)
await value.unmount()
}
},
}
} catch (error) {
states.delete(keys)
throw error
}
}
state.value = mount()
}
++state.i
return state.value
}
}

291
packages/xo-server/src/xo-mixins/file-restore-ng.js
View File

@@ -1,135 +1,5 @@
import defer from 'golike-defer'
import execa from 'execa'
import splitLines from 'split-lines'
import { createParser as createPairsParser } from 'parse-pairs'
import { decorateWith } from '@vates/decorate-with'
import { normalize } from 'path'
import { readdir, rmdir } from 'fs-extra'
import { using } from 'promise-toolbox'
import { ZipFile } from 'yazl'
import { dedupeUnmount } from '../_dedupeUnmount'
import { lvs, pvs } from '../lvm'
import { resolveSubpath, tmpDir } from '../utils'
const compose = (...fns) => value => fns.reduce((value, fn) => fn(value), value)
const dedupeUnmountWithArgs = fn => dedupeUnmount(fn, (...args) => args)
const IGNORED_PARTITION_TYPES = {
// https://github.com/jhermsmeier/node-mbr/blob/master/lib/partition.js#L38
0x05: true,
0x0f: true,
0x15: true,
0x5e: true,
0x5f: true,
0x85: true,
0x91: true,
0x9b: true,
0xc5: true,
0xcf: true,
0xd5: true,
0x82: true, // swap
}
const RE_VHDI = /^vhdi(\d+)$/
async function addDirectory(zip, realPath, metadataPath) {
try {
const files = await readdir(realPath)
await Promise.all(files.map(file => addDirectory(zip, realPath + '/' + file, metadataPath + '/' + file)))
} catch (error) {
if (error == null || error.code !== 'ENOTDIR') {
throw error
}
zip.addFile(realPath, metadataPath)
}
}
const parsePartxLine = createPairsParser({
keyTransform: key => (key === 'UUID' ? 'id' : key.toLowerCase()),
valueTransform: (value, key) => (key === 'start' || key === 'size' || key === 'type' ? +value : value),
})
const listLvmLogicalVolumes = compose(
defer,
dedupeUnmountWithArgs
)(async ($defer, devicePath, partition, results = []) => {
const pv = await mountLvmPhysicalVolume(devicePath, partition)
$defer(pv.unmount)
const lvs = await pvs(['lv_name', 'lv_path', 'lv_size', 'vg_name'], pv.path)
const partitionId = partition !== undefined ? partition.id : ''
lvs.forEach((lv, i) => {
const name = lv.lv_name
if (name !== '') {
results.push({
id: `${partitionId}/${lv.vg_name}/${name}`,
name,
size: lv.lv_size,
})
}
})
return results
})
const mountLvmPhysicalVolume = dedupeUnmountWithArgs(async (devicePath, partition) => {
const args = []
if (partition !== undefined) {
args.push('-o', partition.start * 512)
}
args.push('--show', '-f', devicePath)
const path = (await execa('losetup', args)).stdout.trim()
await execa('pvscan', ['--cache', path])
return {
path,
unmount: async () => {
try {
const vgNames = await pvs('vg_name', path)
await execa('vgchange', ['-an', ...vgNames])
} finally {
await execa('losetup', ['-d', path])
}
},
}
})
const mountPartition = compose(
defer,
dedupeUnmountWithArgs
)(async ($defer, devicePath, partition) => {
const options = ['loop', 'ro']
if (partition !== undefined) {
const { start } = partition
if (start !== undefined) {
options.push(`offset=${start * 512}`)
}
}
const path = await tmpDir()
$defer.onFailure(rmdir, path)
const mount = options =>
execa('mount', [`--options=${options.join(',')}`, `--source=${devicePath}`, `--target=${path}`])
// `norecovery` option is used for ext3/ext4/xfs, if it fails it might be
// another fs, try without
try {
await mount([...options, 'norecovery'])
} catch (error) {
await mount(options)
}
const unmount = async () => {
await execa('umount', ['--lazy', path])
return rmdir(path)
}
$defer.onFailure(unmount)
return { path, unmount }
})
// - [x] list partitions
// - [x] list files in a partition
@@ -164,41 +34,25 @@ export default class BackupNgFileRestore {
})
}
@defer
async fetchBackupNgPartitionFiles($defer, remoteId, diskId, partitionId, paths) {
async fetchBackupNgPartitionFiles(remoteId, diskId, partitionId, paths) {
const app = this._app
const { proxy, url, options } = await app.getRemoteWithCredentials(remoteId)
if (proxy !== undefined) {
return app.callProxyMethod(
proxy,
'backup.fetchPartitionFiles',
{
disk: diskId,
remote: {
url,
options,
const remote = await app.getRemoteWithCredentials(remoteId)
return remote.proxy !== undefined
? app.callProxyMethod(
remote.proxy,
'backup.fetchPartitionFiles',
{
disk: diskId,
remote: {
url: remote.url,
options: remote.options,
},
partition: partitionId,
paths,
},
partition: partitionId,
paths,
},
{ assertType: 'stream' }
)
}
const disk = await this._mountDisk(remoteId, diskId)
$defer.onFailure(disk.unmount)
const partition = await this._mountPartition(disk.path, partitionId)
$defer.onFailure(partition.unmount)
const zip = new ZipFile()
await Promise.all(
paths.map(file =>
addDirectory(zip, resolveSubpath(partition.path, file), normalize('./' + file).replace(/\/+$/, ''))
)
)
zip.end()
return zip.outputStream.on('end', () => partition.unmount().then(disk.unmount))
{ assertType: 'stream' }
)
: using(app.getBackupsRemoteAdapter(remote), adapter => adapter.fetchPartitionFiles(diskId, partitionId, paths))
}
async listBackupNgDiskPartitions(remoteId, diskId) {
@@ -243,115 +97,4 @@ export default class BackupNgFileRestore {
})
: using(app.getBackupsRemoteAdapter(remote), adapter => adapter.listPartitionFiles(diskId, partitionId, path))
}
async _findPartition(devicePath, partitionId) {
const partitions = await this._listPartitions(devicePath, false)
const partition = partitions.find(_ => _.id === partitionId)
if (partition === undefined) {
throw new Error(`partition ${partitionId} not found`)
}
return partition
}
async _listPartitions(devicePath, inspectLvmPv = true) {
const { stdout } = await execa('partx', ['--bytes', '--output=NR,START,SIZE,NAME,UUID,TYPE', '--pairs', devicePath])
const promises = []
const partitions = []
splitLines(stdout).forEach(line => {
const partition = parsePartxLine(line)
const { type } = partition
if (type == null || type in IGNORED_PARTITION_TYPES) {
return
}
if (inspectLvmPv && type === 0x8e) {
promises.push(listLvmLogicalVolumes(devicePath, partition, partitions))
return
}
partitions.push(partition)
})
await Promise.all(promises)
return partitions
}
@decorateWith(dedupeUnmountWithArgs)
@defer
async _mountDisk($defer, remoteId, diskId) {
const handler = await this._app.getRemoteHandler(remoteId)
if (handler._getFilePath === undefined) {
throw new Error(`this remote is not supported`)
}
const diskPath = handler._getFilePath('/' + diskId)
const mountDir = await tmpDir()
$defer.onFailure(rmdir, mountDir)
await execa('vhdimount', [diskPath, mountDir])
const unmount = async () => {
await execa('fusermount', ['-uz', mountDir])
return rmdir(mountDir)
}
$defer.onFailure(unmount)
let max = 0
let maxEntry
const entries = await readdir(mountDir)
entries.forEach(entry => {
const matches = RE_VHDI.exec(entry)
if (matches !== null) {
const value = +matches[1]
if (value > max) {
max = value
maxEntry = entry
}
}
})
if (max === 0) {
throw new Error('no disks found')
}
return {
path: `${mountDir}/${maxEntry}`,
unmount,
}
}
@decorateWith(dedupeUnmountWithArgs)
@defer
async _mountPartition($defer, devicePath, partitionId) {
if (partitionId === undefined) {
return mountPartition(devicePath)
}
if (partitionId.includes('/')) {
const [pvId, vgName, lvName] = partitionId.split('/')
const lvmPartition = pvId !== '' ? await this._findPartition(devicePath, pvId) : undefined
const pv = await mountLvmPhysicalVolume(devicePath, lvmPartition)
const unmountQueue = [pv.unmount]
const unmount = async () => {
let fn
while ((fn = unmountQueue.pop()) !== undefined) {
await fn()
}
}
$defer.onFailure(unmount)
await execa('vgchange', ['-ay', vgName])
unmountQueue.push(() => execa('vgchange', ['-an', vgName]))
const partition = await mountPartition(
(await lvs(['lv_name', 'lv_path'], vgName)).find(_ => _.lv_name === lvName).lv_path
)
unmountQueue.push(partition.unmount)
return { __proto__: partition, unmount }
}
return mountPartition(devicePath, await this._findPartition(devicePath, partitionId))
}
}

2
yarn.lock
View File

@@ -19757,7 +19757,7 @@ yargs@~3.10.0:
decamelize "^1.0.0"
window-size "0.1.0"
yazl@^2.4.3, yazl@^2.5.1:
yazl@^2.5.1:
version "2.5.1"
resolved "https://registry.yarnpkg.com/yazl/-/yazl-2.5.1.tgz#a3d65d3dd659a5b0937850e8609f22fffa2b5c35"
integrity sha512-phENi2PLiHnHb6QBVot+dJnaAZ0xosj7p3fWl+znIjBDlnMI2PsZCJZ306BPTFOaHf5qdDEI8x5qFrSOBN5vrw==