feat(xo-server#fetchBackupNgPartitionFiles): use @xen-orchestra/backups lib (#5606)
This commit is contained in:
badrAZ
GitHub
parent
8a5fe86193
commit
5c9a47b6b7
@ -145,8 +145,7 @@
|
||||
"xo-collection": "^0.4.1",
|
||||
"xo-common": "^0.5.0",
|
||||
"xo-remote-parser": "^0.6.0",
|
||||
"xo-vmdk-to-vhd": "^2.0.0",
|
||||
"yazl": "^2.4.3"
|
||||
"xo-vmdk-to-vhd": "^2.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/cli": "^7.0.0",
|
||||
|
||||
@ -1,45 +0,0 @@
|
||||
import assert from 'assert'
|
||||
import { MultiKeyMap } from '@vates/multi-key-map'
|
||||
|
||||
import ensureArray from './_ensureArray'
|
||||
|
||||
function State() {
|
||||
this.i = 0
|
||||
this.value = undefined
|
||||
}
|
||||
|
||||
export const dedupeUnmount = (fn, keyFn) => {
|
||||
const states = new MultiKeyMap()
|
||||
|
||||
return function () {
|
||||
const keys = ensureArray(keyFn.apply(this, arguments))
|
||||
let state = states.get(keys)
|
||||
if (state === undefined) {
|
||||
state = new State()
|
||||
states.set(keys, state)
|
||||
|
||||
const mount = async () => {
|
||||
try {
|
||||
const value = await fn.apply(this, arguments)
|
||||
return {
|
||||
__proto__: value,
|
||||
async unmount() {
|
||||
assert(state.i > 0)
|
||||
if (--state.i === 0) {
|
||||
states.delete(keys)
|
||||
await value.unmount()
|
||||
}
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
states.delete(keys)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
state.value = mount()
|
||||
}
|
||||
++state.i
|
||||
return state.value
|
||||
}
|
||||
}
|
||||
@ -1,135 +1,5 @@
|
||||
import defer from 'golike-defer'
|
||||
import execa from 'execa'
|
||||
import splitLines from 'split-lines'
|
||||
import { createParser as createPairsParser } from 'parse-pairs'
|
||||
import { decorateWith } from '@vates/decorate-with'
|
||||
import { normalize } from 'path'
|
||||
import { readdir, rmdir } from 'fs-extra'
|
||||
import { using } from 'promise-toolbox'
|
||||
import { ZipFile } from 'yazl'
|
||||
|
||||
import { dedupeUnmount } from '../_dedupeUnmount'
|
||||
import { lvs, pvs } from '../lvm'
|
||||
import { resolveSubpath, tmpDir } from '../utils'
|
||||
|
||||
const compose = (...fns) => value => fns.reduce((value, fn) => fn(value), value)
|
||||
|
||||
const dedupeUnmountWithArgs = fn => dedupeUnmount(fn, (...args) => args)
|
||||
|
||||
const IGNORED_PARTITION_TYPES = {
|
||||
// https://github.com/jhermsmeier/node-mbr/blob/master/lib/partition.js#L38
|
||||
0x05: true,
|
||||
0x0f: true,
|
||||
0x15: true,
|
||||
0x5e: true,
|
||||
0x5f: true,
|
||||
0x85: true,
|
||||
0x91: true,
|
||||
0x9b: true,
|
||||
0xc5: true,
|
||||
0xcf: true,
|
||||
0xd5: true,
|
||||
|
||||
0x82: true, // swap
|
||||
}
|
||||
|
||||
const RE_VHDI = /^vhdi(\d+)$/
|
||||
|
||||
async function addDirectory(zip, realPath, metadataPath) {
|
||||
try {
|
||||
const files = await readdir(realPath)
|
||||
await Promise.all(files.map(file => addDirectory(zip, realPath + '/' + file, metadataPath + '/' + file)))
|
||||
} catch (error) {
|
||||
if (error == null || error.code !== 'ENOTDIR') {
|
||||
throw error
|
||||
}
|
||||
zip.addFile(realPath, metadataPath)
|
||||
}
|
||||
}
|
||||
|
||||
const parsePartxLine = createPairsParser({
|
||||
keyTransform: key => (key === 'UUID' ? 'id' : key.toLowerCase()),
|
||||
valueTransform: (value, key) => (key === 'start' || key === 'size' || key === 'type' ? +value : value),
|
||||
})
|
||||
|
||||
const listLvmLogicalVolumes = compose(
|
||||
defer,
|
||||
dedupeUnmountWithArgs
|
||||
)(async ($defer, devicePath, partition, results = []) => {
|
||||
const pv = await mountLvmPhysicalVolume(devicePath, partition)
|
||||
$defer(pv.unmount)
|
||||
|
||||
const lvs = await pvs(['lv_name', 'lv_path', 'lv_size', 'vg_name'], pv.path)
|
||||
const partitionId = partition !== undefined ? partition.id : ''
|
||||
lvs.forEach((lv, i) => {
|
||||
const name = lv.lv_name
|
||||
if (name !== '') {
|
||||
results.push({
|
||||
id: `${partitionId}/${lv.vg_name}/${name}`,
|
||||
name,
|
||||
size: lv.lv_size,
|
||||
})
|
||||
}
|
||||
})
|
||||
return results
|
||||
})
|
||||
|
||||
const mountLvmPhysicalVolume = dedupeUnmountWithArgs(async (devicePath, partition) => {
|
||||
const args = []
|
||||
if (partition !== undefined) {
|
||||
args.push('-o', partition.start * 512)
|
||||
}
|
||||
args.push('--show', '-f', devicePath)
|
||||
const path = (await execa('losetup', args)).stdout.trim()
|
||||
await execa('pvscan', ['--cache', path])
|
||||
|
||||
return {
|
||||
path,
|
||||
unmount: async () => {
|
||||
try {
|
||||
const vgNames = await pvs('vg_name', path)
|
||||
await execa('vgchange', ['-an', ...vgNames])
|
||||
} finally {
|
||||
await execa('losetup', ['-d', path])
|
||||
}
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
const mountPartition = compose(
|
||||
defer,
|
||||
dedupeUnmountWithArgs
|
||||
)(async ($defer, devicePath, partition) => {
|
||||
const options = ['loop', 'ro']
|
||||
|
||||
if (partition !== undefined) {
|
||||
const { start } = partition
|
||||
if (start !== undefined) {
|
||||
options.push(`offset=${start * 512}`)
|
||||
}
|
||||
}
|
||||
|
||||
const path = await tmpDir()
|
||||
$defer.onFailure(rmdir, path)
|
||||
|
||||
const mount = options =>
|
||||
execa('mount', [`--options=${options.join(',')}`, `--source=${devicePath}`, `--target=${path}`])
|
||||
|
||||
// `norecovery` option is used for ext3/ext4/xfs, if it fails it might be
|
||||
// another fs, try without
|
||||
try {
|
||||
await mount([...options, 'norecovery'])
|
||||
} catch (error) {
|
||||
await mount(options)
|
||||
}
|
||||
const unmount = async () => {
|
||||
await execa('umount', ['--lazy', path])
|
||||
return rmdir(path)
|
||||
}
|
||||
$defer.onFailure(unmount)
|
||||
|
||||
return { path, unmount }
|
||||
})
|
||||
|
||||
// - [x] list partitions
|
||||
// - [x] list files in a partition
|
||||
@ -164,41 +34,25 @@ export default class BackupNgFileRestore {
|
||||
})
|
||||
}
|
||||
|
||||
@defer
|
||||
async fetchBackupNgPartitionFiles($defer, remoteId, diskId, partitionId, paths) {
|
||||
async fetchBackupNgPartitionFiles(remoteId, diskId, partitionId, paths) {
|
||||
const app = this._app
|
||||
const { proxy, url, options } = await app.getRemoteWithCredentials(remoteId)
|
||||
if (proxy !== undefined) {
|
||||
return app.callProxyMethod(
|
||||
proxy,
|
||||
'backup.fetchPartitionFiles',
|
||||
{
|
||||
disk: diskId,
|
||||
remote: {
|
||||
url,
|
||||
options,
|
||||
const remote = await app.getRemoteWithCredentials(remoteId)
|
||||
return remote.proxy !== undefined
|
||||
? app.callProxyMethod(
|
||||
remote.proxy,
|
||||
'backup.fetchPartitionFiles',
|
||||
{
|
||||
disk: diskId,
|
||||
remote: {
|
||||
url: remote.url,
|
||||
options: remote.options,
|
||||
},
|
||||
partition: partitionId,
|
||||
paths,
|
||||
},
|
||||
partition: partitionId,
|
||||
paths,
|
||||
},
|
||||
{ assertType: 'stream' }
|
||||
)
|
||||
}
|
||||
|
||||
const disk = await this._mountDisk(remoteId, diskId)
|
||||
$defer.onFailure(disk.unmount)
|
||||
|
||||
const partition = await this._mountPartition(disk.path, partitionId)
|
||||
$defer.onFailure(partition.unmount)
|
||||
|
||||
const zip = new ZipFile()
|
||||
await Promise.all(
|
||||
paths.map(file =>
|
||||
addDirectory(zip, resolveSubpath(partition.path, file), normalize('./' + file).replace(/\/+$/, ''))
|
||||
)
|
||||
)
|
||||
zip.end()
|
||||
return zip.outputStream.on('end', () => partition.unmount().then(disk.unmount))
|
||||
{ assertType: 'stream' }
|
||||
)
|
||||
: using(app.getBackupsRemoteAdapter(remote), adapter => adapter.fetchPartitionFiles(diskId, partitionId, paths))
|
||||
}
|
||||
|
||||
async listBackupNgDiskPartitions(remoteId, diskId) {
|
||||
@ -243,115 +97,4 @@ export default class BackupNgFileRestore {
|
||||
})
|
||||
: using(app.getBackupsRemoteAdapter(remote), adapter => adapter.listPartitionFiles(diskId, partitionId, path))
|
||||
}
|
||||
|
||||
async _findPartition(devicePath, partitionId) {
|
||||
const partitions = await this._listPartitions(devicePath, false)
|
||||
const partition = partitions.find(_ => _.id === partitionId)
|
||||
if (partition === undefined) {
|
||||
throw new Error(`partition ${partitionId} not found`)
|
||||
}
|
||||
return partition
|
||||
}
|
||||
|
||||
async _listPartitions(devicePath, inspectLvmPv = true) {
|
||||
const { stdout } = await execa('partx', ['--bytes', '--output=NR,START,SIZE,NAME,UUID,TYPE', '--pairs', devicePath])
|
||||
|
||||
const promises = []
|
||||
const partitions = []
|
||||
splitLines(stdout).forEach(line => {
|
||||
const partition = parsePartxLine(line)
|
||||
const { type } = partition
|
||||
if (type == null || type in IGNORED_PARTITION_TYPES) {
|
||||
return
|
||||
}
|
||||
|
||||
if (inspectLvmPv && type === 0x8e) {
|
||||
promises.push(listLvmLogicalVolumes(devicePath, partition, partitions))
|
||||
return
|
||||
}
|
||||
|
||||
partitions.push(partition)
|
||||
})
|
||||
|
||||
await Promise.all(promises)
|
||||
|
||||
return partitions
|
||||
}
|
||||
|
||||
@decorateWith(dedupeUnmountWithArgs)
|
||||
@defer
|
||||
async _mountDisk($defer, remoteId, diskId) {
|
||||
const handler = await this._app.getRemoteHandler(remoteId)
|
||||
if (handler._getFilePath === undefined) {
|
||||
throw new Error(`this remote is not supported`)
|
||||
}
|
||||
|
||||
const diskPath = handler._getFilePath('/' + diskId)
|
||||
const mountDir = await tmpDir()
|
||||
$defer.onFailure(rmdir, mountDir)
|
||||
|
||||
await execa('vhdimount', [diskPath, mountDir])
|
||||
const unmount = async () => {
|
||||
await execa('fusermount', ['-uz', mountDir])
|
||||
return rmdir(mountDir)
|
||||
}
|
||||
$defer.onFailure(unmount)
|
||||
|
||||
let max = 0
|
||||
let maxEntry
|
||||
const entries = await readdir(mountDir)
|
||||
entries.forEach(entry => {
|
||||
const matches = RE_VHDI.exec(entry)
|
||||
if (matches !== null) {
|
||||
const value = +matches[1]
|
||||
if (value > max) {
|
||||
max = value
|
||||
maxEntry = entry
|
||||
}
|
||||
}
|
||||
})
|
||||
if (max === 0) {
|
||||
throw new Error('no disks found')
|
||||
}
|
||||
|
||||
return {
|
||||
path: `${mountDir}/${maxEntry}`,
|
||||
unmount,
|
||||
}
|
||||
}
|
||||
|
||||
@decorateWith(dedupeUnmountWithArgs)
|
||||
@defer
|
||||
async _mountPartition($defer, devicePath, partitionId) {
|
||||
if (partitionId === undefined) {
|
||||
return mountPartition(devicePath)
|
||||
}
|
||||
|
||||
if (partitionId.includes('/')) {
|
||||
const [pvId, vgName, lvName] = partitionId.split('/')
|
||||
const lvmPartition = pvId !== '' ? await this._findPartition(devicePath, pvId) : undefined
|
||||
|
||||
const pv = await mountLvmPhysicalVolume(devicePath, lvmPartition)
|
||||
|
||||
const unmountQueue = [pv.unmount]
|
||||
const unmount = async () => {
|
||||
let fn
|
||||
while ((fn = unmountQueue.pop()) !== undefined) {
|
||||
await fn()
|
||||
}
|
||||
}
|
||||
$defer.onFailure(unmount)
|
||||
|
||||
await execa('vgchange', ['-ay', vgName])
|
||||
unmountQueue.push(() => execa('vgchange', ['-an', vgName]))
|
||||
|
||||
const partition = await mountPartition(
|
||||
(await lvs(['lv_name', 'lv_path'], vgName)).find(_ => _.lv_name === lvName).lv_path
|
||||
)
|
||||
unmountQueue.push(partition.unmount)
|
||||
return { __proto__: partition, unmount }
|
||||
}
|
||||
|
||||
return mountPartition(devicePath, await this._findPartition(devicePath, partitionId))
|
||||
}
|
||||
}
|
||||
|
||||
@ -19757,7 +19757,7 @@ yargs@~3.10.0:
|
||||
decamelize "^1.0.0"
|
||||
window-size "0.1.0"
|
||||
|
||||
yazl@^2.4.3, yazl@^2.5.1:
|
||||
yazl@^2.5.1:
|
||||
version "2.5.1"
|
||||
resolved "https://registry.yarnpkg.com/yazl/-/yazl-2.5.1.tgz#a3d65d3dd659a5b0937850e8609f22fffa2b5c35"
|
||||
integrity sha512-phENi2PLiHnHb6QBVot+dJnaAZ0xosj7p3fWl+znIjBDlnMI2PsZCJZ306BPTFOaHf5qdDEI8x5qFrSOBN5vrw==
|
||||
|
||||
Loading…
Reference in New Issue
Block a user