diff --git a/@vates/fuse-vhd/.npmignore b/@vates/fuse-vhd/.npmignore new file mode 120000 index 000000000..008d1b9b9 --- /dev/null +++ b/@vates/fuse-vhd/.npmignore @@ -0,0 +1 @@ +../../scripts/npmignore \ No newline at end of file diff --git a/@vates/fuse-vhd/index.js b/@vates/fuse-vhd/index.js new file mode 100644 index 000000000..e9e41d12e --- /dev/null +++ b/@vates/fuse-vhd/index.js @@ -0,0 +1,71 @@ +'use strict' + +const LRU = require('lru-cache') +const Fuse = require('fuse-native') +const { VhdSynthetic } = require('vhd-lib') +const { Disposable, fromCallback } = require('promise-toolbox') +const { createLogger } = require('@xen-orchestra/log') + +const { warn } = createLogger('vates:fuse-vhd') + +// build a s stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js +const stat = st => ({ + mtime: st.mtime || new Date(), + atime: st.atime || new Date(), + ctime: st.ctime || new Date(), + size: st.size !== undefined ? st.size : 0, + mode: st.mode === 'dir' ? 16877 : st.mode === 'file' ? 33188 : st.mode === 'link' ? 41453 : st.mode, + uid: st.uid !== undefined ? st.uid : process.getuid(), + gid: st.gid !== undefined ? st.gid : process.getgid(), +}) + +exports.mount = Disposable.factory(async function* mount(handler, diskPath, mountDir) { + const vhd = yield VhdSynthetic.fromVhdChain(handler, diskPath) + + const cache = new LRU({ + max: 16, // each cached block is 2MB in size + }) + await vhd.readBlockAllocationTable() + const fuse = new Fuse(mountDir, { + async readdir(path, cb) { + if (path === '/') { + return cb(null, ['vhd0']) + } + cb(Fuse.ENOENT) + }, + async getattr(path, cb) { + if (path === '/') { + return cb( + null, + stat({ + mode: 'dir', + size: 4096, + }) + ) + } + if (path === '/vhd0') { + return cb( + null, + stat({ + mode: 'file', + size: vhd.footer.currentSize, + }) + ) + } + + cb(Fuse.ENOENT) + }, + read(path, fd, buf, len, pos, cb) { + if (path === '/vhd0') { + return vhd + .readRawData(pos, len, cache, buf) + .then(cb) + } + throw new Error(`read file ${path} not exists`) + }, + }) + return new Disposable( + () => fromCallback(() => fuse.unmount()), + fromCallback(() => fuse.mount()) + ) +}) diff --git a/@vates/fuse-vhd/package.json b/@vates/fuse-vhd/package.json new file mode 100644 index 000000000..01df6b4ce --- /dev/null +++ b/@vates/fuse-vhd/package.json @@ -0,0 +1,30 @@ +{ + "name": "@vates/fuse-vhd", + "version": "0.0.1", + "license": "ISC", + "private": false, + "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/fuse-vhd", + "bugs": "https://github.com/vatesfr/xen-orchestra/issues", + "repository": { + "directory": "@vates/fuse-vhd", + "type": "git", + "url": "https://github.com/vatesfr/xen-orchestra.git" + }, + "author": { + "name": "Vates SAS", + "url": "https://vates.fr" + }, + "engines": { + "node": ">=10.0" + }, + "dependencies": { + "@xen-orchestra/log": "^0.3.0", + "fuse-native": "^2.2.6", + "lru-cache": "^7.14.0", + "promise-toolbox": "^0.21.0", + "vhd-lib": "^4.0.1" + }, + "scripts": { + "postversion": "npm publish --access public" + } +} diff --git a/@xen-orchestra/backups/RemoteAdapter.js b/@xen-orchestra/backups/RemoteAdapter.js index 24722b117..2eb922d60 100644 --- a/@xen-orchestra/backups/RemoteAdapter.js +++ b/@xen-orchestra/backups/RemoteAdapter.js @@ -28,6 +28,9 @@ const { isMetadataFile } = require('./_backupType.js') const { isValidXva } = require('./_isValidXva.js') const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js') const { lvs, pvs } = require('./_lvm.js') +// @todo : this import is marked extraneous , sould be fixed when lib is published +const { mount } = require('@vates/fuse-vhd') +const { asyncEach } = require('@vates/async-each') const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups' exports.DIR_XO_CONFIG_BACKUPS = DIR_XO_CONFIG_BACKUPS @@ -45,8 +48,6 @@ const resolveRelativeFromFile = (file, path) => resolve('/', dirname(file), path const resolveSubpath = (root, path) => resolve(root, `.${resolve('/', path)}`) -const RE_VHDI = /^vhdi(\d+)$/ - async function addDirectory(files, realPath, metadataPath) { const stats = await lstat(realPath) if (stats.isDirectory()) { @@ -75,12 +76,14 @@ const debounceResourceFactory = factory => } class RemoteAdapter { - constructor(handler, { debounceResource = res => res, dirMode, vhdDirectoryCompression } = {}) { + constructor(handler, { debounceResource = res => res, dirMode, vhdDirectoryCompression, useGetDiskLegacy=false } = {}) { this._debounceResource = debounceResource this._dirMode = dirMode this._handler = handler this._vhdDirectoryCompression = vhdDirectoryCompression this._readCacheListVmBackups = synchronized.withKey()(this._readCacheListVmBackups) + this._useGetDiskLegacy = useGetDiskLegacy + } get handler() { @@ -321,7 +324,10 @@ class RemoteAdapter { return this.#useVhdDirectory() } - async *getDisk(diskId) { + + async *#getDiskLegacy(diskId) { + + const RE_VHDI = /^vhdi(\d+)$/ const handler = this._handler const diskPath = handler._getFilePath('/' + diskId) @@ -351,6 +357,20 @@ class RemoteAdapter { } } + async *getDisk(diskId) { + if(this._useGetDiskLegacy){ + yield * this.#getDiskLegacy(diskId) + return + } + const handler = this._handler + // this is a disposable + const mountDir = yield getTmpDir() + // this is also a disposable + yield mount(handler, diskId, mountDir) + // this will yield disk path to caller + yield `${mountDir}/vhd0` + } + // partitionId values: // // - undefined: raw disk @@ -401,22 +421,25 @@ class RemoteAdapter { listPartitionFiles(diskId, partitionId, path) { return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => { path = resolveSubpath(rootPath, path) - const entriesMap = {} - await asyncMap(await readdir(path), async name => { - try { - const stats = await lstat(`${path}/${name}`) - if (stats.isDirectory()) { - entriesMap[name + '/'] = {} - } else if (stats.isFile()) { - entriesMap[name] = {} + await asyncEach( + await readdir(path), + async name => { + try { + const stats = await lstat(`${path}/${name}`) + if (stats.isDirectory()) { + entriesMap[name + '/'] = {} + } else if (stats.isFile()) { + entriesMap[name] = {} + } + } catch (error) { + if (error == null || error.code !== 'ENOENT') { + throw error + } } - } catch (error) { - if (error == null || error.code !== 'ENOENT') { - throw error - } - } - }) + }, + { concurrency: 1 } + ) return entriesMap }) diff --git a/@xen-orchestra/backups/package.json b/@xen-orchestra/backups/package.json index 3dae123a8..81e72c50b 100644 --- a/@xen-orchestra/backups/package.json +++ b/@xen-orchestra/backups/package.json @@ -16,10 +16,12 @@ "postversion": "npm publish --access public" }, "dependencies": { + "@vates/async-each": "^1.0.0", "@vates/cached-dns.lookup": "^1.0.0", "@vates/compose": "^2.1.0", "@vates/decorate-with": "^2.0.0", "@vates/disposable": "^0.1.1", + "@vates/fuse-vhd": "^0.0.1", "@vates/parse-duration": "^0.1.1", "@xen-orchestra/async-map": "^0.1.2", "@xen-orchestra/fs": "^3.1.0", diff --git a/@xen-orchestra/proxy/app/mixins/backups.mjs b/@xen-orchestra/proxy/app/mixins/backups.mjs index fcd756063..b53ce3120 100644 --- a/@xen-orchestra/proxy/app/mixins/backups.mjs +++ b/@xen-orchestra/proxy/app/mixins/backups.mjs @@ -407,6 +407,7 @@ export default class Backups { debounceResource: app.debounceResource.bind(app), dirMode: app.config.get('backups.dirMode'), vhdDirectoryCompression: app.config.get('backups.vhdDirectoryCompression'), + useGetDiskLegacy: app.config.getOptional('backups.useGetDiskLegacy'), }) } diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 0ddd97585..326f318f6 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -7,6 +7,8 @@ > Users must be able to say: “Nice enhancement, I'm eager to test it” +- [Backup/Restore file] Implement File level restore for s3 and encrypted backups (PR [#6409](https://github.com/vatesfr/xen-orchestra/pull/6409)) + - [Backup] Improve listing speed by updating caches instead of regenerating them on backup creation/deletion (PR [#6411](https://github.com/vatesfr/xen-orchestra/pull/6411)) - [Backup] Add `mergeBlockConcurrency` and `writeBlockConcurrency` to allow tuning of backup resources consumptions (PR [#6416](https://github.com/vatesfr/xen-orchestra/pull/6416)) @@ -34,9 +36,11 @@ +- @vates/fuse-vhd major - @xen-orchestra/backups minor - vhd-lib minor - xo-server-auth-saml patch -- xo-web patch +- xo-server minor +- xo-web minor diff --git a/packages/vhd-lib/Vhd/VhdAbstract.js b/packages/vhd-lib/Vhd/VhdAbstract.js index 938ee47a5..76c5e62f0 100644 --- a/packages/vhd-lib/Vhd/VhdAbstract.js +++ b/packages/vhd-lib/Vhd/VhdAbstract.js @@ -360,4 +360,38 @@ exports.VhdAbstract = class VhdAbstract { } return true } + + async readRawData(start, length, cache, buf) { + const header = this.header + const blockSize = header.blockSize + const startBlockId = Math.floor(start / blockSize) + const endBlockId = Math.floor((start + length) / blockSize) + + const startOffset = start % blockSize + let copied = 0 + for (let blockId = startBlockId; blockId <= endBlockId; blockId++) { + let data + if (this.containsBlock(blockId)) { + if (!cache.has(blockId)) { + cache.set( + blockId, + // promise is awaited later, so it won't generate unbounded error + this.readBlock(blockId).then(block => { + return block.data + }) + ) + } + // the cache contains a promise + data = await cache.get(blockId) + } else { + data = Buffer.alloc(blockSize, 0) + } + const offsetStart = blockId === startBlockId ? startOffset : 0 + const offsetEnd = blockId === endBlockId ? (start + length) % blockSize : blockSize + data.copy(buf, copied, offsetStart, offsetEnd) + copied += offsetEnd - offsetStart + } + assert.strictEqual(copied, length, 'invalid length') + return copied + } } diff --git a/packages/xo-server/src/xo-mixins/backups-remote-adapter.mjs b/packages/xo-server/src/xo-mixins/backups-remote-adapter.mjs index 92e01a642..7818c1c29 100644 --- a/packages/xo-server/src/xo-mixins/backups-remote-adapter.mjs +++ b/packages/xo-server/src/xo-mixins/backups-remote-adapter.mjs @@ -22,6 +22,8 @@ export default class BackupsRemoteAdapter { debounceResource: app.debounceResource.bind(app), dirMode: app.config.get('backups.dirMode'), vhdDirectoryCompression: app.config.get('backups.vhdDirectoryCompression'), + // this adapter is also used for file restore + useGetDiskLegacy: app.config.getOptional('backups.useGetDiskLegacy'), }) } } diff --git a/packages/xo-server/src/xo-mixins/remotes.mjs b/packages/xo-server/src/xo-mixins/remotes.mjs index a9fd6af5e..2c666afdd 100644 --- a/packages/xo-server/src/xo-mixins/remotes.mjs +++ b/packages/xo-server/src/xo-mixins/remotes.mjs @@ -13,7 +13,6 @@ import { Remotes } from '../models/remote.mjs' const obfuscateRemote = ({ url, ...remote }) => { const parsedUrl = parse(url) - remote.supportFileRestore = parsedUrl.type !== 's3' remote.url = format(sensitiveValues.obfuscate(parsedUrl)) return remote } diff --git a/packages/xo-web/src/common/intl/locales/es.js b/packages/xo-web/src/common/intl/locales/es.js index d7883e566..948e41fa4 100644 --- a/packages/xo-web/src/common/intl/locales/es.js +++ b/packages/xo-web/src/common/intl/locales/es.js @@ -2676,9 +2676,6 @@ export default { // Original text: 'Click on a VM to display restore options' restoreBackupsInfo: undefined, - // Original text: 'Only the files of Delta Backup which are not on a SMB remote can be restored' - restoreDeltaBackupsInfo: undefined, - // Original text: "Enabled" remoteEnabled: 'activado', diff --git a/packages/xo-web/src/common/intl/locales/fr.js b/packages/xo-web/src/common/intl/locales/fr.js index 4907da4f0..d1c0c12dc 100644 --- a/packages/xo-web/src/common/intl/locales/fr.js +++ b/packages/xo-web/src/common/intl/locales/fr.js @@ -2702,10 +2702,6 @@ export default { // Original text: "Click on a VM to display restore options" restoreBackupsInfo: 'Cliquez sur une VM pour afficher les options de récupération', - // Original text: "Only the files of Delta Backup which are not on a SMB remote can be restored" - restoreDeltaBackupsInfo: - 'Seuls les fichiers de Delta Backup qui ne sont pas sur un emplacement SMB peuvent être restaurés', - // Original text: "Enabled" remoteEnabled: 'activé', diff --git a/packages/xo-web/src/common/intl/locales/it.js b/packages/xo-web/src/common/intl/locales/it.js index f73089f6b..1c58510ff 100644 --- a/packages/xo-web/src/common/intl/locales/it.js +++ b/packages/xo-web/src/common/intl/locales/it.js @@ -3906,9 +3906,6 @@ export default { // Original text: 'Click on a VM to display restore options' restoreBackupsInfo: 'Fare clic su una VM per visualizzare le opzioni di ripristino', - // Original text: 'Only the files of Delta Backup which are not on a SMB remote can be restored' - restoreDeltaBackupsInfo: 'È possibile ripristinare solo i file di Delta Backup che non si trovano su un SMB remoto', - // Original text: 'Enabled' remoteEnabled: 'Abilitato', diff --git a/packages/xo-web/src/common/intl/locales/tr.js b/packages/xo-web/src/common/intl/locales/tr.js index f32de3907..bc1c0b59f 100644 --- a/packages/xo-web/src/common/intl/locales/tr.js +++ b/packages/xo-web/src/common/intl/locales/tr.js @@ -3343,9 +3343,6 @@ export default { // Original text: "Click on a VM to display restore options" restoreBackupsInfo: "Geri getirme seçenekleri için bir VM'e tıklayın", - // Original text: "Only the files of Delta Backup which are not on a SMB remote can be restored" - restoreDeltaBackupsInfo: 'Yalnızca SMB hedefinde olmayan fark yedeklerinden dosya alınabilir', - // Original text: "Enabled" remoteEnabled: 'Etkin', diff --git a/packages/xo-web/src/common/intl/messages.js b/packages/xo-web/src/common/intl/messages.js index 4f6d02785..12b8f1e3d 100644 --- a/packages/xo-web/src/common/intl/messages.js +++ b/packages/xo-web/src/common/intl/messages.js @@ -1636,7 +1636,6 @@ const messages = { getRemote: 'Get remote', noBackups: 'There are no backups!', restoreBackupsInfo: 'Click on a VM to display restore options', - restoreDeltaBackupsInfo: 'Only the files of Delta Backup which are not on a SMB or S3 remote can be restored', remoteEnabled: 'Enabled', remoteDisabled: 'Disabled', enableRemote: 'Enable', diff --git a/packages/xo-web/src/xo-app/backup/file-restore/index.js b/packages/xo-web/src/xo-app/backup/file-restore/index.js index db3f5e86f..cd7f82fbc 100644 --- a/packages/xo-web/src/xo-app/backup/file-restore/index.js +++ b/packages/xo-web/src/xo-app/backup/file-restore/index.js @@ -1,7 +1,6 @@ import _ from 'intl' import ActionButton from 'action-button' import Component from 'base-component' -import Icon from 'icon' import React from 'react' import SortedTable from 'sorted-table' import Upgrade from 'xoa-upgrade' @@ -87,7 +86,7 @@ export default class Restore extends Component { _refreshBackupList = async (_remotes = this.props.remotes, jobs = this.props.jobs) => { const remotes = keyBy( - filter(_remotes, remote => remote.enabled && remote.supportFileRestore), + filter(_remotes, remote => remote.enabled), 'id' ) const backupsByRemote = await listVmBackups(toArray(remotes)) @@ -204,9 +203,6 @@ export default class Restore extends Component { {_('refreshBackupList')} - - {_('restoreDeltaBackupsInfo')} -