Compare commits

...

1 Commits

Author SHA1 Message Date
Julien Fontanet
c249c91186 feat(backups): use zip CLI for file restore
Much faster and memory efficient than the previous implementation.

Downside is that it requires `zip` from being installed on the system.
2023-07-27 17:32:11 +02:00
3 changed files with 13 additions and 32 deletions

View File

@@ -11,7 +11,6 @@ import { mount } from '@vates/fuse-vhd'
import { readdir, lstat } from 'node:fs/promises'
import { synchronized } from 'decorator-synchronized'
import { v4 as uuidv4 } from 'uuid'
import { ZipFile } from 'yazl'
import Disposable from 'promise-toolbox/Disposable'
import fromCallback from 'promise-toolbox/fromCallback'
import fromEvent from 'promise-toolbox/fromEvent'
@@ -30,6 +29,7 @@ import { isValidXva } from './_isValidXva.mjs'
import { listPartitions, LVM_PARTITION_TYPE } from './_listPartitions.mjs'
import { lvs, pvs } from './_lvm.mjs'
import { watchStreamSize } from './_watchStreamSize.mjs'
import { spawn } from 'node:child_process'
export const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
@@ -45,23 +45,6 @@ const resolveRelativeFromFile = (file, path) => resolve('/', dirname(file), path
const makeRelative = path => resolve('/', path).slice(1)
const resolveSubpath = (root, path) => resolve(root, makeRelative(path))
async function addZipEntries(zip, realBasePath, virtualBasePath, relativePaths) {
for (const relativePath of relativePaths) {
const realPath = join(realBasePath, relativePath)
const virtualPath = join(virtualBasePath, relativePath)
const stats = await lstat(realPath)
const { mode, mtime } = stats
const opts = { mode, mtime }
if (stats.isDirectory()) {
zip.addEmptyDirectory(virtualPath, opts)
await addZipEntries(zip, realPath, virtualPath, await readdir(realPath))
} else if (stats.isFile()) {
zip.addFile(realPath, virtualPath, opts)
}
}
}
const createSafeReaddir = (handler, methodName) => (path, options) =>
handler.list(path, options).catch(error => {
if (error?.code !== 'ENOENT') {
@@ -212,10 +195,16 @@ export class RemoteAdapter {
if (format === 'tgz') {
outputStream = tar.c({ cwd: path, gzip: true }, paths.map(makeRelative))
} else if (format === 'zip') {
const zip = new ZipFile()
await addZipEntries(zip, path, '', paths.map(makeRelative))
zip.end()
;({ outputStream } = zip)
// don't use --symlinks due to bug
//
// see https://bugs.launchpad.net/ubuntu/+source/zip/+bug/1892338
const cp = spawn('zip', ['--quiet', '--recurse-paths', '-', ...paths.map(makeRelative)], { cwd: path })
await new Promise((resolve, reject) => {
cp.on('error', reject).on('spawn', resolve)
})
outputStream = cp.stdout
} else {
throw new Error('unsupported format ' + format)
}

View File

@@ -43,8 +43,7 @@
"tar": "^6.1.15",
"uuid": "^9.0.0",
"vhd-lib": "^4.5.0",
"xen-api": "^1.3.3",
"yazl": "^2.5.1"
"xen-api": "^1.3.3"
},
"devDependencies": {
"fs-extra": "^11.1.0",

View File

@@ -6157,7 +6157,7 @@ bser@2.1.1:
dependencies:
node-int64 "^0.4.0"
buffer-crc32@^0.2.13, buffer-crc32@~0.2.3:
buffer-crc32@^0.2.13:
version "0.2.13"
resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"
integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==
@@ -21871,13 +21871,6 @@ yargs@~3.10.0:
decamelize "^1.0.0"
window-size "0.1.0"
yazl@^2.5.1:
version "2.5.1"
resolved "https://registry.yarnpkg.com/yazl/-/yazl-2.5.1.tgz#a3d65d3dd659a5b0937850e8609f22fffa2b5c35"
integrity sha512-phENi2PLiHnHb6QBVot+dJnaAZ0xosj7p3fWl+znIjBDlnMI2PsZCJZ306BPTFOaHf5qdDEI8x5qFrSOBN5vrw==
dependencies:
buffer-crc32 "~0.2.3"
ylru@^1.2.0:
version "1.3.2"
resolved "https://registry.yarnpkg.com/ylru/-/ylru-1.3.2.tgz#0de48017473275a4cbdfc83a1eaf67c01af8a785"