feat: implement file restore on top of FUSE instead of vhdimount (#6409)

It brings file restore to VhdDirectory (and related features like encryption and compression).

@vates/fuse-vhd/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@vates/fuse-vhd/index.js

@@ -0,0 +1,71 @@
+'use strict'
+
+const LRU = require('lru-cache')
+const Fuse = require('fuse-native')
+const { VhdSynthetic } = require('vhd-lib')
+const { Disposable, fromCallback } = require('promise-toolbox')
+const { createLogger } = require('@xen-orchestra/log')
+
+const { warn } = createLogger('vates:fuse-vhd')
+
+// build a s stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js
+const stat = st => ({
+  mtime: st.mtime || new Date(),
+  atime: st.atime || new Date(),
+  ctime: st.ctime || new Date(),
+  size: st.size !== undefined ? st.size : 0,
+  mode: st.mode === 'dir' ? 16877 : st.mode === 'file' ? 33188 : st.mode === 'link' ? 41453 : st.mode,
+  uid: st.uid !== undefined ? st.uid : process.getuid(),
+  gid: st.gid !== undefined ? st.gid : process.getgid(),
+})
+
+exports.mount = Disposable.factory(async function* mount(handler, diskPath, mountDir) {
+  const vhd = yield VhdSynthetic.fromVhdChain(handler, diskPath)
+
+  const cache = new LRU({
+    max: 16, // each cached block is 2MB in size
+  })
+  await vhd.readBlockAllocationTable()
+  const fuse = new Fuse(mountDir, {
+    async readdir(path, cb) {
+      if (path === '/') {
+        return cb(null, ['vhd0'])
+      }
+      cb(Fuse.ENOENT)
+    },
+    async getattr(path, cb) {
+      if (path === '/') {
+        return cb(
+          null,
+          stat({
+            mode: 'dir',
+            size: 4096,
+          })
+        )
+      }
+      if (path === '/vhd0') {
+        return cb(
+          null,
+          stat({
+            mode: 'file',
+            size: vhd.footer.currentSize,
+          })
+        )
+      }
+
+      cb(Fuse.ENOENT)
+    },
+    read(path, fd, buf, len, pos, cb) {
+      if (path === '/vhd0') {
+        return vhd
+          .readRawData(pos, len, cache, buf)
+          .then(cb)
+      }
+      throw new Error(`read file ${path} not exists`)
+    },
+  })
+  return new Disposable(
+    () => fromCallback(() => fuse.unmount()),
+    fromCallback(() => fuse.mount())
+    )
+})

@vates/fuse-vhd/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@vates/fuse-vhd",
+  "version": "0.0.1",
+  "license": "ISC",
+  "private": false,
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/fuse-vhd",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@vates/fuse-vhd",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "engines": {
+    "node": ">=10.0"
+  },
+  "dependencies": {
+    "@xen-orchestra/log": "^0.3.0",
+    "fuse-native": "^2.2.6",
+    "lru-cache": "^7.14.0",
+    "promise-toolbox": "^0.21.0",
+    "vhd-lib": "^4.0.1"
+  },
+  "scripts": {
+    "postversion": "npm publish --access public"
+  }
+}

@xen-orchestra/backups/RemoteAdapter.js

@@ -28,6 +28,9 @@ const { isMetadataFile } = require('./_backupType.js')
 const { isValidXva } = require('./_isValidXva.js')
 const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js')
 const { lvs, pvs } = require('./_lvm.js')
+// @todo : this import is marked extraneous , sould be fixed when lib is published
+const { mount } = require('@vates/fuse-vhd')
+const { asyncEach } = require('@vates/async-each')
 
 const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
 exports.DIR_XO_CONFIG_BACKUPS = DIR_XO_CONFIG_BACKUPS
@@ -45,8 +48,6 @@ const resolveRelativeFromFile = (file, path) => resolve('/', dirname(file), path
 
 const resolveSubpath = (root, path) => resolve(root, `.${resolve('/', path)}`)
 
-const RE_VHDI = /^vhdi(\d+)$/
-
 async function addDirectory(files, realPath, metadataPath) {
   const stats = await lstat(realPath)
   if (stats.isDirectory()) {
@@ -75,12 +76,14 @@ const debounceResourceFactory = factory =>
   }
 
 class RemoteAdapter {
-  constructor(handler, { debounceResource = res => res, dirMode, vhdDirectoryCompression } = {}) {
+  constructor(handler, { debounceResource = res => res, dirMode, vhdDirectoryCompression, useGetDiskLegacy=false } = {}) {
     this._debounceResource = debounceResource
     this._dirMode = dirMode
     this._handler = handler
     this._vhdDirectoryCompression = vhdDirectoryCompression
     this._readCacheListVmBackups = synchronized.withKey()(this._readCacheListVmBackups)
+    this._useGetDiskLegacy = useGetDiskLegacy
+
   }
 
   get handler() {
@@ -321,7 +324,10 @@ class RemoteAdapter {
     return this.#useVhdDirectory()
   }
 
-  async *getDisk(diskId) {
+
+  async *#getDiskLegacy(diskId) {
+
+    const RE_VHDI = /^vhdi(\d+)$/
     const handler = this._handler
 
     const diskPath = handler._getFilePath('/' + diskId)
@@ -351,6 +357,20 @@ class RemoteAdapter {
     }
   }
 
+  async *getDisk(diskId) {
+    if(this._useGetDiskLegacy){
+      yield * this.#getDiskLegacy(diskId)
+      return
+    }
+    const handler = this._handler
+    // this is a disposable
+    const mountDir = yield getTmpDir()
+    // this is also a disposable
+    yield mount(handler, diskId, mountDir)
+    // this will yield disk path to caller
+    yield `${mountDir}/vhd0`
+  }
+
   // partitionId values:
   //
   // - undefined: raw disk
@@ -401,22 +421,25 @@ class RemoteAdapter {
   listPartitionFiles(diskId, partitionId, path) {
     return Disposable.use(this.getPartition(diskId, partitionId), async rootPath => {
       path = resolveSubpath(rootPath, path)
-
       const entriesMap = {}
-      await asyncMap(await readdir(path), async name => {
+      await asyncEach(
-        try {
+        await readdir(path),
-          const stats = await lstat(`${path}/${name}`)
+        async name => {
-          if (stats.isDirectory()) {
+          try {
-            entriesMap[name + '/'] = {}
+            const stats = await lstat(`${path}/${name}`)
-          } else if (stats.isFile()) {
+            if (stats.isDirectory()) {
-            entriesMap[name] = {}
+              entriesMap[name + '/'] = {}
+            } else if (stats.isFile()) {
+              entriesMap[name] = {}
+            }
+          } catch (error) {
+            if (error == null || error.code !== 'ENOENT') {
+              throw error
+            }
           }
-        } catch (error) {
+        },
-          if (error == null || error.code !== 'ENOENT') {
+        { concurrency: 1 }
-            throw error
+      )
-          }
-        }
-      })
 
       return entriesMap
     })

@xen-orchestra/backups/package.json

@@ -16,10 +16,12 @@
     "postversion": "npm publish --access public"
   },
   "dependencies": {
+    "@vates/async-each": "^1.0.0",
     "@vates/cached-dns.lookup": "^1.0.0",
     "@vates/compose": "^2.1.0",
     "@vates/decorate-with": "^2.0.0",
     "@vates/disposable": "^0.1.1",
+    "@vates/fuse-vhd": "^0.0.1",
     "@vates/parse-duration": "^0.1.1",
     "@xen-orchestra/async-map": "^0.1.2",
     "@xen-orchestra/fs": "^3.1.0",

@xen-orchestra/proxy/app/mixins/backups.mjs

@@ -407,6 +407,7 @@ export default class Backups {
       debounceResource: app.debounceResource.bind(app),
       dirMode: app.config.get('backups.dirMode'),
       vhdDirectoryCompression: app.config.get('backups.vhdDirectoryCompression'),
+      useGetDiskLegacy: app.config.getOptional('backups.useGetDiskLegacy'),
     })
   }
 

CHANGELOG.unreleased.md

@@ -7,6 +7,8 @@
 
 > Users must be able to say: “Nice enhancement, I'm eager to test it”
 
+- [Backup/Restore file] Implement File level restore for s3 and encrypted backups (PR [#6409](https://github.com/vatesfr/xen-orchestra/pull/6409))
+
 - [Backup] Improve listing speed by updating caches instead of regenerating them on backup creation/deletion (PR [#6411](https://github.com/vatesfr/xen-orchestra/pull/6411))
 - [Backup] Add `mergeBlockConcurrency` and `writeBlockConcurrency` to allow tuning of backup resources consumptions (PR [#6416](https://github.com/vatesfr/xen-orchestra/pull/6416))
 
@@ -34,9 +36,11 @@
 
 <!--packages-start-->
 
+- @vates/fuse-vhd major
 - @xen-orchestra/backups minor
 - vhd-lib minor
 - xo-server-auth-saml patch
-- xo-web patch
+- xo-server minor
+- xo-web minor
 
 <!--packages-end-->

packages/vhd-lib/Vhd/VhdAbstract.js

@@ -360,4 +360,38 @@ exports.VhdAbstract = class VhdAbstract {
     }
     return true
   }
+
+  async readRawData(start, length, cache, buf) {
+    const header = this.header
+    const blockSize = header.blockSize
+    const startBlockId = Math.floor(start / blockSize)
+    const endBlockId = Math.floor((start + length) / blockSize)
+
+    const startOffset = start % blockSize
+    let copied = 0
+    for (let blockId = startBlockId; blockId <= endBlockId; blockId++) {
+      let data
+      if (this.containsBlock(blockId)) {
+        if (!cache.has(blockId)) {
+          cache.set(
+            blockId,
+            // promise is awaited later, so it won't generate unbounded error
+            this.readBlock(blockId).then(block => {
+              return block.data
+            })
+          )
+        }
+        // the cache contains a promise
+        data = await cache.get(blockId)
+      } else {
+        data = Buffer.alloc(blockSize, 0)
+      }
+      const offsetStart = blockId === startBlockId ? startOffset : 0
+      const offsetEnd = blockId === endBlockId ? (start + length) % blockSize : blockSize
+      data.copy(buf, copied, offsetStart, offsetEnd)
+      copied += offsetEnd - offsetStart
+    }
+    assert.strictEqual(copied, length, 'invalid length')
+    return copied
+  }
 }

packages/xo-server/src/xo-mixins/backups-remote-adapter.mjs

@@ -22,6 +22,8 @@ export default class BackupsRemoteAdapter {
       debounceResource: app.debounceResource.bind(app),
       dirMode: app.config.get('backups.dirMode'),
       vhdDirectoryCompression: app.config.get('backups.vhdDirectoryCompression'),
+      // this adapter is also used for file restore
+      useGetDiskLegacy: app.config.getOptional('backups.useGetDiskLegacy'),
     })
   }
 }

packages/xo-server/src/xo-mixins/remotes.mjs

@@ -13,7 +13,6 @@ import { Remotes } from '../models/remote.mjs'
 
 const obfuscateRemote = ({ url, ...remote }) => {
   const parsedUrl = parse(url)
-  remote.supportFileRestore = parsedUrl.type !== 's3'
   remote.url = format(sensitiveValues.obfuscate(parsedUrl))
   return remote
 }

packages/xo-web/src/common/intl/locales/es.js

@@ -2676,9 +2676,6 @@ export default {
   // Original text: 'Click on a VM to display restore options'
   restoreBackupsInfo: undefined,
 
-  // Original text: 'Only the files of Delta Backup which are not on a SMB remote can be restored'
-  restoreDeltaBackupsInfo: undefined,
-
   // Original text: "Enabled"
   remoteEnabled: 'activado',
 

packages/xo-web/src/common/intl/locales/fr.js

@@ -2702,10 +2702,6 @@ export default {
   // Original text: "Click on a VM to display restore options"
   restoreBackupsInfo: 'Cliquez sur une VM pour afficher les options de récupération',
 
-  // Original text: "Only the files of Delta Backup which are not on a SMB remote can be restored"
-  restoreDeltaBackupsInfo:
-    'Seuls les fichiers de Delta Backup qui ne sont pas sur un emplacement SMB peuvent être restaurés',
-
   // Original text: "Enabled"
   remoteEnabled: 'activé',
 

packages/xo-web/src/common/intl/locales/it.js

@@ -3906,9 +3906,6 @@ export default {
   // Original text: 'Click on a VM to display restore options'
   restoreBackupsInfo: 'Fare clic su una VM per visualizzare le opzioni di ripristino',
 
-  // Original text: 'Only the files of Delta Backup which are not on a SMB remote can be restored'
-  restoreDeltaBackupsInfo: 'È possibile ripristinare solo i file di Delta Backup che non si trovano su un SMB remoto',
-
   // Original text: 'Enabled'
   remoteEnabled: 'Abilitato',
 

packages/xo-web/src/common/intl/locales/tr.js

@@ -3343,9 +3343,6 @@ export default {
   // Original text: "Click on a VM to display restore options"
   restoreBackupsInfo: "Geri getirme seçenekleri için bir VM'e tıklayın",
 
-  // Original text: "Only the files of Delta Backup which are not on a SMB remote can be restored"
-  restoreDeltaBackupsInfo: 'Yalnızca SMB hedefinde olmayan fark yedeklerinden dosya alınabilir',
-
   // Original text: "Enabled"
   remoteEnabled: 'Etkin',
 

packages/xo-web/src/common/intl/messages.js

@@ -1636,7 +1636,6 @@ const messages = {
   getRemote: 'Get remote',
   noBackups: 'There are no backups!',
   restoreBackupsInfo: 'Click on a VM to display restore options',
-  restoreDeltaBackupsInfo: 'Only the files of Delta Backup which are not on a SMB or S3 remote can be restored',
   remoteEnabled: 'Enabled',
   remoteDisabled: 'Disabled',
   enableRemote: 'Enable',

packages/xo-web/src/xo-app/backup/file-restore/index.js

@@ -1,7 +1,6 @@
 import _ from 'intl'
 import ActionButton from 'action-button'
 import Component from 'base-component'
-import Icon from 'icon'
 import React from 'react'
 import SortedTable from 'sorted-table'
 import Upgrade from 'xoa-upgrade'
@@ -87,7 +86,7 @@ export default class Restore extends Component {
 
   _refreshBackupList = async (_remotes = this.props.remotes, jobs = this.props.jobs) => {
     const remotes = keyBy(
-      filter(_remotes, remote => remote.enabled && remote.supportFileRestore),
+      filter(_remotes, remote => remote.enabled),
       'id'
     )
     const backupsByRemote = await listVmBackups(toArray(remotes))
@@ -204,9 +203,6 @@ export default class Restore extends Component {
               {_('refreshBackupList')}
             </ActionButton>
           </div>
-          <em>
-            <Icon icon='info' /> {_('restoreDeltaBackupsInfo')}
-          </em>
           <SortedTable
             actions={this._actions}
             collection={this.state.backupDataByVm}

yarn.lock

@@ -8855,6 +8855,40 @@ functions-have-names@^1.2.2:
   resolved "https://registry.yarnpkg.com/functions-have-names/-/functions-have-names-1.2.3.tgz#0404fe4ee2ba2f607f0e0ec3c80bae994133b834"
   integrity sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==
 
+fuse-native@^2.2.6:
+  version "2.2.6"
+  resolved "https://registry.yarnpkg.com/fuse-native/-/fuse-native-2.2.6.tgz#517224de1158d9acfab465e35b44ad9e9d308c56"
+  integrity sha512-Y5wXd7vUsWWWIIHbjluv7jKZgPZaSVA5YWaW3I5fXIJfcGWL6IRUgoBUveQAq+D8cG9cCiGNahv9CeToccCXrw==
+  dependencies:
+    fuse-shared-library "^1.0.2"
+    nanoresource "^1.3.0"
+    napi-macros "^2.0.0"
+    node-gyp-build "^4.2.0"
+
+fuse-shared-library-darwin@^1.0.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/fuse-shared-library-darwin/-/fuse-shared-library-darwin-1.1.3.tgz#691839bffb5345f27f33eadf321e3a518c97e0dc"
+  integrity sha512-4Q8gMxyMl1+gwHGpiYUoKKpi7xq8WcPo0TvJvjZzHMuCiszouu2GgEs6SJAqPB3LjfmEkl6kPV+2Oluczr0Nig==
+
+fuse-shared-library-linux-arm@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/fuse-shared-library-linux-arm/-/fuse-shared-library-linux-arm-1.0.0.tgz#35c82e9650cf99245c6a3f354639e0aeee1a004d"
+  integrity sha512-Dj4ssxo1/MKGvOsVWRblSRu+o5F5OJTrVPDkjSyGDU2yKvVnIzQSwy1deiWA0qCcS/Q8iJMlZaCpCcZWSwvoug==
+
+fuse-shared-library-linux@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/fuse-shared-library-linux/-/fuse-shared-library-linux-1.0.1.tgz#fd182f0c5e8e46f5c9b0a7639a9db7d48b25c8d5"
+  integrity sha512-07MQRSobrBKwW4D7oKm0gM2TwgvZWb+gC08JdiYDG4KBTncxk9ssqEDiDMKll8hpseZufsY2w1yc/feOu2DPmQ==
+
+fuse-shared-library@^1.0.2:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/fuse-shared-library/-/fuse-shared-library-1.1.1.tgz#988d10390898d1101aebc64d1fdeb8933fc6ce57"
+  integrity sha512-EfgTo/eS1euZFUe7x8KqyA40hV4DwP7kqp1VNZApu2nlPnJv8SanraBE3VXyX7ff41sxw7M0oWY7re3G3wnZVA==
+  dependencies:
+    fuse-shared-library-darwin "^1.0.3"
+    fuse-shared-library-linux "^1.0.1"
+    fuse-shared-library-linux-arm "^1.0.0"
+
 gauge@^3.0.0:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/gauge/-/gauge-3.0.2.tgz#03bf4441c044383908bcfa0656ad91803259b395"
@@ -12343,7 +12377,7 @@ lru-cache@^6.0.0:
   dependencies:
     yallist "^4.0.0"
 
-lru-cache@^7.0.4:
+lru-cache@^7.0.4, lru-cache@^7.14.0:
   version "7.14.0"
   resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.14.0.tgz#21be64954a4680e303a09e9468f880b98a0b3c7f"
   integrity sha512-EIRtP1GrSJny0dqb50QXRUNBxHJhcpxHC++M5tD7RYbvLLn5KVWKsbyswSSqDuU15UFi3bgTQIY8nhDMeF6aDQ==
@@ -12975,7 +13009,14 @@ nanomatch@^1.2.9:
     snapdragon "^0.8.1"
     to-regex "^3.0.1"
 
-napi-macros@~2.0.0:
+nanoresource@^1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/nanoresource/-/nanoresource-1.3.0.tgz#823945d9667ab3e81a8b2591ab8d734552878cd0"
+  integrity sha512-OI5dswqipmlYfyL3k/YMm7mbERlh4Bd1KuKdMHpeoVD1iVxqxaTMKleB4qaA2mbQZ6/zMNSxCXv9M9P/YbqTuQ==
+  dependencies:
+    inherits "^2.0.4"
+
+napi-macros@^2.0.0, napi-macros@~2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/napi-macros/-/napi-macros-2.0.0.tgz#2b6bae421e7b96eb687aa6c77a7858640670001b"
   integrity sha512-A0xLykHtARfueITVDernsAWdtIMbOJgKgcluwENp3AlsKN/PloyO10HtmoqnFAQAcxPkgZN7wdfPfEd0zNGxbg==
@@ -13073,7 +13114,7 @@ node-forge@^1.3.1:
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
   integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==
 
-node-gyp-build@^4.3.0:
+node-gyp-build@^4.2.0, node-gyp-build@^4.3.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.5.0.tgz#7a64eefa0b21112f89f58379da128ac177f20e40"
   integrity sha512-2iGbaQBV+ITgCz76ZEjmhUKAKVf7xfY1sRl4UiKQspfZMH2h06SyhNsnSVy50cwkFQDGLyif6m/6uFXHkOZ6rg==