feat(vhd-cli): add more diagnostic tools

@vates/fuse-vhd/index.js

@@ -1,8 +1,12 @@
 'use strict'
 
 const LRU = require('lru-cache')
+const Fuse = require('fuse-native')
 const { VhdSynthetic } = require('vhd-lib')
 const { Disposable, fromCallback } = require('promise-toolbox')
+const { createLogger } = require('@xen-orchestra/log')
+
+const { warn } = createLogger('vates:fuse-vhd')
 
 // build a s stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js
 const stat = st => ({
@@ -16,7 +20,6 @@ const stat = st => ({
 })
 
 exports.mount = Disposable.factory(async function* mount(handler, diskPath, mountDir) {
-  const Fuse = require('fuse-native')
   const vhd = yield VhdSynthetic.fromVhdChain(handler, diskPath)
 
   const cache = new LRU({
@@ -54,7 +57,9 @@ exports.mount = Disposable.factory(async function* mount(handler, diskPath, moun
     },
     read(path, fd, buf, len, pos, cb) {
       if (path === '/vhd0') {
-        return vhd.readRawData(pos, len, cache, buf).then(cb)
+        return vhd
+          .readRawData(pos, len, cache, buf)
+          .then(cb)
       }
       throw new Error(`read file ${path} not exists`)
     },
@@ -62,5 +67,5 @@ exports.mount = Disposable.factory(async function* mount(handler, diskPath, moun
   return new Disposable(
     () => fromCallback(() => fuse.unmount()),
     fromCallback(() => fuse.mount())
-  )
+    )
 })

@vates/fuse-vhd/package.json

@@ -19,13 +19,11 @@
   },
   "dependencies": {
     "@xen-orchestra/log": "^0.3.0",
+    "fuse-native": "^2.2.6",
     "lru-cache": "^7.14.0",
     "promise-toolbox": "^0.21.0",
     "vhd-lib": "^4.0.1"
   },
-  "optionalDependencies": {
-    "fuse-native": "^2.2.6"
-  },
   "scripts": {
     "postversion": "npm publish --access public"
   }

@xen-orchestra/backups/RemoteAdapter.js

@@ -28,8 +28,10 @@ const { isMetadataFile } = require('./_backupType.js')
 const { isValidXva } = require('./_isValidXva.js')
 const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js')
 const { lvs, pvs } = require('./_lvm.js')
-const { asyncEach } = require('@vates/async-each')
+// @todo : this import is marked extraneous , sould be fixed when lib is published
 const { mount } = require('@vates/fuse-vhd')
+const { asyncEach } = require('@vates/async-each')
+
 const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
 exports.DIR_XO_CONFIG_BACKUPS = DIR_XO_CONFIG_BACKUPS
 
@@ -74,16 +76,14 @@ const debounceResourceFactory = factory =>
   }
 
 class RemoteAdapter {
-  constructor(
-    handler,
-    { debounceResource = res => res, dirMode, vhdDirectoryCompression, useGetDiskLegacy = false } = {}
-  ) {
+  constructor(handler, { debounceResource = res => res, dirMode, vhdDirectoryCompression, useGetDiskLegacy=false } = {}) {
     this._debounceResource = debounceResource
     this._dirMode = dirMode
     this._handler = handler
     this._vhdDirectoryCompression = vhdDirectoryCompression
     this._readCacheListVmBackups = synchronized.withKey()(this._readCacheListVmBackups)
     this._useGetDiskLegacy = useGetDiskLegacy
+
   }
 
   get handler() {
@@ -324,7 +324,9 @@ class RemoteAdapter {
     return this.#useVhdDirectory()
   }
 
+
   async *#getDiskLegacy(diskId) {
+
     const RE_VHDI = /^vhdi(\d+)$/
     const handler = this._handler
 
@@ -356,25 +358,15 @@ class RemoteAdapter {
   }
 
   async *getDisk(diskId) {
-    if (this._useGetDiskLegacy) {
-      yield* this.#getDiskLegacy(diskId)
+    if(this._useGetDiskLegacy){
+      yield * this.#getDiskLegacy(diskId)
       return
     }
     const handler = this._handler
     // this is a disposable
     const mountDir = yield getTmpDir()
-
-    try {
-      // this is also a disposable
-      yield mount(handler, diskId, mountDir)
-    } catch (error) {
-      // fallback in case of missing dependency
-      if (error.code === 'MODULE_NOT_FOUND') {
-        yield* this.#getDiskLegacy(diskId)
-      } else {
-        throw error
-      }
-    }
+    // this is also a disposable
+    yield mount(handler, diskId, mountDir)
     // this will yield disk path to caller
     yield `${mountDir}/vhd0`
   }

packages/vhd-cli/commands/checkChain.js

@@ -0,0 +1,51 @@
+'use strict'
+
+const { Bar } = require('cli-progress')
+const { getHandler } = require('@xen-orchestra/fs')
+const { VhdSynthetic } = require('vhd-lib')
+const { Disposable } = require('promise-toolbox')
+
+async function checkOneVhd(vhd) {
+  await vhd.readBlockAllocationTable()
+  const nBlocks = vhd.header.maxTableEntries
+  const bar = new Bar({
+    format: '[{bar}] {percentage}% | ETA: {eta}s | {value}/{total}',
+  })
+  bar.start(nBlocks, 0)
+  const missings = []
+  for (let blockId = 0; blockId < nBlocks; ++blockId) {
+    bar.update(blockId)
+    try {
+      if (vhd.containsBlock(blockId)) {
+        await vhd.readBlock(blockId)
+      }
+    } catch (error) {
+      console.log('missing block ', blockId)
+      missings.push(blockId)
+      if (missings.length > 1000) {
+        throw new Error('Too much missing blocks')
+      }
+    }
+  }
+  bar.update(nBlocks)
+  bar.stop()
+  return missings
+}
+
+module.exports = async function merge(args) {
+  if (args.length < 1 || args.some(_ => _ === '-h' || _ === '--help')) {
+    return `Usage: ${this.command} <VHD>`
+  }
+
+  const handler = getHandler({ url: 'file:///' })
+
+  await Disposable.use(VhdSynthetic.fromVhdChain(handler, args[0]), async syntheticVhd => {
+    console.log('Check full VHD')
+    const missings = await checkOneVhd(syntheticVhd)
+    if (missings.length > 0) {
+      console.log(`${missings.length} blocks are missing`)
+    } else {
+      console.log('VHD data are ok')
+    }
+  })
+}

packages/vhd-cli/commands/mergeChain.js

@@ -0,0 +1,34 @@
+'use strict'
+
+const { Bar } = require('cli-progress')
+const { getHandler } = require('@xen-orchestra/fs')
+const { mergeVhdChain } = require('vhd-lib/merge')
+const { VhdSynthetic } = require('vhd-lib')
+const { Disposable } = require('promise-toolbox')
+
+module.exports = async function merge(args) {
+  if (args.length < 1 || args.some(_ => _ === '-h' || _ === '--help')) {
+    return `Usage: ${this.command} <VHD>`
+  }
+
+  const handler = getHandler({ url: 'file:///' })
+  let bar
+  await Disposable.use(VhdSynthetic.fromVhdChain(handler, args[0]), async syntheticVhd => {
+    const chainPaths = syntheticVhd.vhds.map(({ _path }) => _path)
+    console.log({ chainPaths })
+    await mergeVhdChain(handler, chainPaths, {
+      onProgress({ done, total }) {
+        if (bar === undefined) {
+          bar = new Bar({
+            format: 'merging [{bar}] {percentage}% | ETA: {eta}s | {value}/{total}',
+          })
+          bar.start(total, done)
+        } else {
+          bar.update(done)
+        }
+      },
+    })
+  })
+  bar.stop()
+  console.log('you must delete the cache.json.gz file to ensure the changes are visible in XO UI')
+}

packages/vhd-lib/Vhd/VhdSynthetic.js

@@ -55,6 +55,10 @@ const VhdSynthetic = class VhdSynthetic extends VhdAbstract {
     return compressionType
   }
 
+  get vhds() {
+    return this.#vhds
+  }
+
   /**
    * @param {Array<VhdAbstract>} vhds the chain of Vhds used to compute this Vhd, from the deepest child (in position 0), to the root (in the last position)
    * only the last one can have any type. Other must have type DISK_TYPES.DIFFERENCING (delta)