feat(vhdDirectory): add check to ensure the vhd directory is complete

@xen-orchestra/backups/_cleanVm.js

@@ -197,7 +197,7 @@ exports.cleanVm = async function cleanVm(
   // remove broken VHDs
   await asyncMap(vhds, async path => {
     try {
-      await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), vhd => {
+      await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), async vhd => {
         if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {
           const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
           vhdParents[path] = parent
@@ -229,6 +229,7 @@ exports.cleanVm = async function cleanVm(
           }
         }
         vhdById.set(UUID.stringify(vhdKept.footer.uuid), vhdKept)
+        await vhd.check()
       })
     } catch (error) {
       vhds.delete(path)

@xen-orchestra/fs/src/s3.js

@@ -259,21 +259,21 @@ export default class S3Handler extends RemoteHandlerAbstract {
     }
   }
 
-  async _list(dir) {
+  async _list(dir, { delimiter = '/' } = {}) {
     let NextContinuationToken
     const uniq = new Set()
     const Prefix = this._makePrefix(dir)
-
     do {
-      const result = await this._s3.send(
-        new ListObjectsV2Command({
-          Bucket: this._bucket,
-          Prefix,
-          Delimiter: '/',
-          // will only return path until delimiters
-          ContinuationToken: NextContinuationToken,
-        })
-      )
+      const command = {
+        Bucket: this._bucket,
+        Prefix,
+        // will only return path until delimiters
+        ContinuationToken: NextContinuationToken,
+      }
+      if (delimiter !== null) {
+        command.Delimiter = delimiter
+      }
+      const result = await this._s3.send(new ListObjectsV2Command(command))
 
       if (result.IsTruncated) {
         warn(`need pagination to browse the directory ${dir} completely`)
@@ -289,7 +289,7 @@ export default class S3Handler extends RemoteHandlerAbstract {
 
       // files
       for (const entry of result.Contents ?? []) {
-        uniq.add(basename(entry.Key))
+        uniq.add(delimiter === null ? entry.Key.substr(Prefix.length) : basename(entry.Key))
       }
     } while (NextContinuationToken !== undefined)
 
@@ -308,7 +308,7 @@ export default class S3Handler extends RemoteHandlerAbstract {
 
   // s3 doesn't have a rename operation, so copy + delete source
   async _rename(oldPath, newPath) {
-    await this.copy(oldPath, newPath)
+    await this._copy(oldPath, newPath)
     await this._s3.send(new DeleteObjectCommand(this._createParams(oldPath)))
   }
 

packages/vhd-lib/Vhd/VhdAbstract.js

@@ -360,4 +360,6 @@ exports.VhdAbstract = class VhdAbstract {
     }
     return true
   }
+
+  async check() {}
 }

packages/vhd-lib/Vhd/VhdDirectory.js

@@ -9,7 +9,7 @@ const assert = require('assert')
 const promisify = require('promise-toolbox/promisify')
 const zlib = require('zlib')
 
-const { debug } = createLogger('vhd-lib:VhdDirectory')
+const { debug, warn } = createLogger('vhd-lib:VhdDirectory')
 
 const NULL_COMPRESSOR = {
   compress: buffer => buffer,
@@ -310,4 +310,27 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
     })
     this.#compressor = getCompressor(chunkFilters[0])
   }
+
+  async check() {
+    await this.readBlockAllocationTable()
+
+    const handler = this._handler
+    // do one massive query instead of one per sub directories
+    const existingBlocks = (await handler._list(`${this._path}/blocks/`, { delimiter: null })).map(path => {
+      const [thousands, units] = path.split('/')
+      return 1000 * parseInt(thousands, 10) + parseInt(units, 10)
+    })
+    existingBlocks.sort((a, b) => a - b)
+    let currentIndex = 0
+    const nBlocks = this.header.maxTableEntries
+
+    for (let blockId = 0; blockId < nBlocks; ++blockId) {
+      if (await this.containsBlock(blockId)) {
+        if (existingBlocks[currentIndex] !== blockId) {
+          warn(`Vhd ${this._path} miss block ${blockId}`, { path: this._path, blockId })
+        }
+        currentIndex++
+      }
+    }
+  }
 }