feat(backups): merge worker implementation

This CLI must be run directly in the directory where the remote is mounted.

It's only compatible with local remote at the moment.

To start the worker:

```js
const MergeWorker = require('@xen-orchestra/backups/merge-worker/index.js')

await MergeWorker.run(remotePath)
```

To register a VM backup dir to be clean (thus merging its unused VHD), create a file in the queue directory containing the VM UUID:

```
> echo cc700fe2-724e-44a5-8663-5f8f88e05e34 > .queue/clean-vm/20211013T142401Z
```

The queue directory is available as `MergeWorker.CLEAN_VM_QUEUE`.

@xen-orchestra/backups/_cleanVm.js

@@ -370,4 +370,9 @@ exports.cleanVm = async function cleanVm(
       }
     }),
   ])
+
+  return {
+    // boolean whether some VHDs were merged (or should be merged)
+    merge: toMerge.length !== 0,
+  }
 }

@xen-orchestra/backups/merge-worker/cli.js

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+
+const { catchGlobalErrors } = require('@xen-orchestra/log/configure.js')
+const { createLogger } = require('@xen-orchestra/log')
+const { getSyncedHandler } = require('@xen-orchestra/fs')
+const { join } = require('path')
+const Disposable = require('promise-toolbox/Disposable')
+const min = require('lodash/min')
+
+const { getVmBackupDir } = require('../_getVmBackupDir.js')
+const { RemoteAdapter } = require('../RemoteAdapter.js')
+
+const { CLEAN_VM_QUEUE } = require('./index.js')
+
+// -------------------------------------------------------------------
+
+catchGlobalErrors(createLogger('xo:backups:mergeWorker'))
+
+const { fatal, info, warn } = createLogger('xo:backups:mergeWorker')
+
+// -------------------------------------------------------------------
+
+const main = Disposable.wrap(async function* main(args) {
+  const handler = yield getSyncedHandler({ url: 'file://' + process.cwd() })
+
+  yield handler.lock(CLEAN_VM_QUEUE)
+
+  const adapter = new RemoteAdapter(handler)
+
+  const listRetry = async () => {
+    const timeoutResolver = resolve => setTimeout(resolve, 10e3)
+    for (let i = 0; i < 10; ++i) {
+      const entries = await handler.list(CLEAN_VM_QUEUE)
+      if (entries.length !== 0) {
+        return entries
+      }
+      await new Promise(timeoutResolver)
+    }
+  }
+
+  let taskFiles
+  while ((taskFiles = await listRetry()) !== undefined) {
+    const taskFileBasename = min(taskFiles)
+    const taskFile = join(CLEAN_VM_QUEUE, '_' + taskFileBasename)
+
+    // move this task to the end
+    await handler.rename(join(CLEAN_VM_QUEUE, taskFileBasename), taskFile)
+    try {
+      const vmDir = getVmBackupDir(String(await handler.readFile(taskFile)))
+      await adapter.cleanVm(vmDir, { merge: true, onLog: info, remove: true })
+
+      handler.unlink(taskFile).catch(error => warn('deleting task failure', { error }))
+    } catch (error) {
+      warn('failure handling task', { error })
+    }
+  }
+})
+
+info('starting')
+main(process.argv.slice(2)).then(
+  () => {
+    info('bye :-)')
+  },
+  error => {
+    fatal(error)
+
+    process.exit(1)
+  }
+)

@xen-orchestra/backups/merge-worker/index.js

@@ -0,0 +1,25 @@
+const { join, resolve } = require('path')
+const { spawn } = require('child_process')
+const { check } = require('proper-lockfile')
+
+const CLEAN_VM_QUEUE = (exports.CLEAN_VM_QUEUE = '/xo-vm-backups/.queue/clean-vm/')
+
+const CLI_PATH = resolve(__dirname, 'cli.js')
+exports.run = async function runMergeWorker(remotePath) {
+  try {
+    // TODO: find a way to pass the acquire the lock and then pass it down the worker
+    if (await check(join(remotePath, CLEAN_VM_QUEUE))) {
+      // already locked, don't start another worker
+      return
+    }
+
+    spawn(CLI_PATH, {
+      cwd: remotePath,
+      detached: true,
+      stdio: 'inherit',
+    }).unref()
+  } catch (error) {
+    // we usually don't want to throw if the merge worker failed to start
+    return error
+  }
+}

@xen-orchestra/backups/package.json

@@ -32,7 +32,8 @@
     "lodash": "^4.17.20",
     "node-zone": "^0.4.0",
     "parse-pairs": "^1.1.0",
-    "promise-toolbox": "^0.19.2",
+    "promise-toolbox": "^0.20.0",
+    "proper-lockfile": "^4.1.2",
     "pump": "^3.0.0",
     "vhd-lib": "^1.2.0",
     "yazl": "^2.5.1"

yarn.lock

@@ -13063,6 +13063,13 @@ promise-toolbox@^0.19.0, promise-toolbox@^0.19.2:
   dependencies:
     make-error "^1.3.2"
 
+promise-toolbox@^0.20.0:
+  version "0.20.0"
+  resolved "https://registry.yarnpkg.com/promise-toolbox/-/promise-toolbox-0.20.0.tgz#af04d7338038c2362b8fb7c27546c57d893bf562"
+  integrity sha512-VXF6waqUheD19yOU7zxsXhw/HCKlXqXwUc4jM8mchtBqZFNA+GHA7dbJsQDLHP4IUpQuTLpCQRd0lCr5z4CqXQ==
+  dependencies:
+    make-error "^1.3.2"
+
 promise@^7.0.1, promise@^7.1.1:
   version "7.3.1"
   resolved "https://registry.yarnpkg.com/promise/-/promise-7.3.1.tgz#064b72602b18f90f29192b8b1bc418ffd1ebd3bf"