feat: immutable backups (#6928)

@xen-orchestra/backups/RemoteAdapter.mjs

@@ -35,6 +35,8 @@ export const DIR_XO_CONFIG_BACKUPS = 'xo-config-backups'
 
 export const DIR_XO_POOL_METADATA_BACKUPS = 'xo-pool-metadata-backups'
 
+const IMMUTABILTY_METADATA_FILENAME = '/immutability.json'
+
 const { debug, warn } = createLogger('xo:backups:RemoteAdapter')
 
 const compareTimestamp = (a, b) => a.timestamp - b.timestamp
@@ -749,10 +751,37 @@ export class RemoteAdapter {
   }
 
   async readVmBackupMetadata(path) {
+    let json
+    let isImmutable = false
+    let remoteIsImmutable = false
+    // if the remote is immutable, check if this metadatas are also immutables
+    try {
+      // this file is not encrypted
+      await this._handler._readFile(IMMUTABILTY_METADATA_FILENAME)
+      remoteIsImmutable = true
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        throw error
+      }
+    }
+
+    try {
+      // this will trigger an EPERM error if the file is immutable
+      json = await this.handler.readFile(path, { flag: 'r+' })
+      // s3 handler don't respect flags
+    } catch (err) {
+      // retry without triggerring immutbaility check ,only on immutable remote
+      if (err.code === 'EPERM' && remoteIsImmutable) {
+        isImmutable = true
+        json = await this._handler.readFile(path, { flag: 'r' })
+      } else {
+        throw err
+      }
+    }
     // _filename is a private field used to compute the backup id
     //
     // it's enumerable to make it cacheable
-    const metadata = { ...JSON.parse(await this._handler.readFile(path)), _filename: path }
+    const metadata = { ...JSON.parse(json), _filename: path, isImmutable }
 
     // backups created on XenServer < 7.1 via JSON in XML-RPC transports have boolean values encoded as integers, which make them unusable with more recent XAPIs
     if (typeof metadata.vm.is_a_template === 'number') {

@xen-orchestra/backups/formatVmBackups.mjs

@@ -31,6 +31,7 @@ function formatVmBackup(backup) {
           }),
 
     id: backup.id,
+    isImmutable: backup.isImmutable,
     jobId: backup.jobId,
     mode: backup.mode,
     scheduleId: backup.scheduleId,

@xen-orchestra/immutable-backups/.USAGE.md

@@ -0,0 +1,10 @@
+### make a remote immutable
+
+launch the `xo-immutable-remote` command. The configuration is stored in the config file.
+This script must be kept running to make file immutable reliably.
+
+### make file mutable
+
+launch the `xo-lift-remote-immutability` cli. The configuration is stored in the config file .
+
+If the config file have a `liftEvery`, this script will contiue to run and check regularly if there are files to update.

@xen-orchestra/immutable-backups/.npmignore

@@ -0,0 +1 @@
+../../scripts/npmignore

@xen-orchestra/immutable-backups/README.md

@@ -0,0 +1,31 @@
+<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
+
+# @xen-orchestra/immutable-backups
+
+## Usage
+
+### make a remote immutable
+
+launch the `xo-immutable-remote` command. The configuration is stored in the config file.
+This script must be kept running to make file immutable reliably.
+
+### make file mutable
+
+launch the `xo-lift-remote-immutability` cli. The configuration is stored in the config file .
+
+If the config file have a `liftEvery`, this script will contiue to run and check regularly if there are files to update.
+
+## Contributions
+
+Contributions are _very_ welcomed, either on the documentation or on
+the code.
+
+You may:
+
+- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
+  you've encountered;
+- fork and create a pull request.
+
+## License
+
+[AGPL-3.0-or-later](https://spdx.org/licenses/AGPL-3.0-or-later) © [Vates SAS](https://vates.fr)

@xen-orchestra/immutable-backups/_cleanXoCache.mjs

@@ -0,0 +1,10 @@
+import fs from 'node:fs/promises'
+import { dirname, join } from 'node:path'
+import isBackupMetadata from './isBackupMetadata.mjs'
+
+export default async path => {
+  if (isBackupMetadata(path)) {
+    // snipe vm metadata cache to force XO to update it
+    await fs.unlink(join(dirname(path), 'cache.json.gz'))
+  }
+}

@xen-orchestra/immutable-backups/_isInVhdDirectory.mjs

@@ -0,0 +1,4 @@
+import { dirname } from 'node:path'
+
+// check if we are handling file directly under a vhd directory ( bat, headr, footer,..)
+export default path => dirname(path).endsWith('.vhd')

@xen-orchestra/immutable-backups/_loadConfig.mjs

@@ -0,0 +1,46 @@
+import { load } from 'app-conf'
+import { homedir } from 'os'
+import { join } from 'node:path'
+import ms from 'ms'
+
+const APP_NAME = 'xo-immutable-backups'
+const APP_DIR = new URL('.', import.meta.url).pathname
+
+export default async function loadConfig() {
+  const config = await load(APP_NAME, {
+    appDir: APP_DIR,
+    ignoreUnknownFormats: true,
+  })
+  if (config.remotes === undefined || config.remotes?.length < 1) {
+    throw new Error(
+      'No remotes are configured in the config file, please add at least one [remotes.<remoteid>]  with a root property pointing to the absolute path of the remote to watch'
+    )
+  }
+  if (config.liftEvery) {
+    config.liftEvery = ms(config.liftEvery)
+  }
+  for (const [remoteId, { indexPath, immutabilityDuration, root }] of Object.entries(config.remotes)) {
+    if (!root) {
+      throw new Error(
+        `Remote ${remoteId} don't have a root property,containing the absolute path to the root of a backup repository `
+      )
+    }
+    if (!immutabilityDuration) {
+      throw new Error(
+        `Remote ${remoteId} don't have a immutabilityDuration property to indicate the minimal duration the backups should be  protected by immutability `
+      )
+    }
+    if (ms(immutabilityDuration) < ms('1d')) {
+      throw new Error(
+        `Remote ${remoteId} immutability duration is smaller than the minimum allowed (1d), current : ${immutabilityDuration}`
+      )
+    }
+    if (!indexPath) {
+      const basePath = indexPath ?? process.env.XDG_DATA_HOME ?? join(homedir(), '.local', 'share')
+      const immutabilityIndexPath = join(basePath, APP_NAME, remoteId)
+      config.remotes[remoteId].indexPath = immutabilityIndexPath
+    }
+    config.remotes[remoteId].immutabilityDuration = ms(immutabilityDuration)
+  }
+  return config
+}

@xen-orchestra/immutable-backups/config.toml

@@ -0,0 +1,14 @@
+
+# how often does the lift immutability script will run to check if 
+# some files need to be made mutable 
+liftEvery =  1h
+
+# you can add as many remote as you want, if you change the id ( here : remote1)
+#[remotes.remote1]
+#root = "/mnt/ssd/vhdblock/" # the absolute path of the root of the backup repository
+#immutabilityDuration = 7d # mandatory
+# optional, default value is false will scan and update the index on start, can be expensive
+#rebuildIndexOnStart = true 
+
+# the index path is optional, default in XDG_DATA_HOME, or if this is not set, in ~/.local/share
+#indexPath = "/var/lib/" # will add automatically the application name immutable-backup 

@xen-orchestra/immutable-backups/directory.mjs

@@ -0,0 +1,21 @@
+import execa from 'execa'
+import { unindexFile, indexFile } from './fileIndex.mjs'
+
+export async function makeImmutable(dirPath, immutabilityCachePath) {
+  if (immutabilityCachePath) {
+    await indexFile(dirPath, immutabilityCachePath)
+  }
+  await execa('chattr', ['+i', '-R', dirPath])
+}
+
+export async function liftImmutability(dirPath, immutabilityCachePath) {
+  if (immutabilityCachePath) {
+    await unindexFile(dirPath, immutabilityCachePath)
+  }
+  await execa('chattr', ['-i', '-R', dirPath])
+}
+
+export async function isImmutable(path) {
+  const { stdout } = await execa('lsattr', ['-d', path])
+  return stdout[4] === 'i'
+}

@xen-orchestra/immutable-backups/directory.spec.mjs

@@ -0,0 +1,31 @@
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import { tmpdir } from 'node:os'
+import * as Directory from './directory.mjs'
+import { rimraf } from 'rimraf'
+
+describe('immutable-backups/file', async () => {
+  it('really lock a directory', async () => {
+    const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests'))
+    const dataDir = path.join(dir, 'data')
+    await fs.mkdir(dataDir)
+    const immutDir = path.join(dir, '.immutable')
+    const filePath = path.join(dataDir, 'test')
+    await fs.writeFile(filePath, 'data')
+    await Directory.makeImmutable(dataDir, immutDir)
+    await assert.rejects(() => fs.writeFile(filePath, 'data'))
+    await assert.rejects(() => fs.appendFile(filePath, 'data'))
+    await assert.rejects(() => fs.unlink(filePath))
+    await assert.rejects(() => fs.rename(filePath, filePath + 'copy'))
+    await assert.rejects(() => fs.writeFile(path.join(dataDir, 'test2'), 'data'))
+    await assert.rejects(() => fs.rename(dataDir, dataDir + 'copy'))
+    await Directory.liftImmutability(dataDir, immutDir)
+    await fs.writeFile(filePath, 'data')
+    await fs.appendFile(filePath, 'data')
+    await fs.unlink(filePath)
+    await fs.rename(dataDir, dataDir + 'copy')
+    await rimraf(dir)
+  })
+})

@xen-orchestra/immutable-backups/doc.md

@@ -0,0 +1,114 @@
+# Imutability
+
+the goal is to make a remote that XO can write, but not modify during the immutability duration set on the remote. That way, it's not possible for XO to delete or encrypt any backup during this period. It protects your backup agains ransomware, at least as long as the attacker does not have a root access to the remote server.
+
+We target `governance` type of immutability, **the local root account of the remote server will be able to lift immutability**.
+
+We use the file system capabilities, they are tested on the protection process start.
+
+It is compatible with encryption at rest made by XO.
+
+## Prerequisites
+
+The commands must be run as root on the remote, or by a user with the `CAP_LINUX_IMMUTABLE` capability . On start, the protect process writes into the remote `imutability.json` file its status and the immutability duration.
+
+the `chattr` and `lsattr` should be installed on the system
+
+## Configuring
+
+this package uses app-conf to store its config. The application name is `xo-immutable-backup`. A sample config file is provided in this package.
+
+## Making a file immutable
+
+when marking a file or a folder immutable, it create an alias file in the `<indexPath>/<DayOfFileCreation>/<sha256(fullpath)>`.
+
+`indexPath` can be defined in the config file, otherwise `XDG_HOME` is used. If not available it goes to `~/.local/share`
+
+This index is used when lifting the immutability of the remote, it will only look at the old enough `<indexPath>/<DayOfFileCreation>/` folders.
+
+## Real time protecting
+
+On start, the watcher will create the index if it does not exists.
+It will also do a checkup to ensure immutability could work on this remote and handle the easiest issues.
+
+The watching process depends on the backup type, since we don't want to make temporary files and cache immutable.
+
+It won't protect files during upload, only when the files have been completly written on disk. Real time, in this case, means "protecting critical files as soon as possible after they are uploaded"
+
+This can be alleviated by :
+
+- Coupling immutability with encryption to ensure the file is not modified
+- Making health check to ensure the data are exactly as the snapshot data
+
+List of protected files :
+
+```js
+const PATHS = [
+  // xo configuration backupq
+  'xo-config-backups/*/*/data',
+  'xo-config-backups/*/*/data.json',
+  'xo-config-backups/*/*/metadata.json',
+  // pool backupq
+  'xo-pool-metadata-backups/*/metadata.json',
+  'xo-pool-metadata-backups/*/data',
+  // vm backups , xo-vm-backups/<vmuuid>/
+  'xo-vm-backups/*/*.json',
+  'xo-vm-backups/*/*.xva',
+  'xo-vm-backups/*/*.xva.checksum',
+  // xo-vm-backups/<vmuuid>/vdis/<jobid>/<vdiUuid>
+  'xo-vm-backups/*/vdis/*/*/*.vhd', // can be an alias or a vhd file
+  // for vhd directory :
+  'xo-vm-backups/*/vdis/*/*/data/*.vhd/bat',
+  'xo-vm-backups/*/vdis/*/*/data/*.vhd/header',
+  'xo-vm-backups/*/vdis/*/*/data/*.vhd/footer',
+]
+```
+
+## Releasing protection on old enough files on a remote
+
+the watcher will periodically check if some file must by unlocked
+
+## Troubleshooting
+
+### some files are still locked
+
+add the `rebuildIndexOnStart` option to the config file
+
+### make remote fully mutable again
+
+- Update the immutability setting with a 0 duration
+- launch the `liftProtection` cli.
+- remove the `protectRemotes` service
+
+### increasing the immutability duration
+
+this will prolong immutable file, but won't protect files that are already out of immutability
+
+### reducing the immutability duration
+
+change the setting, and launch the `liftProtection` cli , or wait for next planed execution
+
+### why are my incremental backups not marked as protected in XO ?
+
+are not marked as protected in XO ?
+
+For incremental backups to be marked as protected in XO, the entire chain must be under protection. To ensure at least 7 days of backups are protected, you need to set the immutability duration and retention at 14 days, the full backup interval at 7 days
+
+That means that if the last backup chain is complete ( 7 backup ) it is completely under protection, and if not, the precedent chain is also under protection. K are key backups, and are delta
+
+```
+Kd Kdddddd Kdddddd K #  8 backups protected, 2 chains
+K Kdddddd Kdddddd Kd #  9 backups protected, 2 chains
+ Kdddddd Kdddddd Kdd # 10 backups protected, 2 chains
+ Kddddd Kdddddd Kddd # 11 backups protected, 2 chains
+ Kdddd Kdddddd Kdddd # 12 backups protected, 2 chains
+ Kddd Kdddddd Kddddd # 13 backups protected, 2 chains
+ Kdd Kdddddd Kdddddd #  7 backups protected, 1 chain since precedent full is now mutable
+Kd Kdddddd Kdddddd K #  8 backups protected, 2 chains
+```
+
+### Why doesn't the protect process start ?
+
+- it should be run as root or by a user with the `CAP_LINUX_IMMUTABLE` capability
+- the underlying file system should support immutability, especially the `chattr` and `lsattr` command
+- logs are in journalctl

@xen-orchestra/immutable-backups/file.mjs

@@ -0,0 +1,24 @@
+import execa from 'execa'
+import { unindexFile, indexFile } from './fileIndex.mjs'
+
+// this work only on linux like systems
+// this could work on windows : https://4sysops.com/archives/set-and-remove-the-read-only-file-attribute-with-powershell/
+
+export async function makeImmutable(path, immutabilityCachePath) {
+  if (immutabilityCachePath) {
+    await indexFile(path, immutabilityCachePath)
+  }
+  await execa('chattr', ['+i', path])
+}
+
+export async function liftImmutability(filePath, immutabilityCachePath) {
+  if (immutabilityCachePath) {
+    await unindexFile(filePath, immutabilityCachePath)
+  }
+  await execa('chattr', ['-i', filePath])
+}
+
+export async function isImmutable(path) {
+  const { stdout } = await execa('lsattr', ['-d', path])
+  return stdout[4] === 'i'
+}

@xen-orchestra/immutable-backups/file.spec.mjs

@@ -0,0 +1,29 @@
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import * as File from './file.mjs'
+import { tmpdir } from 'node:os'
+import { rimraf } from 'rimraf'
+
+describe('immutable-backups/file', async () => {
+  it('really lock a file', async () => {
+    const dir = await fs.mkdtemp(path.join(tmpdir(), 'immutable-backups-tests'))
+    const immutDir = path.join(dir, '.immutable')
+    const filePath = path.join(dir, 'test.ext')
+    await fs.writeFile(filePath, 'data')
+    assert.strictEqual(await File.isImmutable(filePath), false)
+    await File.makeImmutable(filePath, immutDir)
+    assert.strictEqual(await File.isImmutable(filePath), true)
+    await assert.rejects(() => fs.writeFile(filePath, 'data'))
+    await assert.rejects(() => fs.appendFile(filePath, 'data'))
+    await assert.rejects(() => fs.unlink(filePath))
+    await assert.rejects(() => fs.rename(filePath, filePath + 'copy'))
+    await File.liftImmutability(filePath, immutDir)
+    assert.strictEqual(await File.isImmutable(filePath), false)
+    await fs.writeFile(filePath, 'data')
+    await fs.appendFile(filePath, 'data')
+    await fs.unlink(filePath)
+    await rimraf(dir)
+  })
+})

@xen-orchestra/immutable-backups/fileIndex.mjs

@@ -0,0 +1,94 @@
+import { join } from 'node:path'
+import { createHash } from 'node:crypto'
+import fs from 'node:fs/promises'
+import { dirname } from 'path'
+function sha256(content) {
+  return createHash('sha256').update(content).digest('hex')
+}
+
+function formatDate(date) {
+  return date.toISOString().split('T')[0]
+}
+
+async function computeIndexFilePath(path, immutabilityIndexPath) {
+  const stat = await fs.stat(path)
+  const date = new Date(stat.birthtimeMs)
+  const day = formatDate(date)
+  const hash = sha256(path)
+  return join(immutabilityIndexPath, day, hash)
+}
+
+export async function indexFile(path, immutabilityIndexPath) {
+  const indexFilePath = await computeIndexFilePath(path, immutabilityIndexPath)
+  try {
+    await fs.writeFile(indexFilePath, path, { flag: 'wx' })
+  } catch (err) {
+    // missing dir: make it
+    if (err.code === 'ENOENT') {
+      await fs.mkdir(dirname(indexFilePath), { recursive: true })
+      await fs.writeFile(indexFilePath, path)
+    } else if (err.code === 'EPERM') {
+      // an immutable file alreay exist : not a problem if it contains the right target
+      const { size } = await fs.stat(indexFilePath)
+      if (size.length > 1024 * 1024) {
+        throw new Error(`Index file at ${indexFilePath} is too big, ${size} bytes `)
+      }
+      const existingContent = await fs.readFile(indexFilePath, { encoding: 'utf8' })
+      if (existingContent !== path) {
+        throw new Error(`Index file at ${indexFilePath}, should contains ${path}, but contains ${existingContent}`)
+      }
+      throw err
+    } else {
+      throw err
+    }
+  }
+  return indexFilePath
+}
+
+export async function unindexFile(path, immutabilityIndexPath) {
+  try {
+    const cacheFileName = await computeIndexFilePath(path, immutabilityIndexPath)
+    await fs.unlink(cacheFileName)
+  } catch (err) {
+    if (err.code !== 'ENOENT') {
+      throw err
+    }
+  }
+}
+
+export async function* listOlderTargets(immutabilityCachePath, immutabilityDuration) {
+  // walk all dir by day until  the limit day
+  const limitDate = new Date(Date.now() - immutabilityDuration)
+
+  const limitDay = formatDate(limitDate)
+  const dir = await fs.opendir(immutabilityCachePath)
+  for await (const dirent of dir) {
+    if (dirent.isFile()) {
+      continue
+    }
+    // ensure we have a valid date
+    if (isNaN(new Date(dirent.name))) {
+      continue
+    }
+    // recent enough to be kept
+    if (dir.name >= limitDay) {
+      continue
+    }
+    const subDirPath = join(immutabilityDuration, dirent.name)
+    const subdir = await fs.opendir(subDirPath)
+    let nb = 0
+    for await (const hashFileEntry of dir) {
+      const entryFullPath = join(subDirPath, hashFileEntry.name)
+      const targetPath = await fs.readFile(entryFullPath, { encoding: 'utf8' })
+      yield {
+        index: entryFullPath,
+        target: targetPath,
+      }
+      nb++
+    }
+    // cleanup older folder
+    if (nb === 0) {
+      await fs.rmdir(subdir)
+    }
+  }
+}

@xen-orchestra/immutable-backups/isBackupMetadata.mjs

@@ -0,0 +1 @@
+export default path => path.match(/xo-vm-backups\/[^/]+\/[^/]+\.json$/)

@xen-orchestra/immutable-backups/liftProtection.mjs

@@ -0,0 +1,37 @@
+#!/usr/bin/env node
+
+import fs from 'node:fs/promises'
+import * as Directory from './directory.mjs'
+import { createLogger } from '@xen-orchestra/log'
+import { listOlderTargets } from './fileIndex.mjs'
+import cleanXoCache from './_cleanXoCache.mjs'
+import loadConfig from './_loadConfig.mjs'
+
+const { info, warn } = createLogger('xen-orchestra:immutable-backups:liftProtection')
+
+async function liftRemoteImmutability(immutabilityCachePath, immutabilityDuration) {
+  for await (const { index, target } of listOlderTargets(immutabilityCachePath, immutabilityDuration)) {
+    await Directory.liftImmutability(target, immutabilityCachePath)
+    await fs.unlink(index)
+    await cleanXoCache(target)
+  }
+}
+
+async function liftImmutability(remotes) {
+  for (const [remoteId, { indexPath, immutabilityDuration }] of Object.entries(remotes)) {
+    liftRemoteImmutability(indexPath, immutabilityDuration).catch(err =>
+      warn('error during watchRemote', { err, remoteId, indexPath, immutabilityDuration })
+    )
+  }
+}
+
+const { liftEvery, remotes } = await loadConfig()
+
+if (liftEvery > 0) {
+  info('setup watcher for immutability lifting')
+  setInterval(async () => {
+    liftImmutability(remotes)
+  }, liftEvery)
+} else {
+  liftImmutability(remotes)
+}

@xen-orchestra/immutable-backups/package.json

@@ -0,0 +1,41 @@
+{
+  "private": true,
+  "name": "@xen-orchestra/immutable-backups",
+  "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/immutable-backups",
+  "bugs": "https://github.com/vatesfr/xen-orchestra/issues",
+  "repository": {
+    "directory": "@xen-orchestra/immutable-backups",
+    "type": "git",
+    "url": "https://github.com/vatesfr/xen-orchestra.git"
+  },
+  "author": {
+    "name": "Vates SAS",
+    "url": "https://vates.fr"
+  },
+  "bin": {
+    "xo-immutable-remote": "./protectRemotes.mjs",
+    "xo-lift-remote-immutability": "./liftProtection.mjs"
+  },
+  "license": "AGPL-3.0-or-later",
+  "version": "0.0.0",
+  "engines": {
+    "node": ">=14.0.0"
+  },
+  "dependencies": {
+    "@vates/async-each": "^1.0.0",
+    "@xen-orchestra/backups": "^0.44.2",
+    "@xen-orchestra/log": "^0.6.0",
+    "app-conf": "^2.3.0",
+    "chokidar": "^3.5.3",
+    "execa": "^5.0.0",
+    "ms": "^2.1.3",
+    "vhd-lib": "^4.7.0"
+  },
+  "devDependencies": {
+    "rimraf": "^5.0.5",
+    "tap": "^18.6.1"
+  },
+  "scripts": {
+    "test-integration": "tap  *.integ.mjs"
+  }
+}

@xen-orchestra/immutable-backups/protectRemotes.mjs

@@ -0,0 +1,191 @@
+#!/usr/bin/env node
+
+import fs from 'node:fs/promises'
+import * as File from './file.mjs'
+import * as Directory from './directory.mjs'
+import assert from 'node:assert'
+import { dirname, join, sep } from 'node:path'
+import { createLogger } from '@xen-orchestra/log'
+import chokidar from 'chokidar'
+import { indexFile } from './fileIndex.mjs'
+import cleanXoCache from './_cleanXoCache.mjs'
+import loadConfig from './_loadConfig.mjs'
+import isInVhdDirectory from './_isInVhdDirectory.mjs'
+const { debug, info, warn } = createLogger('xen-orchestra:immutable-backups:remote')
+
+async function test(remotePath, indexPath) {
+  await fs.readdir(remotePath)
+
+  const testPath = join(remotePath, '.test-immut')
+  // cleanup
+  try {
+    await File.liftImmutability(testPath, indexPath)
+    await fs.unlink(testPath)
+  } catch (err) {}
+  // can create , modify and delete a file
+  await fs.writeFile(testPath, `test immut ${new Date()}`)
+  await fs.writeFile(testPath, `test immut change 1 ${new Date()}`)
+  await fs.unlink(testPath)
+
+  // cannot modify or delete an immutable file
+  await fs.writeFile(testPath, `test immut ${new Date()}`)
+  await File.makeImmutable(testPath, indexPath)
+  await assert.rejects(fs.writeFile(testPath, `test immut change 2  ${new Date()}`), { code: 'EPERM' })
+  await assert.rejects(fs.unlink(testPath), { code: 'EPERM' })
+  // can modify and delete a file after lifting immutability
+  await File.liftImmutability(testPath, indexPath)
+
+  await fs.writeFile(testPath, `test immut change 3 ${new Date()}`)
+  await fs.unlink(testPath)
+}
+async function handleExistingFile(root, indexPath, path) {
+  try {
+    // a vhd block directory is completly immutable
+    if (isInVhdDirectory(path)) {
+      // this will trigger 3 times per vhd blocks
+      const dir = join(root, dirname(path))
+      if (Directory.isImmutable(dir)) {
+        await indexFile(dir, indexPath)
+      }
+    } else {
+      // other files are immutable a file basis
+      const fullPath = join(root, path)
+      if (File.isImmutable(fullPath)) {
+        await indexFile(fullPath, indexPath)
+      }
+    }
+  } catch (err) {
+    if (err.code !== 'EEXIST' && err.code !== 'EPERM') {
+      // there can be a symbolic link in the tree
+      warn('handleExistingFile', err)
+    }
+  }
+}
+
+async function handleNewFile(root, indexPath, pendingVhds, path) {
+  // with awaitWriteFinish we have complete files here
+  // we can make them immutable
+
+  if (isInVhdDirectory(path)) {
+    // watching a vhd block
+    // wait for header/footer and BAT before making this immutable recursively
+    const splitted = path.split(sep)
+    const vmUuid = splitted[1]
+    const vdiUuid = splitted[4]
+    const uniqPath = `${vmUuid}/${vdiUuid}`
+    const { existing } = pendingVhds.get(uniqPath) ?? {}
+    if (existing === undefined) {
+      pendingVhds.set(uniqPath, { existing: 1, lastModified: Date.now() })
+    } else {
+      // already two of the key files,and we got the last one
+      if (existing === 2) {
+        await Directory.makeImmutable(join(root, dirname(path)), indexPath)
+        pendingVhds.delete(uniqPath)
+      } else {
+        // wait for the other
+        pendingVhds.set(uniqPath, { existing: existing + 1, lastModified: Date.now() })
+      }
+    }
+  } else {
+    const fullFilePath = join(root, path)
+    await File.makeImmutable(fullFilePath, indexPath)
+    await cleanXoCache(fullFilePath)
+  }
+}
+export async function watchRemote(remoteId, { root, immutabilityDuration, rebuildIndexOnStart = false, indexPath }) {
+  // create index directory
+  await fs.mkdir(indexPath, { recursive: true })
+
+  // test if fs and index directories are well configured
+  await test(root, indexPath)
+
+  // add duration and watch status in the metadata.json of the remote
+  const settingPath = join(root, 'immutability.json')
+  try {
+    // this file won't be made mutable by liftimmutability
+    await File.liftImmutability(settingPath)
+  } catch (error) {
+    // file may not exists, and it's not really a problem
+    info('lifting immutability on current settings', error)
+  }
+  await fs.writeFile(
+    settingPath,
+    JSON.stringify({
+      since: Date.now(),
+      immutable: true,
+      duration: immutabilityDuration,
+    })
+  )
+  // no index path in makeImmutable(): the immutability won't be lifted
+  File.makeImmutable(settingPath)
+
+  // we wait for footer/header AND BAT to be written before locking a vhd directory
+  // this map allow us to track the vhd with partial metadata
+  const pendingVhds = new Map()
+  // cleanup pending vhd map periodically
+  setInterval(
+    () => {
+      pendingVhds.forEach(({ lastModified, existing }, path) => {
+        if (Date.now() - lastModified > 60 * 60 * 1000) {
+          pendingVhds.delete(path)
+          warn(`vhd at ${path} is incomplete since ${lastModified}`, { existing, lastModified, path })
+        }
+      })
+    },
+    60 * 60 * 1000
+  )
+
+  // watch the remote for any new VM metadata json file
+  const PATHS = [
+    'xo-config-backups/*/*/data',
+    'xo-config-backups/*/*/data.json',
+    'xo-config-backups/*/*/metadata.json',
+    'xo-pool-metadata-backups/*/metadata.json',
+    'xo-pool-metadata-backups/*/data',
+    // xo-vm-backups/<vmuuid>/
+    'xo-vm-backups/*/*.json',
+    'xo-vm-backups/*/*.xva',
+    'xo-vm-backups/*/*.xva.checksum',
+    // xo-vm-backups/<vmuuid>/vdis/<jobid>/<vdiUuid>
+    'xo-vm-backups/*/vdis/*/*/*.vhd', // can be an alias or a vhd file
+    // for vhd directory :
+    'xo-vm-backups/*/vdis/*/*/data/*.vhd/bat',
+    'xo-vm-backups/*/vdis/*/*/data/*.vhd/header',
+    'xo-vm-backups/*/vdis/*/*/data/*.vhd/footer',
+  ]
+
+  let ready = false
+  const watcher = chokidar.watch(PATHS, {
+    ignored: [
+      /(^|[/\\])\../, // ignore dotfiles
+      /\.lock$/,
+    ],
+    cwd: root,
+    recursive: false, // vhd directory can generate a lot of folder, don't let chokidar choke on this
+    ignoreInitial: !rebuildIndexOnStart,
+    depth: 7,
+    awaitWriteFinish: true,
+  })
+
+  // Add event listeners.
+  watcher
+    .on('add', async path => {
+      debug(`File ${path} has been added ${path.split('/').length}`)
+      if (ready) {
+        await handleNewFile(root, indexPath, pendingVhds, path)
+      } else {
+        await handleExistingFile(root, indexPath, path)
+      }
+    })
+    .on('error', error => warn(`Watcher error: ${error}`))
+    .on('ready', () => {
+      ready = true
+      info('Ready for changes')
+    })
+}
+
+const { remotes } = await loadConfig()
+
+for (const [remoteId, remote] of Object.entries(remotes)) {
+  watchRemote(remoteId, remote).catch(err => warn('error during watchRemote', { err, remoteId, remote }))
+}

CHANGELOG.unreleased.md

@@ -8,6 +8,7 @@
 > Users must be able to say: “Nice enhancement, I'm eager to test it”
 
 - [REST API] New pool action: `create_vm` [#6749](https://github.com/vatesfr/xen-orchestra/issues/6749)
+- [Backup] Implement Backup Repository immutability (PR [#6928](https://github.com/vatesfr/xen-orchestra/pull/6928))
 
 ### Bug fixes
 
@@ -29,8 +30,11 @@
 
 <!--packages-start-->
 
+- @xen-orchestra/backups patch
 - @xen-orchestra/fs patch
+- @xen-orchestra/immutable-backups major
 - xo-cli minor
 - xo-server minor
+- xo-web minor
 
 <!--packages-end-->

packages/xo-web/src/common/render-xo-item.js

@@ -646,6 +646,11 @@ const xoItemToRender = {
       <span className='tag tag-info' style={{ textTransform: 'capitalize' }}>
         {backup.mode === 'delta' ? _('backupIsIncremental') : backup.mode}
       </span>{' '}
+      {backup.isImmutable && (
+        <span className='tag tag-info'>
+          <Icon icon='lock' />
+        </span>
+      )}{' '}
       <span className='tag tag-warning'>{backup.remote.name}</span>{' '}
       {backup.differencingVhds > 0 && (
         <span className='tag tag-info'>