From f618fcdaf8233062fde9cf90daacc5eb2c6b4ccc Mon Sep 17 00:00:00 2001 From: Florent Beauchamp Date: Thu, 16 Jun 2022 17:27:41 +0200 Subject: [PATCH] feat(vhd): implement encryption on vhd directory --- @xen-orchestra/backups/RemoteAdapter.js | 15 +++- @xen-orchestra/backups/_backupWorker.js | 1 + packages/vhd-lib/Vhd/VhdDirectory.js | 79 ++++++------------- packages/vhd-lib/Vhd/_compressors.js | 51 ++++++++++++ packages/vhd-lib/Vhd/_encryptor.integ.spec.js | 22 ++++++ packages/vhd-lib/Vhd/_encryptors.js | 44 +++++++++++ packages/vhd-lib/Vhd/_secretStore.js | 4 + .../vhd-lib/createVhdDirectoryFromStream.js | 13 ++- packages/xo-server/config.toml | 1 + 9 files changed, 168 insertions(+), 62 deletions(-) create mode 100644 packages/vhd-lib/Vhd/_compressors.js create mode 100644 packages/vhd-lib/Vhd/_encryptor.integ.spec.js create mode 100644 packages/vhd-lib/Vhd/_encryptors.js create mode 100644 packages/vhd-lib/Vhd/_secretStore.js diff --git a/@xen-orchestra/backups/RemoteAdapter.js b/@xen-orchestra/backups/RemoteAdapter.js index 605e33354..b5bd21f8c 100644 --- a/@xen-orchestra/backups/RemoteAdapter.js +++ b/@xen-orchestra/backups/RemoteAdapter.js @@ -75,11 +75,15 @@ const debounceResourceFactory = factory => } class RemoteAdapter { - constructor(handler, { debounceResource = res => res, dirMode, vhdDirectoryCompression } = {}) { + constructor( + handler, + { debounceResource = res => res, dirMode, vhdDirectoryCompression, vhdDirectoryEncryption } = {} + ) { this._debounceResource = debounceResource this._dirMode = dirMode this._handler = handler this._vhdDirectoryCompression = vhdDirectoryCompression + this._vhdDirectoryEncryption = vhdDirectoryEncryption this._readCacheListVmBackups = synchronized.withKey()(this._readCacheListVmBackups) } @@ -201,7 +205,9 @@ class RemoteAdapter { const isVhdDirectory = vhd instanceof VhdDirectory return isVhdDirectory - ? this.#useVhdDirectory() && this.#getCompressionType() === vhd.compressionType + ? this.#useVhdDirectory() && + this.#getCompressionType() === vhd.compressionType && + this.#getEncryption() === vhd.encryption : !this.#useVhdDirectory() }) } @@ -291,6 +297,10 @@ class RemoteAdapter { return this._vhdDirectoryCompression } + #getEncryption() { + return this._vhdDirectoryEncryption + } + #useVhdDirectory() { return this.handler.type === 's3' } @@ -576,6 +586,7 @@ class RemoteAdapter { await createVhdDirectoryFromStream(handler, dataPath, input, { concurrency: 16, compression: this.#getCompressionType(), + encryption: this.#getEncryption(), async validator() { await input.task return validator.apply(this, arguments) diff --git a/@xen-orchestra/backups/_backupWorker.js b/@xen-orchestra/backups/_backupWorker.js index b4ac0bd73..ace139625 100644 --- a/@xen-orchestra/backups/_backupWorker.js +++ b/@xen-orchestra/backups/_backupWorker.js @@ -71,6 +71,7 @@ class BackupWorker { debounceResource: this.debounceResource, dirMode: this.#config.dirMode, vhdDirectoryCompression: this.#config.vhdDirectoryCompression, + vhdDirectoryEncryption: this.#config.vhdDirectoryEncryption, }) } finally { await handler.forget() diff --git a/packages/vhd-lib/Vhd/VhdDirectory.js b/packages/vhd-lib/Vhd/VhdDirectory.js index ad70d17ce..32ed6d3ca 100644 --- a/packages/vhd-lib/Vhd/VhdDirectory.js +++ b/packages/vhd-lib/Vhd/VhdDirectory.js @@ -5,58 +5,11 @@ const { createLogger } = require('@xen-orchestra/log') const { fuFooter, fuHeader, checksumStruct } = require('../_structs') const { test, set: setBitmap } = require('../_bitmap') const { VhdAbstract } = require('./VhdAbstract') +const { _getCompressor: getCompressor } = require('./_compressors') +const { _getEncryptor: getEncryptor } = require('./_encryptors') const assert = require('assert') -const promisify = require('promise-toolbox/promisify') -const zlib = require('zlib') - const { debug } = createLogger('vhd-lib:VhdDirectory') -const NULL_COMPRESSOR = { - compress: buffer => buffer, - decompress: buffer => buffer, - baseOptions: {}, -} - -const COMPRESSORS = { - gzip: { - compress: ( - gzip => buffer => - gzip(buffer, { level: zlib.constants.Z_BEST_SPEED }) - )(promisify(zlib.gzip)), - decompress: promisify(zlib.gunzip), - }, - brotli: { - compress: ( - brotliCompress => buffer => - brotliCompress(buffer, { - params: { - [zlib.constants.BROTLI_PARAM_QUALITY]: zlib.constants.BROTLI_MIN_QUALITY, - }, - }) - )(promisify(zlib.brotliCompress)), - decompress: promisify(zlib.brotliDecompress), - }, -} - -// inject identifiers -for (const id of Object.keys(COMPRESSORS)) { - COMPRESSORS[id].id = id -} - -function getCompressor(compressorType) { - if (compressorType === undefined) { - return NULL_COMPRESSOR - } - - const compressor = COMPRESSORS[compressorType] - - if (compressor === undefined) { - throw new Error(`Compression type ${compressorType} is not supported`) - } - - return compressor -} - // =================================================================== // Directory format // @@ -77,10 +30,15 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract { #header footer #compressor + #encryptor + #encryption get compressionType() { return this.#compressor.id } + get encryption() { + return this.#encryption + } set header(header) { this.#header = header @@ -116,9 +74,9 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract { } } - static async create(handler, path, { flags = 'wx+', compression } = {}) { + static async create(handler, path, { flags = 'wx+', compression, encryption } = {}) { await handler.mkdir(path) - const vhd = new VhdDirectory(handler, path, { flags, compression }) + const vhd = new VhdDirectory(handler, path, { flags, compression, encryption }) return { dispose: () => {}, value: vhd, @@ -131,6 +89,8 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract { this._path = path this._opts = opts this.#compressor = getCompressor(opts?.compression) + this.#encryption = opts?.encryption + this.#encryptor = getEncryptor(opts?.encryption) } async readBlockAllocationTable() { @@ -150,7 +110,8 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract { // here we can implement compression and / or crypto const buffer = await this._handler.readFile(this._getChunkPath(partName)) - const uncompressed = await this.#compressor.decompress(buffer) + const decrypted = await this.#encryptor.decrypt(buffer) + const uncompressed = await this.#compressor.decompress(decrypted) return { buffer: uncompressed, } @@ -164,7 +125,8 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract { ) const compressed = await this.#compressor.compress(buffer) - return this._handler.outputFile(this._getChunkPath(partName), compressed, this._opts) + const encrypted = await this.#encryptor.encrypt(compressed) + return this._handler.outputFile(this._getChunkPath(partName), encrypted, this._opts) } // put block in subdirectories to limit impact when doing directory listing @@ -246,7 +208,8 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract { if ( !(child instanceof VhdDirectory) || this._handler !== child._handler || - child.compressionType !== this.compressionType + child.compressionType !== this.compressionType || + child.encryption !== this.encryption ) { return super.coalesceBlock(child, blockId) } @@ -273,11 +236,12 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract { async #writeChunkFilters() { const compressionType = this.compressionType + const encryption = this.encryption const path = this._path + '/chunk-filters.json' - if (compressionType === undefined) { + if (compressionType === undefined && encryption === undefined) { await this._handler.unlink(path) } else { - await this._handler.writeFile(path, JSON.stringify([compressionType]), { flags: 'w' }) + await this._handler.writeFile(path, JSON.stringify([compressionType, encryption]), { flags: 'w' }) } } @@ -288,6 +252,9 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract { } throw error }) + assert(chunkFilters.length, 2) this.#compressor = getCompressor(chunkFilters[0]) + this.#encryption = chunkFilters[1] + this.#encryptor = getEncryptor(chunkFilters[1]) } } diff --git a/packages/vhd-lib/Vhd/_compressors.js b/packages/vhd-lib/Vhd/_compressors.js new file mode 100644 index 000000000..92d7153eb --- /dev/null +++ b/packages/vhd-lib/Vhd/_compressors.js @@ -0,0 +1,51 @@ +'use strict' + +const zlib = require('zlib') +const promisify = require('promise-toolbox/promisify') + +const NULL_COMPRESSOR = { + compress: buffer => buffer, + decompress: buffer => buffer, +} + +const COMPRESSORS = { + gzip: { + compress: ( + gzip => buffer => + gzip(buffer, { level: zlib.constants.Z_BEST_SPEED }) + )(promisify(zlib.gzip)), + decompress: promisify(zlib.gunzip), + }, + brotli: { + compress: ( + brotliCompress => buffer => + brotliCompress(buffer, { + params: { + [zlib.constants.BROTLI_PARAM_QUALITY]: zlib.constants.BROTLI_MIN_QUALITY, + }, + }) + )(promisify(zlib.brotliCompress)), + decompress: promisify(zlib.brotliDecompress), + }, +} + +// inject identifiers +for (const id of Object.keys(COMPRESSORS)) { + COMPRESSORS[id].id = id +} + +function getCompressor(compressorType) { + if (compressorType === undefined) { + return NULL_COMPRESSOR + } + + const compressor = COMPRESSORS[compressorType] + + if (compressor === undefined) { + throw new Error(`Compression type ${compressorType} is not supported`) + } + + return compressor +} + +exports._getCompressor = getCompressor diff --git a/packages/vhd-lib/Vhd/_encryptor.integ.spec.js b/packages/vhd-lib/Vhd/_encryptor.integ.spec.js new file mode 100644 index 000000000..ed8a2e427 --- /dev/null +++ b/packages/vhd-lib/Vhd/_encryptor.integ.spec.js @@ -0,0 +1,22 @@ +'use strict' + +const crypto = require('crypto') +const { _getEncryptor: getEncryptor } = require('./_encryptors') + +/* eslint-env jest */ + +test('can encrypt and decryp AES 256', async () => { + const { encrypt, decrypt } = getEncryptor( + JSON.stringify({ + algorithm: 'aes-256-cbc', + key: crypto.randomBytes(32), + ivLength: 16, + }) + ) + + const buffer = crypto.randomBytes(1024) + + const encrypted = encrypt(buffer) + const decrypted = decrypt(encrypted) + expect(buffer.equals(decrypted)).toEqual(true) +}) diff --git a/packages/vhd-lib/Vhd/_encryptors.js b/packages/vhd-lib/Vhd/_encryptors.js new file mode 100644 index 000000000..a16b23a0b --- /dev/null +++ b/packages/vhd-lib/Vhd/_encryptors.js @@ -0,0 +1,44 @@ +'use strict' + +const crypto = require('crypto') +const secretStore = require('./_secretStore.js') + +function getEncryptor(id = '{}') { + const { algorithm, key, ivLength } = secretStore.get(id) + if (algorithm === undefined) { + return { + id: 'NULL_COMPRESSOR', + algorithm, + key, + ivLength, + encrypt: buffer => buffer, + decrypt: buffer => buffer, + } + } + + function encrypt(buffer) { + const iv = crypto.randomBytes(ivLength) + const cipher = crypto.createCipheriv(algorithm, Buffer.from(key), iv) + const encrypted = cipher.update(buffer) + return Buffer.concat([iv, encrypted, cipher.final()]) + } + + function decrypt(buffer) { + const iv = buffer.slice(0, ivLength) + const encrypted = buffer.slice(ivLength) + const decipher = crypto.createDecipheriv(algorithm, Buffer.from(key), iv) + const decrypted = decipher.update(encrypted) + return Buffer.concat([decrypted, decipher.final()]) + } + + return { + id, + algorithm, + key, + ivLength, + encrypt, + decrypt, + } +} + +exports._getEncryptor = getEncryptor diff --git a/packages/vhd-lib/Vhd/_secretStore.js b/packages/vhd-lib/Vhd/_secretStore.js new file mode 100644 index 000000000..e2eaf9e80 --- /dev/null +++ b/packages/vhd-lib/Vhd/_secretStore.js @@ -0,0 +1,4 @@ +'use strict' + +// @todo : should be moved to his own module +module.exports.get = id => JSON.parse(id || '') || {} diff --git a/packages/vhd-lib/createVhdDirectoryFromStream.js b/packages/vhd-lib/createVhdDirectoryFromStream.js index d95065879..e2a7afe3d 100644 --- a/packages/vhd-lib/createVhdDirectoryFromStream.js +++ b/packages/vhd-lib/createVhdDirectoryFromStream.js @@ -5,8 +5,13 @@ const { VhdDirectory } = require('./Vhd/VhdDirectory.js') const { Disposable } = require('promise-toolbox') const { asyncEach } = require('@vates/async-each') -const buildVhd = Disposable.wrap(async function* (handler, path, inputStream, { concurrency, compression }) { - const vhd = yield VhdDirectory.create(handler, path, { compression }) +const buildVhd = Disposable.wrap(async function* ( + handler, + path, + inputStream, + { concurrency, compression, encryption } +) { + const vhd = yield VhdDirectory.create(handler, path, { compression, encryption }) await asyncEach( parseVhdStream(inputStream), async function (item) { @@ -41,10 +46,10 @@ exports.createVhdDirectoryFromStream = async function createVhdDirectoryFromStre handler, path, inputStream, - { validator, concurrency = 16, compression } = {} + { validator, concurrency = 16, compression, encryption } = {} ) { try { - await buildVhd(handler, path, inputStream, { concurrency, compression }) + await buildVhd(handler, path, inputStream, { concurrency, compression, encryption }) if (validator !== undefined) { await validator.call(this, path) } diff --git a/packages/xo-server/config.toml b/packages/xo-server/config.toml index 1dd098f46..a7d300cfb 100644 --- a/packages/xo-server/config.toml +++ b/packages/xo-server/config.toml @@ -88,6 +88,7 @@ snapshotNameLabelTpl = '[XO Backup {job.name}] {vm.name_label}' listingDebounce = '1 min' vhdDirectoryCompression = 'brotli' +vhdDirectoryEncryption = '{"algorithm": "aes-256-cbc", "key": "45eb3ffe48dd29e7bd04a7941ba425f2" ,"ivLength": 16}' # This is a work-around. #