From a1bcd35e261964dff11baac39f4b8c11f690cc9f Mon Sep 17 00:00:00 2001 From: Florent BEAUCHAMP Date: Fri, 13 May 2022 16:46:22 +0200 Subject: [PATCH] feat(backups/cleanVm): can fully merge VHD chains (#6184) Before this change, `cleanVm` only knew how to merge a single VHD, now, with the help of `VhdSynthetic`, it can merge the whole chain in a single pass. --- @xen-orchestra/backups/RemoteAdapter.js | 35 ++------ @xen-orchestra/backups/_cleanVm.integ.spec.js | 53 ++++++------ @xen-orchestra/backups/_cleanVm.js | 75 +++++++---------- CHANGELOG.unreleased.md | 10 ++- packages/vhd-lib/Vhd/VhdFile.js | 4 +- .../vhd-lib/Vhd/VhdSynthetic.integ.spec.js | 18 ++-- packages/vhd-lib/Vhd/VhdSynthetic.js | 42 +++++++--- packages/vhd-lib/merge.integ.spec.js | 83 +++++++++++++++---- packages/vhd-lib/merge.js | 18 +++- packages/vhd-lib/package.json | 2 +- 10 files changed, 201 insertions(+), 139 deletions(-) diff --git a/@xen-orchestra/backups/RemoteAdapter.js b/@xen-orchestra/backups/RemoteAdapter.js index b1deec4f6..0d947916e 100644 --- a/@xen-orchestra/backups/RemoteAdapter.js +++ b/@xen-orchestra/backups/RemoteAdapter.js @@ -9,7 +9,7 @@ const groupBy = require('lodash/groupBy.js') const pickBy = require('lodash/pickBy.js') const { dirname, join, normalize, resolve } = require('path') const { createLogger } = require('@xen-orchestra/log') -const { Constants, createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib') +const { createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib') const { deduped } = require('@vates/disposable/deduped.js') const { decorateMethodsWith } = require('@vates/decorate-with') const { compose } = require('@vates/compose') @@ -531,46 +531,27 @@ class RemoteAdapter { }) } - async _createSyntheticStream(handler, paths) { - let disposableVhds = [] - - // if it's a path : open all hierarchy of parent - if (typeof paths === 'string') { - let vhd - let vhdPath = paths - do { - const disposable = await openVhd(handler, vhdPath) - vhd = disposable.value - disposableVhds.push(disposable) - vhdPath = resolveRelativeFromFile(vhdPath, vhd.header.parentUnicodeName) - } while (vhd.footer.diskType !== Constants.DISK_TYPES.DYNAMIC) - } else { - // only open the list of path given - disposableVhds = paths.map(path => openVhd(handler, path)) - } - + // open the hierarchy of ancestors until we find a full one + async _createSyntheticStream(handler, path) { + const disposableSynthetic = await VhdSynthetic.fromVhdChain(handler, path) // I don't want the vhds to be disposed on return // but only when the stream is done ( or failed ) - const disposables = await Disposable.all(disposableVhds) - const vhds = disposables.value let disposed = false const disposeOnce = async () => { if (!disposed) { disposed = true - try { - await disposables.dispose() + await disposableSynthetic.dispose() } catch (error) { - warn('_createSyntheticStream: failed to dispose VHDs', { error }) + warn('openVhd: failed to dispose VHDs', { error }) } } } - - const synthetic = new VhdSynthetic(vhds) - await synthetic.readHeaderAndFooter() + const synthetic = disposableSynthetic.value await synthetic.readBlockAllocationTable() const stream = await synthetic.stream() + stream.on('end', disposeOnce) stream.on('close', disposeOnce) stream.on('error', disposeOnce) diff --git a/@xen-orchestra/backups/_cleanVm.integ.spec.js b/@xen-orchestra/backups/_cleanVm.integ.spec.js index f9944e186..735efa7e2 100644 --- a/@xen-orchestra/backups/_cleanVm.integ.spec.js +++ b/@xen-orchestra/backups/_cleanVm.integ.spec.js @@ -5,9 +5,9 @@ const rimraf = require('rimraf') const tmp = require('tmp') const fs = require('fs-extra') +const uuid = require('uuid') const { getHandler } = require('@xen-orchestra/fs') const { pFromCallback } = require('promise-toolbox') -const crypto = require('crypto') const { RemoteAdapter } = require('./RemoteAdapter') const { VHDFOOTER, VHDHEADER } = require('./tests.fixtures.js') const { VhdFile, Constants, VhdDirectory, VhdAbstract } = require('vhd-lib') @@ -34,7 +34,8 @@ afterEach(async () => { await handler.forget() }) -const uniqueId = () => crypto.randomBytes(16).toString('hex') +const uniqueId = () => uuid.v1() +const uniqueIdBuffer = () => Buffer.from(uniqueId(), 'utf-8') async function generateVhd(path, opts = {}) { let vhd @@ -53,10 +54,9 @@ async function generateVhd(path, opts = {}) { } vhd.header = { ...VHDHEADER, ...opts.header } - vhd.footer = { ...VHDFOOTER, ...opts.footer } - vhd.footer.uuid = Buffer.from(crypto.randomBytes(16)) + vhd.footer = { ...VHDFOOTER, ...opts.footer, uuid: uniqueIdBuffer() } - if (vhd.header.parentUnicodeName) { + if (vhd.header.parentUuid) { vhd.footer.diskType = Constants.DISK_TYPES.DIFFERENCING } else { vhd.footer.diskType = Constants.DISK_TYPES.DYNAMIC @@ -91,24 +91,31 @@ test('It remove broken vhd', async () => { }) test('it remove vhd with missing or multiple ancestors', async () => { - // one with a broken parent + // one with a broken parent, should be deleted await generateVhd(`${basePath}/abandonned.vhd`, { header: { parentUnicodeName: 'gone.vhd', - parentUid: Buffer.from(crypto.randomBytes(16)), + parentUuid: uniqueIdBuffer(), }, }) - // one orphan, which is a full vhd, no parent + // one orphan, which is a full vhd, no parent : should stay const orphan = await generateVhd(`${basePath}/orphan.vhd`) - // a child to the orphan + // a child to the orphan in the metadata : should stay await generateVhd(`${basePath}/child.vhd`, { header: { parentUnicodeName: 'orphan.vhd', - parentUid: orphan.footer.uuid, + parentUuid: orphan.footer.uuid, }, }) - + await handler.writeFile( + `metadata.json`, + JSON.stringify({ + mode: 'delta', + vhds: [`${basePath}/child.vhd`, `${basePath}/abandonned.vhd`], + }), + { flags: 'w' } + ) // clean let loggued = '' const onLog = message => { @@ -147,7 +154,7 @@ test('it remove backup meta data referencing a missing vhd in delta backup', asy await generateVhd(`${basePath}/child.vhd`, { header: { parentUnicodeName: 'orphan.vhd', - parentUid: orphan.footer.uuid, + parentUuid: orphan.footer.uuid, }, }) @@ -201,14 +208,14 @@ test('it merges delta of non destroyed chain', async () => { const child = await generateVhd(`${basePath}/child.vhd`, { header: { parentUnicodeName: 'orphan.vhd', - parentUid: orphan.footer.uuid, + parentUuid: orphan.footer.uuid, }, }) // a grand child await generateVhd(`${basePath}/grandchild.vhd`, { header: { parentUnicodeName: 'child.vhd', - parentUid: child.footer.uuid, + parentUuid: child.footer.uuid, }, }) @@ -217,14 +224,12 @@ test('it merges delta of non destroyed chain', async () => { loggued.push(message) } await adapter.cleanVm('/', { remove: true, onLog }) - expect(loggued[0]).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused`) - expect(loggued[1]).toEqual(`incorrect size in metadata: 12000 instead of 209920`) + expect(loggued[0]).toEqual(`incorrect size in metadata: 12000 instead of 209920`) loggued = [] await adapter.cleanVm('/', { remove: true, merge: true, onLog }) - const [unused, merging] = loggued - expect(unused).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused`) - expect(merging).toEqual(`merging /${basePath}/child.vhd into /${basePath}/orphan.vhd`) + const [merging] = loggued + expect(merging).toEqual(`merging 1 children into /${basePath}/orphan.vhd`) const metadata = JSON.parse(await handler.readFile(`metadata.json`)) // size should be the size of children + grand children after the merge @@ -254,7 +259,7 @@ test('it finish unterminated merge ', async () => { const child = await generateVhd(`${basePath}/child.vhd`, { header: { parentUnicodeName: 'orphan.vhd', - parentUid: orphan.footer.uuid, + parentUuid: orphan.footer.uuid, }, }) // a merge in progress file @@ -310,7 +315,7 @@ describe('tests multiple combination ', () => { mode: vhdMode, header: { parentUnicodeName: 'gone.vhd', - parentUid: crypto.randomBytes(16), + parentUuid: uniqueIdBuffer(), }, }) @@ -324,7 +329,7 @@ describe('tests multiple combination ', () => { mode: vhdMode, header: { parentUnicodeName: 'ancestor.vhd' + (useAlias ? '.alias.vhd' : ''), - parentUid: ancestor.footer.uuid, + parentUuid: ancestor.footer.uuid, }, }) // a grand child vhd in metadata @@ -333,7 +338,7 @@ describe('tests multiple combination ', () => { mode: vhdMode, header: { parentUnicodeName: 'child.vhd' + (useAlias ? '.alias.vhd' : ''), - parentUid: child.footer.uuid, + parentUuid: child.footer.uuid, }, }) @@ -348,7 +353,7 @@ describe('tests multiple combination ', () => { mode: vhdMode, header: { parentUnicodeName: 'cleanAncestor.vhd' + (useAlias ? '.alias.vhd' : ''), - parentUid: cleanAncestor.footer.uuid, + parentUuid: cleanAncestor.footer.uuid, }, }) diff --git a/@xen-orchestra/backups/_cleanVm.js b/@xen-orchestra/backups/_cleanVm.js index 8e044675a..9307e2fa4 100644 --- a/@xen-orchestra/backups/_cleanVm.js +++ b/@xen-orchestra/backups/_cleanVm.js @@ -31,66 +31,53 @@ const computeVhdsSize = (handler, vhdPaths) => } ) -// chain is an array of VHDs from child to parent +// chain is [ ancestor, child1, ..., childn] +// 1. Create a VhdSynthetic from all children +// 2. Merge the VhdSynthetic into the ancestor +// 3. Delete all (now) unused VHDs +// 4. Rename the ancestor with the merged data to the latest child // -// the whole chain will be merged into parent, parent will be renamed to child -// and all the others will deleted +// VhdSynthetic +// | +// /‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾\ +// [ ancestor, child1, ...,child n-1, childn ] +// | \___________________/ ^ +// | | | +// | unused VHDs | +// | | +// \___________rename_____________/ + async function mergeVhdChain(chain, { handler, onLog, remove, merge }) { assert(chain.length >= 2) - - let child = chain[0] - const parent = chain[chain.length - 1] - const children = chain.slice(0, -1).reverse() - - chain - .slice(1) - .reverse() - .forEach(parent => { - onLog(`the parent ${parent} of the child ${child} is unused`) - }) + const chainCopy = [...chain] + const parent = chainCopy.pop() + const children = chainCopy if (merge) { - // `mergeVhd` does not work with a stream, either - // - make it accept a stream - // - or create synthetic VHD which is not a stream - if (children.length !== 1) { - // TODO: implement merging multiple children - children.length = 1 - child = children[0] - } - - onLog(`merging ${child} into ${parent}`) + onLog(`merging ${children.length} children into ${parent}`) let done, total const handle = setInterval(() => { if (done !== undefined) { - onLog(`merging ${child}: ${done}/${total}`) + onLog(`merging ${children.join(',')} into ${parent}: ${done}/${total}`) } }, 10e3) - const mergedSize = await mergeVhd( - handler, - parent, - handler, - child, - // children.length === 1 - // ? child - // : await createSyntheticStream(handler, children), - { - onProgress({ done: d, total: t }) { - done = d - total = t - }, - } - ) + const mergedSize = await mergeVhd(handler, parent, handler, children, { + onProgress({ done: d, total: t }) { + done = d + total = t + }, + }) clearInterval(handle) + const mergeTargetChild = children.shift() await Promise.all([ - VhdAbstract.rename(handler, parent, child), - asyncMap(children.slice(0, -1), child => { - onLog(`the VHD ${child} is unused`) + VhdAbstract.rename(handler, parent, mergeTargetChild), + asyncMap(children, child => { + onLog(`the VHD ${child} is already merged`) if (remove) { - onLog(`deleting unused VHD ${child}`) + onLog(`deleting merged VHD ${child}`) return VhdAbstract.unlink(handler, child) } }), diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index bfa22e2fd..951d3bbbf 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -7,6 +7,8 @@ > Users must be able to say: “Nice enhancement, I'm eager to test it” +- [Backup] Merge multiple VHDs at once which will speed up the merging ĥase after reducing the retention of a backup job(PR [#6184](https://github.com/vatesfr/xen-orchestra/pull/6184)) + ### Bug fixes > Users must be able to say: “I had this issue, happy to know it's fixed” @@ -37,8 +39,12 @@ - vhd-cli patch - @xen-orchestra/backups patch - xo-server patch -- xo-vmdk-to-vhd patch +- xo-vmdk-to-vhd minor - @xen-orchestra/upload-ova patch +- @xen-orchestra/backups minor - @xen-orchestra/backups-cli patch -- @xen-orchestra/proxy patch +- @xen-orchestra/proxy minor +- xo-server minor +- xo-web minor + diff --git a/packages/vhd-lib/Vhd/VhdFile.js b/packages/vhd-lib/Vhd/VhdFile.js index 72ebb0642..00781830d 100644 --- a/packages/vhd-lib/Vhd/VhdFile.js +++ b/packages/vhd-lib/Vhd/VhdFile.js @@ -201,9 +201,7 @@ exports.VhdFile = class VhdFile extends VhdAbstract { readBlock(blockId, onlyBitmap = false) { const blockAddr = this._getBatEntry(blockId) - if (blockAddr === BLOCK_UNUSED) { - throw new Error(`no such block ${blockId}`) - } + assert(blockAddr !== BLOCK_UNUSED, `no such block ${blockId}`) return this._read(sectorsToBytes(blockAddr), onlyBitmap ? this.bitmapSize : this.fullBlockSize).then(buf => onlyBitmap diff --git a/packages/vhd-lib/Vhd/VhdSynthetic.integ.spec.js b/packages/vhd-lib/Vhd/VhdSynthetic.integ.spec.js index 8181d5084..ce9182911 100644 --- a/packages/vhd-lib/Vhd/VhdSynthetic.integ.spec.js +++ b/packages/vhd-lib/Vhd/VhdSynthetic.integ.spec.js @@ -9,8 +9,7 @@ const { getSyncedHandler } = require('@xen-orchestra/fs') const { SECTOR_SIZE, PLATFORMS } = require('../_constants') const { createRandomFile, convertFromRawToVhd } = require('../tests/utils') -const { openVhd, chainVhd } = require('..') -const { VhdSynthetic } = require('./VhdSynthetic') +const { openVhd, chainVhd, VhdSynthetic } = require('..') let tempDir = null @@ -40,10 +39,8 @@ test('It can read block and parent locator from a synthetic vhd', async () => { // ensure the two VHD are linked, with the child of type DISK_TYPES.DIFFERENCING await chainVhd(handler, bigVhdFileName, handler, smallVhdFileName, true) - const [smallVhd, bigVhd] = yield Disposable.all([ - openVhd(handler, smallVhdFileName), - openVhd(handler, bigVhdFileName), - ]) + const bigVhd = yield openVhd(handler, bigVhdFileName) + await bigVhd.readBlockAllocationTable() // add parent locato // this will also scramble the block inside the vhd files await bigVhd.writeParentLocator({ @@ -51,7 +48,14 @@ test('It can read block and parent locator from a synthetic vhd', async () => { platformCode: PLATFORMS.W2KU, data: Buffer.from('I am in the big one'), }) - const syntheticVhd = new VhdSynthetic([smallVhd, bigVhd]) + // header changed since thre is a new parent locator + await bigVhd.writeHeader() + // the footer at the end changed since the block have been moved + await bigVhd.writeFooter() + + await bigVhd.readHeaderAndFooter() + + const syntheticVhd = yield VhdSynthetic.open(handler, [smallVhdFileName, bigVhdFileName]) await syntheticVhd.readBlockAllocationTable() expect(syntheticVhd.header.diskType).toEqual(bigVhd.header.diskType) diff --git a/packages/vhd-lib/Vhd/VhdSynthetic.js b/packages/vhd-lib/Vhd/VhdSynthetic.js index 064c8ee2e..39fdde6b1 100644 --- a/packages/vhd-lib/Vhd/VhdSynthetic.js +++ b/packages/vhd-lib/Vhd/VhdSynthetic.js @@ -2,13 +2,16 @@ const UUID = require('uuid') const cloneDeep = require('lodash/cloneDeep.js') +const Disposable = require('promise-toolbox/Disposable') const { asyncMap } = require('@xen-orchestra/async-map') -const { VhdAbstract } = require('./VhdAbstract') -const { DISK_TYPES, FOOTER_SIZE, HEADER_SIZE } = require('../_constants') const assert = require('assert') +const { DISK_TYPES, FOOTER_SIZE, HEADER_SIZE } = require('../_constants') +const { openVhd } = require('../openVhd') +const resolveRelativeFromFile = require('../_resolveRelativeFromFile') +const { VhdAbstract } = require('./VhdAbstract') -exports.VhdSynthetic = class VhdSynthetic extends VhdAbstract { +const VhdSynthetic = class VhdSynthetic extends VhdAbstract { #vhds = [] get header() { @@ -40,13 +43,6 @@ exports.VhdSynthetic = class VhdSynthetic extends VhdAbstract { } } - static async open(vhds) { - const vhd = new VhdSynthetic(vhds) - return { - dispose: () => {}, - value: vhd, - } - } /** * @param {Array} vhds the chain of Vhds used to compute this Vhd, from the deepest child (in position 0), to the root (in the last position) * only the last one can have any type. Other must have type DISK_TYPES.DIFFERENCING (delta) @@ -80,6 +76,8 @@ exports.VhdSynthetic = class VhdSynthetic extends VhdAbstract { async readBlock(blockId, onlyBitmap = false) { const index = this.#vhds.findIndex(vhd => vhd.containsBlock(blockId)) + assert(index !== -1, `no such block ${blockId}`) + // only read the content of the first vhd containing this block return await this.#vhds[index].readBlock(blockId, onlyBitmap) } @@ -88,3 +86,27 @@ exports.VhdSynthetic = class VhdSynthetic extends VhdAbstract { return this.#vhds[this.#vhds.length - 1]._readParentLocatorData(id) } } + +// add decorated static method +VhdSynthetic.fromVhdChain = Disposable.factory(async function* fromVhdChain(handler, childPath) { + let vhdPath = childPath + let vhd + const vhds = [] + do { + vhd = yield openVhd(handler, vhdPath) + vhds.push(vhd) + vhdPath = resolveRelativeFromFile(vhdPath, vhd.header.parentUnicodeName) + } while (vhd.footer.diskType !== DISK_TYPES.DYNAMIC) + + const synthetic = new VhdSynthetic(vhds) + await synthetic.readHeaderAndFooter() + yield synthetic +}) + +VhdSynthetic.open = Disposable.factory(async function* open(handler, paths, opts) { + const synthetic = new VhdSynthetic(yield Disposable.all(paths.map(path => openVhd(handler, path, opts)))) + await synthetic.readHeaderAndFooter() + yield synthetic +}) + +exports.VhdSynthetic = VhdSynthetic diff --git a/packages/vhd-lib/merge.integ.spec.js b/packages/vhd-lib/merge.integ.spec.js index 660521b88..720556935 100644 --- a/packages/vhd-lib/merge.integ.spec.js +++ b/packages/vhd-lib/merge.integ.spec.js @@ -8,7 +8,7 @@ const tmp = require('tmp') const { getHandler } = require('@xen-orchestra/fs') const { pFromCallback } = require('promise-toolbox') -const { VhdFile, chainVhd, mergeVhd: vhdMerge } = require('./index') +const { VhdFile, chainVhd, mergeVhd } = require('./index') const { checkFile, createRandomFile, convertFromRawToVhd } = require('./tests/utils') @@ -27,24 +27,23 @@ afterEach(async () => { test('merge works in normal cases', async () => { const mbOfFather = 8 const mbOfChildren = 4 - const parentRandomFileName = `${tempDir}/randomfile` - const childRandomFileName = `${tempDir}/small_randomfile` - const parentFileName = `${tempDir}/parent.vhd` - const child1FileName = `${tempDir}/child1.vhd` - const handler = getHandler({ url: 'file://' }) + const parentRandomFileName = `randomfile` + const childRandomFileName = `small_randomfile` + const parentFileName = `parent.vhd` + const child1FileName = `child1.vhd` + const handler = getHandler({ url: `file://${tempDir}` }) - await createRandomFile(parentRandomFileName, mbOfFather) - await convertFromRawToVhd(parentRandomFileName, parentFileName) - - await createRandomFile(childRandomFileName, mbOfChildren) - await convertFromRawToVhd(childRandomFileName, child1FileName) + await createRandomFile(`${tempDir}/${parentRandomFileName}`, mbOfFather) + await convertFromRawToVhd(`${tempDir}/${parentRandomFileName}`, `${tempDir}/${parentFileName}`) + await createRandomFile(`${tempDir}/${childRandomFileName}`, mbOfChildren) + await convertFromRawToVhd(`${tempDir}/${childRandomFileName}`, `${tempDir}/${child1FileName}`) await chainVhd(handler, parentFileName, handler, child1FileName, true) // merge - await vhdMerge(handler, parentFileName, handler, child1FileName) + await mergeVhd(handler, parentFileName, handler, child1FileName) // check that vhd is still valid - await checkFile(parentFileName) + await checkFile(`${tempDir}/${parentFileName}`) const parentVhd = new VhdFile(handler, parentFileName) await parentVhd.readHeaderAndFooter() @@ -56,7 +55,7 @@ test('merge works in normal cases', async () => { const blockContent = block.data const file = offset < mbOfChildren * 1024 * 1024 ? childRandomFileName : parentRandomFileName const buffer = Buffer.alloc(blockContent.length) - const fd = await fs.open(file, 'r') + const fd = await fs.open(`${tempDir}/${file}`, 'r') await fs.read(fd, buffer, 0, buffer.length, offset) expect(buffer.equals(blockContent)).toEqual(true) @@ -94,7 +93,7 @@ test('it can resume a merge ', async () => { }) ) // expect merge to fail since child header is not ok - await expect(async () => await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd')).rejects.toThrow() + await expect(async () => await mergeVhd(handler, 'parent.vhd', handler, 'child1.vhd')).rejects.toThrow() await handler.unlink('.parent.vhd.merge.json') await handler.writeFile( @@ -109,7 +108,7 @@ test('it can resume a merge ', async () => { }) ) // expect merge to fail since parent header is not ok - await expect(async () => await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd')).rejects.toThrow() + await expect(async () => await mergeVhd(handler, 'parent.vhd', handler, ['child1.vhd'])).rejects.toThrow() // break the end footer of parent const size = await handler.getSize('parent.vhd') @@ -136,7 +135,7 @@ test('it can resume a merge ', async () => { ) // really merge - await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd') + await mergeVhd(handler, 'parent.vhd', handler, 'child1.vhd') // reload header footer and block allocation table , they should succed await parentVhd.readHeaderAndFooter() @@ -157,3 +156,53 @@ test('it can resume a merge ', async () => { offset += parentVhd.header.blockSize } }) + +test('it merge multiple child in one pass ', async () => { + const mbOfFather = 8 + const mbOfChildren = 6 + const mbOfGrandChildren = 4 + const parentRandomFileName = `${tempDir}/randomfile` + const childRandomFileName = `${tempDir}/small_randomfile` + const grandChildRandomFileName = `${tempDir}/another_small_randomfile` + const parentFileName = `${tempDir}/parent.vhd` + const childFileName = `${tempDir}/child.vhd` + const grandChildFileName = `${tempDir}/grandchild.vhd` + const handler = getHandler({ url: 'file://' }) + await createRandomFile(parentRandomFileName, mbOfFather) + await convertFromRawToVhd(parentRandomFileName, parentFileName) + + await createRandomFile(childRandomFileName, mbOfChildren) + await convertFromRawToVhd(childRandomFileName, childFileName) + await chainVhd(handler, parentFileName, handler, childFileName, true) + + await createRandomFile(grandChildRandomFileName, mbOfGrandChildren) + await convertFromRawToVhd(grandChildRandomFileName, grandChildFileName) + await chainVhd(handler, childFileName, handler, grandChildFileName, true) + + // merge + await mergeVhd(handler, parentFileName, handler, [grandChildFileName, childFileName]) + + // check that vhd is still valid + await checkFile(parentFileName) + + const parentVhd = new VhdFile(handler, parentFileName) + await parentVhd.readHeaderAndFooter() + await parentVhd.readBlockAllocationTable() + + let offset = 0 + // check that the data are the same as source + for await (const block of parentVhd.blocks()) { + const blockContent = block.data + let file = parentRandomFileName + if (offset < mbOfGrandChildren * 1024 * 1024) { + file = grandChildRandomFileName + } else if (offset < mbOfChildren * 1024 * 1024) { + file = childRandomFileName + } + const buffer = Buffer.alloc(blockContent.length) + const fd = await fs.open(file, 'r') + await fs.read(fd, buffer, 0, buffer.length, offset) + expect(buffer.equals(blockContent)).toEqual(true) + offset += parentVhd.header.blockSize + } +}) diff --git a/packages/vhd-lib/merge.js b/packages/vhd-lib/merge.js index b9e857e49..520638cf2 100644 --- a/packages/vhd-lib/merge.js +++ b/packages/vhd-lib/merge.js @@ -13,6 +13,7 @@ const { DISK_TYPES } = require('./_constants') const { Disposable } = require('promise-toolbox') const { asyncEach } = require('@vates/async-each') const { VhdDirectory } = require('./Vhd/VhdDirectory') +const { VhdSynthetic } = require('./Vhd/VhdSynthetic') const { warn } = createLogger('vhd-lib:merge') @@ -27,7 +28,8 @@ function makeThrottledWriter(handler, path, delay) { } } -// Merge vhd child into vhd parent. +// Merge one or multiple vhd child into vhd parent. +// childPath can be array to create a synthetic VHD from multiple VHDs // // TODO: rename the VHD file during the merge module.exports = limitConcurrency(2)(async function merge( @@ -56,16 +58,24 @@ module.exports = limitConcurrency(2)(async function merge( flags: 'r+', checkSecondFooter: mergeState === undefined, }) - const childVhd = yield openVhd(childHandler, childPath) + let childVhd + if (Array.isArray(childPath)) { + childVhd = yield VhdSynthetic.open(childHandler, childPath) + } else { + childVhd = yield openVhd(childHandler, childPath) + } const concurrency = childVhd instanceof VhdDirectory ? 16 : 1 - if (mergeState === undefined) { - assert.strictEqual(childVhd.header.blockSize, parentVhd.header.blockSize) + if (mergeState === undefined) { + // merge should be along a vhd chain + assert.strictEqual(childVhd.header.parentUuid.equals(parentVhd.footer.uuid), true) const parentDiskType = parentVhd.footer.diskType assert(parentDiskType === DISK_TYPES.DIFFERENCING || parentDiskType === DISK_TYPES.DYNAMIC) assert.strictEqual(childVhd.footer.diskType, DISK_TYPES.DIFFERENCING) + assert.strictEqual(childVhd.header.blockSize, parentVhd.header.blockSize) } else { + // vhd should not have changed to resume assert.strictEqual(parentVhd.header.checksum, mergeState.parent.header) assert.strictEqual(childVhd.header.checksum, mergeState.child.header) } diff --git a/packages/vhd-lib/package.json b/packages/vhd-lib/package.json index 133c29f35..8d6ea53a9 100644 --- a/packages/vhd-lib/package.json +++ b/packages/vhd-lib/package.json @@ -30,7 +30,7 @@ }, "devDependencies": { "@xen-orchestra/fs": "^1.0.1", - "execa": "^6.1.0", + "execa": "^5.0.0", "get-stream": "^6.0.0", "rimraf": "^3.0.2", "tmp": "^0.2.1"