diff --git a/.travis.yml b/.travis.yml index 3f1230c14..5c731d960 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,11 @@ node_js: # Use containers. # http://docs.travis-ci.com/user/workers/container-based-infrastructure/ sudo: false +addons: + apt: + packages: + - qemu-utils + - blktap-utils before_install: - curl -o- -L https://yarnpkg.com/install.sh | bash @@ -14,3 +19,7 @@ before_install: cache: yarn: true + +script: + - yarn run test + - yarn run test-integration diff --git a/package.json b/package.json index 041d76ce4..6053127ea 100644 --- a/package.json +++ b/package.json @@ -52,12 +52,13 @@ "build": "scripts/run-script --parallel build", "clean": "scripts/run-script --parallel clean", "dev": "scripts/run-script --parallel dev", - "dev-test": "jest --bail --watch", + "dev-test": "jest --bail --watch \"^(?!.*\\.integ\\.spec\\.js$)\"", "posttest": "scripts/run-script test", "precommit": "scripts/lint-staged", "prepare": "scripts/run-script prepare", "pretest": "eslint --ignore-path .gitignore .", - "test": "jest" + "test": "jest \"^(?!.*\\.integ\\.spec\\.js$)\"", + "test-integration": "jest \".integ\\.spec\\.js$\"" }, "workspaces": [ "@xen-orchestra/*", diff --git a/packages/vhd-cli/package.json b/packages/vhd-cli/package.json index c1ba61d80..64641d93e 100644 --- a/packages/vhd-cli/package.json +++ b/packages/vhd-cli/package.json @@ -26,7 +26,7 @@ "node": ">=4" }, "dependencies": { - "@nraynaud/struct-fu": "^1.0.1", + "struct-fu": "^1.2.0", "@nraynaud/xo-fs": "^0.0.5", "babel-runtime": "^6.22.0", "exec-promise": "^0.7.0" diff --git a/packages/vhd-cli/src/vhd.js b/packages/vhd-cli/src/vhd.js index 8f2098a02..e3278ecee 100644 --- a/packages/vhd-cli/src/vhd.js +++ b/packages/vhd-cli/src/vhd.js @@ -1,5 +1,5 @@ import assert from 'assert' -import fu from '@nraynaud/struct-fu' +import fu from 'struct-fu' import { dirname } from 'path' // =================================================================== diff --git a/packages/xo-remote-parser/package.json b/packages/xo-remote-parser/package.json index 2315cd29e..abd6ce939 100644 --- a/packages/xo-remote-parser/package.json +++ b/packages/xo-remote-parser/package.json @@ -40,7 +40,7 @@ "dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/", "prebuild": "rimraf dist/", "predev": "yarn run prebuild", - "prepublishOnly": "yarn run build" + "prepare": "yarn run build" }, "babel": { "plugins": [ diff --git a/packages/xo-server/package.json b/packages/xo-server/package.json index 4890411ea..06c1fcdb4 100644 --- a/packages/xo-server/package.json +++ b/packages/xo-server/package.json @@ -33,7 +33,6 @@ "dependencies": { "@babel/polyfill": "7.0.0-beta.42", "@marsaud/smb2-promise": "^0.2.1", - "@nraynaud/struct-fu": "^1.0.1", "@xen-orchestra/cron": "^1.0.2", "ajv": "^6.1.1", "app-conf": "^0.5.0", @@ -104,6 +103,7 @@ "split-lines": "^1.1.0", "stack-chain": "^2.0.0", "stoppable": "^1.0.5", + "struct-fu": "^1.2.0", "tar-stream": "^1.5.5", "through2": "^2.0.3", "tmp": "^0.0.33", diff --git a/packages/xo-server/src/vhd-merge.integ.spec.js b/packages/xo-server/src/vhd-merge.integ.spec.js new file mode 100644 index 000000000..c2fc00444 --- /dev/null +++ b/packages/xo-server/src/vhd-merge.integ.spec.js @@ -0,0 +1,284 @@ +/* eslint-env jest */ + +import execa from 'execa' +import fs from 'fs-extra' +import rimraf from 'rimraf' +import { randomBytes } from 'crypto' +import { fromEvent } from 'promise-toolbox' + +import LocalHandler from './remote-handlers/local' +import vhdMerge, { + chainVhd, + createReadStream, + Vhd, + VHD_SECTOR_SIZE, +} from './vhd-merge' +import { pFromCallback, streamToBuffer, tmpDir } from './utils' + +const initialDir = process.cwd() + +jest.setTimeout(10000) + +beforeEach(async () => { + const dir = await tmpDir() + process.chdir(dir) +}) + +afterEach(async () => { + const tmpDir = process.cwd() + process.chdir(initialDir) + await pFromCallback(cb => rimraf(tmpDir, cb)) +}) + +async function createRandomFile (name, sizeMb) { + await execa('bash', [ + '-c', + `< /dev/urandom tr -dc "\\t\\n [:alnum:]" | head -c ${sizeMb}M >${name}`, + ]) +} + +async function checkFile (vhdName) { + await execa('vhd-util', ['check', '-p', '-b', '-t', '-n', vhdName]) +} + +async function recoverRawContent (vhdName, rawName, originalSize) { + await checkFile(vhdName) + await execa('qemu-img', ['convert', '-fvpc', '-Oraw', vhdName, rawName]) + if (originalSize !== undefined) { + await execa('truncate', ['-s', originalSize, rawName]) + } +} + +async function convertFromRawToVhd (rawName, vhdName) { + await execa('qemu-img', ['convert', '-f', 'raw', '-Ovpc', rawName, vhdName]) +} + +test('blocks can be moved', async () => { + const initalSize = 4 + await createRandomFile('randomfile', initalSize) + await convertFromRawToVhd('randomfile', 'randomfile.vhd') + const handler = new LocalHandler({ url: 'file://' + process.cwd() }) + const originalSize = await handler.getSize('randomfile') + const newVhd = new Vhd(handler, 'randomfile.vhd') + await newVhd.readHeaderAndFooter() + await newVhd.readBlockTable() + await newVhd._freeFirstBlockSpace(8000000) + await recoverRawContent('randomfile.vhd', 'recovered', originalSize) + expect(await fs.readFile('recovered')).toEqual( + await fs.readFile('randomfile') + ) +}) + +test('the BAT MSB is not used for sign', async () => { + const randomBuffer = await pFromCallback(cb => + randomBytes(VHD_SECTOR_SIZE, cb) + ) + await execa('qemu-img', ['create', '-fvpc', 'empty.vhd', '1.8T']) + const handler = new LocalHandler({ url: 'file://' + process.cwd() }) + const vhd = new Vhd(handler, 'empty.vhd') + await vhd.readHeaderAndFooter() + await vhd.readBlockTable() + // we want the bit 31 to be on, to prove it's not been used for sign + const hugeWritePositionSectors = Math.pow(2, 31) + 200 + await vhd.writeData(hugeWritePositionSectors, randomBuffer) + await checkFile('empty.vhd') + // here we are moving the first sector very far in the VHD to prove the BAT doesn't use signed int32 + const hugePositionBytes = hugeWritePositionSectors * VHD_SECTOR_SIZE + await vhd._freeFirstBlockSpace(hugePositionBytes) + + // we recover the data manually for speed reasons. + // fs.write() with offset is way faster than qemu-img when there is a 1.5To + // hole before the block of data + const recoveredFile = await fs.open('recovered', 'w') + try { + const vhd2 = new Vhd(handler, 'empty.vhd') + await vhd2.readHeaderAndFooter() + await vhd2.readBlockTable() + for (let i = 0; i < vhd.header.maxTableEntries; i++) { + const entry = vhd._getBatEntry(i) + if (entry !== 0xffffffff) { + const block = (await vhd2._readBlock(i)).data + await fs.write( + recoveredFile, + block, + 0, + block.length, + vhd2.header.blockSize * i + ) + } + } + } finally { + fs.close(recoveredFile) + } + const recovered = await streamToBuffer( + await fs.createReadStream('recovered', { + start: hugePositionBytes, + end: hugePositionBytes + randomBuffer.length - 1, + }) + ) + expect(recovered).toEqual(randomBuffer) +}) + +test('writeData on empty file', async () => { + const mbOfRandom = 3 + await createRandomFile('randomfile', mbOfRandom) + await execa('qemu-img', ['create', '-fvpc', 'empty.vhd', mbOfRandom + 'M']) + const randomData = await fs.readFile('randomfile') + const handler = new LocalHandler({ url: 'file://' + process.cwd() }) + const originalSize = await handler.getSize('randomfile') + const newVhd = new Vhd(handler, 'empty.vhd') + await newVhd.readHeaderAndFooter() + await newVhd.readBlockTable() + await newVhd.writeData(0, randomData) + await recoverRawContent('empty.vhd', 'recovered', originalSize) + expect(await fs.readFile('recovered')).toEqual(randomData) +}) + +test('writeData in 2 non-overlaping operations', async () => { + const mbOfRandom = 3 + await createRandomFile('randomfile', mbOfRandom) + await execa('qemu-img', ['create', '-fvpc', 'empty.vhd', mbOfRandom + 'M']) + const randomData = await fs.readFile('randomfile') + const handler = new LocalHandler({ url: 'file://' + process.cwd() }) + const originalSize = await handler.getSize('randomfile') + const newVhd = new Vhd(handler, 'empty.vhd') + await newVhd.readHeaderAndFooter() + await newVhd.readBlockTable() + const splitPointSectors = 2 + await newVhd.writeData(0, randomData.slice(0, splitPointSectors * 512)) + await newVhd.writeData( + splitPointSectors, + randomData.slice(splitPointSectors * 512) + ) + await recoverRawContent('empty.vhd', 'recovered', originalSize) + expect(await fs.readFile('recovered')).toEqual(randomData) +}) + +test('writeData in 2 overlaping operations', async () => { + const mbOfRandom = 3 + await createRandomFile('randomfile', mbOfRandom) + await execa('qemu-img', ['create', '-fvpc', 'empty.vhd', mbOfRandom + 'M']) + const randomData = await fs.readFile('randomfile') + const handler = new LocalHandler({ url: 'file://' + process.cwd() }) + const originalSize = await handler.getSize('randomfile') + const newVhd = new Vhd(handler, 'empty.vhd') + await newVhd.readHeaderAndFooter() + await newVhd.readBlockTable() + const endFirstWrite = 3 + const startSecondWrite = 2 + await newVhd.writeData(0, randomData.slice(0, endFirstWrite * 512)) + await newVhd.writeData( + startSecondWrite, + randomData.slice(startSecondWrite * 512) + ) + await recoverRawContent('empty.vhd', 'recovered', originalSize) + expect(await fs.readFile('recovered')).toEqual(randomData) +}) + +test('BAT can be extended and blocks moved', async () => { + const initalSize = 4 + await createRandomFile('randomfile', initalSize) + await convertFromRawToVhd('randomfile', 'randomfile.vhd') + const handler = new LocalHandler({ url: 'file://' + process.cwd() }) + const originalSize = await handler.getSize('randomfile') + const newVhd = new Vhd(handler, 'randomfile.vhd') + await newVhd.readHeaderAndFooter() + await newVhd.readBlockTable() + await newVhd.ensureBatSize(2000) + await recoverRawContent('randomfile.vhd', 'recovered', originalSize) + expect(await fs.readFile('recovered')).toEqual( + await fs.readFile('randomfile') + ) +}) + +test('coalesce works with empty parent files', async () => { + const mbOfRandom = 2 + await createRandomFile('randomfile', mbOfRandom) + await convertFromRawToVhd('randomfile', 'randomfile.vhd') + await execa('qemu-img', [ + 'create', + '-fvpc', + 'empty.vhd', + mbOfRandom + 1 + 'M', + ]) + await checkFile('randomfile.vhd') + await checkFile('empty.vhd') + const handler = new LocalHandler({ url: 'file://' + process.cwd() }) + const originalSize = await handler._getSize('randomfile') + await chainVhd(handler, 'empty.vhd', handler, 'randomfile.vhd', true) + await checkFile('randomfile.vhd') + await checkFile('empty.vhd') + await vhdMerge(handler, 'empty.vhd', handler, 'randomfile.vhd') + await recoverRawContent('empty.vhd', 'recovered', originalSize) + expect(await fs.readFile('recovered')).toEqual( + await fs.readFile('randomfile') + ) +}) + +test('coalesce works in normal cases', async () => { + const mbOfRandom = 5 + await createRandomFile('randomfile', mbOfRandom) + await createRandomFile('small_randomfile', Math.ceil(mbOfRandom / 2)) + await execa('qemu-img', [ + 'create', + '-fvpc', + 'parent.vhd', + mbOfRandom + 1 + 'M', + ]) + await convertFromRawToVhd('randomfile', 'child1.vhd') + const handler = new LocalHandler({ url: 'file://' + process.cwd() }) + await execa('vhd-util', ['snapshot', '-n', 'child2.vhd', '-p', 'child1.vhd']) + const vhd = new Vhd(handler, 'child2.vhd') + await vhd.readHeaderAndFooter() + await vhd.readBlockTable() + vhd.footer.creatorApplication = 'xoa' + await vhd.writeFooter() + + const originalSize = await handler._getSize('randomfile') + await chainVhd(handler, 'parent.vhd', handler, 'child1.vhd', true) + await execa('vhd-util', ['check', '-t', '-n', 'child1.vhd']) + await chainVhd(handler, 'child1.vhd', handler, 'child2.vhd', true) + await execa('vhd-util', ['check', '-t', '-n', 'child2.vhd']) + const smallRandom = await fs.readFile('small_randomfile') + const newVhd = new Vhd(handler, 'child2.vhd') + await newVhd.readHeaderAndFooter() + await newVhd.readBlockTable() + await newVhd.writeData(5, smallRandom) + await checkFile('child2.vhd') + await checkFile('child1.vhd') + await checkFile('parent.vhd') + await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd') + await checkFile('parent.vhd') + await chainVhd(handler, 'parent.vhd', handler, 'child2.vhd', true) + await checkFile('child2.vhd') + await vhdMerge(handler, 'parent.vhd', handler, 'child2.vhd') + await checkFile('parent.vhd') + await recoverRawContent( + 'parent.vhd', + 'recovered_from_coalescing', + originalSize + ) + await execa('cp', ['randomfile', 'randomfile2']) + const fd = await fs.open('randomfile2', 'r+') + try { + await fs.write(fd, smallRandom, 0, smallRandom.length, 5 * VHD_SECTOR_SIZE) + } finally { + await fs.close(fd) + } + expect(await fs.readFile('recovered_from_coalescing')).toEqual( + await fs.readFile('randomfile2') + ) +}) + +test('createReadStream passes vhd-util check', async () => { + const initalSize = 4 + await createRandomFile('randomfile', initalSize) + await convertFromRawToVhd('randomfile', 'randomfile.vhd') + const handler = new LocalHandler({ url: 'file://' + process.cwd() }) + const stream = createReadStream(handler, 'randomfile.vhd') + await fromEvent( + stream.pipe(await fs.createWriteStream('recovered.vhd')), + 'finish' + ) + await checkFile('recovered.vhd') +}) diff --git a/packages/xo-server/src/vhd-merge.js b/packages/xo-server/src/vhd-merge.js index 8eb532002..a85c84ec7 100644 --- a/packages/xo-server/src/vhd-merge.js +++ b/packages/xo-server/src/vhd-merge.js @@ -3,8 +3,7 @@ import assert from 'assert' import asyncIteratorToStream from 'async-iterator-to-stream' import concurrency from 'limit-concurrency-decorator' -import fu from '@nraynaud/struct-fu' -import isEqual from 'lodash/isEqual' +import fu from 'struct-fu' import { dirname, relative } from 'path' import { fromEvent } from 'promise-toolbox' @@ -13,7 +12,7 @@ import constantStream from './constant-stream' import { noop, resolveRelativeFromFile, streamToBuffer } from './utils' const VHD_UTIL_DEBUG = 0 -const debug = VHD_UTIL_DEBUG ? str => console.log(`[vhd-util]${str}`) : noop +const debug = VHD_UTIL_DEBUG ? str => console.log(`[vhd-merge]${str}`) : noop // =================================================================== // @@ -28,7 +27,7 @@ const debug = VHD_UTIL_DEBUG ? str => console.log(`[vhd-util]${str}`) : noop // Sizes in bytes. const VHD_FOOTER_SIZE = 512 const VHD_HEADER_SIZE = 1024 -const VHD_SECTOR_SIZE = 512 +export const VHD_SECTOR_SIZE = 512 // Block allocation table entry size. (Block addr) const VHD_ENTRY_SIZE = 4 @@ -40,6 +39,12 @@ const VHD_PLATFORM_CODE_NONE = 0 export const HARD_DISK_TYPE_DYNAMIC = 3 // Full backup. export const HARD_DISK_TYPE_DIFFERENCING = 4 // Delta backup. +export const PLATFORM_NONE = 0 +export const PLATFORM_W2RU = 0x57327275 +export const PLATFORM_W2KU = 0x57326b75 +export const PLATFORM_MAC = 0x4d616320 +export const PLATFORM_MACX = 0x4d616358 + // Other. const BLOCK_UNUSED = 0xffffffff const BIT_MASK = 0x80 @@ -50,28 +55,24 @@ BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0) // =================================================================== +const SIZE_OF_32_BITS = Math.pow(2, 32) +const uint64 = fu.derive( + fu.uint32(2), + number => [Math.floor(number / SIZE_OF_32_BITS), number % SIZE_OF_32_BITS], + _ => _[0] * SIZE_OF_32_BITS + _[1] +) + const fuFooter = fu.struct([ fu.char('cookie', 8), // 0 fu.uint32('features'), // 8 fu.uint32('fileFormatVersion'), // 12 - fu.struct('dataOffset', [ - fu.uint32('high'), // 16 - fu.uint32('low'), // 20 - ]), + uint64('dataOffset'), // offset of the header, should always be 512 fu.uint32('timestamp'), // 24 fu.char('creatorApplication', 4), // 28 fu.uint32('creatorVersion'), // 32 fu.uint32('creatorHostOs'), // 36 - fu.struct('originalSize', [ - // At the creation, current size of the hard disk. - fu.uint32('high'), // 40 - fu.uint32('low'), // 44 - ]), - fu.struct('currentSize', [ - // Current size of the virtual disk. At the creation: currentSize = originalSize. - fu.uint32('high'), // 48 - fu.uint32('low'), // 52 - ]), + uint64('originalSize'), + uint64('currentSize'), fu.struct('diskGeometry', [ fu.uint16('cylinders'), // 56 fu.uint8('heads'), // 58 @@ -87,12 +88,8 @@ const fuFooter = fu.struct([ const fuHeader = fu.struct([ fu.char('cookie', 8), - fu.struct('dataOffset', [fu.uint32('high'), fu.uint32('low')]), - fu.struct('tableOffset', [ - // Absolute byte offset of the Block Allocation Table. - fu.uint32('high'), - fu.uint32('low'), - ]), + fu.uint8('dataOffsetUnused', 8), + uint64('tableOffset'), fu.uint32('headerVersion'), fu.uint32('maxTableEntries'), // Max entries in the Block Allocation Table. fu.uint32('blockSize'), // Block size in bytes. Default (2097152 => 2MB) @@ -108,11 +105,7 @@ const fuHeader = fu.struct([ fu.uint32('platformDataSpace'), fu.uint32('platformDataLength'), fu.uint32('reserved'), - fu.struct('platformDataOffset', [ - // Absolute byte offset of the locator data. - fu.uint32('high'), - fu.uint32('low'), - ]), + uint64('platformDataOffset'), // Absolute byte offset of the locator data. ], VHD_PARENT_LOCATOR_ENTRIES ), @@ -123,16 +116,14 @@ const fuHeader = fu.struct([ // Helpers // =================================================================== -const SIZE_OF_32_BITS = Math.pow(2, 32) -const uint32ToUint64 = fu => fu.high * SIZE_OF_32_BITS + fu.low +const computeBatSize = entries => + sectorsToBytes(sectorsRoundUpNoZero(entries * VHD_ENTRY_SIZE)) // Returns a 32 bits integer corresponding to a Vhd version. const getVhdVersion = (major, minor) => (major << 16) | (minor & 0x0000ffff) // Sectors conversions. -const sectorsRoundUp = bytes => - Math.floor((bytes + VHD_SECTOR_SIZE - 1) / VHD_SECTOR_SIZE) -const sectorsRoundUpNoZero = bytes => sectorsRoundUp(bytes) || 1 +const sectorsRoundUpNoZero = bytes => Math.ceil(bytes / VHD_SECTOR_SIZE) || 1 const sectorsToBytes = sectors => sectors * VHD_SECTOR_SIZE // Check/Set a bit on a vhd map. @@ -163,26 +154,39 @@ const unpackField = (field, buf) => { // Returns the checksum of a raw struct. // The raw struct (footer or header) is altered with the new sum. -function checksumStruct (rawStruct, struct) { +function checksumStruct (buf, struct) { const checksumField = struct.fields.checksum - let sum = 0 - // Reset current sum. - packField(checksumField, 0, rawStruct) - - for (let i = 0, n = struct.size; i < n; i++) { - sum = (sum + rawStruct[i]) & 0xffffffff + // Do not use the stored checksum to compute the new checksum. + const checksumOffset = checksumField.offset + for (let i = 0, n = checksumOffset; i < n; ++i) { + sum += buf[i] + } + for ( + let i = checksumOffset + checksumField.size, n = struct.size; + i < n; + ++i + ) { + sum += buf[i] } - sum = 0xffffffff - sum + sum = ~sum >>> 0 // Write new sum. - packField(checksumField, sum, rawStruct) + packField(checksumField, sum, buf) return sum } +const assertChecksum = (name, buf, struct) => { + const actual = unpackField(struct.fields.checksum, buf) + const expected = checksumStruct(buf, struct) + if (actual !== expected) { + throw new Error(`invalid ${name} checksum ${actual}, expected ${expected}`) + } +} + // =================================================================== // Format: @@ -207,6 +211,10 @@ function checksumStruct (rawStruct, struct) { // - parentLocatorSize(i) = header.parentLocatorEntry[i].platformDataSpace * sectorSize // - sectorSize = 512 export class Vhd { + get batSize () { + return computeBatSize(this.header.maxTableEntries) + } + constructor (handler, path) { this._handler = handler this._path = path @@ -235,17 +243,10 @@ export class Vhd { getEndOfHeaders () { const { header } = this - let end = uint32ToUint64(this.footer.dataOffset) + VHD_HEADER_SIZE - - const blockAllocationTableSize = sectorsToBytes( - sectorsRoundUpNoZero(header.maxTableEntries * VHD_ENTRY_SIZE) - ) + let end = VHD_FOOTER_SIZE + VHD_HEADER_SIZE // Max(end, block allocation table end) - end = Math.max( - end, - uint32ToUint64(header.tableOffset) + blockAllocationTableSize - ) + end = Math.max(end, header.tableOffset + this.batSize) for (let i = 0; i < VHD_PARENT_LOCATOR_ENTRIES; i++) { const entry = header.parentLocatorEntry[i] @@ -253,8 +254,7 @@ export class Vhd { if (entry.platformCode !== VHD_PLATFORM_CODE_NONE) { end = Math.max( end, - uint32ToUint64(entry.platformDataOffset) + - sectorsToBytes(entry.platformDataSpace) + entry.platformDataOffset + sectorsToBytes(entry.platformDataSpace) ) } } @@ -286,21 +286,16 @@ export class Vhd { // Get the beginning (footer + header) of a vhd file. async readHeaderAndFooter () { const buf = await this._read(0, VHD_FOOTER_SIZE + VHD_HEADER_SIZE) + const bufFooter = buf.slice(0, VHD_FOOTER_SIZE) + const bufHeader = buf.slice(VHD_FOOTER_SIZE) - const sum = unpackField(fuFooter.fields.checksum, buf) - const sumToTest = checksumStruct(buf, fuFooter) + assertChecksum('footer', bufFooter, fuFooter) + assertChecksum('header', bufHeader, fuHeader) - // Checksum child & parent. - if (sumToTest !== sum) { - throw new Error( - `Bad checksum in vhd. Expected: ${sum}. Given: ${sumToTest}. (data=${buf.toString( - 'hex' - )})` - ) - } + const footer = (this.footer = fuFooter.unpack(bufFooter)) + assert.strictEqual(footer.dataOffset, VHD_FOOTER_SIZE) - const header = (this.header = fuHeader.unpack(buf.slice(VHD_FOOTER_SIZE))) - this.footer = fuFooter.unpack(buf) + const header = (this.header = fuHeader.unpack(bufHeader)) // Compute the number of sectors in one block. // Default: One block contains 4096 sectors of 512 bytes. @@ -330,13 +325,10 @@ export class Vhd { // Returns a buffer that contains the block allocation table of a vhd file. async readBlockTable () { const { header } = this - - const offset = uint32ToUint64(header.tableOffset) - const size = sectorsToBytes( - sectorsRoundUpNoZero(header.maxTableEntries * VHD_ENTRY_SIZE) + this.blockTable = await this._read( + header.tableOffset, + header.maxTableEntries * VHD_ENTRY_SIZE ) - - this.blockTable = await this._read(offset, size) } // return the first sector (bitmap) of a block @@ -433,71 +425,70 @@ export class Vhd { : fromEvent(data.pipe(stream), 'finish') } - async ensureBatSize (size) { - const { header } = this - - const prevMaxTableEntries = header.maxTableEntries - if (prevMaxTableEntries >= size) { - return - } - - const tableOffset = uint32ToUint64(header.tableOffset) - // extend BAT - const maxTableEntries = (header.maxTableEntries = size) - const batSize = sectorsToBytes( - sectorsRoundUpNoZero(maxTableEntries * VHD_ENTRY_SIZE) - ) - const prevBat = this.blockTable - const bat = (this.blockTable = Buffer.allocUnsafe(batSize)) - prevBat.copy(bat) - bat.fill(BUF_BLOCK_UNUSED, prevBat.length) - debug( - `ensureBatSize: extend in memory BAT ${prevMaxTableEntries} -> ${maxTableEntries}` - ) - - const extendBat = async () => { - debug( - `ensureBatSize: extend in file BAT ${prevMaxTableEntries} -> ${maxTableEntries}` - ) - - return this._write( - constantStream(BUF_BLOCK_UNUSED, maxTableEntries - prevMaxTableEntries), - tableOffset + prevBat.length - ) - } + async _freeFirstBlockSpace (spaceNeededBytes) { try { const { first, firstSector, lastSector } = this._getFirstAndLastBlocks() - if (tableOffset + batSize < sectorsToBytes(firstSector)) { - return Promise.all([extendBat(), this.writeHeader()]) + const tableOffset = this.header.tableOffset + const { batSize } = this + const newMinSector = Math.ceil( + (tableOffset + batSize + spaceNeededBytes) / VHD_SECTOR_SIZE + ) + if ( + tableOffset + batSize + spaceNeededBytes >= + sectorsToBytes(firstSector) + ) { + const { fullBlockSize } = this + const newFirstSector = Math.max( + lastSector + fullBlockSize / VHD_SECTOR_SIZE, + newMinSector + ) + debug( + `freeFirstBlockSpace: move first block ${firstSector} -> ${newFirstSector}` + ) + // copy the first block at the end + const stream = await this._readStream( + sectorsToBytes(firstSector), + fullBlockSize + ) + await this._write(stream, sectorsToBytes(newFirstSector)) + await this._setBatEntry(first, newFirstSector) + await this.writeFooter(true) + spaceNeededBytes -= this.fullBlockSize + if (spaceNeededBytes > 0) { + return this._freeFirstBlockSpace(spaceNeededBytes) + } } - - const { fullBlockSize } = this - const newFirstSector = lastSector + fullBlockSize / VHD_SECTOR_SIZE - debug( - `ensureBatSize: move first block ${firstSector} -> ${newFirstSector}` - ) - - // copy the first block at the end - const stream = await this._readStream( - sectorsToBytes(firstSector), - fullBlockSize - ) - await this._write(stream, sectorsToBytes(newFirstSector)) - await extendBat() - await this._setBatEntry(first, newFirstSector) - await this.writeHeader() - await this.writeFooter() } catch (e) { - if (e.noBlock) { - await extendBat() - await this.writeHeader() - await this.writeFooter() - } else { + if (!e.noBlock) { throw e } } } + async ensureBatSize (entries) { + const { header } = this + const prevMaxTableEntries = header.maxTableEntries + if (prevMaxTableEntries >= entries) { + return + } + + const newBatSize = computeBatSize(entries) + await this._freeFirstBlockSpace(newBatSize - this.batSize) + const maxTableEntries = (header.maxTableEntries = entries) + const prevBat = this.blockTable + const bat = (this.blockTable = Buffer.allocUnsafe(newBatSize)) + prevBat.copy(bat) + bat.fill(BUF_BLOCK_UNUSED, prevMaxTableEntries * VHD_ENTRY_SIZE) + debug( + `ensureBatSize: extend BAT ${prevMaxTableEntries} -> ${maxTableEntries}` + ) + await this._write( + constantStream(BUF_BLOCK_UNUSED, maxTableEntries - prevMaxTableEntries), + header.tableOffset + prevBat.length + ) + await this.writeHeader() + } + // set the first sector (bitmap) of a block _setBatEntry (block, blockSector) { const i = block * VHD_ENTRY_SIZE @@ -507,7 +498,7 @@ export class Vhd { return this._write( blockTable.slice(i, i + VHD_ENTRY_SIZE), - uint32ToUint64(this.header.tableOffset) + i + this.header.tableOffset + i ) } @@ -563,6 +554,9 @@ export class Vhd { if (blockAddr === BLOCK_UNUSED) { blockAddr = await this.createBlock(block.id) + parentBitmap = Buffer.alloc(this.bitmapSize, 0) + } else if (parentBitmap === undefined) { + parentBitmap = (await this._readBlock(block.id, true)).bitmap } const offset = blockAddr + this.sectorsOfBitmap + beginSectorId @@ -629,11 +623,13 @@ export class Vhd { } // Write a context footer. (At the end and beginning of a vhd file.) - async writeFooter () { + async writeFooter (onlyEndFooter = false) { const { footer } = this - const offset = this.getEndOfData() const rawFooter = fuFooter.pack(footer) + const eof = await this._handler.getSize(this._path) + // sometimes the file is longer than anticipated, we still need to put the footer at the end + const offset = Math.max(this.getEndOfData(), eof - rawFooter.length) footer.checksum = checksumStruct(rawFooter, fuFooter) debug( @@ -641,8 +637,9 @@ export class Vhd { footer.checksum }). (data=${rawFooter.toString('hex')})` ) - - await this._write(rawFooter, 0) + if (!onlyEndFooter) { + await this._write(rawFooter, 0) + } await this._write(rawFooter, offset) } @@ -658,6 +655,73 @@ export class Vhd { ) return this._write(rawHeader, offset) } + + async writeData (offsetSectors, buffer) { + const bufferSizeSectors = Math.ceil(buffer.length / VHD_SECTOR_SIZE) + const startBlock = Math.floor(offsetSectors / this.sectorsPerBlock) + const endBufferSectors = offsetSectors + bufferSizeSectors + const lastBlock = Math.ceil(endBufferSectors / this.sectorsPerBlock) - 1 + await this.ensureBatSize(lastBlock) + const blockSizeBytes = this.sectorsPerBlock * VHD_SECTOR_SIZE + const coversWholeBlock = (offsetInBlockSectors, endInBlockSectors) => + offsetInBlockSectors === 0 && endInBlockSectors === this.sectorsPerBlock + + for ( + let currentBlock = startBlock; + currentBlock <= lastBlock; + currentBlock++ + ) { + const offsetInBlockSectors = Math.max( + 0, + offsetSectors - currentBlock * this.sectorsPerBlock + ) + const endInBlockSectors = Math.min( + endBufferSectors - currentBlock * this.sectorsPerBlock, + this.sectorsPerBlock + ) + const startInBuffer = Math.max( + 0, + (currentBlock * this.sectorsPerBlock - offsetSectors) * VHD_SECTOR_SIZE + ) + const endInBuffer = Math.min( + ((currentBlock + 1) * this.sectorsPerBlock - offsetSectors) * + VHD_SECTOR_SIZE, + buffer.length + ) + let inputBuffer + if (coversWholeBlock(offsetInBlockSectors, endInBlockSectors)) { + inputBuffer = buffer.slice(startInBuffer, endInBuffer) + } else { + inputBuffer = Buffer.alloc(blockSizeBytes, 0) + buffer.copy( + inputBuffer, + offsetInBlockSectors * VHD_SECTOR_SIZE, + startInBuffer, + endInBuffer + ) + } + await this.writeBlockSectors( + { id: currentBlock, data: inputBuffer }, + offsetInBlockSectors, + endInBlockSectors + ) + } + await this.writeFooter() + } + + async ensureSpaceForParentLocators (neededSectors) { + const firstLocatorOffset = VHD_FOOTER_SIZE + VHD_HEADER_SIZE + const currentSpace = + Math.floor(this.header.tableOffset / VHD_SECTOR_SIZE) - + firstLocatorOffset / VHD_SECTOR_SIZE + if (currentSpace < neededSectors) { + const deltaSectors = neededSectors - currentSpace + await this._freeFirstBlockSpace(sectorsToBytes(deltaSectors)) + this.header.tableOffset += sectorsToBytes(deltaSectors) + await this._write(this.blockTable, this.header.tableOffset) + } + return firstLocatorOffset + } } // Merge vhd child into vhd parent. @@ -719,9 +783,9 @@ export default concurrency(2)(async function vhdMerge ( const cFooter = childVhd.footer const pFooter = parentVhd.footer - pFooter.currentSize = { ...cFooter.currentSize } + pFooter.currentSize = cFooter.currentSize pFooter.diskGeometry = { ...cFooter.diskGeometry } - pFooter.originalSize = { ...cFooter.originalSize } + pFooter.originalSize = cFooter.originalSize pFooter.timestamp = cFooter.timestamp pFooter.uuid = cFooter.uuid @@ -743,30 +807,51 @@ export async function chainVhd ( parentHandler, parentPath, childHandler, - childPath + childPath, + force = false ) { const parentVhd = new Vhd(parentHandler, parentPath) const childVhd = new Vhd(childHandler, childPath) - await Promise.all([ - parentVhd.readHeaderAndFooter(), - childVhd.readHeaderAndFooter(), - ]) - const { header } = childVhd + await childVhd.readHeaderAndFooter() + const { header, footer } = childVhd - const parentName = relative(dirname(childPath), parentPath) - const parentUuid = parentVhd.footer.uuid - if ( - header.parentUnicodeName !== parentName || - !isEqual(header.parentUuid, parentUuid) - ) { - header.parentUuid = parentUuid - header.parentUnicodeName = parentName - await childVhd.writeHeader() - return true + if (footer.diskType !== HARD_DISK_TYPE_DIFFERENCING) { + if (!force) { + throw new Error('cannot chain disk of type ' + footer.diskType) + } + footer.diskType = HARD_DISK_TYPE_DIFFERENCING } - return false + await Promise.all([ + childVhd.readBlockTable(), + parentVhd.readHeaderAndFooter(), + ]) + + const parentName = relative(dirname(childPath), parentPath) + + header.parentUuid = parentVhd.footer.uuid + header.parentUnicodeName = parentName + + header.parentLocatorEntry[0].platformCode = PLATFORM_W2KU + const encodedFilename = Buffer.from(parentName, 'utf16le') + const dataSpaceSectors = Math.ceil(encodedFilename.length / VHD_SECTOR_SIZE) + const position = await childVhd.ensureSpaceForParentLocators(dataSpaceSectors) + await childVhd._write(encodedFilename, position) + header.parentLocatorEntry[0].platformDataSpace = sectorsToBytes( + dataSpaceSectors + ) + header.parentLocatorEntry[0].platformDataLength = encodedFilename.length + header.parentLocatorEntry[0].platformDataOffset = position + for (let i = 1; i < 8; i++) { + header.parentLocatorEntry[i].platformCode = VHD_PLATFORM_CODE_NONE + header.parentLocatorEntry[i].platformDataSpace = 0 + header.parentLocatorEntry[i].platformDataLength = 0 + header.parentLocatorEntry[i].platformDataOffset = 0 + } + await childVhd.writeHeader() + await childVhd.writeFooter() + return true } export const createReadStream = asyncIteratorToStream(function * (handler, path) { @@ -797,10 +882,7 @@ export const createReadStream = asyncIteratorToStream(function * (handler, path) // TODO: empty parentUuid and parentLocatorEntry-s in header let header = { ...vhd.header, - tableOffset: { - high: 0, - low: 512 + 1024, - }, + tableOffset: 512 + 1024, parentUnicodeName: '', } @@ -815,9 +897,7 @@ export const createReadStream = asyncIteratorToStream(function * (handler, path) const sectorsPerBlock = sectorsPerBlockData + vhd.bitmapSize / VHD_SECTOR_SIZE - const nBlocks = Math.ceil( - uint32ToUint64(footer.currentSize) / header.blockSize - ) + const nBlocks = Math.ceil(footer.currentSize / header.blockSize) const blocksOwner = new Array(nBlocks) for ( diff --git a/scripts/lint-staged b/scripts/lint-staged index be087cf1d..6a562eccf 100755 --- a/scripts/lint-staged +++ b/scripts/lint-staged @@ -10,7 +10,7 @@ const formatFiles = files => { const testFiles = files => run( './node_modules/.bin/jest', - ['--findRelatedTests', '--passWithNoTests'].concat(files) + ['--testRegex=^(?!.*.integ.spec.js$).*.spec.js$', '--findRelatedTests', '--passWithNoTests'].concat(files) ) // ----------------------------------------------------------------------------- diff --git a/yarn.lock b/yarn.lock index 9466bbb6e..20747572e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -701,10 +701,6 @@ dependencies: pako "^1.0.3" -"@nraynaud/struct-fu@^1.0.1": - version "1.0.1" - resolved "https://registry.yarnpkg.com/@nraynaud/struct-fu/-/struct-fu-1.0.1.tgz#059a0588dea50647c3677783692dafdadfcadf97" - "@nraynaud/xo-fs@^0.0.5": version "0.0.5" resolved "https://registry.yarnpkg.com/@nraynaud/xo-fs/-/xo-fs-0.0.5.tgz#0f8c525440909223904b6841a37f4d255baa54b3" @@ -10886,7 +10882,7 @@ strip-json-comments@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" -struct-fu@^1.0.0: +struct-fu@^1.0.0, struct-fu@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/struct-fu/-/struct-fu-1.2.0.tgz#a40b9eb60a41bb341228cff125fde4887daa85ac"