Compare commits

..

2 Commits

Author SHA1 Message Date
Florent Beauchamp
d6d7e87fe5 fix: remove root need for openVhd.integ.spec.js and merge.integ.spec.js 2021-11-12 11:24:00 +01:00
Florent Beauchamp
00f02c795f feat(vhd-lib): tests shouldn't need root access to run 2021-11-10 14:06:24 +01:00
186 changed files with 3609 additions and 7015 deletions

View File

@@ -4,6 +4,7 @@ about: Create a report to help us improve
title: ''
labels: 'status: triaging :triangular_flag_on_post:, type: bug :bug:'
assignees: ''
---
**Describe the bug**
@@ -11,7 +12,6 @@ A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
@@ -24,11 +24,10 @@ A clear and concise description of what you expected to happen.
If applicable, add screenshots to help explain your problem.
**Desktop (please complete the following information):**
- Node: [e.g. 16.12.1]
- xo-server: [e.g. 5.82.3]
- xo-web: [e.g. 5.87.0]
- hypervisor: [e.g. XCP-ng 8.2.0]
- Node: [e.g. 16.12.1]
- xo-server: [e.g. 5.82.3]
- xo-web: [e.g. 5.87.0]
- hypervisor: [e.g. XCP-ng 8.2.0]
**Additional context**
Add any other context about the problem here.

View File

@@ -1 +0,0 @@
../../scripts/npmignore

View File

@@ -1,68 +0,0 @@
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
# @vates/async-each
[![Package Version](https://badgen.net/npm/v/@vates/async-each)](https://npmjs.org/package/@vates/async-each) ![License](https://badgen.net/npm/license/@vates/async-each) [![PackagePhobia](https://badgen.net/bundlephobia/minzip/@vates/async-each)](https://bundlephobia.com/result?p=@vates/async-each) [![Node compatibility](https://badgen.net/npm/node/@vates/async-each)](https://npmjs.org/package/@vates/async-each)
> Run async fn for each item in (async) iterable
## Install
Installation of the [npm package](https://npmjs.org/package/@vates/async-each):
```
> npm install --save @vates/async-each
```
## Usage
### `asyncEach(iterable, iteratee, [opts])`
Executes `iteratee` in order for each value yielded by `iterable`.
Returns a promise wich rejects as soon as a call to `iteratee` throws or a promise returned by it rejects, and which resolves when all promises returned by `iteratee` have resolved.
`iterable` must be an iterable or async iterable.
`iteratee` is called with the same `this` value as `asyncEach`, and with the following arguments:
- `value`: the value yielded by `iterable`
- `index`: the 0-based index for this value
- `iterable`: the iterable itself
`opts` is an object that can contains the following options:
- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
- `signal`: an abort signal to stop the iteration
- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
```js
import { asyncEach } from '@vates/async-each'
const contents = []
await asyncEach(
['foo.txt', 'bar.txt', 'baz.txt'],
async function (filename, i) {
contents[i] = await readFile(filename)
},
{
// reads two files at a time
concurrency: 2,
}
)
```
## Contributions
Contributions are _very_ welcomed, either on the documentation or on
the code.
You may:
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
you've encountered;
- fork and create a pull request.
## License
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)

View File

@@ -1,35 +0,0 @@
### `asyncEach(iterable, iteratee, [opts])`
Executes `iteratee` in order for each value yielded by `iterable`.
Returns a promise wich rejects as soon as a call to `iteratee` throws or a promise returned by it rejects, and which resolves when all promises returned by `iteratee` have resolved.
`iterable` must be an iterable or async iterable.
`iteratee` is called with the same `this` value as `asyncEach`, and with the following arguments:
- `value`: the value yielded by `iterable`
- `index`: the 0-based index for this value
- `iterable`: the iterable itself
`opts` is an object that can contains the following options:
- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
- `signal`: an abort signal to stop the iteration
- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
```js
import { asyncEach } from '@vates/async-each'
const contents = []
await asyncEach(
['foo.txt', 'bar.txt', 'baz.txt'],
async function (filename, i) {
contents[i] = await readFile(filename)
},
{
// reads two files at a time
concurrency: 2,
}
)
```

View File

@@ -1,99 +0,0 @@
'use strict'
const noop = Function.prototype
class AggregateError extends Error {
constructor(errors, message) {
super(message)
this.errors = errors
}
}
exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 1, signal, stopOnError = true } = {}) {
return new Promise((resolve, reject) => {
const it = (iterable[Symbol.iterator] || iterable[Symbol.asyncIterator]).call(iterable)
const errors = []
let running = 0
let index = 0
let onAbort
if (signal !== undefined) {
onAbort = () => {
onRejectedWrapper(new Error('asyncEach aborted'))
}
signal.addEventListener('abort', onAbort)
}
const clean = () => {
onFulfilled = onRejected = noop
if (onAbort !== undefined) {
signal.removeEventListener('abort', onAbort)
}
}
resolve = (resolve =>
function resolveAndClean(value) {
resolve(value)
clean()
})(resolve)
reject = (reject =>
function rejectAndClean(reason) {
reject(reason)
clean()
})(reject)
let onFulfilled = value => {
--running
next()
}
const onFulfilledWrapper = value => onFulfilled(value)
let onRejected = stopOnError
? reject
: error => {
--running
errors.push(error)
next()
}
const onRejectedWrapper = reason => onRejected(reason)
let nextIsRunning = false
let next = async () => {
if (nextIsRunning) {
return
}
nextIsRunning = true
if (running < concurrency) {
const cursor = await it.next()
if (cursor.done) {
next = () => {
if (running === 0) {
if (errors.length === 0) {
resolve()
} else {
reject(new AggregateError(errors))
}
}
}
} else {
++running
try {
const result = iteratee.call(this, cursor.value, index++, iterable)
let then
if (result != null && typeof result === 'object' && typeof (then = result.then) === 'function') {
then.call(result, onFulfilledWrapper, onRejectedWrapper)
} else {
onFulfilled(result)
}
} catch (error) {
onRejected(error)
}
}
nextIsRunning = false
return next()
}
nextIsRunning = false
}
next()
})
}

View File

@@ -1,99 +0,0 @@
'use strict'
/* eslint-env jest */
const { asyncEach } = require('./')
const randomDelay = (max = 10) =>
new Promise(resolve => {
setTimeout(resolve, Math.floor(Math.random() * max + 1))
})
const rejectionOf = p =>
new Promise((resolve, reject) => {
p.then(reject, resolve)
})
describe('asyncEach', () => {
const thisArg = 'qux'
const values = ['foo', 'bar', 'baz']
Object.entries({
'sync iterable': () => values,
'async iterable': async function* () {
for (const value of values) {
await randomDelay()
yield value
}
},
}).forEach(([what, getIterable]) =>
describe('with ' + what, () => {
let iterable
beforeEach(() => {
iterable = getIterable()
})
it('works', async () => {
const iteratee = jest.fn(async () => {})
await asyncEach.call(thisArg, iterable, iteratee)
expect(iteratee.mock.instances).toEqual(Array.from(values, () => thisArg))
expect(iteratee.mock.calls).toEqual(Array.from(values, (value, index) => [value, index, iterable]))
})
;[1, 2, 4].forEach(concurrency => {
it('respects a concurrency of ' + concurrency, async () => {
let running = 0
await asyncEach(
values,
async () => {
++running
expect(running).toBeLessThanOrEqual(concurrency)
await randomDelay()
--running
},
{ concurrency }
)
})
})
it('stops on first error when stopOnError is true', async () => {
const error = new Error()
const iteratee = jest.fn((_, i) => {
if (i === 1) {
throw error
}
})
expect(await rejectionOf(asyncEach(iterable, iteratee, { stopOnError: true }))).toBe(error)
expect(iteratee).toHaveBeenCalledTimes(2)
})
it('rejects AggregateError when stopOnError is false', async () => {
const errors = []
const iteratee = jest.fn(() => {
const error = new Error()
errors.push(error)
throw error
})
const error = await rejectionOf(asyncEach(iterable, iteratee, { stopOnError: false }))
expect(error.errors).toEqual(errors)
expect(iteratee.mock.calls).toEqual(Array.from(values, (value, index) => [value, index, iterable]))
})
it('can be interrupted with an AbortSignal', async () => {
const ac = new AbortController()
const iteratee = jest.fn((_, i) => {
if (i === 1) {
ac.abort()
}
})
await expect(asyncEach(iterable, iteratee, { signal: ac.signal })).rejects.toThrow('asyncEach aborted')
expect(iteratee).toHaveBeenCalledTimes(2)
})
})
)
})

View File

@@ -1,34 +0,0 @@
{
"private": false,
"name": "@vates/async-each",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/async-each",
"description": "Run async fn for each item in (async) iterable",
"keywords": [
"array",
"async",
"collection",
"each",
"for",
"foreach",
"iterable",
"iterator"
],
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
"directory": "@vates/async-each",
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"license": "ISC",
"version": "0.1.0",
"engines": {
"node": ">=8.10"
},
"scripts": {
"postversion": "npm publish --access public"
}
}

View File

@@ -65,23 +65,6 @@ const f = compose(
)
```
Functions can receive extra parameters:
```js
const isIn = (value, min, max) => min <= value && value <= max
// Only compatible when `fns` is passed as an array!
const f = compose([
[add, 2],
[isIn, 3, 10],
])
console.log(f(1))
// → true
```
> Note: if the first function is defined with extra parameters, it will only receive the first value passed to the composed function, instead of all the parameters.
## Contributions
Contributions are _very_ welcomed, either on the documentation or on

View File

@@ -46,20 +46,3 @@ const f = compose(
[add2, mul3]
)
```
Functions can receive extra parameters:
```js
const isIn = (value, min, max) => min <= value && value <= max
// Only compatible when `fns` is passed as an array!
const f = compose([
[add, 2],
[isIn, 3, 10],
])
console.log(f(1))
// → true
```
> Note: if the first function is defined with extra parameters, it will only receive the first value passed to the composed function, instead of all the parameters.

View File

@@ -4,13 +4,11 @@ const defaultOpts = { async: false, right: false }
exports.compose = function compose(opts, fns) {
if (Array.isArray(opts)) {
fns = opts.slice() // don't mutate passed array
fns = opts
opts = defaultOpts
} else if (typeof opts === 'object') {
opts = Object.assign({}, defaultOpts, opts)
if (Array.isArray(fns)) {
fns = fns.slice() // don't mutate passed array
} else {
if (!Array.isArray(fns)) {
fns = Array.prototype.slice.call(arguments, 1)
}
} else {
@@ -22,24 +20,6 @@ exports.compose = function compose(opts, fns) {
if (n === 0) {
throw new TypeError('at least one function must be passed')
}
for (let i = 0; i < n; ++i) {
const entry = fns[i]
if (Array.isArray(entry)) {
const fn = entry[0]
const args = entry.slice()
args[0] = undefined
fns[i] = function composeWithArgs(value) {
args[0] = value
try {
return fn.apply(this, args)
} finally {
args[0] = undefined
}
}
}
}
if (n === 1) {
return fns[0]
}

View File

@@ -14,7 +14,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "2.1.0",
"version": "2.0.0",
"engines": {
"node": ">=7.6"
},

View File

@@ -59,36 +59,6 @@ decorateMethodsWith(Foo, {
The decorated class is returned, so you can export it directly.
To apply multiple transforms to a method, you can either call `decorateMethodsWith` multiple times or use [`@vates/compose`](https://www.npmjs.com/package/@vates/compose):
```js
decorateMethodsWith(Foo, {
bar: compose([
[lodash.debounce, 150]
lodash.curry,
])
})
```
### `perInstance(fn, ...args)`
Helper to decorate the method by instance instead of for the whole class.
This is often necessary for caching or deduplicating calls.
```js
import { perInstance } from '@vates/decorateWith'
class Foo {
@decorateWith(perInstance, lodash.memoize)
bar() {
// body
}
}
```
Because it's a normal function, it can also be used with `decorateMethodsWith`, with `compose` or even by itself.
## Contributions
Contributions are _very_ welcomed, either on the documentation or on

View File

@@ -40,33 +40,3 @@ decorateMethodsWith(Foo, {
```
The decorated class is returned, so you can export it directly.
To apply multiple transforms to a method, you can either call `decorateMethodsWith` multiple times or use [`@vates/compose`](https://www.npmjs.com/package/@vates/compose):
```js
decorateMethodsWith(Foo, {
bar: compose([
[lodash.debounce, 150]
lodash.curry,
])
})
```
### `perInstance(fn, ...args)`
Helper to decorate the method by instance instead of for the whole class.
This is often necessary for caching or deduplicating calls.
```js
import { perInstance } from '@vates/decorateWith'
class Foo {
@decorateWith(perInstance, lodash.memoize)
bar() {
// body
}
}
```
Because it's a normal function, it can also be used with `decorateMethodsWith`, with `compose` or even by itself.

View File

@@ -19,15 +19,3 @@ exports.decorateMethodsWith = function decorateMethodsWith(klass, map) {
}
return klass
}
exports.perInstance = function perInstance(fn, decorator, ...args) {
const map = new WeakMap()
return function () {
let decorated = map.get(this)
if (decorated === undefined) {
decorated = decorator(fn, ...args)
map.set(this, decorated)
}
return decorated.apply(this, arguments)
}
}

View File

@@ -20,7 +20,7 @@
"url": "https://vates.fr"
},
"license": "ISC",
"version": "1.0.0",
"version": "0.1.0",
"engines": {
"node": ">=8.10"
},

View File

@@ -30,7 +30,7 @@
"rimraf": "^3.0.0"
},
"dependencies": {
"@vates/decorate-with": "^1.0.0",
"@vates/decorate-with": "^0.1.0",
"@xen-orchestra/log": "^0.3.0",
"golike-defer": "^0.5.1",
"object-hash": "^2.0.1"

View File

@@ -7,8 +7,8 @@
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"dependencies": {
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/backups": "^0.18.3",
"@xen-orchestra/fs": "^0.19.3",
"@xen-orchestra/backups": "^0.15.1",
"@xen-orchestra/fs": "^0.18.0",
"filenamify": "^4.1.0",
"getopts": "^2.2.5",
"lodash": "^4.17.15",
@@ -27,7 +27,7 @@
"scripts": {
"postversion": "npm publish --access public"
},
"version": "0.6.1",
"version": "0.6.0",
"license": "AGPL-3.0-or-later",
"author": {
"name": "Vates SAS",

View File

@@ -3,20 +3,19 @@ const Disposable = require('promise-toolbox/Disposable.js')
const fromCallback = require('promise-toolbox/fromCallback.js')
const fromEvent = require('promise-toolbox/fromEvent.js')
const pDefer = require('promise-toolbox/defer.js')
const groupBy = require('lodash/groupBy.js')
const { dirname, join, normalize, resolve } = require('path')
const pump = require('pump')
const { basename, dirname, join, normalize, resolve } = require('path')
const { createLogger } = require('@xen-orchestra/log')
const { Constants, createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib')
const { createSyntheticStream, mergeVhd, VhdFile } = require('vhd-lib')
const { deduped } = require('@vates/disposable/deduped.js')
const { execFile } = require('child_process')
const { readdir, stat } = require('fs-extra')
const { v4: uuidv4 } = require('uuid')
const { ZipFile } = require('yazl')
const { BACKUP_DIR } = require('./_getVmBackupDir.js')
const { cleanVm } = require('./_cleanVm.js')
const { getTmpDir } = require('./_getTmpDir.js')
const { isMetadataFile } = require('./_backupType.js')
const { isMetadataFile, isVhdFile } = require('./_backupType.js')
const { isValidXva } = require('./_isValidXva.js')
const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js')
const { lvs, pvs } = require('./_lvm.js')
@@ -68,17 +67,58 @@ const debounceResourceFactory = factory =>
}
class RemoteAdapter {
constructor(handler, { debounceResource = res => res, dirMode, vhdDirectoryCompression } = {}) {
constructor(handler, { debounceResource = res => res, dirMode } = {}) {
this._debounceResource = debounceResource
this._dirMode = dirMode
this._handler = handler
this._vhdDirectoryCompression = vhdDirectoryCompression
}
get handler() {
return this._handler
}
async _deleteVhd(path) {
const handler = this._handler
const vhds = await asyncMapSettled(
await handler.list(dirname(path), {
filter: isVhdFile,
prependDir: true,
}),
async path => {
try {
const vhd = new VhdFile(handler, path)
await vhd.readHeaderAndFooter()
return {
footer: vhd.footer,
header: vhd.header,
path,
}
} catch (error) {
// Do not fail on corrupted VHDs (usually uncleaned temporary files),
// they are probably inconsequent to the backup process and should not
// fail it.
warn(`BackupNg#_deleteVhd ${path}`, { error })
}
}
)
const base = basename(path)
const child = vhds.find(_ => _ !== undefined && _.header.parentUnicodeName === base)
if (child === undefined) {
await handler.unlink(path)
return 0
}
try {
const childPath = child.path
const mergedDataSize = await mergeVhd(handler, path, handler, childPath)
await handler.rename(path, childPath)
return mergedDataSize
} catch (error) {
handler.unlink(path).catch(warn)
throw error
}
}
async _findPartition(devicePath, partitionId) {
const partitions = await listPartitions(devicePath)
const partition = partitions.find(_ => _.id === partitionId)
@@ -192,22 +232,6 @@ class RemoteAdapter {
return files
}
// check if we will be allowed to merge a a vhd created in this adapter
// with the vhd at path `path`
async isMergeableParent(packedParentUid, path) {
return await Disposable.use(openVhd(this.handler, path), vhd => {
// this baseUuid is not linked with this vhd
if (!vhd.footer.uuid.equals(packedParentUid)) {
return false
}
const isVhdDirectory = vhd instanceof VhdDirectory
return isVhdDirectory
? this.#useVhdDirectory() && this.#getCompressionType() === vhd.compressionType
: !this.#useVhdDirectory()
})
}
fetchPartitionFiles(diskId, partitionId, paths) {
const { promise, reject, resolve } = pDefer()
Disposable.use(
@@ -231,7 +255,7 @@ class RemoteAdapter {
const handler = this._handler
// unused VHDs will be detected by `cleanVm`
await asyncMapSettled(backups, ({ _filename }) => VhdAbstract.unlink(handler, _filename))
await asyncMapSettled(backups, ({ _filename }) => handler.unlink(_filename))
}
async deleteMetadataBackup(backupId) {
@@ -261,34 +285,17 @@ class RemoteAdapter {
)
}
deleteVmBackup(file) {
return this.deleteVmBackups([file])
}
async deleteVmBackup(filename) {
const metadata = JSON.parse(String(await this._handler.readFile(filename)))
metadata._filename = filename
async deleteVmBackups(files) {
const { delta, full, ...others } = groupBy(await asyncMap(files, file => this.readVmBackupMetadata(file)), 'mode')
const unsupportedModes = Object.keys(others)
if (unsupportedModes.length !== 0) {
throw new Error('no deleter for backup modes: ' + unsupportedModes.join(', '))
if (metadata.mode === 'delta') {
await this.deleteDeltaVmBackups([metadata])
} else if (metadata.mode === 'full') {
await this.deleteFullVmBackups([metadata])
} else {
throw new Error(`no deleter for backup mode ${metadata.mode}`)
}
await Promise.all([
delta !== undefined && this.deleteDeltaVmBackups(delta),
full !== undefined && this.deleteFullVmBackups(full),
])
}
#getCompressionType() {
return this._vhdDirectoryCompression
}
#useVhdDirectory() {
return this.handler.type === 's3'
}
#useAlias() {
return this.#useVhdDirectory()
}
getDisk = Disposable.factory(this.getDisk)
@@ -347,14 +354,6 @@ class RemoteAdapter {
return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
}
// if we use alias on this remote, we have to name the file alias.vhd
getVhdFileName(baseName) {
if (this.#useAlias()) {
return `${baseName}.alias.vhd`
}
return `${baseName}.vhd`
}
async listAllVmBackups() {
const handler = this._handler
@@ -499,25 +498,6 @@ class RemoteAdapter {
return backups.sort(compareTimestamp)
}
async writeVhd(path, input, { checksum = true, validator = noop } = {}) {
const handler = this._handler
if (this.#useVhdDirectory()) {
const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
await createVhdDirectoryFromStream(handler, dataPath, input, {
concurrency: 16,
compression: this.#getCompressionType(),
async validator() {
await input.task
return validator.apply(this, arguments)
},
})
await VhdAbstract.createAlias(handler, path, dataPath)
} else {
await this.outputStream(path, input, { checksum, validator })
}
}
async outputStream(path, input, { checksum = true, validator = noop } = {}) {
await this._handler.outputStream(path, input, {
checksum,
@@ -529,52 +509,6 @@ class RemoteAdapter {
})
}
async _createSyntheticStream(handler, paths) {
let disposableVhds = []
// if it's a path : open all hierarchy of parent
if (typeof paths === 'string') {
let vhd,
vhdPath = paths
do {
const disposable = await openVhd(handler, vhdPath)
vhd = disposable.value
disposableVhds.push(disposable)
vhdPath = resolveRelativeFromFile(vhdPath, vhd.header.parentUnicodeName)
} while (vhd.footer.diskType !== Constants.DISK_TYPES.DYNAMIC)
} else {
// only open the list of path given
disposableVhds = paths.map(path => openVhd(handler, path))
}
// I don't want the vhds to be disposed on return
// but only when the stream is done ( or failed )
const disposables = await Disposable.all(disposableVhds)
const vhds = disposables.value
let disposed = false
const disposeOnce = async () => {
if (!disposed) {
disposed = true
try {
await disposables.dispose()
} catch (error) {
warn('_createSyntheticStream: failed to dispose VHDs', { error })
}
}
}
const synthetic = new VhdSynthetic(vhds)
await synthetic.readHeaderAndFooter()
await synthetic.readBlockAllocationTable()
const stream = await synthetic.stream()
stream.on('end', disposeOnce)
stream.on('close', disposeOnce)
stream.on('error', disposeOnce)
return stream
}
async readDeltaVmBackup(metadata) {
const handler = this._handler
const { vbds, vdis, vhds, vifs, vm } = metadata
@@ -582,7 +516,7 @@ class RemoteAdapter {
const streams = {}
await asyncMapSettled(Object.keys(vdis), async id => {
streams[`${id}.vhd`] = await this._createSyntheticStream(handler, join(dir, vhds[id]))
streams[`${id}.vhd`] = await createSyntheticStream(handler, join(dir, vhds[id]))
})
return {

View File

@@ -36,11 +36,6 @@ const forkDeltaExport = deltaExport =>
exports.VmBackup = class VmBackup {
constructor({ config, getSnapshotNameLabel, job, remoteAdapters, remotes, schedule, settings, srs, vm }) {
if (vm.other_config['xo:backup:job'] === job.id) {
// otherwise replicated VMs would be matched and replicated again and again
throw new Error('cannot backup a VM created by this very job')
}
this.config = config
this.job = job
this.remoteAdapters = remoteAdapters
@@ -338,16 +333,13 @@ exports.VmBackup = class VmBackup {
const baseUuidToSrcVdi = new Map()
await asyncMap(await baseVm.$getDisks(), async baseRef => {
const [baseUuid, snapshotOf] = await Promise.all([
xapi.getField('VDI', baseRef, 'uuid'),
xapi.getField('VDI', baseRef, 'snapshot_of'),
])
const snapshotOf = await xapi.getField('VDI', baseRef, 'snapshot_of')
const srcVdi = srcVdis[snapshotOf]
if (srcVdi !== undefined) {
baseUuidToSrcVdi.set(baseUuid, srcVdi)
baseUuidToSrcVdi.set(await xapi.getField('VDI', baseRef, 'uuid'), srcVdi)
} else {
debug('ignore snapshot VDI because no longer present on VM', {
vdi: baseUuid,
debug('no base VDI found', {
vdi: srcVdi.uuid,
})
}
})
@@ -359,11 +351,6 @@ exports.VmBackup = class VmBackup {
false
)
if (presentBaseVdis.size === 0) {
debug('no base VM found')
return
}
const fullVdisRequired = new Set()
baseUuidToSrcVdi.forEach((srcVdi, baseUuid) => {
if (presentBaseVdis.has(baseUuid)) {

View File

@@ -70,7 +70,6 @@ class BackupWorker {
yield new RemoteAdapter(handler, {
debounceResource: this.debounceResource,
dirMode: this.#config.dirMode,
vhdDirectoryCompression: this.#config.vhdDirectoryCompression,
})
} finally {
await handler.forget()

View File

@@ -1,407 +0,0 @@
/* eslint-env jest */
const rimraf = require('rimraf')
const tmp = require('tmp')
const fs = require('fs-extra')
const { getHandler } = require('@xen-orchestra/fs')
const { pFromCallback } = require('promise-toolbox')
const crypto = require('crypto')
const { RemoteAdapter } = require('./RemoteAdapter')
const { VHDFOOTER, VHDHEADER } = require('./tests.fixtures.js')
const { VhdFile, Constants, VhdDirectory, VhdAbstract } = require('vhd-lib')
let tempDir, adapter, handler, jobId, vdiId, basePath
jest.setTimeout(60000)
beforeEach(async () => {
tempDir = await pFromCallback(cb => tmp.dir(cb))
handler = getHandler({ url: `file://${tempDir}` })
await handler.sync()
adapter = new RemoteAdapter(handler)
jobId = uniqueId()
vdiId = uniqueId()
basePath = `vdis/${jobId}/${vdiId}`
await fs.mkdirp(`${tempDir}/${basePath}`)
})
afterEach(async () => {
await pFromCallback(cb => rimraf(tempDir, cb))
await handler.forget()
})
const uniqueId = () => crypto.randomBytes(16).toString('hex')
async function generateVhd(path, opts = {}) {
let vhd
const dataPath = opts.useAlias ? path + '.data' : path
if (opts.mode === 'directory') {
await handler.mkdir(dataPath)
vhd = new VhdDirectory(handler, dataPath)
} else {
const fd = await handler.openFile(dataPath, 'wx')
vhd = new VhdFile(handler, fd)
}
vhd.header = { ...VHDHEADER, ...opts.header }
vhd.footer = { ...VHDFOOTER, ...opts.footer }
vhd.footer.uuid = Buffer.from(crypto.randomBytes(16))
if (vhd.header.parentUnicodeName) {
vhd.footer.diskType = Constants.DISK_TYPES.DIFFERENCING
} else {
vhd.footer.diskType = Constants.DISK_TYPES.DYNAMIC
}
if (opts.useAlias === true) {
await VhdAbstract.createAlias(handler, path + '.alias.vhd', dataPath)
}
await vhd.writeBlockAllocationTable()
await vhd.writeHeader()
await vhd.writeFooter()
return vhd
}
test('It remove broken vhd', async () => {
// todo also tests a directory and an alias
await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
expect((await handler.list(basePath)).length).toEqual(1)
let loggued = ''
const onLog = message => {
loggued += message
}
await adapter.cleanVm('/', { remove: false, onLog })
expect(loggued).toEqual(`error while checking the VHD with path /${basePath}/notReallyAVhd.vhd`)
// not removed
expect((await handler.list(basePath)).length).toEqual(1)
// really remove it
await adapter.cleanVm('/', { remove: true, onLog })
expect((await handler.list(basePath)).length).toEqual(0)
})
test('it remove vhd with missing or multiple ancestors', async () => {
// one with a broken parent
await generateVhd(`${basePath}/abandonned.vhd`, {
header: {
parentUnicodeName: 'gone.vhd',
parentUid: Buffer.from(crypto.randomBytes(16)),
},
})
// one orphan, which is a full vhd, no parent
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan
await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUid: orphan.footer.uuid,
},
})
// clean
let loggued = ''
const onLog = message => {
loggued += message + '\n'
}
await adapter.cleanVm('/', { remove: true, onLog })
const deletedOrphanVhd = loggued.match(/deleting orphan VHD/g) || []
expect(deletedOrphanVhd.length).toEqual(1) // only one vhd should have been deleted
const deletedAbandonnedVhd = loggued.match(/abandonned.vhd is missing/g) || []
expect(deletedAbandonnedVhd.length).toEqual(1) // and it must be abandonned.vhd
// we don't test the filew on disk, since they will all be marker as unused and deleted without a metadata.json file
})
test('it remove backup meta data referencing a missing vhd in delta backup', async () => {
// create a metadata file marking child and orphan as ok
await handler.writeFile(
`metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [
`${basePath}/orphan.vhd`,
`${basePath}/child.vhd`,
// abandonned.json is not here
],
})
)
await generateVhd(`${basePath}/abandonned.vhd`)
// one orphan, which is a full vhd, no parent
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan
await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUid: orphan.footer.uuid,
},
})
let loggued = ''
const onLog = message => {
loggued += message + '\n'
}
await adapter.cleanVm('/', { remove: true, onLog })
let matched = loggued.match(/deleting unused VHD /g) || []
expect(matched.length).toEqual(1) // only one vhd should have been deleted
matched = loggued.match(/abandonned.vhd is unused/g) || []
expect(matched.length).toEqual(1) // and it must be abandonned.vhd
// a missing vhd cause clean to remove all vhds
await handler.writeFile(
`metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [
`${basePath}/deleted.vhd`, // in metadata but not in vhds
`${basePath}/orphan.vhd`,
`${basePath}/child.vhd`,
// abandonned.json is not here
],
}),
{ flags: 'w' }
)
loggued = ''
await adapter.cleanVm('/', { remove: true, onLog })
matched = loggued.match(/deleting unused VHD /g) || []
expect(matched.length).toEqual(2) // all vhds (orphan and child ) should have been deleted
})
test('it merges delta of non destroyed chain', async () => {
await handler.writeFile(
`metadata.json`,
JSON.stringify({
mode: 'delta',
size: 12000, // a size too small
vhds: [
`${basePath}/grandchild.vhd`, // grand child should not be merged
`${basePath}/child.vhd`,
// orphan is not here, he should be merged in child
],
})
)
// one orphan, which is a full vhd, no parent
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan
const child = await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUid: orphan.footer.uuid,
},
})
// a grand child
await generateVhd(`${basePath}/grandchild.vhd`, {
header: {
parentUnicodeName: 'child.vhd',
parentUid: child.footer.uuid,
},
})
let loggued = []
const onLog = message => {
loggued.push(message)
}
await adapter.cleanVm('/', { remove: true, onLog })
expect(loggued[0]).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused`)
expect(loggued[1]).toEqual(`incorrect size in metadata: 12000 instead of 209920`)
loggued = []
await adapter.cleanVm('/', { remove: true, merge: true, onLog })
const [unused, merging] = loggued
expect(unused).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused`)
expect(merging).toEqual(`merging /${basePath}/child.vhd into /${basePath}/orphan.vhd`)
const metadata = JSON.parse(await handler.readFile(`metadata.json`))
// size should be the size of children + grand children after the merge
expect(metadata.size).toEqual(209920)
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
// only check deletion
const remainingVhds = await handler.list(basePath)
expect(remainingVhds.length).toEqual(2)
expect(remainingVhds.includes('child.vhd')).toEqual(true)
expect(remainingVhds.includes('grandchild.vhd')).toEqual(true)
})
test('it finish unterminated merge ', async () => {
await handler.writeFile(
`metadata.json`,
JSON.stringify({
mode: 'delta',
size: undefined,
vhds: [
`${basePath}/orphan.vhd`, // grand child should not be merged
`${basePath}/child.vhd`,
// orphan is not here, he should be merged in child
],
})
)
// one orphan, which is a full vhd, no parent
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
// a child to the orphan
const child = await generateVhd(`${basePath}/child.vhd`, {
header: {
parentUnicodeName: 'orphan.vhd',
parentUid: orphan.footer.uuid,
},
})
// a merge in progress file
await handler.writeFile(
`${basePath}/.orphan.vhd.merge.json`,
JSON.stringify({
parent: {
header: orphan.header.checksum,
},
child: {
header: child.header.checksum,
},
})
)
// a unfinished merging
await adapter.cleanVm('/', { remove: true, merge: true })
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
// only check deletion
const remainingVhds = await handler.list(basePath)
expect(remainingVhds.length).toEqual(1)
expect(remainingVhds.includes('child.vhd')).toEqual(true)
})
// each of the vhd can be a file, a directory, an alias to a file or an alias to a directory
// the message an resulting files should be identical to the output with vhd files which is tested independantly
describe('tests mulitple combination ', () => {
for (const useAlias of [true, false]) {
for (const vhdMode of ['file', 'directory']) {
test(`alias : ${useAlias}, mode: ${vhdMode}`, async () => {
// a broken VHD
const brokenVhdDataPath = basePath + useAlias ? 'broken.data' : 'broken.vhd'
if (vhdMode === 'directory') {
await handler.mkdir(brokenVhdDataPath)
} else {
await handler.writeFile(brokenVhdDataPath, 'notreallyavhd')
}
if (useAlias) {
await VhdAbstract.createAlias(handler, 'broken.alias.vhd', brokenVhdDataPath)
}
// a vhd non referenced in metada
await generateVhd(`${basePath}/nonreference.vhd`, { useAlias, mode: vhdMode })
// an abandonded delta vhd without its parent
await generateVhd(`${basePath}/abandonned.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'gone.vhd',
parentUid: crypto.randomBytes(16),
},
})
// an ancestor of a vhd present in metadata
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, {
useAlias,
mode: vhdMode,
})
const child = await generateVhd(`${basePath}/child.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'ancestor.vhd' + (useAlias ? '.alias.vhd' : ''),
parentUid: ancestor.footer.uuid,
},
})
// a grand child vhd in metadata
await generateVhd(`${basePath}/grandchild.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'child.vhd' + (useAlias ? '.alias.vhd' : ''),
parentUid: child.footer.uuid,
},
})
// an older parent that was merging in clean
const cleanAncestor = await generateVhd(`${basePath}/cleanAncestor.vhd`, {
useAlias,
mode: vhdMode,
})
// a clean vhd in metadata
const clean = await generateVhd(`${basePath}/clean.vhd`, {
useAlias,
mode: vhdMode,
header: {
parentUnicodeName: 'cleanAncestor.vhd' + (useAlias ? '.alias.vhd' : ''),
parentUid: cleanAncestor.footer.uuid,
},
})
await handler.writeFile(
`${basePath}/.cleanAncestor.vhd${useAlias ? '.alias.vhd' : ''}.merge.json`,
JSON.stringify({
parent: {
header: cleanAncestor.header.checksum,
},
child: {
header: clean.header.checksum,
},
})
)
// the metadata file
await handler.writeFile(
`metadata.json`,
JSON.stringify({
mode: 'delta',
vhds: [
`${basePath}/grandchild.vhd` + (useAlias ? '.alias.vhd' : ''), // grand child should not be merged
`${basePath}/child.vhd` + (useAlias ? '.alias.vhd' : ''),
`${basePath}/clean.vhd` + (useAlias ? '.alias.vhd' : ''),
],
})
)
await adapter.cleanVm('/', { remove: true, merge: true })
const metadata = JSON.parse(await handler.readFile(`metadata.json`))
// size should be the size of children + grand children + clean after the merge
expect(metadata.size).toEqual(vhdMode === 'file' ? 314880 : undefined)
// broken vhd, non referenced, abandonned should be deleted ( alias and data)
// ancestor and child should be merged
// grand child and clean vhd should not have changed
const survivors = await handler.list(basePath)
// console.log(survivors)
if (useAlias) {
// the goal of the alias : do not move a full folder
expect(survivors).toContain('ancestor.vhd.data')
expect(survivors).toContain('grandchild.vhd.data')
expect(survivors).toContain('cleanAncestor.vhd.data')
expect(survivors).toContain('clean.vhd.alias.vhd')
expect(survivors).toContain('child.vhd.alias.vhd')
expect(survivors).toContain('grandchild.vhd.alias.vhd')
expect(survivors.length).toEqual(6)
} else {
expect(survivors).toContain('clean.vhd')
expect(survivors).toContain('child.vhd')
expect(survivors).toContain('grandchild.vhd')
expect(survivors.length).toEqual(3)
}
})
}
}
})
test('it cleans orphan merge states ', async () => {
await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
await adapter.cleanVm('/', { remove: true })
expect(await handler.list(basePath)).toEqual([])
})

View File

@@ -1,32 +1,13 @@
const assert = require('assert')
const sum = require('lodash/sum')
const { asyncMap } = require('@xen-orchestra/async-map')
const { Constants, mergeVhd, openVhd, VhdAbstract, VhdFile } = require('vhd-lib')
const { VhdFile, mergeVhd } = require('vhd-lib')
const { dirname, resolve } = require('path')
const { DISK_TYPES } = Constants
const { DISK_TYPE_DIFFERENCING } = require('vhd-lib/dist/_constants.js')
const { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } = require('./_backupType.js')
const { limitConcurrency } = require('limit-concurrency-decorator')
const { Task } = require('./Task.js')
const { Disposable } = require('promise-toolbox')
// checking the size of a vhd directory is costly
// 1 Http Query per 1000 blocks
// we only check size of all the vhd are VhdFiles
function shouldComputeVhdsSize(vhds) {
return vhds.every(vhd => vhd instanceof VhdFile)
}
const computeVhdsSize = (handler, vhdPaths) =>
Disposable.use(
vhdPaths.map(vhdPath => openVhd(handler, vhdPath)),
async vhds => {
if (shouldComputeVhdsSize(vhds)) {
const sizes = await asyncMap(vhds, vhd => vhd.getSize())
return sum(sizes)
}
}
)
// chain is an array of VHDs from child to parent
//
@@ -84,12 +65,12 @@ async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
clearInterval(handle)
await Promise.all([
VhdAbstract.rename(handler, parent, child),
handler.rename(parent, child),
asyncMap(children.slice(0, -1), child => {
onLog(`the VHD ${child} is unused`)
if (remove) {
onLog(`deleting unused VHD ${child}`)
return VhdAbstract.unlink(handler, child)
return handler.unlink(child)
}
}),
])
@@ -100,10 +81,10 @@ async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
const noop = Function.prototype
const INTERRUPTED_VHDS_REG = /^\.(.+)\.merge.json$/
const INTERRUPTED_VHDS_REG = /^(?:(.+)\/)?\.(.+)\.merge.json$/
const listVhds = async (handler, vmDir) => {
const vhds = new Set()
const interruptedVhds = new Map()
const vhds = []
const interruptedVhds = new Set()
await asyncMap(
await handler.list(`${vmDir}/vdis`, {
@@ -118,14 +99,16 @@ const listVhds = async (handler, vmDir) => {
async vdiDir => {
const list = await handler.list(vdiDir, {
filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file),
prependDir: true,
})
list.forEach(file => {
const res = INTERRUPTED_VHDS_REG.exec(file)
if (res === null) {
vhds.add(`${vdiDir}/${file}`)
vhds.push(file)
} else {
interruptedVhds.set(`${vdiDir}/${res[1]}`, `${vdiDir}/${file}`)
const [, dir, file] = res
interruptedVhds.add(`${dir}/${file}`)
}
})
}
@@ -145,81 +128,62 @@ exports.cleanVm = async function cleanVm(
const handler = this._handler
const vhdsToJSons = new Set()
const vhds = new Set()
const vhdParents = { __proto__: null }
const vhdChildren = { __proto__: null }
const { vhds, interruptedVhds } = await listVhds(handler, vmDir)
const vhdsList = await listVhds(handler, vmDir)
// remove broken VHDs
await asyncMap(vhds, async path => {
await asyncMap(vhdsList.vhds, async path => {
try {
await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), vhd => {
if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {
const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
vhdParents[path] = parent
if (parent in vhdChildren) {
const error = new Error('this script does not support multiple VHD children')
error.parent = parent
error.child1 = vhdChildren[parent]
error.child2 = path
throw error // should we throw?
}
vhdChildren[parent] = path
const vhd = new VhdFile(handler, path)
await vhd.readHeaderAndFooter(!vhdsList.interruptedVhds.has(path))
vhds.add(path)
if (vhd.footer.diskType === DISK_TYPE_DIFFERENCING) {
const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
vhdParents[path] = parent
if (parent in vhdChildren) {
const error = new Error('this script does not support multiple VHD children')
error.parent = parent
error.child1 = vhdChildren[parent]
error.child2 = path
throw error // should we throw?
}
})
vhdChildren[parent] = path
}
} catch (error) {
vhds.delete(path)
onLog(`error while checking the VHD with path ${path}`, { error })
if (error?.code === 'ERR_ASSERTION' && remove) {
onLog(`deleting broken ${path}`)
return VhdAbstract.unlink(handler, path)
await handler.unlink(path)
}
}
})
// remove interrupted merge states for missing VHDs
for (const interruptedVhd of interruptedVhds.keys()) {
if (!vhds.has(interruptedVhd)) {
const statePath = interruptedVhds.get(interruptedVhd)
interruptedVhds.delete(interruptedVhd)
onLog('orphan merge state', {
mergeStatePath: statePath,
missingVhdPath: interruptedVhd,
})
if (remove) {
onLog(`deleting orphan merge state ${statePath}`)
await handler.unlink(statePath)
}
}
}
// @todo : add check for data folder of alias not referenced in a valid alias
// remove VHDs with missing ancestors
{
const deletions = []
// return true if the VHD has been deleted or is missing
const deleteIfOrphan = vhdPath => {
const parent = vhdParents[vhdPath]
const deleteIfOrphan = vhd => {
const parent = vhdParents[vhd]
if (parent === undefined) {
return
}
// no longer needs to be checked
delete vhdParents[vhdPath]
delete vhdParents[vhd]
deleteIfOrphan(parent)
if (!vhds.has(parent)) {
vhds.delete(vhdPath)
vhds.delete(vhd)
onLog(`the parent ${parent} of the VHD ${vhdPath} is missing`)
onLog(`the parent ${parent} of the VHD ${vhd} is missing`)
if (remove) {
onLog(`deleting orphan VHD ${vhdPath}`)
deletions.push(VhdAbstract.unlink(handler, vhdPath))
onLog(`deleting orphan VHD ${vhd}`)
deletions.push(handler.unlink(vhd))
}
}
}
@@ -235,7 +199,7 @@ exports.cleanVm = async function cleanVm(
await Promise.all(deletions)
}
const jsons = new Set()
const jsons = []
const xvas = new Set()
const xvaSums = []
const entries = await handler.list(vmDir, {
@@ -243,7 +207,7 @@ exports.cleanVm = async function cleanVm(
})
entries.forEach(path => {
if (isMetadataFile(path)) {
jsons.add(path)
jsons.push(path)
} else if (isXvaFile(path)) {
xvas.add(path)
} else if (isXvaSumFile(path)) {
@@ -265,25 +229,22 @@ exports.cleanVm = async function cleanVm(
// compile the list of unused XVAs and VHDs, and remove backup metadata which
// reference a missing XVA/VHD
await asyncMap(jsons, async json => {
let metadata
try {
metadata = JSON.parse(await handler.readFile(json))
} catch (error) {
onLog(`failed to read metadata file ${json}`, { error })
jsons.delete(json)
return
}
const metadata = JSON.parse(await handler.readFile(json))
const { mode } = metadata
let size
if (mode === 'full') {
const linkedXva = resolve('/', vmDir, metadata.xva)
if (xvas.has(linkedXva)) {
unusedXvas.delete(linkedXva)
size = await handler.getSize(linkedXva).catch(error => {
onLog(`failed to get size of ${json}`, { error })
})
} else {
onLog(`the XVA linked to the metadata ${json} is missing`)
if (remove) {
onLog(`deleting incomplete backup ${json}`)
jsons.delete(json)
await handler.unlink(json)
}
}
@@ -293,24 +254,38 @@ exports.cleanVm = async function cleanVm(
return Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
})()
const missingVhds = linkedVhds.filter(_ => !vhds.has(_))
// FIXME: find better approach by keeping as much of the backup as
// possible (existing disks) even if one disk is missing
if (missingVhds.length === 0) {
if (linkedVhds.every(_ => vhds.has(_))) {
linkedVhds.forEach(_ => unusedVhds.delete(_))
linkedVhds.forEach(path => {
vhdsToJSons[path] = json
size = await asyncMap(linkedVhds, vhd => handler.getSize(vhd)).then(sum, error => {
onLog(`failed to get size of ${json}`, { error })
})
} else {
onLog(`Some VHDs linked to the metadata ${json} are missing`, { missingVhds })
onLog(`Some VHDs linked to the metadata ${json} are missing`)
if (remove) {
onLog(`deleting incomplete backup ${json}`)
jsons.delete(json)
await handler.unlink(json)
}
}
}
const metadataSize = metadata.size
if (size !== undefined && metadataSize !== size) {
onLog(`incorrect size in metadata: ${metadataSize ?? 'none'} instead of ${size}`)
// don't update if the the stored size is greater than found files,
// it can indicates a problem
if (fixMetadata && (metadataSize === undefined || metadataSize < size)) {
try {
metadata.size = size
await handler.writeFile(json, JSON.stringify(metadata), { flags: 'w' })
} catch (error) {
onLog(`failed to update size in backup metadata ${json}`, { error })
}
}
}
})
// TODO: parallelize by vm/job/vdi
@@ -349,7 +324,7 @@ exports.cleanVm = async function cleanVm(
onLog(`the VHD ${vhd} is unused`)
if (remove) {
onLog(`deleting unused VHD ${vhd}`)
unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
unusedVhdsDeletion.push(handler.unlink(vhd))
}
}
@@ -358,9 +333,9 @@ exports.cleanVm = async function cleanVm(
})
// merge interrupted VHDs
for (const parent of interruptedVhds.keys()) {
vhdsList.interruptedVhds.forEach(parent => {
vhdChainsToMerge[parent] = [vhdChildren[parent], parent]
}
})
Object.values(vhdChainsToMerge).forEach(chain => {
if (chain !== undefined) {
@@ -369,15 +344,9 @@ exports.cleanVm = async function cleanVm(
})
}
const metadataWithMergedVhd = {}
const doMerge = async () => {
await asyncMap(toMerge, async chain => {
const merged = await limitedMergeVhdChain(chain, { handler, onLog, remove, merge })
if (merged !== undefined) {
const metadataPath = vhdsToJSons[chain[0]] // all the chain should have the same metada file
metadataWithMergedVhd[metadataPath] = true
}
})
const doMerge = () => {
const promise = asyncMap(toMerge, async chain => limitedMergeVhdChain(chain, { handler, onLog, remove, merge }))
return merge ? promise.then(sizes => ({ size: sum(sizes) })) : promise
}
await Promise.all([
@@ -402,52 +371,6 @@ exports.cleanVm = async function cleanVm(
}),
])
// update size for delta metadata with merged VHD
// check for the other that the size is the same as the real file size
await asyncMap(jsons, async metadataPath => {
const metadata = JSON.parse(await handler.readFile(metadataPath))
let fileSystemSize
const merged = metadataWithMergedVhd[metadataPath] !== undefined
const { mode, size, vhds, xva } = metadata
try {
if (mode === 'full') {
// a full backup : check size
const linkedXva = resolve('/', vmDir, xva)
fileSystemSize = await handler.getSize(linkedXva)
} else if (mode === 'delta') {
const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
fileSystemSize = await computeVhdsSize(handler, linkedVhds)
// the size is not computed in some cases (e.g. VhdDirectory)
if (fileSystemSize === undefined) {
return
}
// don't warn if the size has changed after a merge
if (!merged && fileSystemSize !== size) {
onLog(`incorrect size in metadata: ${size ?? 'none'} instead of ${fileSystemSize}`)
}
}
} catch (error) {
onLog(`failed to get size of ${metadataPath}`, { error })
return
}
// systematically update size after a merge
if ((merged || fixMetadata) && size !== fileSystemSize) {
metadata.size = fileSystemSize
try {
await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })
} catch (error) {
onLog(`failed to update size in backup metadata ${metadataPath} after merge`, { error })
}
}
})
return {
// boolean whether some VHDs were merged (or should be merged)
merge: toMerge.length !== 0,

View File

@@ -1,24 +1,11 @@
const assert = require('assert')
const COMPRESSED_MAGIC_NUMBERS = [
const isGzipFile = async (handler, fd) => {
// https://tools.ietf.org/html/rfc1952.html#page-5
Buffer.from('1F8B', 'hex'),
const magicNumber = Buffer.allocUnsafe(2)
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames
Buffer.from('28B52FFD', 'hex'),
]
const MAGIC_NUMBER_MAX_LENGTH = Math.max(...COMPRESSED_MAGIC_NUMBERS.map(_ => _.length))
const isCompressedFile = async (handler, fd) => {
const header = Buffer.allocUnsafe(MAGIC_NUMBER_MAX_LENGTH)
assert.strictEqual((await handler.read(fd, header, 0)).bytesRead, header.length)
for (const magicNumber of COMPRESSED_MAGIC_NUMBERS) {
if (magicNumber.compare(header, 0, magicNumber.length) === 0) {
return true
}
}
return false
assert.strictEqual((await handler.read(fd, magicNumber, 0)).bytesRead, magicNumber.length)
return magicNumber[0] === 31 && magicNumber[1] === 139
}
// TODO: better check?
@@ -56,8 +43,8 @@ async function isValidXva(path) {
return false
}
return (await isCompressedFile(handler, fd))
? true // compressed files cannot be validated at this time
return (await isGzipFile(handler, fd))
? true // gzip files cannot be validated at this time
: await isValidTar(handler, size, fd)
} finally {
handler.closeFile(fd).catch(noop)

View File

@@ -8,7 +8,7 @@
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"version": "0.18.3",
"version": "0.15.1",
"engines": {
"node": ">=14.6"
},
@@ -16,11 +16,11 @@
"postversion": "npm publish --access public"
},
"dependencies": {
"@vates/compose": "^2.1.0",
"@vates/compose": "^2.0.0",
"@vates/disposable": "^0.1.1",
"@vates/parse-duration": "^0.1.1",
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/fs": "^0.19.3",
"@xen-orchestra/fs": "^0.18.0",
"@xen-orchestra/log": "^0.3.0",
"@xen-orchestra/template": "^0.1.0",
"compare-versions": "^4.0.1",
@@ -35,12 +35,11 @@
"promise-toolbox": "^0.20.0",
"proper-lockfile": "^4.1.2",
"pump": "^3.0.0",
"uuid": "^8.3.2",
"vhd-lib": "^3.0.0",
"vhd-lib": "^1.3.0",
"yazl": "^2.5.1"
},
"peerDependencies": {
"@xen-orchestra/xapi": "^0.8.5"
"@xen-orchestra/xapi": "^0.8.0"
},
"license": "AGPL-3.0-or-later",
"author": {

View File

@@ -1,92 +0,0 @@
// a valid footer of a 2
exports.VHDFOOTER = {
cookie: 'conectix',
features: 2,
fileFormatVersion: 65536,
dataOffset: 512,
timestamp: 0,
creatorApplication: 'caml',
creatorVersion: 1,
creatorHostOs: 0,
originalSize: 53687091200,
currentSize: 53687091200,
diskGeometry: { cylinders: 25700, heads: 16, sectorsPerTrackCylinder: 255 },
diskType: 3,
checksum: 4294962945,
uuid: Buffer.from('d8dbcad85265421e8b298d99c2eec551', 'utf-8'),
saved: '',
hidden: '',
reserved: '',
}
exports.VHDHEADER = {
cookie: 'cxsparse',
dataOffset: undefined,
tableOffset: 2048,
headerVersion: 65536,
maxTableEntries: 25600,
blockSize: 2097152,
checksum: 4294964241,
parentUuid: null,
parentTimestamp: 0,
reserved1: 0,
parentUnicodeName: '',
parentLocatorEntry: [
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
{
platformCode: 0,
platformDataSpace: 0,
platformDataLength: 0,
reserved: 0,
platformDataOffset: 0,
},
],
reserved2: '',
}

View File

@@ -3,7 +3,7 @@ const map = require('lodash/map.js')
const mapValues = require('lodash/mapValues.js')
const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
const { asyncMap } = require('@xen-orchestra/async-map')
const { chainVhd, checkVhdChain, openVhd, VhdAbstract } = require('vhd-lib')
const { chainVhd, checkVhdChain, VhdFile } = require('vhd-lib')
const { createLogger } = require('@xen-orchestra/log')
const { dirname } = require('path')
@@ -16,7 +16,6 @@ const { MixinBackupWriter } = require('./_MixinBackupWriter.js')
const { AbstractDeltaWriter } = require('./_AbstractDeltaWriter.js')
const { checkVhd } = require('./_checkVhd.js')
const { packUuid } = require('./_packUuid.js')
const { Disposable } = require('promise-toolbox')
const { warn } = createLogger('xo:backups:DeltaBackupWriter')
@@ -24,7 +23,6 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
async checkBaseVdis(baseUuidToSrcVdi) {
const { handler } = this._adapter
const backup = this._backup
const adapter = this._adapter
const backupDir = getVmBackupDir(backup.vm.uuid)
const vdisDir = `${backupDir}/vdis/${backup.job.id}`
@@ -36,21 +34,16 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
filter: _ => _[0] !== '.' && _.endsWith('.vhd'),
prependDir: true,
})
const packedBaseUuid = packUuid(baseUuid)
await asyncMap(vhds, async path => {
try {
await checkVhdChain(handler, path)
// Warning, this should not be written as found = found || await adapter.isMergeableParent(packedBaseUuid, path)
//
// since all the checks of a path are done in parallel, found would be containing
// only the last answer of isMergeableParent which is probably not the right one
// this led to the support tickets https://help.vates.fr/#ticket/zoom/4751 , 4729, 4665 and 4300
const isMergeable = await adapter.isMergeableParent(packedBaseUuid, path)
found = found || isMergeable
const vhd = new VhdFile(handler, path)
await vhd.readHeaderAndFooter()
found = found || vhd.footer.uuid.equals(packUuid(baseUuid))
} catch (error) {
warn('checkBaseVdis', { error })
await ignoreErrors.call(VhdAbstract.unlink(handler, path))
await ignoreErrors.call(handler.unlink(path))
}
})
} catch (error) {
@@ -151,7 +144,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
// don't do delta for it
vdi.uuid
: vdi.$snapshot_of$uuid
}/${adapter.getVhdFileName(basename)}`
}/${basename}.vhd`
)
const metadataFilename = `${backupDir}/${basename}.json`
@@ -195,7 +188,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
await checkVhd(handler, parentPath)
}
await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
await adapter.outputStream(path, deltaExport.streams[`${id}.vhd`], {
// no checksum for VHDs, because they will be invalidated by
// merges and chainings
checksum: false,
@@ -207,11 +200,11 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
}
// set the correct UUID in the VHD
await Disposable.use(openVhd(handler, path), async vhd => {
vhd.footer.uuid = packUuid(vdi.uuid)
await vhd.readBlockAllocationTable() // required by writeFooter()
await vhd.writeFooter()
})
const vhd = new VhdFile(handler, path)
await vhd.readHeaderAndFooter()
vhd.footer.uuid = packUuid(vdi.uuid)
await vhd.readBlockAllocationTable() // required by writeFooter()
await vhd.writeFooter()
})
)
return {

View File

@@ -21,18 +21,10 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
this.#vmBackupDir = getVmBackupDir(this._backup.vm.uuid)
}
async _cleanVm(options) {
try {
return await this._adapter.cleanVm(this.#vmBackupDir, {
...options,
fixMetadata: true,
onLog: warn,
lock: false,
})
} catch (error) {
warn(error)
return {}
}
_cleanVm(options) {
return this._adapter
.cleanVm(this.#vmBackupDir, { ...options, fixMetadata: true, onLog: warn, lock: false })
.catch(warn)
}
async beforeBackup() {
@@ -51,13 +43,7 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
// merge worker only compatible with local remotes
const { handler } = this._adapter
if (merge && !disableMergeWorker && typeof handler._getRealPath === 'function') {
const taskFile =
join(MergeWorker.CLEAN_VM_QUEUE, formatFilenameDate(new Date())) +
'-' +
// add a random suffix to avoid collision in case multiple tasks are created at the same second
Math.random().toString(36).slice(2)
await handler.outputFile(taskFile, this._backup.vm.uuid)
await handler.outputFile(join(MergeWorker.CLEAN_VM_QUEUE, formatFilenameDate(new Date())), this._backup.vm.uuid)
const remotePath = handler._getRealPath()
await MergeWorker.run(remotePath)
}

View File

@@ -1,6 +1,5 @@
const openVhd = require('vhd-lib').openVhd
const Disposable = require('promise-toolbox/Disposable')
const Vhd = require('vhd-lib').VhdFile
exports.checkVhd = async function checkVhd(handler, path) {
await Disposable.use(openVhd(handler, path), () => {})
await new Vhd(handler, path).readHeaderAndFooter()
}

View File

@@ -1,7 +1,7 @@
{
"private": false,
"name": "@xen-orchestra/fs",
"version": "0.19.3",
"version": "0.18.0",
"license": "AGPL-3.0-or-later",
"description": "The File System for Xen Orchestra backups.",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
@@ -33,7 +33,7 @@
"proper-lockfile": "^4.1.2",
"readable-stream": "^3.0.6",
"through2": "^4.0.2",
"xo-remote-parser": "^0.8.0"
"xo-remote-parser": "^0.7.0"
},
"devDependencies": {
"@babel/cli": "^7.0.0",

View File

@@ -76,7 +76,6 @@ export default class RemoteHandlerAbstract {
const sharedLimit = limitConcurrency(options.maxParallelOperations ?? DEFAULT_MAX_PARALLEL_OPERATIONS)
this.closeFile = sharedLimit(this.closeFile)
this.copy = sharedLimit(this.copy)
this.getInfo = sharedLimit(this.getInfo)
this.getSize = sharedLimit(this.getSize)
this.list = sharedLimit(this.list)
@@ -308,17 +307,6 @@ export default class RemoteHandlerAbstract {
return p
}
async copy(oldPath, newPath, { checksum = false } = {}) {
oldPath = normalizePath(oldPath)
newPath = normalizePath(newPath)
let p = timeout.call(this._copy(oldPath, newPath), this._timeout)
if (checksum) {
p = Promise.all([p, this._copy(checksumFile(oldPath), checksumFile(newPath))])
}
return p
}
async rmdir(dir) {
await timeout.call(this._rmdir(normalizePath(dir)).catch(ignoreEnoent), this._timeout)
}
@@ -531,9 +519,6 @@ export default class RemoteHandlerAbstract {
async _rename(oldPath, newPath) {
throw new Error('Not implemented')
}
async _copy(oldPath, newPath) {
throw new Error('Not implemented')
}
async _rmdir(dir) {
throw new Error('Not implemented')

View File

@@ -33,10 +33,6 @@ export default class LocalHandler extends RemoteHandlerAbstract {
return fs.close(fd)
}
async _copy(oldPath, newPath) {
return fs.copy(this._getFilePath(oldPath), this._getFilePath(newPath))
}
async _createReadStream(file, options) {
if (typeof file === 'string') {
const stream = fs.createReadStream(this._getFilePath(file), options)

View File

@@ -1,7 +1,6 @@
import aws from '@sullux/aws-sdk'
import assert from 'assert'
import http from 'http'
import https from 'https'
import { parse } from 'xo-remote-parser'
import RemoteHandlerAbstract from './abstract'
@@ -17,7 +16,7 @@ const IDEAL_FRAGMENT_SIZE = Math.ceil(MAX_OBJECT_SIZE / MAX_PARTS_COUNT) // the
export default class S3Handler extends RemoteHandlerAbstract {
constructor(remote, _opts) {
super(remote)
const { allowUnauthorized, host, path, username, password, protocol, region } = parse(remote.url)
const { host, path, username, password, protocol, region } = parse(remote.url)
const params = {
accessKeyId: username,
apiVersion: '2006-03-01',
@@ -30,13 +29,8 @@ export default class S3Handler extends RemoteHandlerAbstract {
},
}
if (protocol === 'http') {
params.httpOptions.agent = new http.Agent({ keepAlive: true })
params.httpOptions.agent = new http.Agent()
params.sslEnabled = false
} else if (protocol === 'https') {
params.httpOptions.agent = new https.Agent({
rejectUnauthorized: !allowUnauthorized,
keepAlive: true,
})
}
if (region !== undefined) {
params.region = region
@@ -57,27 +51,6 @@ export default class S3Handler extends RemoteHandlerAbstract {
return { Bucket: this._bucket, Key: this._dir + file }
}
async _copy(oldPath, newPath) {
const size = await this._getSize(oldPath)
const multipartParams = await this._s3.createMultipartUpload({ ...this._createParams(newPath) })
const param2 = { ...multipartParams, CopySource: `/${this._bucket}/${this._dir}${oldPath}` }
try {
const parts = []
let start = 0
while (start < size) {
const range = `bytes=${start}-${Math.min(start + MAX_PART_SIZE, size) - 1}`
const partParams = { ...param2, PartNumber: parts.length + 1, CopySourceRange: range }
const upload = await this._s3.uploadPartCopy(partParams)
parts.push({ ETag: upload.CopyPartResult.ETag, PartNumber: partParams.PartNumber })
start += MAX_PART_SIZE
}
await this._s3.completeMultipartUpload({ ...multipartParams, MultipartUpload: { Parts: parts } })
} catch (e) {
await this._s3.abortMultipartUpload(multipartParams)
throw e
}
}
async _isNotEmptyDir(path) {
const result = await this._s3.listObjectsV2({
Bucket: this._bucket,
@@ -152,30 +125,16 @@ export default class S3Handler extends RemoteHandlerAbstract {
const splitPrefix = splitPath(prefix)
const result = await this._s3.listObjectsV2({
Bucket: this._bucket,
Prefix: splitPrefix.join('/') + '/', // need slash at the end with the use of delimiters
Delimiter: '/', // will only return path until delimiters
Prefix: splitPrefix.join('/'),
})
if (result.isTruncated) {
const error = new Error('more than 1000 objects, unsupported in this implementation')
error.dir = dir
throw error
}
const uniq = []
// sub directories
for (const entry of result.CommonPrefixes) {
const line = splitPath(entry.Prefix)
uniq.push(line[line.length - 1])
}
// files
const uniq = new Set()
for (const entry of result.Contents) {
const line = splitPath(entry.Key)
uniq.push(line[line.length - 1])
if (line.length > splitPrefix.length) {
uniq.add(line[splitPrefix.length])
}
}
return uniq
return [...uniq]
}
async _mkdir(path) {
@@ -188,9 +147,25 @@ export default class S3Handler extends RemoteHandlerAbstract {
// nothing to do, directories do not exist, they are part of the files' path
}
// s3 doesn't have a rename operation, so copy + delete source
async _rename(oldPath, newPath) {
await this.copy(oldPath, newPath)
const size = await this._getSize(oldPath)
const multipartParams = await this._s3.createMultipartUpload({ ...this._createParams(newPath) })
const param2 = { ...multipartParams, CopySource: `/${this._bucket}/${this._dir}${oldPath}` }
try {
const parts = []
let start = 0
while (start < size) {
const range = `bytes=${start}-${Math.min(start + MAX_PART_SIZE, size) - 1}`
const partParams = { ...param2, PartNumber: parts.length + 1, CopySourceRange: range }
const upload = await this._s3.uploadPartCopy(partParams)
parts.push({ ETag: upload.CopyPartResult.ETag, PartNumber: partParams.PartNumber })
start += MAX_PART_SIZE
}
await this._s3.completeMultipartUpload({ ...multipartParams, MultipartUpload: { Parts: parts } })
} catch (e) {
await this._s3.abortMultipartUpload(multipartParams)
throw e
}
await this._s3.deleteObject(this._createParams(oldPath))
}
@@ -236,9 +211,9 @@ export default class S3Handler extends RemoteHandlerAbstract {
// nothing to do, directories do not exist, they are part of the files' path
}
// reimplement _rmtree to handle efficiantly path with more than 1000 entries in trees
// reimplement _rmTree to handle efficiantly path with more than 1000 entries in trees
// @todo : use parallel processing for unlink
async _rmtree(path) {
async _rmTree(path) {
let NextContinuationToken
do {
const result = await this._s3.listObjectsV2({
@@ -246,16 +221,11 @@ export default class S3Handler extends RemoteHandlerAbstract {
Prefix: this._dir + path + '/',
ContinuationToken: NextContinuationToken,
})
NextContinuationToken = result.isTruncated ? result.NextContinuationToken : undefined
for (const { Key } of result.Contents) {
// _unlink will add the prefix, but Key contains everything
// also we don't need to check if we delete a directory, since the list only return files
await this._s3.deleteObject({
Bucket: this._bucket,
Key,
})
NextContinuationToken = result.isTruncated ? null : result.NextContinuationToken
for (const path of result.Contents) {
await this._unlink(path)
}
} while (NextContinuationToken !== undefined)
} while (NextContinuationToken !== null)
}
async _write(file, buffer, position) {

View File

@@ -19,7 +19,7 @@
"node": ">=6"
},
"dependencies": {
"bind-property-descriptor": "^2.0.0",
"bind-property-descriptor": "^1.0.0",
"lodash": "^4.17.21"
},
"scripts": {

View File

@@ -13,7 +13,7 @@ module.exports = class Config {
const watchers = (this._watchers = new Set())
app.hooks.on('start', async () => {
app.hooks.once(
app.hooks.on(
'stop',
await watch({ appDir, appName, ignoreUnknownFormats: true }, (error, config) => {
if (error != null) {
@@ -32,7 +32,7 @@ module.exports = class Config {
get(path) {
const value = get(this._config, path)
if (value === undefined) {
throw new TypeError('missing config entry: ' + path)
throw new TypeError('missing config entry: ' + value)
}
return value
}

View File

@@ -14,7 +14,7 @@
"url": "https://vates.fr"
},
"license": "AGPL-3.0-or-later",
"version": "0.1.2",
"version": "0.1.1",
"engines": {
"node": ">=12"
},
@@ -22,7 +22,7 @@
"@vates/parse-duration": "^0.1.1",
"@xen-orchestra/emit-async": "^0.1.0",
"@xen-orchestra/log": "^0.3.0",
"app-conf": "^1.0.0",
"app-conf": "^0.9.0",
"lodash": "^4.17.21"
},
"scripts": {

View File

@@ -29,7 +29,7 @@
"@iarna/toml": "^2.2.0",
"@vates/read-chunk": "^0.1.2",
"ansi-colors": "^4.1.1",
"app-conf": "^1.0.0",
"app-conf": "^0.9.0",
"content-type": "^1.0.4",
"cson-parser": "^4.0.7",
"getopts": "^2.2.3",

View File

@@ -20,7 +20,6 @@ keepAliveInterval = 10e3
dirMode = 0o700
disableMergeWorker = false
snapshotNameLabelTpl = '[XO Backup {job.name}] {vm.name_label}'
vhdDirectoryCompression = 'brotli'
[backups.defaultSettings]
reportWhen = 'failure'
@@ -88,20 +87,3 @@ ignoreNobakVdis = false
maxUncoalescedVdis = 1
watchEvents = ['network', 'PIF', 'pool', 'SR', 'task', 'VBD', 'VDI', 'VIF', 'VM']
#compact mode
[reverseProxies]
# '/http/' = 'http://localhost:8081/'
#The target can have a path ( like `http://target/sub/directory/`),
# parameters (`?param=one`) and hash (`#jwt:32154`) that are automatically added to all queries transfered by the proxy.
# If a parameter is present in the configuration and in the query, only the config parameter is transferred.
# '/another' = http://hiddenServer:8765/path/
# And use the extended mode when required
# The additionnal options of a proxy's configuraiton's section are used to instantiate the `https` Agent(respectively the `http`).
# A notable option is `rejectUnauthorized` which allow to connect to a HTTPS backend with an invalid/ self signed certificate
#[reverseProxies.'/https/']
# target = 'https://localhost:8080/'
# rejectUnauthorized = false

View File

@@ -1,7 +1,7 @@
{
"private": true,
"name": "@xen-orchestra/proxy",
"version": "0.17.3",
"version": "0.15.2",
"license": "AGPL-3.0-or-later",
"description": "XO Proxy used to remotely execute backup jobs",
"keywords": [
@@ -22,33 +22,32 @@
"xo-proxy": "dist/index.mjs"
},
"engines": {
"node": ">=14.18"
"node": ">=14.13"
},
"dependencies": {
"@iarna/toml": "^2.2.0",
"@koa/router": "^10.0.0",
"@vates/compose": "^2.1.0",
"@vates/decorate-with": "^1.0.0",
"@vates/compose": "^2.0.0",
"@vates/decorate-with": "^0.1.0",
"@vates/disposable": "^0.1.1",
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/backups": "^0.18.3",
"@xen-orchestra/fs": "^0.19.3",
"@xen-orchestra/backups": "^0.15.1",
"@xen-orchestra/fs": "^0.18.0",
"@xen-orchestra/log": "^0.3.0",
"@xen-orchestra/mixin": "^0.1.0",
"@xen-orchestra/mixins": "^0.1.2",
"@xen-orchestra/mixins": "^0.1.1",
"@xen-orchestra/self-signed": "^0.1.0",
"@xen-orchestra/xapi": "^0.8.5",
"@xen-orchestra/xapi": "^0.8.0",
"ajv": "^8.0.3",
"app-conf": "^1.0.0",
"app-conf": "^0.9.0",
"async-iterator-to-stream": "^1.1.0",
"fs-extra": "^10.0.0",
"get-stream": "^6.0.0",
"getopts": "^2.2.3",
"golike-defer": "^0.5.1",
"http-server-plus": "^0.11.0",
"http2-proxy": "^5.0.53",
"json-rpc-protocol": "^0.13.1",
"jsonrpc-websocket-client": "^0.7.2",
"jsonrpc-websocket-client": "^0.6.0",
"koa": "^2.5.1",
"koa-compress": "^5.0.1",
"koa-helmet": "^5.1.0",
@@ -58,7 +57,7 @@
"promise-toolbox": "^0.20.0",
"source-map-support": "^0.5.16",
"stoppable": "^1.0.6",
"xdg-basedir": "^5.1.0",
"xdg-basedir": "^4.0.0",
"xen-api": "^0.35.1",
"xo-common": "^0.7.0"
},

View File

@@ -14,7 +14,7 @@ import { createLogger } from '@xen-orchestra/log'
const { debug, warn } = createLogger('xo:proxy:api')
const ndJsonStream = asyncIteratorToStream(async function* (responseId, iterable) {
const ndJsonStream = asyncIteratorToStream(async function*(responseId, iterable) {
try {
let cursor, iterator
try {
@@ -45,14 +45,14 @@ export default class Api {
constructor(app, { appVersion, httpServer }) {
this._ajv = new Ajv({ allErrors: true })
this._methods = { __proto__: null }
const PREFIX = '/api/v1'
const router = new Router({ prefix: PREFIX }).post('/', async ctx => {
const router = new Router({ prefix: '/api/v1' }).post('/', async ctx => {
// Before Node 13.0 there was an inactivity timeout of 2 mins, which may
// not be enough for the API.
ctx.req.setTimeout(0)
const profile = await app.authentication.findProfile({
authenticationToken: ctx.cookies.get('authenticationToken'),
authenticationToken: ctx.cookies.get('authenticationToken')
})
if (profile === undefined) {
ctx.status = 401
@@ -102,7 +102,6 @@ export default class Api {
// breaks, send some data every 10s to keep it opened.
const stopTimer = clearInterval.bind(
undefined,
// @to check : can this add space inside binary data ?
setInterval(() => stream.push(' '), keepAliveInterval)
)
stream.on('end', stopTimer).on('error', stopTimer)
@@ -119,19 +118,12 @@ export default class Api {
.use(router.routes())
.use(router.allowedMethods())
const callback = koa.callback()
httpServer.on('request', (req, res) => {
// only answers to query to the root url of this mixin
// do it before giving the request to Koa to ensure it's not modified
if (req.url.startsWith(PREFIX)) {
callback(req, res)
}
})
httpServer.on('request', koa.callback())
this.addMethods({
system: {
getMethodsInfo: [
function* () {
function*() {
const methods = this._methods
for (const name in methods) {
const { description, params = {} } = methods[name]
@@ -139,25 +131,25 @@ export default class Api {
}
}.bind(this),
{
description: 'returns the signatures of all available API methods',
},
description: 'returns the signatures of all available API methods'
}
],
getServerVersion: [
() => appVersion,
{
description: 'returns the version of xo-server',
},
description: 'returns the version of xo-server'
}
],
listMethods: [
function* () {
function*() {
const methods = this._methods
for (const name in methods) {
yield name
}
}.bind(this),
{
description: 'returns the name of all available API methods',
},
description: 'returns the name of all available API methods'
}
],
methodSignature: [
({ method: name }) => {
@@ -172,14 +164,14 @@ export default class Api {
{
description: 'returns the signature of an API method',
params: {
method: { type: 'string' },
},
},
],
method: { type: 'string' }
}
}
]
},
test: {
range: [
function* ({ start = 0, stop, step }) {
function*({ start = 0, stop, step }) {
if (step === undefined) {
step = start > stop ? -1 : 1
}
@@ -197,11 +189,11 @@ export default class Api {
params: {
start: { optional: true, type: 'number' },
step: { optional: true, type: 'number' },
stop: { type: 'number' },
},
},
],
},
stop: { type: 'number' }
}
}
]
}
})
}
@@ -228,7 +220,7 @@ export default class Api {
return required
}),
type: 'object',
type: 'object'
})
const m = params => {

View File

@@ -1,6 +1,5 @@
import assert from 'assert'
import fse from 'fs-extra'
import { xdgConfig } from 'xdg-basedir'
import xdg from 'xdg-basedir'
import { createLogger } from '@xen-orchestra/log'
import { execFileSync } from 'child_process'
@@ -11,48 +10,33 @@ const { warn } = createLogger('xo:proxy:authentication')
const isValidToken = t => typeof t === 'string' && t.length !== 0
export default class Authentication {
#token
constructor(_, { appName, config: { authenticationToken: token } }) {
if (!isValidToken(token)) {
token = JSON.parse(execFileSync('xenstore-read', ['vm-data/xo-proxy-authenticationToken']))
constructor(app, { appName, config: { authenticationToken: token } }) {
const setToken = ({ token }) => {
assert(isValidToken(token), 'invalid authentication token: ' + token)
// save this token in the automatically handled conf file
fse.outputFileSync(
// this file must take precedence over normal user config
`${xdgConfig}/${appName}/config.z-auto.json`,
JSON.stringify({ authenticationToken: token }),
{ mode: 0o600 }
)
this.#token = token
}
if (isValidToken(token)) {
this.#token = token
} else {
setToken({ token: JSON.parse(execFileSync('xenstore-read', ['vm-data/xo-proxy-authenticationToken'])) })
if (!isValidToken(token)) {
throw new Error('missing authenticationToken in configuration')
}
try {
// save this token in the automatically handled conf file
fse.outputFileSync(
// this file must take precedence over normal user config
`${xdg.config}/${appName}/config.z-auto.json`,
JSON.stringify({ authenticationToken: token }),
{ mode: 0o600 }
)
execFileSync('xenstore-rm', ['vm-data/xo-proxy-authenticationToken'])
} catch (error) {
warn('failed to remove token from XenStore', { error })
}
}
app.api.addMethod('authentication.setToken', setToken, {
description: 'change the authentication token used by this XO Proxy',
params: {
token: {
type: 'string',
minLength: 1,
},
},
})
this._token = token
}
async findProfile(credentials) {
if (credentials?.authenticationToken === this.#token) {
if (credentials?.authenticationToken === this._token) {
return new Profile()
}
}

View File

@@ -164,17 +164,6 @@ export default class Backups {
},
},
],
deleteVmBackups: [
({ filenames, remote }) =>
Disposable.use(this.getAdapter(remote), adapter => adapter.deleteVmBackups(filenames)),
{
description: 'delete VM backups',
params: {
filenames: { type: 'array', items: { type: 'string' } },
remote: { type: 'object' },
},
},
],
fetchPartitionFiles: [
({ disk: diskId, remote, partition: partitionId, paths }) =>
Disposable.use(this.getAdapter(remote), adapter => adapter.fetchPartitionFiles(diskId, partitionId, paths)),
@@ -414,7 +403,6 @@ export default class Backups {
return new RemoteAdapter(yield app.remotes.getHandler(remote), {
debounceResource: app.debounceResource.bind(app),
dirMode: app.config.get('backups.dirMode'),
vhdDirectoryCompression: app.config.get('backups.vhdDirectoryCompression'),
})
}

View File

@@ -1,120 +0,0 @@
import { urlToHttpOptions } from 'url'
import proxy from 'http2-proxy'
function removeSlash(str) {
return str.replace(/^\/|\/$/g, '')
}
function mergeUrl(relative, base) {
const res = new URL(base)
const relativeUrl = new URL(relative, base)
res.pathname = relativeUrl.pathname
relativeUrl.searchParams.forEach((value, name) => {
// we do not allow to modify params already specified by config
if (!res.searchParams.has(name)) {
res.searchParams.append(name, value)
}
})
res.hash = relativeUrl.hash.length > 0 ? relativeUrl.hash : res.hash
return res
}
export function backendToLocalPath(basePath, target, backendUrl) {
// keep redirect url relative to local server
const localPath = `${basePath}/${backendUrl.pathname.substring(target.pathname.length)}${backendUrl.search}${
backendUrl.hash
}`
return localPath
}
export function localToBackendUrl(basePath, target, localPath) {
let localPathWithoutBase = removeSlash(localPath).substring(basePath.length)
localPathWithoutBase = './' + removeSlash(localPathWithoutBase)
const url = mergeUrl(localPathWithoutBase, target)
return url
}
export default class ReverseProxy {
constructor(app, { httpServer }) {
app.config.watch('reverseProxies', proxies => {
this._proxies = Object.keys(proxies)
.sort((a, b) => b.length - a.length)
.map(path => {
let config = proxies[path]
if (typeof config === 'string') {
config = { target: config }
}
config.path = '/proxy/v1/' + removeSlash(path) + '/'
return config
})
})
httpServer.on('request', (req, res) => this._proxy(req, res))
httpServer.on('upgrade', (req, socket, head) => this._upgrade(req, socket, head))
}
_getConfigFromRequest(req) {
return this._proxies.find(({ path }) => req.url.startsWith(path))
}
_proxy(req, res) {
const config = this._getConfigFromRequest(req)
if (config === undefined) {
res.writeHead(404)
res.end('404')
return
}
const url = new URL(config.target)
const targetUrl = localToBackendUrl(config.path, url, req.originalUrl || req.url)
proxy.web(req, res, {
...urlToHttpOptions(targetUrl),
...config.options,
onReq: (req, { headers }) => {
headers['x-forwarded-for'] = req.socket.remoteAddress
headers['x-forwarded-proto'] = req.socket.encrypted ? 'https' : 'http'
if (req.headers.host !== undefined) {
headers['x-forwarded-host'] = req.headers.host
}
},
onRes: (req, res, proxyRes) => {
// rewrite redirect to pass through this proxy
if (proxyRes.statusCode === 301 || proxyRes.statusCode === 302) {
// handle relative/ absolute location
const redirectTargetLocation = new URL(proxyRes.headers.location, url)
// this proxy should only allow communication between known hosts. Don't open it too much
if (redirectTargetLocation.hostname !== url.hostname || redirectTargetLocation.protocol !== url.protocol) {
throw new Error(`Can't redirect from ${url.hostname} to ${redirectTargetLocation.hostname} `)
}
res.writeHead(proxyRes.statusCode, {
...proxyRes.headers,
location: backendToLocalPath(config.path, url, redirectTargetLocation),
})
res.end()
return
}
// pass through the answer of the remote server
res.writeHead(proxyRes.statusCode, proxyRes.headers)
// pass through content
proxyRes.pipe(res)
},
})
}
_upgrade(req, socket, head) {
const config = this._getConfigFromRequest(req)
if (config === undefined) {
return
}
const { path, target, options } = config
const targetUrl = localToBackendUrl(path, target, req.originalUrl || req.url)
proxy.ws(req, socket, head, {
...urlToHttpOptions(targetUrl),
...options,
})
}
}

View File

@@ -1,19 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIDETCCAfkCFHXO1U7YJHI61bPNhYDvyBNJYH4LMA0GCSqGSIb3DQEBCwUAMEUx
CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjIwMTEwMTI0MTU4WhcNNDkwNTI3MTI0
MTU4WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE
CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC
AQ8AMIIBCgKCAQEA1jMLdHuZu2R1fETyB2iRect1alwv76clp/7A8tx4zNaVA9qB
BcHbI83mkozuyrXpsEUblTvvcWkheBPAvWD4gj0eWSDSiuf0edcIS6aky+Lr/n0T
W/vL5kVNrgPTlsO8OyQcXjDeuUOR1xDWIa8G71Ynd6wtATB7oXe7kaV/Z6b2fENr
4wlW0YEDnMHik59c9jXDshhQYDlErwZsSyHuLwkC7xuYO26SUW9fPcHJA3uOfxeG
BrCxMuSMOJtdmslRWhLCjbk0PT12OYCCRlvuTvPHa8N57GEQbi4xAu+XATgO1DUm
Dq/oCSj0TcWUXXOykN/PAC2cjIyqkU2e7orGaQIDAQABMA0GCSqGSIb3DQEBCwUA
A4IBAQCTshhF3V5WVhnpFGHd+tPfeHmUVrUnbC+xW7fSeWpamNmTjHb7XB6uDR0O
DGswhEitbbSOsCiwz4/zpfE3/3+X07O8NPbdHTVHCei6D0uyegEeWQ2HoocfZs3X
8CORe8TItuvQAevV17D0WkGRoJGVAOiKo+izpjI55QXQ+FjkJ0bfl1iksnUJk0+I
ZNmRRNjNyOxo7NAzomSBHfJ5rDE+E440F2uvXIE9OIwHRiq6FGvQmvGijPeeP5J0
LzcSK98jfINFSsA/Wn5vWE+gfH9ySD2G3r2cDTS904T77PNiYH+cNSP6ujtmNzvK
Bgoa3jXZPRBi82TUOb2jj5DB33bg
-----END CERTIFICATE-----

View File

@@ -1,27 +0,0 @@
-----BEGIN RSA PRIVATE KEY-----
MIIEpAIBAAKCAQEA1jMLdHuZu2R1fETyB2iRect1alwv76clp/7A8tx4zNaVA9qB
BcHbI83mkozuyrXpsEUblTvvcWkheBPAvWD4gj0eWSDSiuf0edcIS6aky+Lr/n0T
W/vL5kVNrgPTlsO8OyQcXjDeuUOR1xDWIa8G71Ynd6wtATB7oXe7kaV/Z6b2fENr
4wlW0YEDnMHik59c9jXDshhQYDlErwZsSyHuLwkC7xuYO26SUW9fPcHJA3uOfxeG
BrCxMuSMOJtdmslRWhLCjbk0PT12OYCCRlvuTvPHa8N57GEQbi4xAu+XATgO1DUm
Dq/oCSj0TcWUXXOykN/PAC2cjIyqkU2e7orGaQIDAQABAoIBAQC65uVq6WLWGa1O
FtbdUggGL1svyGrngYChGvB/uZMKoX57U1DbljDCCCrV23WNmbfkYBjWWervmZ1j
qlC2roOJGQ1/Fd3A6O7w1YnegPUxFrt3XunijE55iiVi3uHknryDGlpKcfgVzfjW
oVFHKPMzKYjcqnbGn+hwlwoq5y7JYFTOa57/dZbyommbodRyy9Dpn0OES0grQqwR
VD1amrQ7XJhukcxQgYPuDc/jM3CuowoBsv9f+Q2zsPgr6CpHxxLLIs+kt8NQJT9v
neg/pm8ojcwOa9qoILdtu6ue7ee3VE9cFnB1vutxS1+MPeI5wgTJjaYrgPCMxXBM
2LdJJEmBAoGBAPA6LpuU1vv5R3x66hzenSk4LS1fj24K0WuBdTwFvzQmCr70oKdo
Yywxt+ZkBw5aEtzQlB8GewolHobDJrzxMorU+qEXX3jP2BIPDVQl2orfjr03Yyus
s5mYS/Qa6Zf1yObrjulTNm8oTn1WaG3TIvi8c5DyG2OK28N/9oMI1XGRAoGBAORD
YKyII/S66gZsJSf45qmrhq1hHuVt1xae5LUPP6lVD+MCCAmuoJnReV8fc9h7Dvgd
YPMINkWUTePFr3o4p1mh2ZC7ldczgDn6X4TldY2J3Zg47xJa5hL0L6JL4NiCGRIE
FV5rLJxkGh/DDBfmC9hQQ6Yg6cHvyewso5xVnBtZAoGAI+OdWPMIl0ZrrqYyWbPM
aP8SiMfRBtCo7tW9bQUyxpi0XEjxw3Dt+AlJfysMftFoJgMnTedK9H4NLHb1T579
PQ6KjwyN39+1WSVUiXDKUJsLmSswLrMzdcvx9PscUO6QYCdrB2K+LCcqasFBAr9b
ZyvIXCw/eUSihneUnYjxUnECgYAoPgCzKiU8ph9QFozOaUExNH4/3tl1lVHQOR8V
FKUik06DtP35xwGlXJrLPF5OEhPnhjZrYk0/IxBAUb/ICmjmknQq4gdes0Ot9QgW
A+Yfl+irR45ObBwXx1kGgd4YDYeh93pU9QweXj+Ezfw50mLQNgZXKYJMoJu2uX/2
tdkZsQKBgQCTfDcW8qBntI6V+3Gh+sIThz+fjdv5+qT54heO4EHadc98ykEZX0M1
sCWJiAQWM/zWXcsTndQDgDsvo23jpoulVPDitSEISp5gSe9FEN2njsVVID9h1OIM
f30s5kwcJoiV9kUCya/BFtuS7kbuQfAyPU0v3I+lUey6VCW6A83OTg==
-----END RSA PRIVATE KEY-----

View File

@@ -1,60 +0,0 @@
import { createServer as creatServerHttps } from 'https'
import { createServer as creatServerHttp } from 'http'
import { WebSocketServer } from 'ws'
import fs from 'fs'
const httpsServer = creatServerHttps({
key: fs.readFileSync('key.pem'),
cert: fs.readFileSync('cert.pem'),
})
const httpServer = creatServerHttp()
const wss = new WebSocketServer({ noServer: true, perMessageDeflate: false })
function upgrade(request, socket, head) {
const { pathname } = new URL(request.url)
// web socket server only on /foo url
if (pathname === '/foo') {
wss.handleUpgrade(request, socket, head, function done(ws) {
wss.emit('connection', ws, request)
ws.on('message', function message(data) {
ws.send(data)
})
})
} else {
socket.destroy()
}
}
function httpHandler(req, res) {
switch (req.url) {
case '/index.html':
res.end('hi')
return
case '/redirect':
res.writeHead(301, {
Location: 'index.html',
})
res.end()
return
case '/chainRedirect':
res.writeHead(301, {
Location: '/redirect',
})
res.end()
return
default:
res.writeHead(404)
res.end()
}
}
httpsServer.on('upgrade', upgrade)
httpServer.on('upgrade', upgrade)
httpsServer.on('request', httpHandler)
httpServer.on('request', httpHandler)
httpsServer.listen(8080)
httpServer.listen(8081)

View File

@@ -1,123 +0,0 @@
import ReverseProxy, { backendToLocalPath, localToBackendUrl } from '../dist/app/mixins/reverseProxy.mjs'
import { deepEqual, strictEqual } from 'assert'
function makeApp(reverseProxies) {
return {
config: {
get: () => reverseProxies,
},
}
}
const app = makeApp({
https: {
target: 'https://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
oneOption: true,
},
http: 'http://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
})
// test localToBackendUrl
const expectedLocalToRemote = {
https: [
{
local: '/proxy/v1/https/',
remote: 'https://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
},
{
local: '/proxy/v1/https/sub',
remote: 'https://localhost:8080/remotePath/sub?baseParm=1#one=2&another=3',
},
{
local: '/proxy/v1/https/sub/index.html',
remote: 'https://localhost:8080/remotePath/sub/index.html?baseParm=1#one=2&another=3',
},
{
local: '/proxy/v1/https/sub?param=1',
remote: 'https://localhost:8080/remotePath/sub?baseParm=1&param=1#one=2&another=3',
},
{
local: '/proxy/v1/https/sub?baseParm=willbeoverwritten&param=willstay',
remote: 'https://localhost:8080/remotePath/sub?baseParm=1&param=willstay#one=2&another=3',
},
{
local: '/proxy/v1/https/sub?param=1#another=willoverwrite',
remote: 'https://localhost:8080/remotePath/sub?baseParm=1&param=1#another=willoverwrite',
},
],
}
const proxy = new ReverseProxy(app, { httpServer: { on: () => {} } })
for (const proxyId in expectedLocalToRemote) {
for (const { local, remote } of expectedLocalToRemote[proxyId]) {
const config = proxy._getConfigFromRequest({ url: local })
const url = new URL(config.target)
strictEqual(localToBackendUrl(config.path, url, local).href, remote, 'error converting to backend')
}
}
// test backendToLocalPath
const expectedRemoteToLocal = {
https: [
{
local: '/proxy/v1/https/',
remote: 'https://localhost:8080/remotePath/',
},
{
local: '/proxy/v1/https/sub/index.html',
remote: '/remotePath/sub/index.html',
},
{
local: '/proxy/v1/https/?baseParm=1#one=2&another=3',
remote: '?baseParm=1#one=2&another=3',
},
{
local: '/proxy/v1/https/sub?baseParm=1#one=2&another=3',
remote: 'https://localhost:8080/remotePath/sub?baseParm=1#one=2&another=3',
},
],
}
for (const proxyId in expectedRemoteToLocal) {
for (const { local, remote } of expectedRemoteToLocal[proxyId]) {
const config = proxy._getConfigFromRequest({ url: local })
const targetUrl = new URL('https://localhost:8080/remotePath/?baseParm=1#one=2&another=3')
const remoteUrl = new URL(remote, targetUrl)
strictEqual(backendToLocalPath(config.path, targetUrl, remoteUrl), local, 'error converting to local')
}
}
// test _getConfigFromRequest
const expectedConfig = [
{
local: '/proxy/v1/http/other',
config: {
target: 'http://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
options: {},
path: '/proxy/v1/http',
},
},
{
local: '/proxy/v1/http',
config: undefined,
},
{
local: '/proxy/v1/other',
config: undefined,
},
{
local: '/proxy/v1/https/',
config: {
target: 'https://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
options: {
oneOption: true,
},
path: '/proxy/v1/https',
},
},
]
for (const { local, config } of expectedConfig) {
deepEqual(proxy._getConfigFromRequest({ url: local }), config)
}

View File

@@ -44,8 +44,8 @@
"pw": "^0.0.4",
"strip-indent": "^3.0.0",
"xdg-basedir": "^4.0.0",
"xo-lib": "^0.11.1",
"xo-vmdk-to-vhd": "^2.0.3"
"xo-lib": "^0.10.1",
"xo-vmdk-to-vhd": "^2.0.0"
},
"devDependencies": {
"@babel/cli": "^7.0.0",

View File

@@ -1,6 +1,6 @@
{
"name": "@xen-orchestra/xapi",
"version": "0.8.5",
"version": "0.8.0",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/xapi",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
"repository": {
@@ -38,7 +38,7 @@
"prepublishOnly": "yarn run build"
},
"dependencies": {
"@vates/decorate-with": "^1.0.0",
"@vates/decorate-with": "^0.1.0",
"@xen-orchestra/async-map": "^0.1.2",
"@xen-orchestra/log": "^0.3.0",
"d3-time-format": "^3.0.0",

View File

@@ -15,18 +15,16 @@ exports.VDI_FORMAT_VHD = 'vhd'
// xapi.call('host.get_servertime', host.$ref) for example
exports.formatDateTime = utcFormat('%Y%m%dT%H:%M:%SZ')
const dateTimeParsers = ['%Y%m%dT%H:%M:%SZ', '%Y-%m-%dT%H:%M:%S.%LZ'].map(utcParse)
const parseDateTimeHelper = utcParse('%Y%m%dT%H:%M:%SZ')
exports.parseDateTime = function (str, defaultValue) {
for (const parser of dateTimeParsers) {
const date = parser(str)
if (date !== null) {
return date.getTime()
const date = parseDateTimeHelper(str)
if (date === null) {
if (arguments.length > 1) {
return defaultValue
}
throw new RangeError(`unable to parse XAPI datetime ${JSON.stringify(str)}`)
}
if (arguments.length > 1) {
return defaultValue
}
throw new RangeError(`unable to parse XAPI datetime ${JSON.stringify(str)}`)
return date.getTime()
}
const hasProps = o => {
@@ -100,16 +98,18 @@ function removeWatcher(predicate, cb) {
class Xapi extends Base {
constructor({
callRetryWhenTooManyPendingTasks = { delay: 5e3, tries: 10 },
callRetryWhenTooManyPendingTasks,
ignoreNobakVdis,
maxUncoalescedVdis,
vdiDestroyRetryWhenInUse = { delay: 5e3, tries: 10 },
vdiDestroyRetryWhenInUse,
...opts
}) {
assert.notStrictEqual(ignoreNobakVdis, undefined)
super(opts)
this._callRetryWhenTooManyPendingTasks = {
delay: 5e3,
tries: 10,
...callRetryWhenTooManyPendingTasks,
onRetry,
when: { code: 'TOO_MANY_PENDING_TASKS' },
@@ -117,6 +117,8 @@ class Xapi extends Base {
this._ignoreNobakVdis = ignoreNobakVdis
this._maxUncoalescedVdis = maxUncoalescedVdis
this._vdiDestroyRetryWhenInUse = {
delay: 5e3,
retries: 10,
...vdiDestroyRetryWhenInUse,
onRetry,
when: { code: 'VDI_IN_USE' },

View File

@@ -1,17 +0,0 @@
/* eslint-env jest */
const { parseDateTime } = require('./')
describe('parseDateTime()', () => {
it('parses legacy XAPI format', () => {
expect(parseDateTime('20220106T21:25:15Z')).toBe(1641504315000)
})
it('parses ISO 8601 with millisconds format', () => {
expect(parseDateTime('2022-01-06T17:32:35.000Z')).toBe(1641490355000)
})
it('throws when value cannot be parsed', () => {
expect(() => parseDateTime('foo bar')).toThrow('unable to parse XAPI datetime "foo bar"')
})
})

View File

@@ -60,7 +60,7 @@ module.exports = class Vm {
try {
vdi = await this[vdiRefOrUuid.startsWith('OpaqueRef:') ? 'getRecord' : 'getRecordByUuid']('VDI', vdiRefOrUuid)
} catch (error) {
warn('_assertHealthyVdiChain, could not fetch VDI', { error })
warn(error)
return
}
cache[vdi.$ref] = vdi
@@ -81,7 +81,7 @@ module.exports = class Vm {
try {
vdi = await this.getRecord('VDI', vdiRef)
} catch (error) {
warn('_assertHealthyVdiChain, could not fetch VDI', { error })
warn(error)
return
}
cache[vdiRef] = vdi
@@ -99,7 +99,6 @@ module.exports = class Vm {
// should coalesce
const children = childrenMap[vdi.uuid]
if (
children !== undefined && // unused unmanaged VDI, will be GC-ed
children.length === 1 &&
!children[0].managed && // some SRs do not coalesce the leaf
tolerance-- <= 0
@@ -167,7 +166,7 @@ module.exports = class Vm {
memory_static_min,
name_description,
name_label,
NVRAM,
// NVRAM, // experimental
order,
other_config = {},
PCI_bus = '',
@@ -256,7 +255,6 @@ module.exports = class Vm {
is_vmss_snapshot,
name_description,
name_label,
NVRAM,
order,
reference_label,
shutdown_delay,

View File

@@ -1,152 +1,9 @@
## **5.64.0** (2021-10-29)
# ChangeLog
## **5.66.2** (2022-01-05)
<img id="latest" src="https://badgen.net/badge/channel/latest/yellow" alt="Channel: latest" />
### Bug fixes
- [Import/Disk] Fix `JSON.parse` and `createReadableSparseStream is not a function` errors [#6068](https://github.com/vatesfr/xen-orchestra/issues/6068)
- [Backup] Fix delta backup are almost always full backup instead of differentials [Forum#5256](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it/69) [Forum#5371](https://xcp-ng.org/forum/topic/5371/delta-backup-changes-in-5-66) (PR [#6075](https://github.com/vatesfr/xen-orchestra/pull/6075))
### Released packages
- vhd-lib 3.0.0
- xo-vmdk-to-vhd 2.0.3
- @xen-orchestra/backups 0.18.3
- @xen-orchestra/proxy 0.17.3
- xo-server 5.86.3
- xo-web 5.91.2
## **5.66.1** (2021-12-23)
### Bug fixes
- [Dashboard/Health] Fix `error has occured` when a pool has no default SR
- [Delta Backup] Fix unnecessary full backup when not using S3 [Forum #5371](https://xcp-ng.org/forum/topic/5371/delta-backup-changes-in-5-66)d (PR [#6070](https://github.com/vatesfr/xen-orchestra/pull/6070))
- [Backup] Fix incorrect warnings `incorrect size [...] instead of undefined`
### Released packages
- @xen-orchestra/backups 0.18.2
- @xen-orchestra/proxy 0.17.2
- xo-server 5.86.2
- xo-web 5.91.1
## **5.66.0** (2021-12-21)
### Enhancements
- [About] Show commit instead of version numbers for source users (PR [#6045](https://github.com/vatesfr/xen-orchestra/pull/6045))
- [Health] Display default SRs that aren't shared [#5871](https://github.com/vatesfr/xen-orchestra/issues/5871) (PR [#6033](https://github.com/vatesfr/xen-orchestra/pull/6033))
- [Pool,VM/advanced] Ability to change the suspend SR [#4163](https://github.com/vatesfr/xen-orchestra/issues/4163) (PR [#6044](https://github.com/vatesfr/xen-orchestra/pull/6044))
- [Home/VMs/Backup filter] Filter out VMs in disabled backup jobs (PR [#6037](https://github.com/vatesfr/xen-orchestra/pull/6037))
- [Rolling Pool Update] Automatically disable High Availability during the update [#5711](https://github.com/vatesfr/xen-orchestra/issues/5711) (PR [#6057](https://github.com/vatesfr/xen-orchestra/pull/6057))
- [Delta Backup on S3] Compress blocks by default ([Brotli](https://en.wikipedia.org/wiki/Brotli)) which reduces remote usage and increase backup speed (PR [#5932](https://github.com/vatesfr/xen-orchestra/pull/5932))
### Bug fixes
- [Tables/actions] Fix collapsed actions being clickable despite being disabled (PR [#6023](https://github.com/vatesfr/xen-orchestra/pull/6023))
- [Backup] Remove incorrect size warning following a merge [Forum #5727](https://xcp-ng.org/forum/topic/4769/warnings-showing-in-system-logs-following-each-backup-job/4) (PR [#6010](https://github.com/vatesfr/xen-orchestra/pull/6010))
- [Delta Backup] Preserve UEFI boot parameters [#6054](https://github.com/vatesfr/xen-orchestra/issues/6054) [Forum #5319](https://xcp-ng.org/forum/topic/5319/bug-uefi-boot-parameters-not-preserved-with-delta-backups)
### Released packages
- @xen-orchestra/mixins 0.1.2
- @xen-orchestra/xapi 0.8.5
- vhd-lib 2.1.0
- xo-vmdk-to-vhd 2.0.2
- @xen-orchestra/backups 0.18.1
- @xen-orchestra/proxy 0.17.1
- xo-server 5.86.1
- xo-web 5.91.0
## **5.65.3** (2021-12-20)
<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
### Bug fixes
- [Continuous Replication] Fix `could not find the base VM`
- [Backup/Smart mode] Always ignore replicated VMs created by the current job
- [Backup] Fix `Unexpected end of JSON input` during merge step
- [Backup] Fix stuck jobs when using S3 remotes (PR [#6067](https://github.com/vatesfr/xen-orchestra/pull/6067))
### Released packages
- @xen-orchestra/fs 0.19.3
- vhd-lib 2.0.4
- @xen-orchestra/backups 0.17.1
- xo-server 5.85.1
## **5.65.2** (2021-12-10)
### Bug fixes
- [Backup] Fix `handler.rmTree` is not a function (Forum [5256](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it/29) PR [#6041](https://github.com/vatesfr/xen-orchestra/pull/6041) )
- [Backup] Fix `EEXIST` in logs when multiple merge tasks are created at the same time ([Forum #5301](https://xcp-ng.org/forum/topic/5301/warnings-errors-in-journalctl))
- [Backup] Fix missing backup on restore (Forum [5256](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it/29) (PR [#6048](https://github.com/vatesfr/xen-orchestra/pull/6048))
### Released packages
- @xen-orchestra/fs 0.19.2
- vhd-lib 2.0.3
- @xen-orchestra/backups 0.16.2
- xo-server 5.84.3
- @xen-orchestra/proxy 0.15.5
## **5.65.1** (2021-12-03)
### Bug fixes
- [Delta Backup Restoration] Fix assertion error [Forum #5257](https://xcp-ng.org/forum/topic/5257/problems-building-from-source/16)
- [Delta Backup Restoration] `TypeError: this disposable has already been disposed` [Forum #5257](https://xcp-ng.org/forum/topic/5257/problems-building-from-source/20)
- [Backups] Fix: `Error: Chaining alias is forbidden xo-vm-backups/..alias.vhd to xo-vm-backups/....alias.vhd` when backuping a file to s3 [Forum #5226](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it)
- [Delta Backup Restoration] `VDI_IO_ERROR(Device I/O errors)` [Forum #5727](https://xcp-ng.org/forum/topic/5257/problems-building-from-source/4) (PR [#6031](https://github.com/vatesfr/xen-orchestra/pull/6031))
- [Delta Backup] Fix `Cannot read property 'uuid' of undefined` when a VDI has been removed from a backed up VM (PR [#6034](https://github.com/vatesfr/xen-orchestra/pull/6034))
### Released packages
- @vates/compose 2.1.0
- vhd-lib 2.0.2
- xo-vmdk-to-vhd 2.0.1
- @xen-orchestra/backups 0.16.1
- @xen-orchestra/proxy 0.15.4
- xo-server 5.84.2
## **5.65.0** (2021-11-30)
### Highlights
- [VM] Ability to export a snapshot's memory (PR [#6015](https://github.com/vatesfr/xen-orchestra/pull/6015))
- [Cloud config] Ability to create a network cloud config template and reuse it in the VM creation [#5931](https://github.com/vatesfr/xen-orchestra/issues/5931) (PR [#5979](https://github.com/vatesfr/xen-orchestra/pull/5979))
- [Backup/logs] identify XAPI errors (PR [#6001](https://github.com/vatesfr/xen-orchestra/pull/6001))
- [lite] Highlight selected VM (PR [#5939](https://github.com/vatesfr/xen-orchestra/pull/5939))
### Enhancements
- [S3] Ability to authorize self signed certificates for S3 remote (PR [#5961](https://github.com/vatesfr/xen-orchestra/pull/5961))
### Bug fixes
- [Import/VM] Fix the import of OVA files (PR [#5976](https://github.com/vatesfr/xen-orchestra/pull/5976))
### Released packages
- @vates/async-each 0.1.0
- xo-remote-parser 0.8.4
- @xen-orchestra/fs 0.19.0
- @xen-orchestra/xapi patch
- vhd-lib 2.0.1
- @xen-orchestra/backups 0.16.0
- xo-lib 0.11.1
- @xen-orchestra/proxy 0.15.3
- xo-server 5.84.1
- vhd-cli 0.6.0
- xo-web 5.90.0
## **5.64.0** (2021-10-29)
## Highlights
- [Netbox] Support older versions of Netbox and prevent "active is not a valid choice" error [#5898](https://github.com/vatesfr/xen-orchestra/issues/5898) (PR [#5946](https://github.com/vatesfr/xen-orchestra/pull/5946))
@@ -154,8 +11,8 @@
- [Host] Handle evacuation failure during host shutdown (PR [#5966](https://github.com/vatesfr/xen-orchestra/pull/#5966))
- [Menu] Notify user when proxies need to be upgraded (PR [#5930](https://github.com/vatesfr/xen-orchestra/pull/5930))
- [Servers] Ability to use an HTTP proxy between XO and a server (PR [#5958](https://github.com/vatesfr/xen-orchestra/pull/5958))
- [VM/export] Ability to copy the export URL (PR [#5948](https://github.com/vatesfr/xen-orchestra/pull/5948))
- [Pool/advanced] Ability to define network for importing/exporting VMs/VDIs (PR [#5957](https://github.com/vatesfr/xen-orchestra/pull/5957))
- [VM/export] Ability to copy the export URL (PR [#5948](https://github.com/vatesfr/xen-orchestra/pull/5948))
- [Pool/advanced] Ability to define network for importing/exporting VMs/VDIs (PR [#5957](https://github.com/vatesfr/xen-orchestra/pull/5957))
- [Host/advanced] Add button to enable/disable the host (PR [#5952](https://github.com/vatesfr/xen-orchestra/pull/5952))
- [Backups] Enable merge worker by default
@@ -188,11 +45,13 @@
## **5.63.0** (2021-09-30)
<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
### Highlights
- [Backup] Go back to previous page instead of going to the overview after editing a job: keeps current filters and page (PR [#5913](https://github.com/vatesfr/xen-orchestra/pull/5913))
- [Health] Do not take into consideration duplicated MAC addresses from CR VMs (PR [#5916](https://github.com/vatesfr/xen-orchestra/pull/5916))
- [Health] Ability to filter duplicated MAC addresses by running VMs (PR [#5917](https://github.com/vatesfr/xen-orchestra/pull/5917))
- [Health] Do not take into consideration duplicated MAC addresses from CR VMs (PR [#5916](https://github.com/vatesfr/xen-orchestra/pull/5916))
- [Health] Ability to filter duplicated MAC addresses by running VMs (PR [#5917](https://github.com/vatesfr/xen-orchestra/pull/5917))
- [Tables] Move the search bar and pagination to the top of the table (PR [#5914](https://github.com/vatesfr/xen-orchestra/pull/5914))
- [Netbox] Handle nested prefixes by always assigning an IP to the smallest prefix it matches (PR [#5908](https://github.com/vatesfr/xen-orchestra/pull/5908))
@@ -459,7 +318,7 @@
- [Proxy] _Redeploy_ now works when the bound VM is missing
- [VM template] Fix confirmation modal doesn't appear on deleting a default template (PR [#5644](https://github.com/vatesfr/xen-orchestra/pull/5644))
- [OVA VM Import] Fix imported VMs all having the same MAC addresses
- [Disk import] Fix `an error has occurred` when importing wrong format or corrupted files [#5663](https://github.com/vatesfr/xen-orchestra/issues/5663) (PR [#5683](https://github.com/vatesfr/xen-orchestra/pull/5683))
- [Disk import] Fix `an error has occurred` when importing wrong format or corrupted files [#5663](https://github.com/vatesfr/xen-orchestra/issues/5663) (PR [#5683](https://github.com/vatesfr/xen-orchestra/pull/5683))
### Released packages

View File

@@ -7,15 +7,11 @@
> Users must be able to say: “Nice enhancement, I'm eager to test it”
- Limit number of concurrent VM migrations per pool to `3` [#6065](https://github.com/vatesfr/xen-orchestra/issues/6065) (PR [#6076](https://github.com/vatesfr/xen-orchestra/pull/6076))
Can be changed in `xo-server`'s configuration file: `xapiOptions.vmMigrationConcurrency`
- [Proxy] Now ships a reverse proxy [PR#6072](https://github.com/vatesfr/xen-orchestra/pull/6072)
### Bug fixes
> Users must be able to say: “I had this issue, happy to know it's fixed”
- [Backup] Detect and clear orphan merge states, fix `ENOENT` errors (PR [#6087](https://github.com/vatesfr/xen-orchestra/pull/6087))
[Import/VM] Fix the import of OVA files (PR [#5976](https://github.com/vatesfr/xen-orchestra/pull/5976))
### Packages to release
@@ -34,7 +30,7 @@
>
> In case of conflict, the highest (lowest in previous list) `$version` wins.
- @xen-orchestra/backups minor
- @xen-orchestra/backups-cli minor
- @xen-orchestra/proxy minor
- xo-server minor
- @xen-orchestra/fs minor
- vhd-lib minor
- xo-server patch
- vhd-cli minor

View File

@@ -341,16 +341,6 @@ XO will try to find the right prefix for each IP address. If it can't find a pre
- Generate a token:
- Go to Admin > Tokens > Add token
- Create a token with "Write enabled"
- The owner of the token must have at least the following permissions:
- View permissions on:
- extras > custom-fields
- ipam > prefixes
- All permissions on:
- ipam > ip-addresses
- virtualization > cluster-types
- virtualization > clusters
- virtualization > interfaces
- virtualization > virtual-machines
- Add a UUID custom field (for **Netbox 2.x**):
- Got to Admin > Custom fields > Add custom field
- Create a custom field called "uuid" (lower case!)

View File

@@ -26,12 +26,6 @@ Each backups' job execution is identified by a `runId`. You can find this `runId
![](./assets/log-runId.png)
## Exclude disks
During a backup job, you can avoid saving all disks of the VM. To do that is trivial: just edit the VM disk name and add `[NOBAK]` before the current name, eg: `data-disk` will become `[NOBAK] data-disk` (with a space or not, doesn't matter).
The disks marked with `[NOBAK]` will be now ignored in all following backups.
## Schedule
:::tip

View File

@@ -43,6 +43,12 @@ Just go into your "Backup" view, and select Delta Backup. Then, it's the same as
Unlike other types of backup jobs which delete the associated snapshot when the job is done and it has been exported, delta backups always keep a snapshot of every VM in the backup job, and uses it for the delta. Do not delete these snapshots!
## Exclude disks
During a delta backup job, you can avoid saving all disks of the VM. To do that is trivial: just edit the VM disk name and add `[NOBAK]` before the current name, eg: `data-disk` will become `[NOBAK] data-disk` (with a space or not, doesn't matter).
The disks marked with `[NOBAK]` will be now ignored in all following backups.
## Delta backup initial seed
If you don't want to do an initial full directly toward the destination, you can create a local delta backup first, then transfer the files to your destination.

View File

@@ -103,9 +103,9 @@ In that case, you already set the password for `xoa` user. If you forgot it, see
### Manually deployed
If you connect via SSH or console for the first time without using our [web deploy form](https://xen-orchestra.com/#!/xoa), be aware **there is NO default password set for security reasons**. To set it, you need to connect to your host to find the XOA VM UUID (eg via `xe vm-list`).
If you connect via SSH or console for the first time without using our [web deploy form](https://xen-orchestra.com/#!/xoa), be aware **there's NO default password set for security reasons**. To set it, you need to connect to your host to find the XOA VM UUID (eg via `xe vm-list`).
Next, you can replace `<UUID>` with the UUID you found previously, and `<password>` with your password:
Then replace `<UUID>` with the previously find UUID, and `<password>` with your password:
```
xe vm-param-set uuid=<UUID> xenstore-data:vm-data/system-account-xoa-password=<password>
@@ -115,9 +115,7 @@ xe vm-param-set uuid=<UUID> xenstore-data:vm-data/system-account-xoa-password=<p
Don't forget to use quotes for your password, eg: `xenstore-data:vm-data/system-account-xoa-password='MyPassW0rd!'`
:::
Finally, you must reboot the VM to implement the changes.
You can now connect with the `xoa` username and password you defined in the previous command, eg with `ssh xoa@<XOA IP ADDRESS>`.
Then, you could connect with `xoa` username and the password you defined in the previous command, eg with `ssh xoa@<XOA IP ADDRESS>`.
### Using sudo

View File

@@ -19,7 +19,7 @@
"handlebars": "^4.7.6",
"husky": "^4.2.5",
"jest": "^27.3.1",
"lint-staged": "^12.0.3",
"lint-staged": "^11.1.2",
"lodash": "^4.17.4",
"prettier": "^2.0.5",
"promise-toolbox": "^0.20.0",
@@ -46,6 +46,7 @@
],
"^(value-matcher)$": "$1/src",
"^(vhd-cli)$": "$1/src",
"^(vhd-lib)$": "$1/src",
"^(xo-[^/]+)$": [
"$1/src",
"$1"

View File

@@ -1,7 +1,7 @@
{
"private": false,
"name": "vhd-cli",
"version": "0.6.0",
"version": "0.5.0",
"license": "ISC",
"description": "Tools to read/create and merge VHD files",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-cli",
@@ -24,12 +24,11 @@
"node": ">=8.10"
},
"dependencies": {
"@xen-orchestra/fs": "^0.19.3",
"@xen-orchestra/fs": "^0.18.0",
"cli-progress": "^3.1.0",
"exec-promise": "^0.7.0",
"getopts": "^2.2.3",
"human-format": "^0.11.0",
"vhd-lib": "^3.0.0"
"vhd-lib": "^1.3.0"
},
"devDependencies": {
"@babel/cli": "^7.0.0",

View File

@@ -1,34 +1,6 @@
import { Constants, VhdFile } from 'vhd-lib'
import { VhdFile } from 'vhd-lib'
import { getHandler } from '@xen-orchestra/fs'
import { resolve } from 'path'
import humanFormat from 'human-format'
import invert from 'lodash/invert.js'
const { PLATFORMS } = Constants
const DISK_TYPES_MAP = invert(Constants.DISK_TYPES)
const PLATFORMS_MAP = invert(PLATFORMS)
const MAPPERS = {
bytes: humanFormat.bytes,
date: _ => (_ !== 0 ? new Date(_) : 0),
diskType: _ => DISK_TYPES_MAP[_],
platform: _ => PLATFORMS_MAP[_],
}
function mapProperties(object, mapping) {
const result = { ...object }
for (const prop of Object.keys(mapping)) {
const value = object[prop]
if (value !== undefined) {
let mapper = mapping[prop]
if (typeof mapper === 'string') {
mapper = MAPPERS[mapper]
}
result[prop] = mapper(value)
}
}
return result
}
export default async args => {
const vhd = new VhdFile(getHandler({ url: 'file:///' }), resolve(args[0]))
@@ -40,26 +12,6 @@ export default async args => {
await vhd.readHeaderAndFooter(false)
}
console.log(
mapProperties(vhd.footer, {
currentSize: 'bytes',
diskType: 'diskType',
originalSize: 'bytes',
timestamp: 'date',
})
)
console.log(
mapProperties(vhd.header, {
blockSize: 'bytes',
parentTimestamp: 'date',
parentLocatorEntry: _ =>
_.filter(_ => _.platformCode !== PLATFORMS.NONE) // hide empty
.map(_ =>
mapProperties(_, {
platformCode: 'platform',
})
),
})
)
console.log(vhd.header)
console.log(vhd.footer)
}

View File

@@ -1,18 +1,13 @@
import { openVhd } from 'vhd-lib'
import { getSyncedHandler } from '@xen-orchestra/fs'
import { createContentStream } from 'vhd-lib'
import { getHandler } from '@xen-orchestra/fs'
import { resolve } from 'path'
import { writeStream } from '../_utils'
import { Disposable } from 'promise-toolbox'
export default async args => {
if (args.length < 2 || args.some(_ => _ === '-h' || _ === '--help')) {
return `Usage: ${this.command} <input VHD> [<output raw>]`
}
await Disposable.use(async function* () {
const handler = getSyncedHandler({ url: 'file:///' })
const vhd = openVhd(handler, resolve(args[0]))
await writeStream(vhd.rawContent())
})
await writeStream(createContentStream(getHandler({ url: 'file:///' }), resolve(args[0])), args[1])
}

View File

@@ -0,0 +1 @@
module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))

View File

@@ -1,275 +0,0 @@
/* eslint-env jest */
const rimraf = require('rimraf')
const tmp = require('tmp')
const fs = require('fs-extra')
const { getSyncedHandler } = require('@xen-orchestra/fs')
const { Disposable, pFromCallback } = require('promise-toolbox')
const { openVhd } = require('../index')
const { checkFile, createRandomFile, convertFromRawToVhd, createRandomVhdDirectory } = require('../tests/utils')
const { VhdAbstract } = require('./VhdAbstract')
const { BLOCK_UNUSED, FOOTER_SIZE, HEADER_SIZE, PLATFORMS, SECTOR_SIZE } = require('../_constants')
const { unpackHeader, unpackFooter } = require('./_utils')
let tempDir
jest.setTimeout(60000)
beforeEach(async () => {
tempDir = await pFromCallback(cb => tmp.dir(cb))
})
afterEach(async () => {
await pFromCallback(cb => rimraf(tempDir, cb))
})
const streamToBuffer = stream => {
let buffer = Buffer.alloc(0)
return new Promise((resolve, reject) => {
stream.on('data', data => (buffer = Buffer.concat([buffer, data])))
stream.on('end', () => resolve(buffer))
})
}
test('It creates an alias', async () => {
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: 'file://' + tempDir })
const aliasPath = `alias/alias.alias.vhd`
const aliasFsPath = `${tempDir}/${aliasPath}`
await fs.mkdirp(`${tempDir}/alias`)
const testOneCombination = async ({ targetPath, targetContent }) => {
await VhdAbstract.createAlias(handler, aliasPath, targetPath)
// alias file is created
expect(await fs.exists(aliasFsPath)).toEqual(true)
// content is the target path relative to the alias location
const content = await fs.readFile(aliasFsPath, 'utf-8')
expect(content).toEqual(targetContent)
// create alias fails if alias already exists, remove it before next loop step
await fs.unlink(aliasFsPath)
}
const combinations = [
{ targetPath: `targets.vhd`, targetContent: `../targets.vhd` },
{ targetPath: `alias/targets.vhd`, targetContent: `targets.vhd` },
{ targetPath: `alias/sub/targets.vhd`, targetContent: `sub/targets.vhd` },
{ targetPath: `sibling/targets.vhd`, targetContent: `../sibling/targets.vhd` },
]
for (const { targetPath, targetContent } of combinations) {
await testOneCombination({ targetPath, targetContent })
}
})
})
test('alias must have *.alias.vhd extension', async () => {
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: 'file:///' })
const aliasPath = `${tempDir}/invalidalias.vhd`
const targetPath = `${tempDir}/targets.vhd`
expect(async () => await VhdAbstract.createAlias(handler, aliasPath, targetPath)).rejects.toThrow()
expect(await fs.exists(aliasPath)).toEqual(false)
})
})
test('alias must not be chained', async () => {
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: 'file:///' })
const aliasPath = `${tempDir}/valid.alias.vhd`
const targetPath = `${tempDir}/an.other.valid.alias.vhd`
expect(async () => await VhdAbstract.createAlias(handler, aliasPath, targetPath)).rejects.toThrow()
expect(await fs.exists(aliasPath)).toEqual(false)
})
})
test('It rename and unlink a VHDFile', async () => {
const initalSize = 4
const rawFileName = `${tempDir}/randomfile`
await createRandomFile(rawFileName, initalSize)
const vhdFileName = `${tempDir}/randomfile.vhd`
await convertFromRawToVhd(rawFileName, vhdFileName)
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: 'file:///' })
const { size } = await fs.stat(vhdFileName)
const targetFileName = `${tempDir}/renamed.vhd`
await VhdAbstract.rename(handler, vhdFileName, targetFileName)
expect(await fs.exists(vhdFileName)).toEqual(false)
const { size: renamedSize } = await fs.stat(targetFileName)
expect(size).toEqual(renamedSize)
await VhdAbstract.unlink(handler, targetFileName)
expect(await fs.exists(targetFileName)).toEqual(false)
})
})
test('It rename and unlink a VhdDirectory', async () => {
const initalSize = 4
const vhdDirectory = `${tempDir}/randomfile.dir`
await createRandomVhdDirectory(vhdDirectory, initalSize)
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: 'file:///' })
const vhd = yield openVhd(handler, vhdDirectory)
expect(vhd.header.cookie).toEqual('cxsparse')
expect(vhd.footer.cookie).toEqual('conectix')
const targetFileName = `${tempDir}/renamed.vhd`
// it should clean an existing directory
await fs.mkdir(targetFileName)
await fs.writeFile(`${targetFileName}/dummy`, 'I exists')
await VhdAbstract.rename(handler, vhdDirectory, targetFileName)
expect(await fs.exists(vhdDirectory)).toEqual(false)
expect(await fs.exists(targetFileName)).toEqual(true)
expect(await fs.exists(`${targetFileName}/dummy`)).toEqual(false)
await VhdAbstract.unlink(handler, targetFileName)
expect(await fs.exists(targetFileName)).toEqual(false)
})
})
test('It create , rename and unlink alias', async () => {
const initalSize = 4
const rawFileName = `${tempDir}/randomfile`
await createRandomFile(rawFileName, initalSize)
const vhdFileName = `${tempDir}/randomfile.vhd`
await convertFromRawToVhd(rawFileName, vhdFileName)
const aliasFileName = `${tempDir}/aliasFileName.alias.vhd`
const aliasFileNameRenamed = `${tempDir}/aliasFileNameRenamed.alias.vhd`
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: 'file:///' })
await VhdAbstract.createAlias(handler, aliasFileName, vhdFileName)
expect(await fs.exists(aliasFileName)).toEqual(true)
expect(await fs.exists(vhdFileName)).toEqual(true)
await VhdAbstract.rename(handler, aliasFileName, aliasFileNameRenamed)
expect(await fs.exists(aliasFileName)).toEqual(false)
expect(await fs.exists(vhdFileName)).toEqual(true)
expect(await fs.exists(aliasFileNameRenamed)).toEqual(true)
await VhdAbstract.unlink(handler, aliasFileNameRenamed)
expect(await fs.exists(aliasFileName)).toEqual(false)
expect(await fs.exists(vhdFileName)).toEqual(false)
expect(await fs.exists(aliasFileNameRenamed)).toEqual(false)
})
})
test('it can create a vhd stream', async () => {
const initialNbBlocks = 3
const initalSize = initialNbBlocks * 2
const rawFileName = `${tempDir}/randomfile`
await createRandomFile(rawFileName, initalSize)
const vhdFileName = `${tempDir}/vhd.vhd`
await convertFromRawToVhd(rawFileName, vhdFileName)
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: 'file://' + tempDir })
const vhd = yield openVhd(handler, 'vhd.vhd')
await vhd.readBlockAllocationTable()
const parentLocatorBase = Buffer.from('a file path, not aligned', 'utf16le')
const aligned = Buffer.alloc(SECTOR_SIZE, 0)
parentLocatorBase.copy(aligned)
await vhd.writeParentLocator({
id: 0,
platformCode: PLATFORMS.W2KU,
data: parentLocatorBase,
})
await vhd.writeFooter()
const stream = vhd.stream()
// read all the stream into a buffer
const buffer = await streamToBuffer(stream)
const length = buffer.length
const bufFooter = buffer.slice(0, FOOTER_SIZE)
// footer is still valid
expect(() => unpackFooter(bufFooter)).not.toThrow()
const footer = unpackFooter(bufFooter)
// header is still valid
const bufHeader = buffer.slice(FOOTER_SIZE, HEADER_SIZE + FOOTER_SIZE)
expect(() => unpackHeader(bufHeader, footer)).not.toThrow()
// 1 deleted block should be in ouput
let start = FOOTER_SIZE + HEADER_SIZE + vhd.batSize
const parentLocatorData = buffer.slice(start, start + SECTOR_SIZE)
expect(parentLocatorData.equals(aligned)).toEqual(true)
start += SECTOR_SIZE // parent locator
expect(length).toEqual(start + initialNbBlocks * vhd.fullBlockSize + FOOTER_SIZE)
expect(stream.length).toEqual(buffer.length)
// blocks
const blockBuf = Buffer.alloc(vhd.sectorsPerBlock * SECTOR_SIZE, 0)
for (let i = 0; i < initialNbBlocks; i++) {
const blockDataStart = start + i * vhd.fullBlockSize + 512 /* block bitmap */
const blockDataEnd = blockDataStart + vhd.sectorsPerBlock * SECTOR_SIZE
const content = buffer.slice(blockDataStart, blockDataEnd)
await handler.read('randomfile', blockBuf, i * vhd.sectorsPerBlock * SECTOR_SIZE)
expect(content.equals(blockBuf)).toEqual(true)
}
// footer
const endFooter = buffer.slice(length - FOOTER_SIZE)
expect(bufFooter).toEqual(endFooter)
await handler.writeFile('out.vhd', buffer)
// check that the vhd is still valid
await checkFile(`${tempDir}/out.vhd`)
})
})
it('can stream content', async () => {
const initalSizeMb = 5 // 2 block and an half
const initialNbBlocks = Math.ceil(initalSizeMb / 2)
const initialByteSize = initalSizeMb * 1024 * 1024
const rawFileName = `${tempDir}/randomfile`
await createRandomFile(rawFileName, initalSizeMb)
const vhdFileName = `${tempDir}/vhd.vhd`
await convertFromRawToVhd(rawFileName, vhdFileName)
const bat = Buffer.alloc(512)
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: 'file://' + tempDir })
const vhd = yield openVhd(handler, 'vhd.vhd')
// mark first block as unused
await handler.read('vhd.vhd', bat, vhd.header.tableOffset)
bat.writeUInt32BE(BLOCK_UNUSED, 0)
await handler.write('vhd.vhd', bat, vhd.header.tableOffset)
// read our modified block allocation table
await vhd.readBlockAllocationTable()
const stream = vhd.rawContent()
const buffer = await streamToBuffer(stream)
// qemu can modify size, to align it to geometry
// check that data didn't change
const blockDataLength = vhd.sectorsPerBlock * SECTOR_SIZE
// first block should be empty
const EMPTY = Buffer.alloc(blockDataLength, 0)
const firstBlock = buffer.slice(0, blockDataLength)
// using buffer1 toEquals buffer2 make jest crash trying to stringify it on failure
expect(firstBlock.equals(EMPTY)).toEqual(true)
let remainingLength = initialByteSize - blockDataLength // already checked the first block
for (let i = 1; i < initialNbBlocks; i++) {
// last block will be truncated
const blockSize = Math.min(blockDataLength, remainingLength - blockDataLength)
const blockDataStart = i * blockDataLength // first block have been deleted
const blockDataEnd = blockDataStart + blockSize
const content = buffer.slice(blockDataStart, blockDataEnd)
const blockBuf = Buffer.alloc(blockSize, 0)
await handler.read('randomfile', blockBuf, i * blockDataLength)
expect(content.equals(blockBuf)).toEqual(true)
remainingLength -= blockSize
}
})
})

View File

@@ -1,114 +0,0 @@
/* eslint-env jest */
const rimraf = require('rimraf')
const tmp = require('tmp')
const fs = require('fs-extra')
const { getHandler, getSyncedHandler } = require('@xen-orchestra/fs')
const { Disposable, pFromCallback } = require('promise-toolbox')
const { openVhd, VhdDirectory } = require('../')
const { createRandomFile, convertFromRawToVhd, convertToVhdDirectory } = require('../tests/utils')
let tempDir = null
jest.setTimeout(60000)
beforeEach(async () => {
tempDir = await pFromCallback(cb => tmp.dir(cb))
})
afterEach(async () => {
await pFromCallback(cb => rimraf(tempDir, cb))
})
test('Can coalesce block', async () => {
const initalSize = 4
const parentrawFileName = `${tempDir}/randomfile`
const parentFileName = `${tempDir}/parent.vhd`
const parentDirectoryName = `${tempDir}/parent.dir.vhd`
await createRandomFile(parentrawFileName, initalSize)
await convertFromRawToVhd(parentrawFileName, parentFileName)
await convertToVhdDirectory(parentrawFileName, parentFileName, parentDirectoryName)
const childrawFileName = `${tempDir}/randomfile`
const childFileName = `${tempDir}/childFile.vhd`
await createRandomFile(childrawFileName, initalSize)
await convertFromRawToVhd(childrawFileName, childFileName)
const childRawDirectoryName = `${tempDir}/randomFile2.vhd`
const childDirectoryFileName = `${tempDir}/childDirFile.vhd`
const childDirectoryName = `${tempDir}/childDir.vhd`
await createRandomFile(childRawDirectoryName, initalSize)
await convertFromRawToVhd(childRawDirectoryName, childDirectoryFileName)
await convertToVhdDirectory(childRawDirectoryName, childDirectoryFileName, childDirectoryName)
await Disposable.use(async function* () {
const handler = getHandler({ url: 'file://' })
const parentVhd = yield openVhd(handler, parentDirectoryName, { flags: 'w' })
await parentVhd.readBlockAllocationTable()
const childFileVhd = yield openVhd(handler, childFileName)
await childFileVhd.readBlockAllocationTable()
const childDirectoryVhd = yield openVhd(handler, childDirectoryName)
await childDirectoryVhd.readBlockAllocationTable()
await parentVhd.coalesceBlock(childFileVhd, 0)
await parentVhd.writeFooter()
await parentVhd.writeBlockAllocationTable()
let parentBlockData = (await parentVhd.readBlock(0)).data
let childBlockData = (await childFileVhd.readBlock(0)).data
expect(parentBlockData.equals(childBlockData)).toEqual(true)
await parentVhd.coalesceBlock(childDirectoryVhd, 0)
await parentVhd.writeFooter()
await parentVhd.writeBlockAllocationTable()
parentBlockData = (await parentVhd.readBlock(0)).data
childBlockData = (await childDirectoryVhd.readBlock(0)).data
expect(parentBlockData).toEqual(childBlockData)
})
})
test('compressed blocks and metadata works', async () => {
const initalSize = 4
const rawFileName = `${tempDir}/randomfile`
const vhdName = `${tempDir}/parent.vhd`
await createRandomFile(rawFileName, initalSize)
await convertFromRawToVhd(rawFileName, vhdName)
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
const vhd = yield openVhd(handler, 'parent.vhd')
await vhd.readBlockAllocationTable()
const compressedVhd = yield VhdDirectory.create(handler, 'compressed.vhd', { compression: 'gzip' })
compressedVhd.header = vhd.header
compressedVhd.footer = vhd.footer
for await (const block of vhd.blocks()) {
await compressedVhd.writeEntireBlock(block)
}
await Promise
.all[(await compressedVhd.writeHeader(), await compressedVhd.writeFooter(), await compressedVhd.writeBlockAllocationTable())]
// compressed vhd have a metadata file
expect(await fs.exists(`${tempDir}/compressed.vhd/metadata.json`)).toEqual(true)
const metada = JSON.parse(await handler.readFile('compressed.vhd/metadata.json'))
expect(metada.compression.type).toEqual('gzip')
expect(metada.compression.options.level).toEqual(1)
// compressed vhd should not be broken
await compressedVhd.readHeaderAndFooter()
await compressedVhd.readBlockAllocationTable()
// check that footer and header are not modified
expect(compressedVhd.footer).toEqual(vhd.footer)
expect(compressedVhd.header).toEqual(vhd.header)
// their block content should not have changed
let counter = 0
for await (const block of compressedVhd.blocks()) {
const source = await vhd.readBlock(block.id)
expect(source.data.equals(block.data)).toEqual(true)
counter++
}
// neither the number of blocks
expect(counter).toEqual(2)
})
})

View File

@@ -1,83 +0,0 @@
/* eslint-env jest */
const rimraf = require('rimraf')
const tmp = require('tmp')
const { Disposable, pFromCallback } = require('promise-toolbox')
const { getSyncedHandler } = require('@xen-orchestra/fs')
const { SECTOR_SIZE, PLATFORMS } = require('../_constants')
const { createRandomFile, convertFromRawToVhd } = require('../tests/utils')
const { openVhd, chainVhd } = require('..')
const { VhdSynthetic } = require('./VhdSynthetic')
let tempDir = null
jest.setTimeout(60000)
beforeEach(async () => {
tempDir = await pFromCallback(cb => tmp.dir(cb))
})
afterEach(async () => {
await pFromCallback(cb => rimraf(tempDir, cb))
})
test('It can read block and parent locator from a synthetic vhd', async () => {
const bigRawFileName = `/bigrandomfile`
await createRandomFile(`${tempDir}/${bigRawFileName}`, 8)
const bigVhdFileName = `/bigrandomfile.vhd`
await convertFromRawToVhd(`${tempDir}/${bigRawFileName}`, `${tempDir}/${bigVhdFileName}`)
const smallRawFileName = `/smallrandomfile`
await createRandomFile(`${tempDir}/${smallRawFileName}`, 4)
const smallVhdFileName = `/smallrandomfile.vhd`
await convertFromRawToVhd(`${tempDir}/${smallRawFileName}`, `${tempDir}/${smallVhdFileName}`)
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
// ensure the two VHD are linked, with the child of type DISK_TYPES.DIFFERENCING
await chainVhd(handler, bigVhdFileName, handler, smallVhdFileName, true)
const [smallVhd, bigVhd] = yield Disposable.all([
openVhd(handler, smallVhdFileName),
openVhd(handler, bigVhdFileName),
])
// add parent locato
// this will also scramble the block inside the vhd files
await bigVhd.writeParentLocator({
id: 0,
platformCode: PLATFORMS.W2KU,
data: Buffer.from('I am in the big one'),
})
const syntheticVhd = new VhdSynthetic([smallVhd, bigVhd])
await syntheticVhd.readBlockAllocationTable()
expect(syntheticVhd.header.diskType).toEqual(bigVhd.header.diskType)
expect(syntheticVhd.header.parentTimestamp).toEqual(bigVhd.header.parentTimestamp)
// first two block should be from small
const buf = Buffer.alloc(syntheticVhd.sectorsPerBlock * SECTOR_SIZE, 0)
let content = (await syntheticVhd.readBlock(0)).data
await handler.read(smallRawFileName, buf, 0)
expect(content).toEqual(buf)
content = (await syntheticVhd.readBlock(1)).data
await handler.read(smallRawFileName, buf, buf.length)
expect(content).toEqual(buf)
// the next one from big
content = (await syntheticVhd.readBlock(2)).data
await handler.read(bigRawFileName, buf, buf.length * 2)
expect(content).toEqual(buf)
content = (await syntheticVhd.readBlock(3)).data
await handler.read(bigRawFileName, buf, buf.length * 3)
expect(content).toEqual(buf)
// the parent locator should the one of the root vhd
const parentLocator = await syntheticVhd.readParentLocator(0)
expect(parentLocator.platformCode).toEqual(PLATFORMS.W2KU)
expect(Buffer.from(parentLocator.data, 'utf-8').toString()).toEqual('I am in the big one')
})
})

View File

@@ -1,88 +0,0 @@
const UUID = require('uuid')
const cloneDeep = require('lodash/cloneDeep.js')
const { asyncMap } = require('@xen-orchestra/async-map')
const { VhdAbstract } = require('./VhdAbstract')
const { DISK_TYPES, FOOTER_SIZE, HEADER_SIZE } = require('../_constants')
const assert = require('assert')
exports.VhdSynthetic = class VhdSynthetic extends VhdAbstract {
#vhds = []
get header() {
// this the VHD we want to synthetize
const vhd = this.#vhds[0]
// this is the root VHD
const rootVhd = this.#vhds[this.#vhds.length - 1]
// data of our synthetic VHD
// TODO: set parentLocatorEntry-s in header
return {
...vhd.header,
parentLocatorEntry: cloneDeep(rootVhd.header.parentLocatorEntry),
tableOffset: FOOTER_SIZE + HEADER_SIZE,
parentTimestamp: rootVhd.header.parentTimestamp,
parentUnicodeName: rootVhd.header.parentUnicodeName,
parentUuid: rootVhd.header.parentUuid,
}
}
get footer() {
// this is the root VHD
const rootVhd = this.#vhds[this.#vhds.length - 1]
return {
...this.#vhds[0].footer,
dataOffset: FOOTER_SIZE,
diskType: rootVhd.footer.diskType,
}
}
static async open(vhds) {
const vhd = new VhdSynthetic(vhds)
return {
dispose: () => {},
value: vhd,
}
}
/**
* @param {Array<VhdAbstract>} vhds the chain of Vhds used to compute this Vhd, from the deepest child (in position 0), to the root (in the last position)
* only the last one can have any type. Other must have type DISK_TYPES.DIFFERENCING (delta)
*/
constructor(vhds) {
assert(vhds.length > 0)
super()
this.#vhds = vhds
}
async readBlockAllocationTable() {
await asyncMap(this.#vhds, vhd => vhd.readBlockAllocationTable())
}
containsBlock(blockId) {
return this.#vhds.some(vhd => vhd.containsBlock(blockId))
}
async readHeaderAndFooter() {
const vhds = this.#vhds
await asyncMap(vhds, vhd => vhd.readHeaderAndFooter())
for (let i = 0, n = vhds.length - 1; i < n; ++i) {
const child = vhds[i]
const parent = vhds[i + 1]
assert.strictEqual(child.footer.diskType, DISK_TYPES.DIFFERENCING)
assert.strictEqual(UUID.stringify(child.header.parentUuid), UUID.stringify(parent.footer.uuid))
}
}
async readBlock(blockId, onlyBitmap = false) {
const index = this.#vhds.findIndex(vhd => vhd.containsBlock(blockId))
// only read the content of the first vhd containing this block
return await this.#vhds[index].readBlock(blockId, onlyBitmap)
}
_readParentLocatorData(id) {
return this.#vhds[this.#vhds.length - 1]._readParentLocatorData(id)
}
}

View File

@@ -1,67 +0,0 @@
const assert = require('assert')
const { BLOCK_UNUSED, SECTOR_SIZE } = require('../_constants')
const { fuFooter, fuHeader, checksumStruct, unpackField } = require('../_structs')
const checkFooter = require('../checkFooter')
const checkHeader = require('../_checkHeader')
const computeBatSize = entries => sectorsToBytes(sectorsRoundUpNoZero(entries * 4))
exports.computeBatSize = computeBatSize
const computeSectorsPerBlock = blockSize => blockSize / SECTOR_SIZE
exports.computeSectorsPerBlock = computeSectorsPerBlock
// one bit per sector
const computeBlockBitmapSize = blockSize => computeSectorsPerBlock(blockSize) >>> 3
exports.computeBlockBitmapSize = computeBlockBitmapSize
const computeFullBlockSize = blockSize => blockSize + SECTOR_SIZE * computeSectorOfBitmap(blockSize)
exports.computeFullBlockSize = computeFullBlockSize
const computeSectorOfBitmap = blockSize => sectorsRoundUpNoZero(computeBlockBitmapSize(blockSize))
exports.computeSectorOfBitmap = computeSectorOfBitmap
// Sectors conversions.
const sectorsRoundUpNoZero = bytes => Math.ceil(bytes / SECTOR_SIZE) || 1
exports.sectorsRoundUpNoZero = sectorsRoundUpNoZero
const sectorsToBytes = sectors => sectors * SECTOR_SIZE
exports.sectorsToBytes = sectorsToBytes
const assertChecksum = (name, buf, struct) => {
const actual = unpackField(struct.fields.checksum, buf)
const expected = checksumStruct(buf, struct)
assert.strictEqual(actual, expected, `invalid ${name} checksum ${actual}, expected ${expected}`)
}
exports.assertChecksum = assertChecksum
// unused block as buffer containing a uint32BE
const BUF_BLOCK_UNUSED = Buffer.allocUnsafe(4)
BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
exports.BUF_BLOCK_UNUSED = BUF_BLOCK_UNUSED
/**
* Check and parse the header buffer to build an header object
*
* @param {Buffer} bufHeader
* @param {Object} footer
* @returns {Object} the parsed header
*/
exports.unpackHeader = (bufHeader, footer) => {
assertChecksum('header', bufHeader, fuHeader)
const header = fuHeader.unpack(bufHeader)
checkHeader(header, footer)
return header
}
/**
* Check and parse the footer buffer to build a footer object
*
* @param {Buffer} bufHeader
* @param {Object} footer
* @returns {Object} the parsed footer
*/
exports.unpackFooter = bufFooter => {
assertChecksum('footer', bufFooter, fuFooter)
const footer = fuFooter.unpack(bufFooter)
checkFooter(footer)
return footer
}

View File

@@ -1,7 +0,0 @@
const MASK = 0x80
exports.set = (map, bit) => {
map[bit >> 3] |= MASK >> (bit & 7)
}
exports.test = (map, bit) => ((map[bit >> 3] << (bit & 7)) & MASK) !== 0

View File

@@ -1,40 +0,0 @@
exports.BLOCK_UNUSED = 0xffffffff
// This lib has been extracted from the Xen Orchestra project.
exports.CREATOR_APPLICATION = 'xo '
// Sizes in bytes.
exports.FOOTER_SIZE = 512
exports.HEADER_SIZE = 1024
exports.SECTOR_SIZE = 512
exports.DEFAULT_BLOCK_SIZE = 0x00200000 // from the spec
exports.FOOTER_COOKIE = 'conectix'
exports.HEADER_COOKIE = 'cxsparse'
exports.DISK_TYPES = {
__proto__: null,
FIXED: 2,
DYNAMIC: 3,
DIFFERENCING: 4,
}
exports.PARENT_LOCATOR_ENTRIES = 8
exports.PLATFORMS = {
__proto__: null,
NONE: 0,
WI2R: 0x57693272,
WI2K: 0x5769326b,
W2RU: 0x57327275,
W2KU: 0x57326b75,
MAC: 0x4d616320,
MACX: 0x4d616358,
}
exports.FILE_FORMAT_VERSION = 1 << 16
exports.HEADER_VERSION = 1 << 16
exports.ALIAS_MAX_PATH_LENGTH = 1024

View File

@@ -1 +0,0 @@
module.exports = Function.prototype

View File

@@ -1,64 +0,0 @@
/* eslint-env jest */
const rimraf = require('rimraf')
const tmp = require('tmp')
const { getSyncedHandler } = require('@xen-orchestra/fs')
const { Disposable, pFromCallback } = require('promise-toolbox')
const { isVhdAlias, resolveAlias } = require('./_resolveAlias')
const { ALIAS_MAX_PATH_LENGTH } = require('./_constants')
let tempDir
jest.setTimeout(60000)
beforeEach(async () => {
tempDir = await pFromCallback(cb => tmp.dir(cb))
})
afterEach(async () => {
await pFromCallback(cb => rimraf(tempDir, cb))
})
test('is vhd alias recognize only *.alias.vhd files', () => {
expect(isVhdAlias('filename.alias.vhd')).toEqual(true)
expect(isVhdAlias('alias.vhd')).toEqual(false)
expect(isVhdAlias('filename.vhd')).toEqual(false)
expect(isVhdAlias('filename.alias.vhd.other')).toEqual(false)
})
test('resolve return the path in argument for a non alias file ', async () => {
expect(await resolveAlias(null, 'filename.vhd')).toEqual('filename.vhd')
})
test('resolve get the path of the target file for an alias', async () => {
await Disposable.use(async function* () {
// same directory
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
const alias = `alias.alias.vhd`
await handler.writeFile(alias, 'target.vhd')
await expect(await resolveAlias(handler, alias)).toEqual(`target.vhd`)
// different directory
await handler.mkdir(`sub`)
await handler.writeFile(alias, 'sub/target.vhd', { flags: 'w' })
await expect(await resolveAlias(handler, alias)).toEqual(`sub/target.vhd`)
})
})
test('resolve throws an error an alias to an alias', async () => {
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
const alias = `alias.alias.vhd`
const target = `target.alias.vhd`
await handler.writeFile(alias, target)
await expect(async () => await resolveAlias(handler, alias)).rejects.toThrow(Error)
})
})
test('resolve throws an error on a file too big ', async () => {
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
await handler.writeFile('toobig.alias.vhd', Buffer.alloc(ALIAS_MAX_PATH_LENGTH + 1, 0))
await expect(async () => await resolveAlias(handler, 'toobig.alias.vhd')).rejects.toThrow(Error)
})
})

View File

@@ -1,25 +0,0 @@
const { ALIAS_MAX_PATH_LENGTH } = require('./_constants')
const resolveRelativeFromFile = require('./_resolveRelativeFromFile')
function isVhdAlias(filename) {
return filename.endsWith('.alias.vhd')
}
exports.isVhdAlias = isVhdAlias
exports.resolveAlias = async function resolveAlias(handler, filename) {
if (!isVhdAlias(filename)) {
return filename
}
const size = await handler.getSize(filename)
if (size > ALIAS_MAX_PATH_LENGTH) {
// seems reasonnable for a relative path
throw new Error(`The alias file ${filename} is too big (${size} bytes)`)
}
const aliasContent = (await handler.readFile(filename)).toString().trim()
// also handle circular references and unreasonnably long chains
if (isVhdAlias(aliasContent)) {
throw new Error(`Chaining alias is forbidden ${filename} to ${aliasContent}`)
}
// the target is relative to the alias location
return resolveRelativeFromFile(filename, aliasContent)
}

View File

@@ -1,30 +0,0 @@
const { dirname, relative } = require('path')
const { openVhd } = require('./openVhd')
const { DISK_TYPES } = require('./_constants')
const { Disposable } = require('promise-toolbox')
module.exports = async function chain(parentHandler, parentPath, childHandler, childPath, force = false) {
await Disposable.use(
[openVhd(parentHandler, parentPath), openVhd(childHandler, childPath)],
async ([parentVhd, childVhd]) => {
await childVhd.readHeaderAndFooter()
const { header, footer } = childVhd
if (footer.diskType !== DISK_TYPES.DIFFERENCING) {
if (!force) {
throw new Error('cannot chain disk of type ' + footer.diskType)
}
footer.diskType = DISK_TYPES.DIFFERENCING
}
await childVhd.readBlockAllocationTable()
const parentName = relative(dirname(childPath), parentPath)
header.parentUuid = parentVhd.footer.uuid
header.parentUnicodeName = parentName
await childVhd.setUniqueParentLocator(parentName)
await childVhd.writeHeader()
await childVhd.writeFooter()
}
)
}

View File

@@ -1,14 +0,0 @@
const { openVhd } = require('./openVhd')
const resolveRelativeFromFile = require('./_resolveRelativeFromFile')
const { DISK_TYPES } = require('./_constants')
const { Disposable } = require('promise-toolbox')
module.exports = async function checkChain(handler, path) {
await Disposable.use(function* () {
let vhd
do {
vhd = yield openVhd(handler, path)
path = resolveRelativeFromFile(path, vhd.header.parentUnicodeName)
} while (vhd.footer.diskType !== DISK_TYPES.DYNAMIC)
})
}

View File

@@ -1,11 +0,0 @@
const assert = require('assert')
const { DISK_TYPES, FILE_FORMAT_VERSION, FOOTER_COOKIE, FOOTER_SIZE } = require('./_constants')
module.exports = footer => {
assert.strictEqual(footer.cookie, FOOTER_COOKIE)
assert.strictEqual(footer.dataOffset, FOOTER_SIZE)
assert.strictEqual(footer.fileFormatVersion, FILE_FORMAT_VERSION)
assert(footer.originalSize <= footer.currentSize)
assert(footer.diskType === DISK_TYPES.DIFFERENCING || footer.diskType === DISK_TYPES.DYNAMIC)
}

View File

@@ -1,54 +0,0 @@
const { parseVhdStream } = require('./parseVhdStream.js')
const { VhdDirectory } = require('./Vhd/VhdDirectory.js')
const { Disposable } = require('promise-toolbox')
const { asyncEach } = require('@vates/async-each')
const buildVhd = Disposable.wrap(async function* (handler, path, inputStream, { concurrency, compression }) {
const vhd = yield VhdDirectory.create(handler, path, { compression })
await asyncEach(
parseVhdStream(inputStream),
async function (item) {
switch (item.type) {
case 'footer':
vhd.footer = item.footer
break
case 'header':
vhd.header = item.header
break
case 'parentLocator':
await vhd.writeParentLocator({ ...item, data: item.buffer })
break
case 'block':
await vhd.writeEntireBlock(item)
break
case 'bat':
// it exists but I don't care
break
default:
throw new Error(`unhandled type of block generated by parser : ${item.type} while generating ${path}`)
}
},
{
concurrency,
}
)
await Promise.all([vhd.writeFooter(), vhd.writeHeader(), vhd.writeBlockAllocationTable()])
})
exports.createVhdDirectoryFromStream = async function createVhdDirectoryFromStream(
handler,
path,
inputStream,
{ validator, concurrency = 16, compression } = {}
) {
try {
await buildVhd(handler, path, inputStream, { concurrency, compression })
if (validator !== undefined) {
await validator.call(this, path)
}
} catch (error) {
// cleanup on error
await handler.rmtree(path)
throw error
}
}

View File

@@ -1,16 +0,0 @@
exports.chainVhd = require('./chain')
exports.checkFooter = require('./checkFooter')
exports.checkVhdChain = require('./checkChain')
exports.createReadableRawStream = require('./createReadableRawStream')
exports.createReadableSparseStream = require('./createReadableSparseStream')
exports.createVhdStreamWithLength = require('./createVhdStreamWithLength')
exports.createVhdDirectoryFromStream = require('./createVhdDirectoryFromStream').createVhdDirectoryFromStream
exports.mergeVhd = require('./merge')
exports.peekFooterFromVhdStream = require('./peekFooterFromVhdStream')
exports.openVhd = require('./openVhd').openVhd
exports.parseVhdStream = require('./parseVhdStream').parseVhdStream
exports.VhdAbstract = require('./Vhd/VhdAbstract').VhdAbstract
exports.VhdDirectory = require('./Vhd/VhdDirectory').VhdDirectory
exports.VhdFile = require('./Vhd/VhdFile').VhdFile
exports.VhdSynthetic = require('./Vhd/VhdSynthetic').VhdSynthetic
exports.Constants = require('./_constants')

View File

@@ -1,157 +0,0 @@
/* eslint-env jest */
const fs = require('fs-extra')
const rimraf = require('rimraf')
const tmp = require('tmp')
const { getHandler } = require('@xen-orchestra/fs')
const { pFromCallback } = require('promise-toolbox')
const { VhdFile, chainVhd, mergeVhd: vhdMerge } = require('./index')
const { checkFile, createRandomFile, convertFromRawToVhd } = require('./tests/utils')
let tempDir = null
jest.setTimeout(60000)
beforeEach(async () => {
tempDir = await pFromCallback(cb => tmp.dir(cb))
})
afterEach(async () => {
await pFromCallback(cb => rimraf(tempDir, cb))
})
test('merge works in normal cases', async () => {
const mbOfFather = 8
const mbOfChildren = 4
const parentRandomFileName = `${tempDir}/randomfile`
const childRandomFileName = `${tempDir}/small_randomfile`
const parentFileName = `${tempDir}/parent.vhd`
const child1FileName = `${tempDir}/child1.vhd`
const handler = getHandler({ url: 'file://' })
await createRandomFile(parentRandomFileName, mbOfFather)
await convertFromRawToVhd(parentRandomFileName, parentFileName)
await createRandomFile(childRandomFileName, mbOfChildren)
await convertFromRawToVhd(childRandomFileName, child1FileName)
await chainVhd(handler, parentFileName, handler, child1FileName, true)
// merge
await vhdMerge(handler, parentFileName, handler, child1FileName)
// check that vhd is still valid
await checkFile(parentFileName)
const parentVhd = new VhdFile(handler, parentFileName)
await parentVhd.readHeaderAndFooter()
await parentVhd.readBlockAllocationTable()
let offset = 0
// check that the data are the same as source
for await (const block of parentVhd.blocks()) {
const blockContent = block.data
const file = offset < mbOfChildren * 1024 * 1024 ? childRandomFileName : parentRandomFileName
const buffer = Buffer.alloc(blockContent.length)
const fd = await fs.open(file, 'r')
await fs.read(fd, buffer, 0, buffer.length, offset)
expect(buffer.equals(blockContent)).toEqual(true)
offset += parentVhd.header.blockSize
}
})
test('it can resume a merge ', async () => {
const mbOfFather = 8
const mbOfChildren = 4
const parentRandomFileName = `${tempDir}/randomfile`
const childRandomFileName = `${tempDir}/small_randomfile`
const handler = getHandler({ url: `file://${tempDir}` })
await createRandomFile(`${tempDir}/randomfile`, mbOfFather)
await convertFromRawToVhd(`${tempDir}/randomfile`, `${tempDir}/parent.vhd`)
const parentVhd = new VhdFile(handler, 'parent.vhd')
await parentVhd.readHeaderAndFooter()
await createRandomFile(`${tempDir}/small_randomfile`, mbOfChildren)
await convertFromRawToVhd(`${tempDir}/small_randomfile`, `${tempDir}/child1.vhd`)
await chainVhd(handler, 'parent.vhd', handler, 'child1.vhd', true)
const childVhd = new VhdFile(handler, 'child1.vhd')
await childVhd.readHeaderAndFooter()
await handler.writeFile(
'.parent.vhd.merge.json',
JSON.stringify({
parent: {
header: parentVhd.header.checksum,
},
child: {
header: 'NOT CHILD HEADER ',
},
})
)
// expect merge to fail since child header is not ok
await expect(async () => await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd')).rejects.toThrow()
await handler.unlink('.parent.vhd.merge.json')
await handler.writeFile(
'.parent.vhd.merge.json',
JSON.stringify({
parent: {
header: 'NOT PARENT HEADER',
},
child: {
header: childVhd.header.checksum,
},
})
)
// expect merge to fail since parent header is not ok
await expect(async () => await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd')).rejects.toThrow()
// break the end footer of parent
const size = await handler.getSize('parent.vhd')
const fd = await handler.openFile('parent.vhd', 'r+')
const buffer = Buffer.alloc(512, 0)
// add a fake footer at the end
handler.write(fd, buffer, size)
await handler.closeFile(fd)
// check vhd should fail
await expect(async () => await parentVhd.readHeaderAndFooter()).rejects.toThrow()
await handler.unlink('.parent.vhd.merge.json')
await handler.writeFile(
'.parent.vhd.merge.json',
JSON.stringify({
parent: {
header: parentVhd.header.checksum,
},
child: {
header: childVhd.header.checksum,
},
currentBlock: 1,
})
)
// really merge
await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd')
// reload header footer and block allocation table , they should succed
await parentVhd.readHeaderAndFooter()
await parentVhd.readBlockAllocationTable()
let offset = 0
// check that the data are the same as source
for await (const block of parentVhd.blocks()) {
const blockContent = block.data
// first block is marked as already merged, should not be modified
// second block should come from children
// then two block only in parent
const file = block.id === 1 ? childRandomFileName : parentRandomFileName
const buffer = Buffer.alloc(blockContent.length)
const fd = await fs.open(file, 'r')
await fs.read(fd, buffer, 0, buffer.length, offset)
expect(buffer.equals(blockContent)).toEqual(true)
offset += parentVhd.header.blockSize
}
})

View File

@@ -1,127 +0,0 @@
// TODO: remove once completely merged in vhd.js
const assert = require('assert')
const noop = require('./_noop')
const { createLogger } = require('@xen-orchestra/log')
const { limitConcurrency } = require('limit-concurrency-decorator')
const { openVhd } = require('./openVhd')
const { basename, dirname } = require('path')
const { DISK_TYPES } = require('./_constants')
const { Disposable } = require('promise-toolbox')
const { warn } = createLogger('vhd-lib:merge')
// Merge vhd child into vhd parent.
//
// TODO: rename the VHD file during the merge
module.exports = limitConcurrency(2)(async function merge(
parentHandler,
parentPath,
childHandler,
childPath,
{ onProgress = noop } = {}
) {
const mergeStatePath = dirname(parentPath) + '/' + '.' + basename(parentPath) + '.merge.json'
return await Disposable.use(async function* () {
let mergeState = await parentHandler
.readFile(mergeStatePath)
.then(content => {
const state = JSON.parse(content)
// ensure the correct merge will be continued
assert.strictEqual(parentVhd.header.checksum, state.parent.header)
assert.strictEqual(childVhd.header.checksum, state.child.header)
return state
})
.catch(error => {
if (error.code !== 'ENOENT') {
warn('problem while checking the merge state', { error })
}
})
// during merging, the end footer of the parent can be overwritten by new blocks
// we should use it as a way to check vhd health
const parentVhd = yield openVhd(parentHandler, parentPath, {
flags: 'r+',
checkSecondFooter: mergeState === undefined,
})
const childVhd = yield openVhd(childHandler, childPath)
if (mergeState === undefined) {
assert.strictEqual(childVhd.header.blockSize, parentVhd.header.blockSize)
const parentDiskType = parentVhd.footer.diskType
assert(parentDiskType === DISK_TYPES.DIFFERENCING || parentDiskType === DISK_TYPES.DYNAMIC)
assert.strictEqual(childVhd.footer.diskType, DISK_TYPES.DIFFERENCING)
}
// Read allocation table of child/parent.
await Promise.all([parentVhd.readBlockAllocationTable(), childVhd.readBlockAllocationTable()])
const { maxTableEntries } = childVhd.header
if (mergeState === undefined) {
await parentVhd.ensureBatSize(childVhd.header.maxTableEntries)
mergeState = {
child: { header: childVhd.header.checksum },
parent: { header: parentVhd.header.checksum },
currentBlock: 0,
mergedDataSize: 0,
}
// finds first allocated block for the 2 following loops
while (mergeState.currentBlock < maxTableEntries && !childVhd.containsBlock(mergeState.currentBlock)) {
++mergeState.currentBlock
}
}
// counts number of allocated blocks
let nBlocks = 0
for (let block = mergeState.currentBlock; block < maxTableEntries; block++) {
if (childVhd.containsBlock(block)) {
nBlocks += 1
}
}
onProgress({ total: nBlocks, done: 0 })
// merges blocks
for (let i = 0; i < nBlocks; ++i, ++mergeState.currentBlock) {
while (!childVhd.containsBlock(mergeState.currentBlock)) {
++mergeState.currentBlock
}
await parentHandler.writeFile(mergeStatePath, JSON.stringify(mergeState), { flags: 'w' }).catch(warn)
mergeState.mergedDataSize += await parentVhd.coalesceBlock(childVhd, mergeState.currentBlock)
onProgress({
total: nBlocks,
done: i + 1,
})
}
// some blocks could have been created or moved in parent : write bat
await parentVhd.writeBlockAllocationTable()
const cFooter = childVhd.footer
const pFooter = parentVhd.footer
pFooter.currentSize = cFooter.currentSize
pFooter.diskGeometry = { ...cFooter.diskGeometry }
pFooter.originalSize = cFooter.originalSize
pFooter.timestamp = cFooter.timestamp
pFooter.uuid = cFooter.uuid
// necessary to update values and to recreate the footer after block
// creation
await parentVhd.writeFooter()
// should be a disposable
parentHandler.unlink(mergeStatePath).catch(warn)
return mergeState.mergedDataSize
})
})

View File

@@ -1,15 +0,0 @@
const { resolveAlias } = require('./_resolveAlias')
const { VhdDirectory } = require('./Vhd/VhdDirectory.js')
const { VhdFile } = require('./Vhd/VhdFile.js')
exports.openVhd = async function openVhd(handler, path, opts) {
const resolved = await resolveAlias(handler, path)
try {
return await VhdFile.open(handler, resolved, opts)
} catch (e) {
if (e.code !== 'EISDIR') {
throw e
}
return await VhdDirectory.open(handler, resolved, opts)
}
}

View File

@@ -1,7 +1,7 @@
{
"private": false,
"name": "vhd-lib",
"version": "3.0.0",
"version": "1.3.0",
"license": "AGPL-3.0-or-later",
"description": "Primitives for VHD file handling",
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-lib",
@@ -11,11 +11,11 @@
"type": "git",
"url": "https://github.com/vatesfr/xen-orchestra.git"
},
"main": "dist/",
"engines": {
"node": ">=12"
"node": ">=10"
},
"dependencies": {
"@vates/async-each": "^0.1.0",
"@vates/read-chunk": "^0.1.2",
"@xen-orchestra/log": "^0.3.0",
"async-iterator-to-stream": "^1.0.2",
@@ -27,12 +27,25 @@
"uuid": "^8.3.1"
},
"devDependencies": {
"@xen-orchestra/fs": "^0.19.3",
"@babel/cli": "^7.0.0",
"@babel/core": "^7.0.0",
"@babel/preset-env": "^7.0.0",
"@xen-orchestra/fs": "^0.18.0",
"babel-plugin-lodash": "^3.3.2",
"cross-env": "^7.0.2",
"execa": "^5.0.0",
"get-stream": "^6.0.0",
"readable-stream": "^3.0.6",
"rimraf": "^3.0.0",
"tmp": "^0.2.1"
},
"scripts": {
"build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
"clean": "rimraf dist/",
"dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
"prebuild": "yarn run clean",
"predev": "yarn run clean",
"prepublishOnly": "yarn run build",
"postversion": "npm publish"
},
"author": {

View File

@@ -1,134 +0,0 @@
const { BLOCK_UNUSED, FOOTER_SIZE, HEADER_SIZE, SECTOR_SIZE } = require('./_constants')
const { readChunk } = require('@vates/read-chunk')
const assert = require('assert')
const { unpackFooter, unpackHeader, computeFullBlockSize } = require('./Vhd/_utils')
const cappedBufferConcat = (buffers, maxSize) => {
let buffer = Buffer.concat(buffers)
if (buffer.length > maxSize) {
buffer = buffer.slice(buffer.length - maxSize)
}
return buffer
}
exports.parseVhdStream = async function* parseVhdStream(stream) {
let bytesRead = 0
// handle empty space between elements
// ensure we read stream in order
async function read(offset, size) {
assert(bytesRead <= offset, `offset is ${offset} but we already read ${bytesRead} bytes`)
if (bytesRead < offset) {
// empty spaces
await read(bytesRead, offset - bytesRead)
}
const buf = await readChunk(stream, size)
assert.strictEqual(buf.length, size, `read ${buf.length} instead of ${size}`)
bytesRead += size
return buf
}
const bufFooter = await read(0, FOOTER_SIZE)
const footer = unpackFooter(bufFooter)
yield { type: 'footer', footer, offset: 0 }
const bufHeader = await read(FOOTER_SIZE, HEADER_SIZE)
const header = unpackHeader(bufHeader, footer)
yield { type: 'header', header, offset: SECTOR_SIZE }
const blockSize = header.blockSize
assert.strictEqual(blockSize % SECTOR_SIZE, 0)
const fullBlockSize = computeFullBlockSize(blockSize)
const bitmapSize = fullBlockSize - blockSize
const index = []
for (const parentLocatorId in header.parentLocatorEntry) {
const parentLocatorEntry = header.parentLocatorEntry[parentLocatorId]
// empty parent locator entry, does not exist in the content
if (parentLocatorEntry.platformDataSpace === 0) {
continue
}
index.push({
...parentLocatorEntry,
type: 'parentLocator',
offset: parentLocatorEntry.platformDataOffset,
size: parentLocatorEntry.platformDataLength,
id: parentLocatorId,
})
}
const batOffset = header.tableOffset
const batSize = Math.max(1, Math.ceil((header.maxTableEntries * 4) / SECTOR_SIZE)) * SECTOR_SIZE
index.push({
type: 'bat',
offset: batOffset,
size: batSize,
})
// sometimes some parent locator are before the BAT
index.sort((a, b) => a.offset - b.offset)
while (index.length > 0) {
const item = index.shift()
const buffer = await read(item.offset, item.size)
item.buffer = buffer
const { type } = item
if (type === 'bat') {
// found the BAT : read it and add block to index
let blockCount = 0
for (let blockCounter = 0; blockCounter < header.maxTableEntries; blockCounter++) {
const batEntrySector = buffer.readUInt32BE(blockCounter * 4)
// unallocated block, no need to export it
if (batEntrySector !== BLOCK_UNUSED) {
const batEntryBytes = batEntrySector * SECTOR_SIZE
// ensure the block is not before the bat
assert.ok(batEntryBytes >= batOffset + batSize)
index.push({
type: 'block',
id: blockCounter,
offset: batEntryBytes,
size: fullBlockSize,
})
blockCount++
}
}
// sort again index to ensure block and parent locator are in the right order
index.sort((a, b) => a.offset - b.offset)
item.blockCount = blockCount
} else if (type === 'block') {
item.bitmap = buffer.slice(0, bitmapSize)
item.data = buffer.slice(bitmapSize)
}
yield item
}
/**
* the second footer is at filesize - 512 , there can be empty spaces between last block
* and the start of the footer
*
* we read till the end of the stream, and use the last 512 bytes as the footer
*/
const bufFooterEnd = await readLastSector(stream)
assert(bufFooter.equals(bufFooterEnd), 'footer1 !== footer2')
}
function readLastSector(stream) {
return new Promise((resolve, reject) => {
let bufFooterEnd = Buffer.alloc(0)
stream.on('data', chunk => {
if (chunk.length > 0) {
bufFooterEnd = cappedBufferConcat([bufFooterEnd, chunk], SECTOR_SIZE)
}
})
stream.on('end', () => resolve(bufFooterEnd))
stream.on('error', reject)
})
}

View File

@@ -1,11 +0,0 @@
const { readChunk } = require('@vates/read-chunk')
const { FOOTER_SIZE } = require('./_constants')
const { fuFooter } = require('./_structs')
module.exports = async function peekFooterFromStream(stream) {
const footerBuffer = await readChunk(stream, FOOTER_SIZE)
const footer = fuFooter.unpack(footerBuffer)
stream.unshift(footerBuffer)
return footer
}

View File

@@ -0,0 +1,138 @@
/* eslint-env jest */
import rimraf from 'rimraf'
import tmp from 'tmp'
import fs from 'fs-extra'
import { getSyncedHandler } from '@xen-orchestra/fs'
import { Disposable, pFromCallback } from 'promise-toolbox'
import { openVhd } from '../index'
import { createRandomFile, convertFromRawToVhd, createRandomVhdDirectory } from '../tests/utils'
import { VhdAbstract } from './VhdAbstract'
let tempDir
jest.setTimeout(60000)
beforeEach(async () => {
tempDir = await pFromCallback(cb => tmp.dir(cb))
})
afterEach(async () => {
await pFromCallback(cb => rimraf(tempDir, cb))
})
test('It creates an alias', async () => {
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: 'file://' + tempDir })
const aliasPath = `alias/alias.alias.vhd`
const aliasFsPath = `${tempDir}/${aliasPath}`
await fs.mkdirp(`${tempDir}/alias`)
const testOneCombination = async ({ targetPath, targetContent }) => {
await VhdAbstract.createAlias(handler, aliasPath, targetPath)
// alias file is created
expect(await fs.exists(aliasFsPath)).toEqual(true)
// content is the target path relative to the alias location
const content = await fs.readFile(aliasFsPath, 'utf-8')
expect(content).toEqual(targetContent)
// create alias fails if alias already exists, remove it before next loop step
await fs.unlink(aliasFsPath)
}
const combinations = [
{ targetPath: `targets.vhd`, targetContent: `../targets.vhd` },
{ targetPath: `alias/targets.vhd`, targetContent: `targets.vhd` },
{ targetPath: `alias/sub/targets.vhd`, targetContent: `sub/targets.vhd` },
{ targetPath: `sibling/targets.vhd`, targetContent: `../sibling/targets.vhd` },
]
for (const { targetPath, targetContent } of combinations) {
await testOneCombination({ targetPath, targetContent })
}
})
})
test('alias must have *.alias.vhd extension', async () => {
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
const aliasPath = 'invalidalias.vhd'
const targetPath = 'targets.vhd'
expect(async () => await VhdAbstract.createAlias(handler, aliasPath, targetPath)).rejects.toThrow()
expect(await fs.exists(aliasPath)).toEqual(false)
})
})
test('alias must not be chained', async () => {
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
const aliasPath = 'valid.alias.vhd'
const targetPath = 'an.other.valid.alias.vhd'
expect(async () => await VhdAbstract.createAlias(handler, aliasPath, targetPath)).rejects.toThrow()
expect(await fs.exists(aliasPath)).toEqual(false)
})
})
test('It rename and unlink a VHDFile', async () => {
const initalSize = 4
const rawFileName = `${tempDir}/randomfile`
await createRandomFile(rawFileName, initalSize)
await convertFromRawToVhd(rawFileName, `${tempDir}/randomfile.vhd`)
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
const { size } = await fs.stat(`${tempDir}/randomfile.vhd`)
await VhdAbstract.rename(handler, 'randomfile.vhd', 'renamed.vhd')
expect(await fs.exists(`${tempDir}/randomfile.vhd`)).toEqual(false)
const { size: renamedSize } = await fs.stat(`${tempDir}/renamed.vhd`)
expect(size).toEqual(renamedSize)
await VhdAbstract.unlink(handler, 'renamed.vhd')
expect(await fs.exists(`${tempDir}/renamed.vhd`)).toEqual(false)
})
})
test('It rename and unlink a VhdDirectory', async () => {
const initalSize = 4
const vhdDirectory = `${tempDir}/randomfile.dir`
await createRandomVhdDirectory(vhdDirectory, initalSize)
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
const vhd = yield openVhd(handler, 'randomfile.dir')
expect(vhd.header.cookie).toEqual('cxsparse')
expect(vhd.footer.cookie).toEqual('conectix')
await VhdAbstract.rename(handler, 'randomfile.dir', 'renamed.vhd')
expect(await fs.exists(`${tempDir}/randomfile.dir`)).toEqual(false)
await VhdAbstract.unlink(handler, `renamed.vhd`)
expect(await fs.exists(`${tempDir}/renamed.vhd`)).toEqual(false)
})
})
test('It create , rename and unlink alias', async () => {
const initalSize = 4
const rawFileName = `${tempDir}/randomfile`
await createRandomFile(rawFileName, initalSize)
const vhdFileName = `${tempDir}/randomfile.vhd`
await convertFromRawToVhd(rawFileName, vhdFileName)
const aliasFileName = `${tempDir}/aliasFileName.alias.vhd`
const aliasFileNameRenamed = `${tempDir}/aliasFileNameRenamed.alias.vhd`
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
await VhdAbstract.createAlias(handler, 'aliasFileName.alias.vhd', 'randomfile.vhd')
expect(await fs.exists(aliasFileName)).toEqual(true)
expect(await fs.exists(vhdFileName)).toEqual(true)
await VhdAbstract.rename(handler, 'aliasFileName.alias.vhd', 'aliasFileNameRenamed.alias.vhd')
expect(await fs.exists(aliasFileName)).toEqual(false)
expect(await fs.exists(vhdFileName)).toEqual(true)
expect(await fs.exists(aliasFileNameRenamed)).toEqual(true)
await VhdAbstract.unlink(handler, 'aliasFileNameRenamed.alias.vhd')
expect(await fs.exists(aliasFileName)).toEqual(false)
expect(await fs.exists(vhdFileName)).toEqual(false)
expect(await fs.exists(aliasFileNameRenamed)).toEqual(false)
})
})

View File

@@ -1,48 +1,29 @@
const {
computeBatSize,
computeFullBlockSize,
computeSectorOfBitmap,
computeSectorsPerBlock,
sectorsToBytes,
} = require('./_utils')
const {
ALIAS_MAX_PATH_LENGTH,
PLATFORMS,
SECTOR_SIZE,
PARENT_LOCATOR_ENTRIES,
FOOTER_SIZE,
HEADER_SIZE,
BLOCK_UNUSED,
} = require('../_constants')
const assert = require('assert')
const path = require('path')
const asyncIteratorToStream = require('async-iterator-to-stream')
const { checksumStruct, fuFooter, fuHeader } = require('../_structs')
const { isVhdAlias, resolveAlias } = require('../_resolveAlias')
import { computeBatSize, sectorsRoundUpNoZero, sectorsToBytes } from './_utils'
import { PLATFORM_NONE, SECTOR_SIZE, PLATFORM_W2KU, PARENT_LOCATOR_ENTRIES } from '../_constants'
import { resolveAlias, isVhdAlias } from '../_resolveAlias'
exports.VhdAbstract = class VhdAbstract {
get bitmapSize() {
return sectorsToBytes(this.sectorsOfBitmap)
}
import assert from 'assert'
import path from 'path'
get fullBlockSize() {
return computeFullBlockSize(this.header.blockSize)
}
get sectorsOfBitmap() {
return computeSectorOfBitmap(this.header.blockSize)
}
get sectorsPerBlock() {
return computeSectorsPerBlock(this.header.blockSize)
}
export class VhdAbstract {
#header
bitmapSize
footer
fullBlockSize
sectorsOfBitmap
sectorsPerBlock
get header() {
throw new Error('get header is not implemented')
assert.notStrictEqual(this.#header, undefined, `header must be read before it's used`)
return this.#header
}
get footer() {
throw new Error('get footer not implemented')
set header(header) {
this.#header = header
this.sectorsPerBlock = header.blockSize / SECTOR_SIZE
this.sectorsOfBitmap = sectorsRoundUpNoZero(this.sectorsPerBlock >> 3)
this.fullBlockSize = sectorsToBytes(this.sectorsOfBitmap + this.sectorsPerBlock)
this.bitmapSize = sectorsToBytes(this.sectorsOfBitmap)
}
/**
@@ -102,10 +83,8 @@ exports.VhdAbstract = class VhdAbstract {
*
* @returns {number} the merged data size
*/
async coalesceBlock(child, blockId) {
const block = await child.readBlock(blockId)
await this.writeEntireBlock(block)
return block.data.length
coalesceBlock(child, blockId) {
throw new Error(`coalescing the block ${blockId} from ${child} is not implemented`)
}
/**
@@ -138,7 +117,7 @@ exports.VhdAbstract = class VhdAbstract {
return computeBatSize(this.header.maxTableEntries)
}
async writeParentLocator({ id, platformCode = PLATFORMS.NONE, data = Buffer.alloc(0) }) {
async writeParentLocator({ id, platformCode = PLATFORM_NONE, data = Buffer.alloc(0) }) {
assert(id >= 0, 'parent Locator id must be a positive number')
assert(id < PARENT_LOCATOR_ENTRIES, `parent Locator id must be less than ${PARENT_LOCATOR_ENTRIES}`)
@@ -147,7 +126,7 @@ exports.VhdAbstract = class VhdAbstract {
const entry = this.header.parentLocatorEntry[id]
const dataSpaceSectors = Math.ceil(data.length / SECTOR_SIZE)
entry.platformCode = platformCode
entry.platformDataSpace = dataSpaceSectors
entry.platformDataSpace = dataSpaceSectors * SECTOR_SIZE
entry.platformDataLength = data.length
}
@@ -167,14 +146,14 @@ exports.VhdAbstract = class VhdAbstract {
async setUniqueParentLocator(fileNameString) {
await this.writeParentLocator({
id: 0,
platformCode: PLATFORMS.W2KU,
platformCode: PLATFORM_W2KU,
data: Buffer.from(fileNameString, 'utf16le'),
})
for (let i = 1; i < PARENT_LOCATOR_ENTRIES; i++) {
await this.writeParentLocator({
id: i,
platformCode: PLATFORMS.NONE,
platformCode: PLATFORM_NONE,
data: Buffer.alloc(0),
})
}
@@ -190,10 +169,6 @@ exports.VhdAbstract = class VhdAbstract {
}
static async rename(handler, sourcePath, targetPath) {
try {
// delete target if it already exists
await VhdAbstract.unlink(handler, targetPath)
} catch (e) {}
await handler.rename(sourcePath, targetPath)
}
@@ -227,109 +202,6 @@ exports.VhdAbstract = class VhdAbstract {
const aliasDir = path.dirname(path.resolve('/', aliasPath))
// only store the relative path from alias to target
const relativePathToTarget = path.relative(aliasDir, path.resolve('/', targetPath))
if (relativePathToTarget.length > ALIAS_MAX_PATH_LENGTH) {
throw new Error(
`Alias relative path ${relativePathToTarget} is too long : ${relativePathToTarget.length} chars, max is ${ALIAS_MAX_PATH_LENGTH}`
)
}
await handler.writeFile(aliasPath, relativePathToTarget)
}
stream() {
const { footer, batSize } = this
const { ...header } = this.header // copy since we don't ant to modifiy the current header
const rawFooter = fuFooter.pack(footer)
checksumStruct(rawFooter, fuFooter)
// update them in header
// update checksum in header
let offset = FOOTER_SIZE + HEADER_SIZE + batSize
const rawHeader = fuHeader.pack(header)
checksumStruct(rawHeader, fuHeader)
// add parentlocator size
for (let i = 0; i < PARENT_LOCATOR_ENTRIES; i++) {
header.parentLocatorEntry[i] = {
...header.parentLocatorEntry[i],
platformDataOffset: offset,
}
offset += header.parentLocatorEntry[i].platformDataSpace * SECTOR_SIZE
}
assert.strictEqual(offset % SECTOR_SIZE, 0)
const bat = Buffer.allocUnsafe(batSize)
let offsetSector = offset / SECTOR_SIZE
const blockSizeInSectors = this.fullBlockSize / SECTOR_SIZE
let fileSize = offsetSector * SECTOR_SIZE + FOOTER_SIZE /* the footer at the end */
// compute BAT , blocks starts after parent locator entries
for (let i = 0; i < header.maxTableEntries; i++) {
if (this.containsBlock(i)) {
bat.writeUInt32BE(offsetSector, i * 4)
offsetSector += blockSizeInSectors
fileSize += this.fullBlockSize
} else {
bat.writeUInt32BE(BLOCK_UNUSED, i * 4)
}
}
const self = this
async function* iterator() {
yield rawFooter
yield rawHeader
yield bat
// yield parent locator
for (let i = 0; i < PARENT_LOCATOR_ENTRIES; i++) {
const space = header.parentLocatorEntry[i].platformDataSpace * SECTOR_SIZE
if (space > 0) {
const data = (await self.readParentLocator(i)).data
// align data to a sector
const buffer = Buffer.alloc(space, 0)
data.copy(buffer)
yield buffer
}
}
// yield all blocks
// since contains() can be costly for synthetic vhd, use the computed bat
for (let i = 0; i < header.maxTableEntries; i++) {
if (bat.readUInt32BE(i * 4) !== BLOCK_UNUSED) {
const block = await self.readBlock(i)
yield block.buffer
}
}
// yield footer again
yield rawFooter
}
const stream = asyncIteratorToStream(iterator())
stream.length = fileSize
return stream
}
rawContent() {
const { header, footer } = this
const { blockSize } = header
const self = this
async function* iterator() {
const nBlocks = header.maxTableEntries
let remainingSize = footer.currentSize
const EMPTY = Buffer.alloc(blockSize, 0)
for (let blockId = 0; blockId < nBlocks; ++blockId) {
let buffer = self.containsBlock(blockId) ? (await self.readBlock(blockId)).data : EMPTY
// the last block can be truncated since raw size is not a multiple of blockSize
buffer = remainingSize < blockSize ? buffer.slice(0, remainingSize) : buffer
remainingSize -= blockSize
yield buffer
}
}
const stream = asyncIteratorToStream(iterator())
stream.length = footer.currentSize
return stream
}
}

View File

@@ -1,67 +1,15 @@
const { unpackHeader, unpackFooter, sectorsToBytes } = require('./_utils')
const { createLogger } = require('@xen-orchestra/log')
const { fuFooter, fuHeader, checksumStruct } = require('../_structs')
const { test, set: setBitmap } = require('../_bitmap')
const { VhdAbstract } = require('./VhdAbstract')
const assert = require('assert')
const promisify = require('promise-toolbox/promisify')
const zlib = require('zlib')
import { buildHeader, buildFooter } from './_utils'
import { createLogger } from '@xen-orchestra/log'
import { fuFooter, fuHeader, checksumStruct } from '../_structs'
import { test, set as setBitmap } from '../_bitmap'
import { VhdAbstract } from './VhdAbstract'
import assert from 'assert'
const { debug } = createLogger('vhd-lib:VhdDirectory')
const NULL_COMPRESSOR = {
compress: buffer => buffer,
decompress: buffer => buffer,
baseOptions: {},
}
const COMPRESSORS = {
gzip: {
compress: (
gzip => buffer =>
gzip(buffer, { level: zlib.constants.Z_BEST_SPEED })
)(promisify(zlib.gzip)),
decompress: promisify(zlib.gunzip),
},
brotli: {
compress: (
brotliCompress => buffer =>
brotliCompress(buffer, {
params: {
[zlib.constants.BROTLI_PARAM_QUALITY]: zlib.constants.BROTLI_MIN_QUALITY,
},
})
)(promisify(zlib.brotliCompress)),
decompress: promisify(zlib.brotliDecompress),
},
}
// inject identifiers
for (const id of Object.keys(COMPRESSORS)) {
COMPRESSORS[id].id = id
}
function getCompressor(compressorType) {
if (compressorType === undefined) {
return NULL_COMPRESSOR
}
const compressor = COMPRESSORS[compressorType]
if (compressor === undefined) {
throw new Error(`Compression type ${compressorType} is not supported`)
}
return compressor
}
// ===================================================================
// Directory format
// <path>
// ├─ chunk-filters.json
// │ Ordered array of filters that have been applied before writing chunks.
// │ These filters needs to be applied in reverse order to read them.
// │
// ├─ header // raw content of the header
// ├─ footer // raw content of the footer
// ├─ bat // bit array. A zero bit indicates at a position that this block is not present
@@ -70,24 +18,16 @@ function getCompressor(compressorType) {
// └─ <the first to {blockId.length -3} numbers of blockId >
// └─ <the three last numbers of blockID > // block content.
exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
export class VhdDirectory extends VhdAbstract {
#uncheckedBlockTable
#header
footer
#compressor
get compressionType() {
return this.#compressor.id
}
set header(header) {
this.#header = header
super.header = header
this.#blockTable = Buffer.alloc(header.maxTableEntries)
}
get header() {
assert.notStrictEqual(this.#header, undefined, `header must be read before it's used`)
return this.#header
return super.header
}
get #blockTable() {
@@ -99,8 +39,8 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
this.#uncheckedBlockTable = blockTable
}
static async open(handler, path, { flags = 'r+' } = {}) {
const vhd = new VhdDirectory(handler, path, { flags })
static async open(handler, path) {
const vhd = new VhdDirectory(handler, path)
// openning a file for reading does not trigger EISDIR as long as we don't really read from it :
// https://man7.org/linux/man-pages/man2/open.2.html
@@ -114,21 +54,19 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
}
}
static async create(handler, path, { flags = 'wx+', compression } = {}) {
static async create(handler, path) {
await handler.mkdir(path)
const vhd = new VhdDirectory(handler, path, { flags, compression })
const vhd = new VhdDirectory(handler, path)
return {
dispose: () => {},
value: vhd,
}
}
constructor(handler, path, opts) {
constructor(handler, path) {
super()
this._handler = handler
this._path = path
this._opts = opts
this.#compressor = getCompressor(opts?.compression)
}
async readBlockAllocationTable() {
@@ -140,29 +78,34 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
return test(this.#blockTable, blockId)
}
_getChunkPath(partName) {
getChunkPath(partName) {
return this._path + '/' + partName
}
async _readChunk(partName) {
// here we can implement compression and / or crypto
const buffer = await this._handler.readFile(this._getChunkPath(partName))
const buffer = await this._handler.readFile(this.getChunkPath(partName))
const uncompressed = await this.#compressor.decompress(buffer)
return {
buffer: uncompressed,
buffer: Buffer.from(buffer),
}
}
async _writeChunk(partName, buffer) {
assert.notStrictEqual(
this._opts?.flags,
'r',
`Can't write a chunk ${partName} in ${this._path} with read permission`
)
assert(Buffer.isBuffer(buffer))
// here we can implement compression and / or crypto
const compressed = await this.#compressor.compress(buffer)
return this._handler.outputFile(this._getChunkPath(partName), compressed, this._opts)
// chunks can be in sub directories : create direcotries if necessary
const pathParts = partName.split('/')
let currentPath = this._path
// the last one is the file name
for (let i = 0; i < pathParts.length - 1; i++) {
currentPath += '/' + pathParts[i]
await this._handler.mkdir(currentPath)
}
return this._handler.writeFile(this.getChunkPath(partName), buffer)
}
// put block in subdirectories to limit impact when doing directory listing
@@ -173,12 +116,10 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
}
async readHeaderAndFooter() {
// we need to know if thre is compression before reading headers
await this.#readChunkFilters()
const { buffer: bufHeader } = await this._readChunk('header')
const { buffer: bufFooter } = await this._readChunk('footer')
const footer = unpackFooter(bufFooter)
const header = unpackHeader(bufHeader, footer)
const footer = buildFooter(bufFooter)
const header = buildHeader(bufHeader, footer)
this.footer = footer
this.header = header
@@ -211,13 +152,12 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
await this._writeChunk('footer', rawFooter)
}
async writeHeader() {
writeHeader() {
const { header } = this
const rawHeader = fuHeader.pack(header)
header.checksum = checksumStruct(rawHeader, fuHeader)
debug(`Write header (checksum=${header.checksum}). (data=${rawHeader.toString('hex')})`)
await this._writeChunk('header', rawHeader)
await this.#writeChunkFilters()
return this._writeChunk('header', rawHeader)
}
writeBlockAllocationTable() {
@@ -227,22 +167,11 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
return this._writeChunk('bat', this.#blockTable)
}
// only works if data are in the same handler
// only works if data are in the same bucket
// and if the full block is modified in child ( which is the case whit xcp)
// and if the compression type is same on both sides
async coalesceBlock(child, blockId) {
if (
!(child instanceof VhdDirectory) ||
this._handler !== child._handler ||
child.compressionType !== this.compressionType
) {
return super.coalesceBlock(child, blockId)
}
await this._handler.copy(
child._getChunkPath(child._getBlockPath(blockId)),
this._getChunkPath(this._getBlockPath(blockId))
)
return sectorsToBytes(this.sectorsPerBlock)
coalesceBlock(child, blockId) {
this._handler.copy(child.getChunkPath(blockId), this.getChunkPath(blockId))
}
async writeEntireBlock(block) {
@@ -258,24 +187,4 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
await this._writeChunk('parentLocatorEntry' + id, data)
this.header.parentLocatorEntry[id].platformDataOffset = 0
}
async #writeChunkFilters() {
const compressionType = this.compressionType
const path = this._path + '/chunk-filters.json'
if (compressionType === undefined) {
await this._handler.unlink(path)
} else {
await this._handler.writeFile(path, JSON.stringify([compressionType]))
}
}
async #readChunkFilters() {
const chunkFilters = await this._handler.readFile(this._path + '/chunk-filters.json').then(JSON.parse, error => {
if (error.code === 'ENOENT') {
return []
}
throw error
})
this.#compressor = getCompressor(chunkFilters[0])
}
}

View File

@@ -1,25 +1,18 @@
/* eslint-env jest */
const execa = require('execa')
const fs = require('fs-extra')
const getStream = require('get-stream')
const rimraf = require('rimraf')
const tmp = require('tmp')
const { getHandler } = require('@xen-orchestra/fs')
const { Disposable, pFromCallback } = require('promise-toolbox')
const { randomBytes } = require('crypto')
import execa from 'execa'
import fs from 'fs-extra'
import getStream from 'get-stream'
import rimraf from 'rimraf'
import tmp from 'tmp'
import { getHandler } from '@xen-orchestra/fs'
import { pFromCallback } from 'promise-toolbox'
import { randomBytes } from 'crypto'
const { VhdFile } = require('./VhdFile')
const { openVhd } = require('../openVhd')
import { VhdFile } from './VhdFile'
const { SECTOR_SIZE } = require('../_constants')
const {
checkFile,
createRandomFile,
convertFromRawToVhd,
convertToVhdDirectory,
recoverRawContent,
} = require('../tests/utils')
import { SECTOR_SIZE } from '../_constants'
import { checkFile, createRandomFile, convertFromRawToVhd, recoverRawContent } from '../tests/utils'
let tempDir = null
@@ -33,29 +26,6 @@ afterEach(async () => {
await pFromCallback(cb => rimraf(tempDir, cb))
})
test('respect the checkSecondFooter flag', async () => {
const initalSize = 0
const rawFileName = `${tempDir}/randomfile`
await createRandomFile(rawFileName, initalSize)
const vhdFileName = `${tempDir}/randomfile.vhd`
await convertFromRawToVhd(rawFileName, vhdFileName)
const handler = getHandler({ url: `file://${tempDir}` })
const size = await handler.getSize('randomfile.vhd')
const fd = await handler.openFile('randomfile.vhd', 'r+')
const buffer = Buffer.alloc(512, 0)
// add a fake footer at the end
handler.write(fd, buffer, size)
await handler.closeFile(fd)
// not using openVhd to be able to call readHeaderAndFooter separatly
const vhd = new VhdFile(handler, 'randomfile.vhd')
await expect(async () => await vhd.readHeaderAndFooter()).rejects.toThrow()
await expect(async () => await vhd.readHeaderAndFooter(true)).rejects.toThrow()
await expect(await vhd.readHeaderAndFooter(false)).toEqual(undefined)
})
test('blocks can be moved', async () => {
const initalSize = 4
const rawFileName = `${tempDir}/randomfile`
@@ -88,7 +58,6 @@ test('the BAT MSB is not used for sign', async () => {
// here we are moving the first sector very far in the VHD to prove the BAT doesn't use signed int32
const hugePositionBytes = hugeWritePositionSectors * SECTOR_SIZE
await vhd._freeFirstBlockSpace(hugePositionBytes)
await vhd.writeFooter()
// we recover the data manually for speed reasons.
// fs.write() with offset is way faster than qemu-img when there is a 1.5To
@@ -193,45 +162,3 @@ test('BAT can be extended and blocks moved', async () => {
await recoverRawContent(vhdFileName, recoveredFileName, originalSize)
expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(rawFileName))
})
test('Can coalesce block', async () => {
const initalSize = 4
const parentrawFileName = `${tempDir}/randomfile`
const parentFileName = `${tempDir}/parent.vhd`
await createRandomFile(parentrawFileName, initalSize)
await convertFromRawToVhd(parentrawFileName, parentFileName)
const childrawFileName = `${tempDir}/randomfile`
const childFileName = `${tempDir}/childFile.vhd`
await createRandomFile(childrawFileName, initalSize)
await convertFromRawToVhd(childrawFileName, childFileName)
const childRawDirectoryName = `${tempDir}/randomFile2.vhd`
const childDirectoryFileName = `${tempDir}/childDirFile.vhd`
const childDirectoryName = `${tempDir}/childDir.vhd`
await createRandomFile(childRawDirectoryName, initalSize)
await convertFromRawToVhd(childRawDirectoryName, childDirectoryFileName)
await convertToVhdDirectory(childRawDirectoryName, childDirectoryFileName, childDirectoryName)
await Disposable.use(async function* () {
const handler = getHandler({ url: 'file://' })
const parentVhd = yield openVhd(handler, parentFileName, { flags: 'r+' })
await parentVhd.readBlockAllocationTable()
const childFileVhd = yield openVhd(handler, childFileName)
await childFileVhd.readBlockAllocationTable()
const childDirectoryVhd = yield openVhd(handler, childDirectoryName)
await childDirectoryVhd.readBlockAllocationTable()
await parentVhd.coalesceBlock(childFileVhd, 0)
await parentVhd.writeFooter()
await parentVhd.writeBlockAllocationTable()
let parentBlockData = (await parentVhd.readBlock(0)).data
let childBlockData = (await childFileVhd.readBlock(0)).data
expect(parentBlockData).toEqual(childBlockData)
await parentVhd.coalesceBlock(childDirectoryVhd, 0)
await parentVhd.writeFooter()
await parentVhd.writeBlockAllocationTable()
parentBlockData = (await parentVhd.readBlock(0)).data
childBlockData = (await childDirectoryVhd.readBlock(0)).data
expect(parentBlockData).toEqual(childBlockData)
})
})

View File

@@ -1,18 +1,18 @@
const {
import {
BLOCK_UNUSED,
FOOTER_SIZE,
HEADER_SIZE,
PLATFORMS,
PLATFORM_NONE,
SECTOR_SIZE,
PARENT_LOCATOR_ENTRIES,
} = require('../_constants')
const { computeBatSize, sectorsToBytes, unpackHeader, unpackFooter, BUF_BLOCK_UNUSED } = require('./_utils')
const { createLogger } = require('@xen-orchestra/log')
const { fuFooter, fuHeader, checksumStruct } = require('../_structs')
const { set: mapSetBit } = require('../_bitmap')
const { VhdAbstract } = require('./VhdAbstract')
const assert = require('assert')
const getFirstAndLastBlocks = require('../_getFirstAndLastBlocks')
} from '../_constants'
import { computeBatSize, sectorsToBytes, buildHeader, buildFooter, BUF_BLOCK_UNUSED } from './_utils'
import { createLogger } from '@xen-orchestra/log'
import { fuFooter, fuHeader, checksumStruct } from '../_structs'
import { set as mapSetBit, test as mapTestBit } from '../_bitmap'
import { VhdAbstract } from './VhdAbstract'
import assert from 'assert'
import getFirstAndLastBlocks from '../_getFirstAndLastBlocks'
const { debug } = createLogger('vhd-lib:VhdFile')
@@ -50,10 +50,8 @@ const { debug } = createLogger('vhd-lib:VhdFile')
// - parentLocatorSize(i) = header.parentLocatorEntry[i].platformDataSpace * sectorSize
// - sectorSize = 512
exports.VhdFile = class VhdFile extends VhdAbstract {
export class VhdFile extends VhdAbstract {
#uncheckedBlockTable
#header
footer
get #blockTable() {
assert.notStrictEqual(this.#uncheckedBlockTable, undefined, 'Block table must be initialized before access')
@@ -69,7 +67,7 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
}
set header(header) {
this.#header = header
super.header = header
const size = this.batSize
this.#blockTable = Buffer.alloc(size)
for (let i = 0; i < this.header.maxTableEntries; i++) {
@@ -77,26 +75,26 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
}
}
get header() {
return this.#header
return super.header
}
static async open(handler, path, { flags, checkSecondFooter = true } = {}) {
const fd = await handler.openFile(path, flags ?? 'r+')
static async open(handler, path) {
const fd = await handler.openFile(path, 'r+')
const vhd = new VhdFile(handler, fd)
// openning a file for reading does not trigger EISDIR as long as we don't really read from it :
// https://man7.org/linux/man-pages/man2/open.2.html
// EISDIR pathname refers to a directory and the access requested
// involved writing (that is, O_WRONLY or O_RDWR is set).
// reading the header ensure we have a well formed file immediatly
await vhd.readHeaderAndFooter(checkSecondFooter)
await vhd.readHeaderAndFooter()
return {
dispose: () => handler.closeFile(fd),
value: vhd,
}
}
static async create(handler, path, { flags } = {}) {
const fd = await handler.openFile(path, flags ?? 'wx')
static async create(handler, path) {
const fd = await handler.openFile(path, 'wx')
const vhd = new VhdFile(handler, fd)
return {
dispose: () => handler.closeFile(fd),
@@ -131,7 +129,7 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
for (let i = 0; i < PARENT_LOCATOR_ENTRIES; i++) {
const entry = header.parentLocatorEntry[i]
if (entry.platformCode !== PLATFORMS.NONE) {
if (entry.platformCode !== PLATFORM_NONE) {
end = Math.max(end, entry.platformDataOffset + sectorsToBytes(entry.platformDataSpace))
}
}
@@ -179,8 +177,8 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
const bufFooter = buf.slice(0, FOOTER_SIZE)
const bufHeader = buf.slice(FOOTER_SIZE)
const footer = unpackFooter(bufFooter)
const header = unpackHeader(bufHeader, footer)
const footer = buildFooter(bufFooter)
const header = buildHeader(bufHeader, footer)
if (checkSecondFooter) {
const size = await this._handler.getSize(this._path)
@@ -345,6 +343,47 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
)
}
async coalesceBlock(child, blockId) {
const block = await child.readBlock(blockId)
const { bitmap, data } = block
debug(`coalesceBlock block=${blockId}`)
// For each sector of block data...
const { sectorsPerBlock } = child
let parentBitmap = null
for (let i = 0; i < sectorsPerBlock; i++) {
// If no changes on one sector, skip.
if (!mapTestBit(bitmap, i)) {
continue
}
let endSector = i + 1
// Count changed sectors.
while (endSector < sectorsPerBlock && mapTestBit(bitmap, endSector)) {
++endSector
}
// Write n sectors into parent.
debug(`coalesceBlock: write sectors=${i}...${endSector}`)
const isFullBlock = i === 0 && endSector === sectorsPerBlock
if (isFullBlock) {
await this.writeEntireBlock(block)
} else {
if (parentBitmap === null) {
parentBitmap = (await this.readBlock(blockId, true)).bitmap
}
await this._writeBlockSectors(block, i, endSector, parentBitmap)
}
i = endSector
}
// Return the merged data size
return data.length
}
// Write a context footer. (At the end and beginning of a vhd file.)
async writeFooter(onlyEndFooter = false) {
const { footer } = this
@@ -423,7 +462,7 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
async _readParentLocatorData(parentLocatorId) {
const { platformDataOffset, platformDataLength } = this.header.parentLocatorEntry[parentLocatorId]
if (platformDataLength > 0) {
return await this._read(platformDataOffset, platformDataLength)
return (await this._read(platformDataOffset, platformDataLength)).buffer
}
return Buffer.alloc(0)
}
@@ -435,8 +474,7 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
// reset offset if data is empty
header.parentLocatorEntry[parentLocatorId].platformDataOffset = 0
} else {
const space = header.parentLocatorEntry[parentLocatorId].platformDataSpace * SECTOR_SIZE
if (data.length <= space) {
if (data.length <= header.parentLocatorEntry[parentLocatorId].platformDataSpace) {
// new parent locator length is smaller than available space : keep it in place
position = header.parentLocatorEntry[parentLocatorId].platformDataOffset
} else {
@@ -451,7 +489,7 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
// move the first(s) block(s) at the end of the data
// move the parent locator to the precedent position of the first block
const { firstSector } = firstAndLastBlocks
await this._freeFirstBlockSpace(space)
await this._freeFirstBlockSpace(header.parentLocatorEntry[parentLocatorId].platformDataSpace)
position = sectorsToBytes(firstSector)
}
}
@@ -459,8 +497,4 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
header.parentLocatorEntry[parentLocatorId].platformDataOffset = position
}
}
async getSize() {
return await this._handler.getSize(this._path)
}
}

View File

@@ -0,0 +1,52 @@
import assert from 'assert'
import { BLOCK_UNUSED, SECTOR_SIZE } from '../_constants'
import { fuFooter, fuHeader, checksumStruct, unpackField } from '../_structs'
import checkFooter from '../checkFooter'
import checkHeader from '../_checkHeader'
export const computeBatSize = entries => sectorsToBytes(sectorsRoundUpNoZero(entries * 4))
// Sectors conversions.
export const sectorsRoundUpNoZero = bytes => Math.ceil(bytes / SECTOR_SIZE) || 1
export const sectorsToBytes = sectors => sectors * SECTOR_SIZE
export const assertChecksum = (name, buf, struct) => {
const actual = unpackField(struct.fields.checksum, buf)
const expected = checksumStruct(buf, struct)
assert.strictEqual(actual, expected, `invalid ${name} checksum ${actual}, expected ${expected}`)
}
// unused block as buffer containing a uint32BE
export const BUF_BLOCK_UNUSED = Buffer.allocUnsafe(4)
BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
/**
* Check and parse the header buffer to build an header object
*
* @param {Buffer} bufHeader
* @param {Object} footer
* @returns {Object} the parsed header
*/
export const buildHeader = (bufHeader, footer) => {
assertChecksum('header', bufHeader, fuHeader)
const header = fuHeader.unpack(bufHeader)
checkHeader(header, footer)
return header
}
/**
* Check and parse the footer buffer to build a footer object
*
* @param {Buffer} bufHeader
* @param {Object} footer
* @returns {Object} the parsed footer
*/
export const buildFooter = bufFooter => {
assertChecksum('footer', bufFooter, fuFooter)
const footer = fuFooter.unpack(bufFooter)
checkFooter(footer)
return footer
}

View File

@@ -0,0 +1,7 @@
const MASK = 0x80
export const set = (map, bit) => {
map[bit >> 3] |= MASK >> (bit & 7)
}
export const test = (map, bit) => ((map[bit >> 3] << (bit & 7)) & MASK) !== 0

View File

@@ -1,8 +1,8 @@
const assert = require('assert')
import assert from 'assert'
const { HEADER_COOKIE, HEADER_VERSION, SECTOR_SIZE } = require('./_constants')
import { HEADER_COOKIE, HEADER_VERSION, SECTOR_SIZE } from './_constants'
module.exports = (header, footer) => {
export default (header, footer) => {
assert.strictEqual(header.cookie, HEADER_COOKIE)
assert.strictEqual(header.dataOffset, undefined)
assert.strictEqual(header.headerVersion, HEADER_VERSION)

View File

@@ -1,6 +1,6 @@
const { SECTOR_SIZE } = require('./_constants')
import { SECTOR_SIZE } from './_constants'
module.exports = function computeGeometryForSize(size) {
export default function computeGeometryForSize(size) {
const totalSectors = Math.min(Math.ceil(size / 512), 65535 * 16 * 255)
let sectorsPerTrackCylinder
let heads

View File

@@ -0,0 +1,30 @@
export const BLOCK_UNUSED = 0xffffffff
// This lib has been extracted from the Xen Orchestra project.
export const CREATOR_APPLICATION = 'xo '
// Sizes in bytes.
export const FOOTER_SIZE = 512
export const HEADER_SIZE = 1024
export const SECTOR_SIZE = 512
export const DEFAULT_BLOCK_SIZE = 0x00200000 // from the spec
export const FOOTER_COOKIE = 'conectix'
export const HEADER_COOKIE = 'cxsparse'
export const DISK_TYPE_FIXED = 2
export const DISK_TYPE_DYNAMIC = 3
export const DISK_TYPE_DIFFERENCING = 4
export const PARENT_LOCATOR_ENTRIES = 8
export const PLATFORM_NONE = 0
export const PLATFORM_WI2R = 0x57693272
export const PLATFORM_WI2K = 0x5769326b
export const PLATFORM_W2RU = 0x57327275
export const PLATFORM_W2KU = 0x57326b75
export const PLATFORM_MAC = 0x4d616320
export const PLATFORM_MACX = 0x4d616358
export const FILE_FORMAT_VERSION = 1 << 16
export const HEADER_VERSION = 1 << 16

View File

@@ -1,5 +1,5 @@
/* eslint-env jest */
const { createFooter } = require('./_createFooterHeader')
import { createFooter } from './_createFooterHeader'
test('createFooter() does not crash', () => {
createFooter(104448, Math.floor(Date.now() / 1000), {

View File

@@ -1,20 +1,20 @@
const { v4: generateUuid } = require('uuid')
import { v4 as generateUuid } from 'uuid'
const { checksumStruct, fuFooter, fuHeader } = require('./_structs')
const {
import { checksumStruct, fuFooter, fuHeader } from './_structs'
import {
CREATOR_APPLICATION,
DEFAULT_BLOCK_SIZE: VHD_BLOCK_SIZE_BYTES,
DISK_TYPES,
DEFAULT_BLOCK_SIZE as VHD_BLOCK_SIZE_BYTES,
DISK_TYPE_FIXED,
FILE_FORMAT_VERSION,
FOOTER_COOKIE,
FOOTER_SIZE,
HEADER_COOKIE,
HEADER_SIZE,
HEADER_VERSION,
PLATFORMS,
} = require('./_constants')
PLATFORM_WI2K,
} from './_constants'
exports.createFooter = function createFooter(size, timestamp, geometry, dataOffset, diskType = DISK_TYPES.FIXED) {
export function createFooter(size, timestamp, geometry, dataOffset, diskType = DISK_TYPE_FIXED) {
const footer = fuFooter.pack({
cookie: FOOTER_COOKIE,
features: 2,
@@ -22,7 +22,7 @@ exports.createFooter = function createFooter(size, timestamp, geometry, dataOffs
dataOffset,
timestamp,
creatorApplication: CREATOR_APPLICATION,
creatorHostOs: PLATFORMS.WI2K, // it looks like everybody is using Wi2k
creatorHostOs: PLATFORM_WI2K, // it looks like everybody is using Wi2k
originalSize: size,
currentSize: size,
diskGeometry: geometry,
@@ -33,7 +33,7 @@ exports.createFooter = function createFooter(size, timestamp, geometry, dataOffs
return footer
}
exports.createHeader = function createHeader(
export function createHeader(
maxTableEntries,
tableOffset = HEADER_SIZE + FOOTER_SIZE,
blockSize = VHD_BLOCK_SIZE_BYTES

View File

@@ -1,10 +1,10 @@
const assert = require('assert')
import assert from 'assert'
const { BLOCK_UNUSED } = require('./_constants')
import { BLOCK_UNUSED } from './_constants'
// get the identifiers and first sectors of the first and last block
// in the file
module.exports = bat => {
export default bat => {
const n = bat.length
if (n === 0) {
return

View File

@@ -0,0 +1 @@
export default Function.prototype

View File

@@ -0,0 +1,66 @@
/* eslint-env jest */
import rimraf from 'rimraf'
import tmp from 'tmp'
import { getSyncedHandler } from '@xen-orchestra/fs'
import { Disposable, pFromCallback } from 'promise-toolbox'
import { isVhdAlias, resolveAlias } from './_resolveAlias'
let tempDir
jest.setTimeout(60000)
beforeEach(async () => {
tempDir = await pFromCallback(cb => tmp.dir(cb))
})
afterEach(async () => {
await pFromCallback(cb => rimraf(tempDir, cb))
})
test('is vhd alias recognize only *.alias.vhd files', () => {
expect(isVhdAlias('filename.alias.vhd')).toEqual(true)
expect(isVhdAlias('alias.vhd')).toEqual(false)
expect(isVhdAlias('filename.vhd')).toEqual(false)
expect(isVhdAlias('filename.alias.vhd.other')).toEqual(false)
})
test('resolve return the path in argument for a non alias file ', async () => {
expect(await resolveAlias(null, 'filename.vhd')).toEqual('filename.vhd')
})
test('resolve get the path of the target file for an alias', async () => {
await Disposable.use(async function* () {
// same directory
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
await handler.mkdir(`alias`)
const aliasPath = 'alias/alias.alias.vhd'
const testOneCombination = async ({ targetPath, targetContent }) => {
await handler.writeFile(aliasPath, targetPath, { flags: 'w' })
const resolved = await resolveAlias(handler, aliasPath)
expect(resolved).toEqual(targetContent)
await handler.unlink(aliasPath)
}
// the alias contain the relative path to the file. The resolved values is the full path from the root of the remote
const combinations = [
{ targetPath: `../targets.vhd`, targetContent: `targets.vhd` },
{ targetPath: `targets.vhd`, targetContent: `alias/targets.vhd` },
{ targetPath: `sub/targets.vhd`, targetContent: `alias/sub/targets.vhd` },
{ targetPath: `../sibling/targets.vhd`, targetContent: `sibling/targets.vhd` },
]
for (const { targetPath, targetContent } of combinations) {
await testOneCombination({ targetPath, targetContent })
}
})
})
test('resolve throws an error an alias to an alias', async () => {
await Disposable.use(async function* () {
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
const alias = 'alias.alias.vhd'
const target = 'target.alias.vhd'
await handler.writeFile(alias, target)
expect(async () => await resolveAlias(handler, alias)).rejects.toThrow(Error)
})
})

View File

@@ -0,0 +1,18 @@
import resolveRelativeFromFile from './_resolveRelativeFromFile'
export function isVhdAlias(filename) {
return filename.endsWith('.alias.vhd')
}
export async function resolveAlias(handler, filename) {
if (!isVhdAlias(filename)) {
return filename
}
const aliasContent = (await handler.readFile(filename)).toString().trim()
// also handle circular references and unreasonnably long chains
if (isVhdAlias(aliasContent)) {
throw new Error(`Chaining alias is forbidden ${filename} to ${aliasContent}`)
}
// the target is relative to the alias location
return resolveRelativeFromFile(filename, aliasContent)
}

Some files were not shown because too many files have changed in this diff Show More