Compare commits
2 Commits
xapi-suppo
...
florent-fi
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d6d7e87fe5 | ||
|
|
00f02c795f |
11
.github/ISSUE_TEMPLATE/bug_report.md
vendored
11
.github/ISSUE_TEMPLATE/bug_report.md
vendored
@@ -4,6 +4,7 @@ about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: 'status: triaging :triangular_flag_on_post:, type: bug :bug:'
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
@@ -11,7 +12,6 @@ A clear and concise description of what the bug is.
|
||||
|
||||
**To Reproduce**
|
||||
Steps to reproduce the behavior:
|
||||
|
||||
1. Go to '...'
|
||||
2. Click on '....'
|
||||
3. Scroll down to '....'
|
||||
@@ -24,11 +24,10 @@ A clear and concise description of what you expected to happen.
|
||||
If applicable, add screenshots to help explain your problem.
|
||||
|
||||
**Desktop (please complete the following information):**
|
||||
|
||||
- Node: [e.g. 16.12.1]
|
||||
- xo-server: [e.g. 5.82.3]
|
||||
- xo-web: [e.g. 5.87.0]
|
||||
- hypervisor: [e.g. XCP-ng 8.2.0]
|
||||
- Node: [e.g. 16.12.1]
|
||||
- xo-server: [e.g. 5.82.3]
|
||||
- xo-web: [e.g. 5.87.0]
|
||||
- hypervisor: [e.g. XCP-ng 8.2.0]
|
||||
|
||||
**Additional context**
|
||||
Add any other context about the problem here.
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
../../scripts/npmignore
|
||||
@@ -1,68 +0,0 @@
|
||||
<!-- DO NOT EDIT MANUALLY, THIS FILE HAS BEEN GENERATED -->
|
||||
|
||||
# @vates/async-each
|
||||
|
||||
[](https://npmjs.org/package/@vates/async-each)  [](https://bundlephobia.com/result?p=@vates/async-each) [](https://npmjs.org/package/@vates/async-each)
|
||||
|
||||
> Run async fn for each item in (async) iterable
|
||||
|
||||
## Install
|
||||
|
||||
Installation of the [npm package](https://npmjs.org/package/@vates/async-each):
|
||||
|
||||
```
|
||||
> npm install --save @vates/async-each
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### `asyncEach(iterable, iteratee, [opts])`
|
||||
|
||||
Executes `iteratee` in order for each value yielded by `iterable`.
|
||||
|
||||
Returns a promise wich rejects as soon as a call to `iteratee` throws or a promise returned by it rejects, and which resolves when all promises returned by `iteratee` have resolved.
|
||||
|
||||
`iterable` must be an iterable or async iterable.
|
||||
|
||||
`iteratee` is called with the same `this` value as `asyncEach`, and with the following arguments:
|
||||
|
||||
- `value`: the value yielded by `iterable`
|
||||
- `index`: the 0-based index for this value
|
||||
- `iterable`: the iterable itself
|
||||
|
||||
`opts` is an object that can contains the following options:
|
||||
|
||||
- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
|
||||
- `signal`: an abort signal to stop the iteration
|
||||
- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
|
||||
|
||||
```js
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
|
||||
const contents = []
|
||||
await asyncEach(
|
||||
['foo.txt', 'bar.txt', 'baz.txt'],
|
||||
async function (filename, i) {
|
||||
contents[i] = await readFile(filename)
|
||||
},
|
||||
{
|
||||
// reads two files at a time
|
||||
concurrency: 2,
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
## Contributions
|
||||
|
||||
Contributions are _very_ welcomed, either on the documentation or on
|
||||
the code.
|
||||
|
||||
You may:
|
||||
|
||||
- report any [issue](https://github.com/vatesfr/xen-orchestra/issues)
|
||||
you've encountered;
|
||||
- fork and create a pull request.
|
||||
|
||||
## License
|
||||
|
||||
[ISC](https://spdx.org/licenses/ISC) © [Vates SAS](https://vates.fr)
|
||||
@@ -1,35 +0,0 @@
|
||||
### `asyncEach(iterable, iteratee, [opts])`
|
||||
|
||||
Executes `iteratee` in order for each value yielded by `iterable`.
|
||||
|
||||
Returns a promise wich rejects as soon as a call to `iteratee` throws or a promise returned by it rejects, and which resolves when all promises returned by `iteratee` have resolved.
|
||||
|
||||
`iterable` must be an iterable or async iterable.
|
||||
|
||||
`iteratee` is called with the same `this` value as `asyncEach`, and with the following arguments:
|
||||
|
||||
- `value`: the value yielded by `iterable`
|
||||
- `index`: the 0-based index for this value
|
||||
- `iterable`: the iterable itself
|
||||
|
||||
`opts` is an object that can contains the following options:
|
||||
|
||||
- `concurrency`: a number which indicates the maximum number of parallel call to `iteratee`, defaults to `1`
|
||||
- `signal`: an abort signal to stop the iteration
|
||||
- `stopOnError`: wether to stop iteration of first error, or wait for all calls to finish and throw an `AggregateError`, defaults to `true`
|
||||
|
||||
```js
|
||||
import { asyncEach } from '@vates/async-each'
|
||||
|
||||
const contents = []
|
||||
await asyncEach(
|
||||
['foo.txt', 'bar.txt', 'baz.txt'],
|
||||
async function (filename, i) {
|
||||
contents[i] = await readFile(filename)
|
||||
},
|
||||
{
|
||||
// reads two files at a time
|
||||
concurrency: 2,
|
||||
}
|
||||
)
|
||||
```
|
||||
@@ -1,99 +0,0 @@
|
||||
'use strict'
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
class AggregateError extends Error {
|
||||
constructor(errors, message) {
|
||||
super(message)
|
||||
this.errors = errors
|
||||
}
|
||||
}
|
||||
|
||||
exports.asyncEach = function asyncEach(iterable, iteratee, { concurrency = 1, signal, stopOnError = true } = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const it = (iterable[Symbol.iterator] || iterable[Symbol.asyncIterator]).call(iterable)
|
||||
const errors = []
|
||||
let running = 0
|
||||
let index = 0
|
||||
|
||||
let onAbort
|
||||
if (signal !== undefined) {
|
||||
onAbort = () => {
|
||||
onRejectedWrapper(new Error('asyncEach aborted'))
|
||||
}
|
||||
signal.addEventListener('abort', onAbort)
|
||||
}
|
||||
|
||||
const clean = () => {
|
||||
onFulfilled = onRejected = noop
|
||||
if (onAbort !== undefined) {
|
||||
signal.removeEventListener('abort', onAbort)
|
||||
}
|
||||
}
|
||||
|
||||
resolve = (resolve =>
|
||||
function resolveAndClean(value) {
|
||||
resolve(value)
|
||||
clean()
|
||||
})(resolve)
|
||||
reject = (reject =>
|
||||
function rejectAndClean(reason) {
|
||||
reject(reason)
|
||||
clean()
|
||||
})(reject)
|
||||
|
||||
let onFulfilled = value => {
|
||||
--running
|
||||
next()
|
||||
}
|
||||
const onFulfilledWrapper = value => onFulfilled(value)
|
||||
|
||||
let onRejected = stopOnError
|
||||
? reject
|
||||
: error => {
|
||||
--running
|
||||
errors.push(error)
|
||||
next()
|
||||
}
|
||||
const onRejectedWrapper = reason => onRejected(reason)
|
||||
|
||||
let nextIsRunning = false
|
||||
let next = async () => {
|
||||
if (nextIsRunning) {
|
||||
return
|
||||
}
|
||||
nextIsRunning = true
|
||||
if (running < concurrency) {
|
||||
const cursor = await it.next()
|
||||
if (cursor.done) {
|
||||
next = () => {
|
||||
if (running === 0) {
|
||||
if (errors.length === 0) {
|
||||
resolve()
|
||||
} else {
|
||||
reject(new AggregateError(errors))
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
++running
|
||||
try {
|
||||
const result = iteratee.call(this, cursor.value, index++, iterable)
|
||||
let then
|
||||
if (result != null && typeof result === 'object' && typeof (then = result.then) === 'function') {
|
||||
then.call(result, onFulfilledWrapper, onRejectedWrapper)
|
||||
} else {
|
||||
onFulfilled(result)
|
||||
}
|
||||
} catch (error) {
|
||||
onRejected(error)
|
||||
}
|
||||
}
|
||||
nextIsRunning = false
|
||||
return next()
|
||||
}
|
||||
nextIsRunning = false
|
||||
}
|
||||
next()
|
||||
})
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
'use strict'
|
||||
|
||||
/* eslint-env jest */
|
||||
|
||||
const { asyncEach } = require('./')
|
||||
|
||||
const randomDelay = (max = 10) =>
|
||||
new Promise(resolve => {
|
||||
setTimeout(resolve, Math.floor(Math.random() * max + 1))
|
||||
})
|
||||
|
||||
const rejectionOf = p =>
|
||||
new Promise((resolve, reject) => {
|
||||
p.then(reject, resolve)
|
||||
})
|
||||
|
||||
describe('asyncEach', () => {
|
||||
const thisArg = 'qux'
|
||||
const values = ['foo', 'bar', 'baz']
|
||||
|
||||
Object.entries({
|
||||
'sync iterable': () => values,
|
||||
'async iterable': async function* () {
|
||||
for (const value of values) {
|
||||
await randomDelay()
|
||||
yield value
|
||||
}
|
||||
},
|
||||
}).forEach(([what, getIterable]) =>
|
||||
describe('with ' + what, () => {
|
||||
let iterable
|
||||
beforeEach(() => {
|
||||
iterable = getIterable()
|
||||
})
|
||||
|
||||
it('works', async () => {
|
||||
const iteratee = jest.fn(async () => {})
|
||||
|
||||
await asyncEach.call(thisArg, iterable, iteratee)
|
||||
|
||||
expect(iteratee.mock.instances).toEqual(Array.from(values, () => thisArg))
|
||||
expect(iteratee.mock.calls).toEqual(Array.from(values, (value, index) => [value, index, iterable]))
|
||||
})
|
||||
;[1, 2, 4].forEach(concurrency => {
|
||||
it('respects a concurrency of ' + concurrency, async () => {
|
||||
let running = 0
|
||||
|
||||
await asyncEach(
|
||||
values,
|
||||
async () => {
|
||||
++running
|
||||
expect(running).toBeLessThanOrEqual(concurrency)
|
||||
await randomDelay()
|
||||
--running
|
||||
},
|
||||
{ concurrency }
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
it('stops on first error when stopOnError is true', async () => {
|
||||
const error = new Error()
|
||||
const iteratee = jest.fn((_, i) => {
|
||||
if (i === 1) {
|
||||
throw error
|
||||
}
|
||||
})
|
||||
|
||||
expect(await rejectionOf(asyncEach(iterable, iteratee, { stopOnError: true }))).toBe(error)
|
||||
expect(iteratee).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('rejects AggregateError when stopOnError is false', async () => {
|
||||
const errors = []
|
||||
const iteratee = jest.fn(() => {
|
||||
const error = new Error()
|
||||
errors.push(error)
|
||||
throw error
|
||||
})
|
||||
|
||||
const error = await rejectionOf(asyncEach(iterable, iteratee, { stopOnError: false }))
|
||||
expect(error.errors).toEqual(errors)
|
||||
expect(iteratee.mock.calls).toEqual(Array.from(values, (value, index) => [value, index, iterable]))
|
||||
})
|
||||
|
||||
it('can be interrupted with an AbortSignal', async () => {
|
||||
const ac = new AbortController()
|
||||
const iteratee = jest.fn((_, i) => {
|
||||
if (i === 1) {
|
||||
ac.abort()
|
||||
}
|
||||
})
|
||||
|
||||
await expect(asyncEach(iterable, iteratee, { signal: ac.signal })).rejects.toThrow('asyncEach aborted')
|
||||
expect(iteratee).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
})
|
||||
)
|
||||
})
|
||||
@@ -1,34 +0,0 @@
|
||||
{
|
||||
"private": false,
|
||||
"name": "@vates/async-each",
|
||||
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/async-each",
|
||||
"description": "Run async fn for each item in (async) iterable",
|
||||
"keywords": [
|
||||
"array",
|
||||
"async",
|
||||
"collection",
|
||||
"each",
|
||||
"for",
|
||||
"foreach",
|
||||
"iterable",
|
||||
"iterator"
|
||||
],
|
||||
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
|
||||
"repository": {
|
||||
"directory": "@vates/async-each",
|
||||
"type": "git",
|
||||
"url": "https://github.com/vatesfr/xen-orchestra.git"
|
||||
},
|
||||
"author": {
|
||||
"name": "Vates SAS",
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"license": "ISC",
|
||||
"version": "0.1.0",
|
||||
"engines": {
|
||||
"node": ">=8.10"
|
||||
},
|
||||
"scripts": {
|
||||
"postversion": "npm publish --access public"
|
||||
}
|
||||
}
|
||||
@@ -65,23 +65,6 @@ const f = compose(
|
||||
)
|
||||
```
|
||||
|
||||
Functions can receive extra parameters:
|
||||
|
||||
```js
|
||||
const isIn = (value, min, max) => min <= value && value <= max
|
||||
|
||||
// Only compatible when `fns` is passed as an array!
|
||||
const f = compose([
|
||||
[add, 2],
|
||||
[isIn, 3, 10],
|
||||
])
|
||||
|
||||
console.log(f(1))
|
||||
// → true
|
||||
```
|
||||
|
||||
> Note: if the first function is defined with extra parameters, it will only receive the first value passed to the composed function, instead of all the parameters.
|
||||
|
||||
## Contributions
|
||||
|
||||
Contributions are _very_ welcomed, either on the documentation or on
|
||||
|
||||
@@ -46,20 +46,3 @@ const f = compose(
|
||||
[add2, mul3]
|
||||
)
|
||||
```
|
||||
|
||||
Functions can receive extra parameters:
|
||||
|
||||
```js
|
||||
const isIn = (value, min, max) => min <= value && value <= max
|
||||
|
||||
// Only compatible when `fns` is passed as an array!
|
||||
const f = compose([
|
||||
[add, 2],
|
||||
[isIn, 3, 10],
|
||||
])
|
||||
|
||||
console.log(f(1))
|
||||
// → true
|
||||
```
|
||||
|
||||
> Note: if the first function is defined with extra parameters, it will only receive the first value passed to the composed function, instead of all the parameters.
|
||||
|
||||
@@ -4,13 +4,11 @@ const defaultOpts = { async: false, right: false }
|
||||
|
||||
exports.compose = function compose(opts, fns) {
|
||||
if (Array.isArray(opts)) {
|
||||
fns = opts.slice() // don't mutate passed array
|
||||
fns = opts
|
||||
opts = defaultOpts
|
||||
} else if (typeof opts === 'object') {
|
||||
opts = Object.assign({}, defaultOpts, opts)
|
||||
if (Array.isArray(fns)) {
|
||||
fns = fns.slice() // don't mutate passed array
|
||||
} else {
|
||||
if (!Array.isArray(fns)) {
|
||||
fns = Array.prototype.slice.call(arguments, 1)
|
||||
}
|
||||
} else {
|
||||
@@ -22,24 +20,6 @@ exports.compose = function compose(opts, fns) {
|
||||
if (n === 0) {
|
||||
throw new TypeError('at least one function must be passed')
|
||||
}
|
||||
|
||||
for (let i = 0; i < n; ++i) {
|
||||
const entry = fns[i]
|
||||
if (Array.isArray(entry)) {
|
||||
const fn = entry[0]
|
||||
const args = entry.slice()
|
||||
args[0] = undefined
|
||||
fns[i] = function composeWithArgs(value) {
|
||||
args[0] = value
|
||||
try {
|
||||
return fn.apply(this, args)
|
||||
} finally {
|
||||
args[0] = undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (n === 1) {
|
||||
return fns[0]
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"license": "ISC",
|
||||
"version": "2.1.0",
|
||||
"version": "2.0.0",
|
||||
"engines": {
|
||||
"node": ">=7.6"
|
||||
},
|
||||
|
||||
@@ -59,36 +59,6 @@ decorateMethodsWith(Foo, {
|
||||
|
||||
The decorated class is returned, so you can export it directly.
|
||||
|
||||
To apply multiple transforms to a method, you can either call `decorateMethodsWith` multiple times or use [`@vates/compose`](https://www.npmjs.com/package/@vates/compose):
|
||||
|
||||
```js
|
||||
decorateMethodsWith(Foo, {
|
||||
bar: compose([
|
||||
[lodash.debounce, 150]
|
||||
lodash.curry,
|
||||
])
|
||||
})
|
||||
```
|
||||
|
||||
### `perInstance(fn, ...args)`
|
||||
|
||||
Helper to decorate the method by instance instead of for the whole class.
|
||||
|
||||
This is often necessary for caching or deduplicating calls.
|
||||
|
||||
```js
|
||||
import { perInstance } from '@vates/decorateWith'
|
||||
|
||||
class Foo {
|
||||
@decorateWith(perInstance, lodash.memoize)
|
||||
bar() {
|
||||
// body
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Because it's a normal function, it can also be used with `decorateMethodsWith`, with `compose` or even by itself.
|
||||
|
||||
## Contributions
|
||||
|
||||
Contributions are _very_ welcomed, either on the documentation or on
|
||||
|
||||
@@ -40,33 +40,3 @@ decorateMethodsWith(Foo, {
|
||||
```
|
||||
|
||||
The decorated class is returned, so you can export it directly.
|
||||
|
||||
To apply multiple transforms to a method, you can either call `decorateMethodsWith` multiple times or use [`@vates/compose`](https://www.npmjs.com/package/@vates/compose):
|
||||
|
||||
```js
|
||||
decorateMethodsWith(Foo, {
|
||||
bar: compose([
|
||||
[lodash.debounce, 150]
|
||||
lodash.curry,
|
||||
])
|
||||
})
|
||||
```
|
||||
|
||||
### `perInstance(fn, ...args)`
|
||||
|
||||
Helper to decorate the method by instance instead of for the whole class.
|
||||
|
||||
This is often necessary for caching or deduplicating calls.
|
||||
|
||||
```js
|
||||
import { perInstance } from '@vates/decorateWith'
|
||||
|
||||
class Foo {
|
||||
@decorateWith(perInstance, lodash.memoize)
|
||||
bar() {
|
||||
// body
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Because it's a normal function, it can also be used with `decorateMethodsWith`, with `compose` or even by itself.
|
||||
|
||||
@@ -19,15 +19,3 @@ exports.decorateMethodsWith = function decorateMethodsWith(klass, map) {
|
||||
}
|
||||
return klass
|
||||
}
|
||||
|
||||
exports.perInstance = function perInstance(fn, decorator, ...args) {
|
||||
const map = new WeakMap()
|
||||
return function () {
|
||||
let decorated = map.get(this)
|
||||
if (decorated === undefined) {
|
||||
decorated = decorator(fn, ...args)
|
||||
map.set(this, decorated)
|
||||
}
|
||||
return decorated.apply(this, arguments)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"license": "ISC",
|
||||
"version": "1.0.0",
|
||||
"version": "0.1.0",
|
||||
"engines": {
|
||||
"node": ">=8.10"
|
||||
},
|
||||
|
||||
@@ -30,7 +30,7 @@
|
||||
"rimraf": "^3.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@vates/decorate-with": "^1.0.0",
|
||||
"@vates/decorate-with": "^0.1.0",
|
||||
"@xen-orchestra/log": "^0.3.0",
|
||||
"golike-defer": "^0.5.1",
|
||||
"object-hash": "^2.0.1"
|
||||
|
||||
@@ -7,8 +7,8 @@
|
||||
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
|
||||
"dependencies": {
|
||||
"@xen-orchestra/async-map": "^0.1.2",
|
||||
"@xen-orchestra/backups": "^0.18.3",
|
||||
"@xen-orchestra/fs": "^0.19.3",
|
||||
"@xen-orchestra/backups": "^0.15.1",
|
||||
"@xen-orchestra/fs": "^0.18.0",
|
||||
"filenamify": "^4.1.0",
|
||||
"getopts": "^2.2.5",
|
||||
"lodash": "^4.17.15",
|
||||
@@ -27,7 +27,7 @@
|
||||
"scripts": {
|
||||
"postversion": "npm publish --access public"
|
||||
},
|
||||
"version": "0.6.1",
|
||||
"version": "0.6.0",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"author": {
|
||||
"name": "Vates SAS",
|
||||
|
||||
@@ -3,20 +3,19 @@ const Disposable = require('promise-toolbox/Disposable.js')
|
||||
const fromCallback = require('promise-toolbox/fromCallback.js')
|
||||
const fromEvent = require('promise-toolbox/fromEvent.js')
|
||||
const pDefer = require('promise-toolbox/defer.js')
|
||||
const groupBy = require('lodash/groupBy.js')
|
||||
const { dirname, join, normalize, resolve } = require('path')
|
||||
const pump = require('pump')
|
||||
const { basename, dirname, join, normalize, resolve } = require('path')
|
||||
const { createLogger } = require('@xen-orchestra/log')
|
||||
const { Constants, createVhdDirectoryFromStream, openVhd, VhdAbstract, VhdDirectory, VhdSynthetic } = require('vhd-lib')
|
||||
const { createSyntheticStream, mergeVhd, VhdFile } = require('vhd-lib')
|
||||
const { deduped } = require('@vates/disposable/deduped.js')
|
||||
const { execFile } = require('child_process')
|
||||
const { readdir, stat } = require('fs-extra')
|
||||
const { v4: uuidv4 } = require('uuid')
|
||||
const { ZipFile } = require('yazl')
|
||||
|
||||
const { BACKUP_DIR } = require('./_getVmBackupDir.js')
|
||||
const { cleanVm } = require('./_cleanVm.js')
|
||||
const { getTmpDir } = require('./_getTmpDir.js')
|
||||
const { isMetadataFile } = require('./_backupType.js')
|
||||
const { isMetadataFile, isVhdFile } = require('./_backupType.js')
|
||||
const { isValidXva } = require('./_isValidXva.js')
|
||||
const { listPartitions, LVM_PARTITION_TYPE } = require('./_listPartitions.js')
|
||||
const { lvs, pvs } = require('./_lvm.js')
|
||||
@@ -68,17 +67,58 @@ const debounceResourceFactory = factory =>
|
||||
}
|
||||
|
||||
class RemoteAdapter {
|
||||
constructor(handler, { debounceResource = res => res, dirMode, vhdDirectoryCompression } = {}) {
|
||||
constructor(handler, { debounceResource = res => res, dirMode } = {}) {
|
||||
this._debounceResource = debounceResource
|
||||
this._dirMode = dirMode
|
||||
this._handler = handler
|
||||
this._vhdDirectoryCompression = vhdDirectoryCompression
|
||||
}
|
||||
|
||||
get handler() {
|
||||
return this._handler
|
||||
}
|
||||
|
||||
async _deleteVhd(path) {
|
||||
const handler = this._handler
|
||||
const vhds = await asyncMapSettled(
|
||||
await handler.list(dirname(path), {
|
||||
filter: isVhdFile,
|
||||
prependDir: true,
|
||||
}),
|
||||
async path => {
|
||||
try {
|
||||
const vhd = new VhdFile(handler, path)
|
||||
await vhd.readHeaderAndFooter()
|
||||
return {
|
||||
footer: vhd.footer,
|
||||
header: vhd.header,
|
||||
path,
|
||||
}
|
||||
} catch (error) {
|
||||
// Do not fail on corrupted VHDs (usually uncleaned temporary files),
|
||||
// they are probably inconsequent to the backup process and should not
|
||||
// fail it.
|
||||
warn(`BackupNg#_deleteVhd ${path}`, { error })
|
||||
}
|
||||
}
|
||||
)
|
||||
const base = basename(path)
|
||||
const child = vhds.find(_ => _ !== undefined && _.header.parentUnicodeName === base)
|
||||
if (child === undefined) {
|
||||
await handler.unlink(path)
|
||||
return 0
|
||||
}
|
||||
|
||||
try {
|
||||
const childPath = child.path
|
||||
const mergedDataSize = await mergeVhd(handler, path, handler, childPath)
|
||||
await handler.rename(path, childPath)
|
||||
return mergedDataSize
|
||||
} catch (error) {
|
||||
handler.unlink(path).catch(warn)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async _findPartition(devicePath, partitionId) {
|
||||
const partitions = await listPartitions(devicePath)
|
||||
const partition = partitions.find(_ => _.id === partitionId)
|
||||
@@ -192,22 +232,6 @@ class RemoteAdapter {
|
||||
return files
|
||||
}
|
||||
|
||||
// check if we will be allowed to merge a a vhd created in this adapter
|
||||
// with the vhd at path `path`
|
||||
async isMergeableParent(packedParentUid, path) {
|
||||
return await Disposable.use(openVhd(this.handler, path), vhd => {
|
||||
// this baseUuid is not linked with this vhd
|
||||
if (!vhd.footer.uuid.equals(packedParentUid)) {
|
||||
return false
|
||||
}
|
||||
|
||||
const isVhdDirectory = vhd instanceof VhdDirectory
|
||||
return isVhdDirectory
|
||||
? this.#useVhdDirectory() && this.#getCompressionType() === vhd.compressionType
|
||||
: !this.#useVhdDirectory()
|
||||
})
|
||||
}
|
||||
|
||||
fetchPartitionFiles(diskId, partitionId, paths) {
|
||||
const { promise, reject, resolve } = pDefer()
|
||||
Disposable.use(
|
||||
@@ -231,7 +255,7 @@ class RemoteAdapter {
|
||||
const handler = this._handler
|
||||
|
||||
// unused VHDs will be detected by `cleanVm`
|
||||
await asyncMapSettled(backups, ({ _filename }) => VhdAbstract.unlink(handler, _filename))
|
||||
await asyncMapSettled(backups, ({ _filename }) => handler.unlink(_filename))
|
||||
}
|
||||
|
||||
async deleteMetadataBackup(backupId) {
|
||||
@@ -261,34 +285,17 @@ class RemoteAdapter {
|
||||
)
|
||||
}
|
||||
|
||||
deleteVmBackup(file) {
|
||||
return this.deleteVmBackups([file])
|
||||
}
|
||||
async deleteVmBackup(filename) {
|
||||
const metadata = JSON.parse(String(await this._handler.readFile(filename)))
|
||||
metadata._filename = filename
|
||||
|
||||
async deleteVmBackups(files) {
|
||||
const { delta, full, ...others } = groupBy(await asyncMap(files, file => this.readVmBackupMetadata(file)), 'mode')
|
||||
|
||||
const unsupportedModes = Object.keys(others)
|
||||
if (unsupportedModes.length !== 0) {
|
||||
throw new Error('no deleter for backup modes: ' + unsupportedModes.join(', '))
|
||||
if (metadata.mode === 'delta') {
|
||||
await this.deleteDeltaVmBackups([metadata])
|
||||
} else if (metadata.mode === 'full') {
|
||||
await this.deleteFullVmBackups([metadata])
|
||||
} else {
|
||||
throw new Error(`no deleter for backup mode ${metadata.mode}`)
|
||||
}
|
||||
|
||||
await Promise.all([
|
||||
delta !== undefined && this.deleteDeltaVmBackups(delta),
|
||||
full !== undefined && this.deleteFullVmBackups(full),
|
||||
])
|
||||
}
|
||||
|
||||
#getCompressionType() {
|
||||
return this._vhdDirectoryCompression
|
||||
}
|
||||
|
||||
#useVhdDirectory() {
|
||||
return this.handler.type === 's3'
|
||||
}
|
||||
|
||||
#useAlias() {
|
||||
return this.#useVhdDirectory()
|
||||
}
|
||||
|
||||
getDisk = Disposable.factory(this.getDisk)
|
||||
@@ -347,14 +354,6 @@ class RemoteAdapter {
|
||||
return yield this._getPartition(devicePath, await this._findPartition(devicePath, partitionId))
|
||||
}
|
||||
|
||||
// if we use alias on this remote, we have to name the file alias.vhd
|
||||
getVhdFileName(baseName) {
|
||||
if (this.#useAlias()) {
|
||||
return `${baseName}.alias.vhd`
|
||||
}
|
||||
return `${baseName}.vhd`
|
||||
}
|
||||
|
||||
async listAllVmBackups() {
|
||||
const handler = this._handler
|
||||
|
||||
@@ -499,25 +498,6 @@ class RemoteAdapter {
|
||||
return backups.sort(compareTimestamp)
|
||||
}
|
||||
|
||||
async writeVhd(path, input, { checksum = true, validator = noop } = {}) {
|
||||
const handler = this._handler
|
||||
|
||||
if (this.#useVhdDirectory()) {
|
||||
const dataPath = `${dirname(path)}/data/${uuidv4()}.vhd`
|
||||
await createVhdDirectoryFromStream(handler, dataPath, input, {
|
||||
concurrency: 16,
|
||||
compression: this.#getCompressionType(),
|
||||
async validator() {
|
||||
await input.task
|
||||
return validator.apply(this, arguments)
|
||||
},
|
||||
})
|
||||
await VhdAbstract.createAlias(handler, path, dataPath)
|
||||
} else {
|
||||
await this.outputStream(path, input, { checksum, validator })
|
||||
}
|
||||
}
|
||||
|
||||
async outputStream(path, input, { checksum = true, validator = noop } = {}) {
|
||||
await this._handler.outputStream(path, input, {
|
||||
checksum,
|
||||
@@ -529,52 +509,6 @@ class RemoteAdapter {
|
||||
})
|
||||
}
|
||||
|
||||
async _createSyntheticStream(handler, paths) {
|
||||
let disposableVhds = []
|
||||
|
||||
// if it's a path : open all hierarchy of parent
|
||||
if (typeof paths === 'string') {
|
||||
let vhd,
|
||||
vhdPath = paths
|
||||
do {
|
||||
const disposable = await openVhd(handler, vhdPath)
|
||||
vhd = disposable.value
|
||||
disposableVhds.push(disposable)
|
||||
vhdPath = resolveRelativeFromFile(vhdPath, vhd.header.parentUnicodeName)
|
||||
} while (vhd.footer.diskType !== Constants.DISK_TYPES.DYNAMIC)
|
||||
} else {
|
||||
// only open the list of path given
|
||||
disposableVhds = paths.map(path => openVhd(handler, path))
|
||||
}
|
||||
|
||||
// I don't want the vhds to be disposed on return
|
||||
// but only when the stream is done ( or failed )
|
||||
const disposables = await Disposable.all(disposableVhds)
|
||||
const vhds = disposables.value
|
||||
|
||||
let disposed = false
|
||||
const disposeOnce = async () => {
|
||||
if (!disposed) {
|
||||
disposed = true
|
||||
|
||||
try {
|
||||
await disposables.dispose()
|
||||
} catch (error) {
|
||||
warn('_createSyntheticStream: failed to dispose VHDs', { error })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const synthetic = new VhdSynthetic(vhds)
|
||||
await synthetic.readHeaderAndFooter()
|
||||
await synthetic.readBlockAllocationTable()
|
||||
const stream = await synthetic.stream()
|
||||
stream.on('end', disposeOnce)
|
||||
stream.on('close', disposeOnce)
|
||||
stream.on('error', disposeOnce)
|
||||
return stream
|
||||
}
|
||||
|
||||
async readDeltaVmBackup(metadata) {
|
||||
const handler = this._handler
|
||||
const { vbds, vdis, vhds, vifs, vm } = metadata
|
||||
@@ -582,7 +516,7 @@ class RemoteAdapter {
|
||||
|
||||
const streams = {}
|
||||
await asyncMapSettled(Object.keys(vdis), async id => {
|
||||
streams[`${id}.vhd`] = await this._createSyntheticStream(handler, join(dir, vhds[id]))
|
||||
streams[`${id}.vhd`] = await createSyntheticStream(handler, join(dir, vhds[id]))
|
||||
})
|
||||
|
||||
return {
|
||||
|
||||
@@ -36,11 +36,6 @@ const forkDeltaExport = deltaExport =>
|
||||
|
||||
exports.VmBackup = class VmBackup {
|
||||
constructor({ config, getSnapshotNameLabel, job, remoteAdapters, remotes, schedule, settings, srs, vm }) {
|
||||
if (vm.other_config['xo:backup:job'] === job.id) {
|
||||
// otherwise replicated VMs would be matched and replicated again and again
|
||||
throw new Error('cannot backup a VM created by this very job')
|
||||
}
|
||||
|
||||
this.config = config
|
||||
this.job = job
|
||||
this.remoteAdapters = remoteAdapters
|
||||
@@ -338,16 +333,13 @@ exports.VmBackup = class VmBackup {
|
||||
|
||||
const baseUuidToSrcVdi = new Map()
|
||||
await asyncMap(await baseVm.$getDisks(), async baseRef => {
|
||||
const [baseUuid, snapshotOf] = await Promise.all([
|
||||
xapi.getField('VDI', baseRef, 'uuid'),
|
||||
xapi.getField('VDI', baseRef, 'snapshot_of'),
|
||||
])
|
||||
const snapshotOf = await xapi.getField('VDI', baseRef, 'snapshot_of')
|
||||
const srcVdi = srcVdis[snapshotOf]
|
||||
if (srcVdi !== undefined) {
|
||||
baseUuidToSrcVdi.set(baseUuid, srcVdi)
|
||||
baseUuidToSrcVdi.set(await xapi.getField('VDI', baseRef, 'uuid'), srcVdi)
|
||||
} else {
|
||||
debug('ignore snapshot VDI because no longer present on VM', {
|
||||
vdi: baseUuid,
|
||||
debug('no base VDI found', {
|
||||
vdi: srcVdi.uuid,
|
||||
})
|
||||
}
|
||||
})
|
||||
@@ -359,11 +351,6 @@ exports.VmBackup = class VmBackup {
|
||||
false
|
||||
)
|
||||
|
||||
if (presentBaseVdis.size === 0) {
|
||||
debug('no base VM found')
|
||||
return
|
||||
}
|
||||
|
||||
const fullVdisRequired = new Set()
|
||||
baseUuidToSrcVdi.forEach((srcVdi, baseUuid) => {
|
||||
if (presentBaseVdis.has(baseUuid)) {
|
||||
|
||||
@@ -70,7 +70,6 @@ class BackupWorker {
|
||||
yield new RemoteAdapter(handler, {
|
||||
debounceResource: this.debounceResource,
|
||||
dirMode: this.#config.dirMode,
|
||||
vhdDirectoryCompression: this.#config.vhdDirectoryCompression,
|
||||
})
|
||||
} finally {
|
||||
await handler.forget()
|
||||
|
||||
@@ -1,407 +0,0 @@
|
||||
/* eslint-env jest */
|
||||
|
||||
const rimraf = require('rimraf')
|
||||
const tmp = require('tmp')
|
||||
const fs = require('fs-extra')
|
||||
const { getHandler } = require('@xen-orchestra/fs')
|
||||
const { pFromCallback } = require('promise-toolbox')
|
||||
const crypto = require('crypto')
|
||||
const { RemoteAdapter } = require('./RemoteAdapter')
|
||||
const { VHDFOOTER, VHDHEADER } = require('./tests.fixtures.js')
|
||||
const { VhdFile, Constants, VhdDirectory, VhdAbstract } = require('vhd-lib')
|
||||
|
||||
let tempDir, adapter, handler, jobId, vdiId, basePath
|
||||
|
||||
jest.setTimeout(60000)
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await pFromCallback(cb => tmp.dir(cb))
|
||||
handler = getHandler({ url: `file://${tempDir}` })
|
||||
await handler.sync()
|
||||
adapter = new RemoteAdapter(handler)
|
||||
jobId = uniqueId()
|
||||
vdiId = uniqueId()
|
||||
basePath = `vdis/${jobId}/${vdiId}`
|
||||
await fs.mkdirp(`${tempDir}/${basePath}`)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await pFromCallback(cb => rimraf(tempDir, cb))
|
||||
await handler.forget()
|
||||
})
|
||||
|
||||
const uniqueId = () => crypto.randomBytes(16).toString('hex')
|
||||
|
||||
async function generateVhd(path, opts = {}) {
|
||||
let vhd
|
||||
|
||||
const dataPath = opts.useAlias ? path + '.data' : path
|
||||
if (opts.mode === 'directory') {
|
||||
await handler.mkdir(dataPath)
|
||||
vhd = new VhdDirectory(handler, dataPath)
|
||||
} else {
|
||||
const fd = await handler.openFile(dataPath, 'wx')
|
||||
vhd = new VhdFile(handler, fd)
|
||||
}
|
||||
|
||||
vhd.header = { ...VHDHEADER, ...opts.header }
|
||||
vhd.footer = { ...VHDFOOTER, ...opts.footer }
|
||||
vhd.footer.uuid = Buffer.from(crypto.randomBytes(16))
|
||||
|
||||
if (vhd.header.parentUnicodeName) {
|
||||
vhd.footer.diskType = Constants.DISK_TYPES.DIFFERENCING
|
||||
} else {
|
||||
vhd.footer.diskType = Constants.DISK_TYPES.DYNAMIC
|
||||
}
|
||||
|
||||
if (opts.useAlias === true) {
|
||||
await VhdAbstract.createAlias(handler, path + '.alias.vhd', dataPath)
|
||||
}
|
||||
|
||||
await vhd.writeBlockAllocationTable()
|
||||
await vhd.writeHeader()
|
||||
await vhd.writeFooter()
|
||||
return vhd
|
||||
}
|
||||
|
||||
test('It remove broken vhd', async () => {
|
||||
// todo also tests a directory and an alias
|
||||
|
||||
await handler.writeFile(`${basePath}/notReallyAVhd.vhd`, 'I AM NOT A VHD')
|
||||
expect((await handler.list(basePath)).length).toEqual(1)
|
||||
let loggued = ''
|
||||
const onLog = message => {
|
||||
loggued += message
|
||||
}
|
||||
await adapter.cleanVm('/', { remove: false, onLog })
|
||||
expect(loggued).toEqual(`error while checking the VHD with path /${basePath}/notReallyAVhd.vhd`)
|
||||
// not removed
|
||||
expect((await handler.list(basePath)).length).toEqual(1)
|
||||
// really remove it
|
||||
await adapter.cleanVm('/', { remove: true, onLog })
|
||||
expect((await handler.list(basePath)).length).toEqual(0)
|
||||
})
|
||||
|
||||
test('it remove vhd with missing or multiple ancestors', async () => {
|
||||
// one with a broken parent
|
||||
await generateVhd(`${basePath}/abandonned.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'gone.vhd',
|
||||
parentUid: Buffer.from(crypto.randomBytes(16)),
|
||||
},
|
||||
})
|
||||
|
||||
// one orphan, which is a full vhd, no parent
|
||||
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
|
||||
// a child to the orphan
|
||||
await generateVhd(`${basePath}/child.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'orphan.vhd',
|
||||
parentUid: orphan.footer.uuid,
|
||||
},
|
||||
})
|
||||
|
||||
// clean
|
||||
let loggued = ''
|
||||
const onLog = message => {
|
||||
loggued += message + '\n'
|
||||
}
|
||||
await adapter.cleanVm('/', { remove: true, onLog })
|
||||
|
||||
const deletedOrphanVhd = loggued.match(/deleting orphan VHD/g) || []
|
||||
expect(deletedOrphanVhd.length).toEqual(1) // only one vhd should have been deleted
|
||||
const deletedAbandonnedVhd = loggued.match(/abandonned.vhd is missing/g) || []
|
||||
expect(deletedAbandonnedVhd.length).toEqual(1) // and it must be abandonned.vhd
|
||||
|
||||
// we don't test the filew on disk, since they will all be marker as unused and deleted without a metadata.json file
|
||||
})
|
||||
|
||||
test('it remove backup meta data referencing a missing vhd in delta backup', async () => {
|
||||
// create a metadata file marking child and orphan as ok
|
||||
await handler.writeFile(
|
||||
`metadata.json`,
|
||||
JSON.stringify({
|
||||
mode: 'delta',
|
||||
vhds: [
|
||||
`${basePath}/orphan.vhd`,
|
||||
`${basePath}/child.vhd`,
|
||||
// abandonned.json is not here
|
||||
],
|
||||
})
|
||||
)
|
||||
|
||||
await generateVhd(`${basePath}/abandonned.vhd`)
|
||||
|
||||
// one orphan, which is a full vhd, no parent
|
||||
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
|
||||
|
||||
// a child to the orphan
|
||||
await generateVhd(`${basePath}/child.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'orphan.vhd',
|
||||
parentUid: orphan.footer.uuid,
|
||||
},
|
||||
})
|
||||
|
||||
let loggued = ''
|
||||
const onLog = message => {
|
||||
loggued += message + '\n'
|
||||
}
|
||||
await adapter.cleanVm('/', { remove: true, onLog })
|
||||
let matched = loggued.match(/deleting unused VHD /g) || []
|
||||
expect(matched.length).toEqual(1) // only one vhd should have been deleted
|
||||
matched = loggued.match(/abandonned.vhd is unused/g) || []
|
||||
expect(matched.length).toEqual(1) // and it must be abandonned.vhd
|
||||
|
||||
// a missing vhd cause clean to remove all vhds
|
||||
await handler.writeFile(
|
||||
`metadata.json`,
|
||||
JSON.stringify({
|
||||
mode: 'delta',
|
||||
vhds: [
|
||||
`${basePath}/deleted.vhd`, // in metadata but not in vhds
|
||||
`${basePath}/orphan.vhd`,
|
||||
`${basePath}/child.vhd`,
|
||||
// abandonned.json is not here
|
||||
],
|
||||
}),
|
||||
{ flags: 'w' }
|
||||
)
|
||||
loggued = ''
|
||||
await adapter.cleanVm('/', { remove: true, onLog })
|
||||
matched = loggued.match(/deleting unused VHD /g) || []
|
||||
expect(matched.length).toEqual(2) // all vhds (orphan and child ) should have been deleted
|
||||
})
|
||||
|
||||
test('it merges delta of non destroyed chain', async () => {
|
||||
await handler.writeFile(
|
||||
`metadata.json`,
|
||||
JSON.stringify({
|
||||
mode: 'delta',
|
||||
size: 12000, // a size too small
|
||||
vhds: [
|
||||
`${basePath}/grandchild.vhd`, // grand child should not be merged
|
||||
`${basePath}/child.vhd`,
|
||||
// orphan is not here, he should be merged in child
|
||||
],
|
||||
})
|
||||
)
|
||||
|
||||
// one orphan, which is a full vhd, no parent
|
||||
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
|
||||
// a child to the orphan
|
||||
const child = await generateVhd(`${basePath}/child.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'orphan.vhd',
|
||||
parentUid: orphan.footer.uuid,
|
||||
},
|
||||
})
|
||||
// a grand child
|
||||
await generateVhd(`${basePath}/grandchild.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'child.vhd',
|
||||
parentUid: child.footer.uuid,
|
||||
},
|
||||
})
|
||||
|
||||
let loggued = []
|
||||
const onLog = message => {
|
||||
loggued.push(message)
|
||||
}
|
||||
await adapter.cleanVm('/', { remove: true, onLog })
|
||||
expect(loggued[0]).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused`)
|
||||
expect(loggued[1]).toEqual(`incorrect size in metadata: 12000 instead of 209920`)
|
||||
|
||||
loggued = []
|
||||
await adapter.cleanVm('/', { remove: true, merge: true, onLog })
|
||||
const [unused, merging] = loggued
|
||||
expect(unused).toEqual(`the parent /${basePath}/orphan.vhd of the child /${basePath}/child.vhd is unused`)
|
||||
expect(merging).toEqual(`merging /${basePath}/child.vhd into /${basePath}/orphan.vhd`)
|
||||
|
||||
const metadata = JSON.parse(await handler.readFile(`metadata.json`))
|
||||
// size should be the size of children + grand children after the merge
|
||||
expect(metadata.size).toEqual(209920)
|
||||
|
||||
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
|
||||
// only check deletion
|
||||
const remainingVhds = await handler.list(basePath)
|
||||
expect(remainingVhds.length).toEqual(2)
|
||||
expect(remainingVhds.includes('child.vhd')).toEqual(true)
|
||||
expect(remainingVhds.includes('grandchild.vhd')).toEqual(true)
|
||||
})
|
||||
|
||||
test('it finish unterminated merge ', async () => {
|
||||
await handler.writeFile(
|
||||
`metadata.json`,
|
||||
JSON.stringify({
|
||||
mode: 'delta',
|
||||
size: undefined,
|
||||
vhds: [
|
||||
`${basePath}/orphan.vhd`, // grand child should not be merged
|
||||
`${basePath}/child.vhd`,
|
||||
// orphan is not here, he should be merged in child
|
||||
],
|
||||
})
|
||||
)
|
||||
|
||||
// one orphan, which is a full vhd, no parent
|
||||
const orphan = await generateVhd(`${basePath}/orphan.vhd`)
|
||||
// a child to the orphan
|
||||
const child = await generateVhd(`${basePath}/child.vhd`, {
|
||||
header: {
|
||||
parentUnicodeName: 'orphan.vhd',
|
||||
parentUid: orphan.footer.uuid,
|
||||
},
|
||||
})
|
||||
// a merge in progress file
|
||||
await handler.writeFile(
|
||||
`${basePath}/.orphan.vhd.merge.json`,
|
||||
JSON.stringify({
|
||||
parent: {
|
||||
header: orphan.header.checksum,
|
||||
},
|
||||
child: {
|
||||
header: child.header.checksum,
|
||||
},
|
||||
})
|
||||
)
|
||||
|
||||
// a unfinished merging
|
||||
await adapter.cleanVm('/', { remove: true, merge: true })
|
||||
// merging is already tested in vhd-lib, don't retest it here (and theses vhd are as empty as my stomach at 12h12)
|
||||
|
||||
// only check deletion
|
||||
const remainingVhds = await handler.list(basePath)
|
||||
expect(remainingVhds.length).toEqual(1)
|
||||
expect(remainingVhds.includes('child.vhd')).toEqual(true)
|
||||
})
|
||||
|
||||
// each of the vhd can be a file, a directory, an alias to a file or an alias to a directory
|
||||
// the message an resulting files should be identical to the output with vhd files which is tested independantly
|
||||
|
||||
describe('tests mulitple combination ', () => {
|
||||
for (const useAlias of [true, false]) {
|
||||
for (const vhdMode of ['file', 'directory']) {
|
||||
test(`alias : ${useAlias}, mode: ${vhdMode}`, async () => {
|
||||
// a broken VHD
|
||||
const brokenVhdDataPath = basePath + useAlias ? 'broken.data' : 'broken.vhd'
|
||||
if (vhdMode === 'directory') {
|
||||
await handler.mkdir(brokenVhdDataPath)
|
||||
} else {
|
||||
await handler.writeFile(brokenVhdDataPath, 'notreallyavhd')
|
||||
}
|
||||
if (useAlias) {
|
||||
await VhdAbstract.createAlias(handler, 'broken.alias.vhd', brokenVhdDataPath)
|
||||
}
|
||||
|
||||
// a vhd non referenced in metada
|
||||
await generateVhd(`${basePath}/nonreference.vhd`, { useAlias, mode: vhdMode })
|
||||
// an abandonded delta vhd without its parent
|
||||
await generateVhd(`${basePath}/abandonned.vhd`, {
|
||||
useAlias,
|
||||
mode: vhdMode,
|
||||
header: {
|
||||
parentUnicodeName: 'gone.vhd',
|
||||
parentUid: crypto.randomBytes(16),
|
||||
},
|
||||
})
|
||||
// an ancestor of a vhd present in metadata
|
||||
const ancestor = await generateVhd(`${basePath}/ancestor.vhd`, {
|
||||
useAlias,
|
||||
mode: vhdMode,
|
||||
})
|
||||
const child = await generateVhd(`${basePath}/child.vhd`, {
|
||||
useAlias,
|
||||
mode: vhdMode,
|
||||
header: {
|
||||
parentUnicodeName: 'ancestor.vhd' + (useAlias ? '.alias.vhd' : ''),
|
||||
parentUid: ancestor.footer.uuid,
|
||||
},
|
||||
})
|
||||
// a grand child vhd in metadata
|
||||
await generateVhd(`${basePath}/grandchild.vhd`, {
|
||||
useAlias,
|
||||
mode: vhdMode,
|
||||
header: {
|
||||
parentUnicodeName: 'child.vhd' + (useAlias ? '.alias.vhd' : ''),
|
||||
parentUid: child.footer.uuid,
|
||||
},
|
||||
})
|
||||
|
||||
// an older parent that was merging in clean
|
||||
const cleanAncestor = await generateVhd(`${basePath}/cleanAncestor.vhd`, {
|
||||
useAlias,
|
||||
mode: vhdMode,
|
||||
})
|
||||
// a clean vhd in metadata
|
||||
const clean = await generateVhd(`${basePath}/clean.vhd`, {
|
||||
useAlias,
|
||||
mode: vhdMode,
|
||||
header: {
|
||||
parentUnicodeName: 'cleanAncestor.vhd' + (useAlias ? '.alias.vhd' : ''),
|
||||
parentUid: cleanAncestor.footer.uuid,
|
||||
},
|
||||
})
|
||||
|
||||
await handler.writeFile(
|
||||
`${basePath}/.cleanAncestor.vhd${useAlias ? '.alias.vhd' : ''}.merge.json`,
|
||||
JSON.stringify({
|
||||
parent: {
|
||||
header: cleanAncestor.header.checksum,
|
||||
},
|
||||
child: {
|
||||
header: clean.header.checksum,
|
||||
},
|
||||
})
|
||||
)
|
||||
|
||||
// the metadata file
|
||||
await handler.writeFile(
|
||||
`metadata.json`,
|
||||
JSON.stringify({
|
||||
mode: 'delta',
|
||||
vhds: [
|
||||
`${basePath}/grandchild.vhd` + (useAlias ? '.alias.vhd' : ''), // grand child should not be merged
|
||||
`${basePath}/child.vhd` + (useAlias ? '.alias.vhd' : ''),
|
||||
`${basePath}/clean.vhd` + (useAlias ? '.alias.vhd' : ''),
|
||||
],
|
||||
})
|
||||
)
|
||||
await adapter.cleanVm('/', { remove: true, merge: true })
|
||||
|
||||
const metadata = JSON.parse(await handler.readFile(`metadata.json`))
|
||||
// size should be the size of children + grand children + clean after the merge
|
||||
expect(metadata.size).toEqual(vhdMode === 'file' ? 314880 : undefined)
|
||||
|
||||
// broken vhd, non referenced, abandonned should be deleted ( alias and data)
|
||||
// ancestor and child should be merged
|
||||
// grand child and clean vhd should not have changed
|
||||
const survivors = await handler.list(basePath)
|
||||
// console.log(survivors)
|
||||
if (useAlias) {
|
||||
// the goal of the alias : do not move a full folder
|
||||
expect(survivors).toContain('ancestor.vhd.data')
|
||||
expect(survivors).toContain('grandchild.vhd.data')
|
||||
expect(survivors).toContain('cleanAncestor.vhd.data')
|
||||
expect(survivors).toContain('clean.vhd.alias.vhd')
|
||||
expect(survivors).toContain('child.vhd.alias.vhd')
|
||||
expect(survivors).toContain('grandchild.vhd.alias.vhd')
|
||||
expect(survivors.length).toEqual(6)
|
||||
} else {
|
||||
expect(survivors).toContain('clean.vhd')
|
||||
expect(survivors).toContain('child.vhd')
|
||||
expect(survivors).toContain('grandchild.vhd')
|
||||
expect(survivors.length).toEqual(3)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
test('it cleans orphan merge states ', async () => {
|
||||
await handler.writeFile(`${basePath}/.orphan.vhd.merge.json`, '')
|
||||
|
||||
await adapter.cleanVm('/', { remove: true })
|
||||
|
||||
expect(await handler.list(basePath)).toEqual([])
|
||||
})
|
||||
@@ -1,32 +1,13 @@
|
||||
const assert = require('assert')
|
||||
const sum = require('lodash/sum')
|
||||
const { asyncMap } = require('@xen-orchestra/async-map')
|
||||
const { Constants, mergeVhd, openVhd, VhdAbstract, VhdFile } = require('vhd-lib')
|
||||
const { VhdFile, mergeVhd } = require('vhd-lib')
|
||||
const { dirname, resolve } = require('path')
|
||||
const { DISK_TYPES } = Constants
|
||||
const { DISK_TYPE_DIFFERENCING } = require('vhd-lib/dist/_constants.js')
|
||||
const { isMetadataFile, isVhdFile, isXvaFile, isXvaSumFile } = require('./_backupType.js')
|
||||
const { limitConcurrency } = require('limit-concurrency-decorator')
|
||||
|
||||
const { Task } = require('./Task.js')
|
||||
const { Disposable } = require('promise-toolbox')
|
||||
|
||||
// checking the size of a vhd directory is costly
|
||||
// 1 Http Query per 1000 blocks
|
||||
// we only check size of all the vhd are VhdFiles
|
||||
function shouldComputeVhdsSize(vhds) {
|
||||
return vhds.every(vhd => vhd instanceof VhdFile)
|
||||
}
|
||||
|
||||
const computeVhdsSize = (handler, vhdPaths) =>
|
||||
Disposable.use(
|
||||
vhdPaths.map(vhdPath => openVhd(handler, vhdPath)),
|
||||
async vhds => {
|
||||
if (shouldComputeVhdsSize(vhds)) {
|
||||
const sizes = await asyncMap(vhds, vhd => vhd.getSize())
|
||||
return sum(sizes)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
// chain is an array of VHDs from child to parent
|
||||
//
|
||||
@@ -84,12 +65,12 @@ async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
|
||||
clearInterval(handle)
|
||||
|
||||
await Promise.all([
|
||||
VhdAbstract.rename(handler, parent, child),
|
||||
handler.rename(parent, child),
|
||||
asyncMap(children.slice(0, -1), child => {
|
||||
onLog(`the VHD ${child} is unused`)
|
||||
if (remove) {
|
||||
onLog(`deleting unused VHD ${child}`)
|
||||
return VhdAbstract.unlink(handler, child)
|
||||
return handler.unlink(child)
|
||||
}
|
||||
}),
|
||||
])
|
||||
@@ -100,10 +81,10 @@ async function mergeVhdChain(chain, { handler, onLog, remove, merge }) {
|
||||
|
||||
const noop = Function.prototype
|
||||
|
||||
const INTERRUPTED_VHDS_REG = /^\.(.+)\.merge.json$/
|
||||
const INTERRUPTED_VHDS_REG = /^(?:(.+)\/)?\.(.+)\.merge.json$/
|
||||
const listVhds = async (handler, vmDir) => {
|
||||
const vhds = new Set()
|
||||
const interruptedVhds = new Map()
|
||||
const vhds = []
|
||||
const interruptedVhds = new Set()
|
||||
|
||||
await asyncMap(
|
||||
await handler.list(`${vmDir}/vdis`, {
|
||||
@@ -118,14 +99,16 @@ const listVhds = async (handler, vmDir) => {
|
||||
async vdiDir => {
|
||||
const list = await handler.list(vdiDir, {
|
||||
filter: file => isVhdFile(file) || INTERRUPTED_VHDS_REG.test(file),
|
||||
prependDir: true,
|
||||
})
|
||||
|
||||
list.forEach(file => {
|
||||
const res = INTERRUPTED_VHDS_REG.exec(file)
|
||||
if (res === null) {
|
||||
vhds.add(`${vdiDir}/${file}`)
|
||||
vhds.push(file)
|
||||
} else {
|
||||
interruptedVhds.set(`${vdiDir}/${res[1]}`, `${vdiDir}/${file}`)
|
||||
const [, dir, file] = res
|
||||
interruptedVhds.add(`${dir}/${file}`)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -145,81 +128,62 @@ exports.cleanVm = async function cleanVm(
|
||||
|
||||
const handler = this._handler
|
||||
|
||||
const vhdsToJSons = new Set()
|
||||
const vhds = new Set()
|
||||
const vhdParents = { __proto__: null }
|
||||
const vhdChildren = { __proto__: null }
|
||||
|
||||
const { vhds, interruptedVhds } = await listVhds(handler, vmDir)
|
||||
const vhdsList = await listVhds(handler, vmDir)
|
||||
|
||||
// remove broken VHDs
|
||||
await asyncMap(vhds, async path => {
|
||||
await asyncMap(vhdsList.vhds, async path => {
|
||||
try {
|
||||
await Disposable.use(openVhd(handler, path, { checkSecondFooter: !interruptedVhds.has(path) }), vhd => {
|
||||
if (vhd.footer.diskType === DISK_TYPES.DIFFERENCING) {
|
||||
const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
|
||||
vhdParents[path] = parent
|
||||
if (parent in vhdChildren) {
|
||||
const error = new Error('this script does not support multiple VHD children')
|
||||
error.parent = parent
|
||||
error.child1 = vhdChildren[parent]
|
||||
error.child2 = path
|
||||
throw error // should we throw?
|
||||
}
|
||||
vhdChildren[parent] = path
|
||||
const vhd = new VhdFile(handler, path)
|
||||
await vhd.readHeaderAndFooter(!vhdsList.interruptedVhds.has(path))
|
||||
vhds.add(path)
|
||||
if (vhd.footer.diskType === DISK_TYPE_DIFFERENCING) {
|
||||
const parent = resolve('/', dirname(path), vhd.header.parentUnicodeName)
|
||||
vhdParents[path] = parent
|
||||
if (parent in vhdChildren) {
|
||||
const error = new Error('this script does not support multiple VHD children')
|
||||
error.parent = parent
|
||||
error.child1 = vhdChildren[parent]
|
||||
error.child2 = path
|
||||
throw error // should we throw?
|
||||
}
|
||||
})
|
||||
vhdChildren[parent] = path
|
||||
}
|
||||
} catch (error) {
|
||||
vhds.delete(path)
|
||||
onLog(`error while checking the VHD with path ${path}`, { error })
|
||||
if (error?.code === 'ERR_ASSERTION' && remove) {
|
||||
onLog(`deleting broken ${path}`)
|
||||
return VhdAbstract.unlink(handler, path)
|
||||
await handler.unlink(path)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// remove interrupted merge states for missing VHDs
|
||||
for (const interruptedVhd of interruptedVhds.keys()) {
|
||||
if (!vhds.has(interruptedVhd)) {
|
||||
const statePath = interruptedVhds.get(interruptedVhd)
|
||||
interruptedVhds.delete(interruptedVhd)
|
||||
|
||||
onLog('orphan merge state', {
|
||||
mergeStatePath: statePath,
|
||||
missingVhdPath: interruptedVhd,
|
||||
})
|
||||
if (remove) {
|
||||
onLog(`deleting orphan merge state ${statePath}`)
|
||||
await handler.unlink(statePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// @todo : add check for data folder of alias not referenced in a valid alias
|
||||
|
||||
// remove VHDs with missing ancestors
|
||||
{
|
||||
const deletions = []
|
||||
|
||||
// return true if the VHD has been deleted or is missing
|
||||
const deleteIfOrphan = vhdPath => {
|
||||
const parent = vhdParents[vhdPath]
|
||||
const deleteIfOrphan = vhd => {
|
||||
const parent = vhdParents[vhd]
|
||||
if (parent === undefined) {
|
||||
return
|
||||
}
|
||||
|
||||
// no longer needs to be checked
|
||||
delete vhdParents[vhdPath]
|
||||
delete vhdParents[vhd]
|
||||
|
||||
deleteIfOrphan(parent)
|
||||
|
||||
if (!vhds.has(parent)) {
|
||||
vhds.delete(vhdPath)
|
||||
vhds.delete(vhd)
|
||||
|
||||
onLog(`the parent ${parent} of the VHD ${vhdPath} is missing`)
|
||||
onLog(`the parent ${parent} of the VHD ${vhd} is missing`)
|
||||
if (remove) {
|
||||
onLog(`deleting orphan VHD ${vhdPath}`)
|
||||
deletions.push(VhdAbstract.unlink(handler, vhdPath))
|
||||
onLog(`deleting orphan VHD ${vhd}`)
|
||||
deletions.push(handler.unlink(vhd))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -235,7 +199,7 @@ exports.cleanVm = async function cleanVm(
|
||||
await Promise.all(deletions)
|
||||
}
|
||||
|
||||
const jsons = new Set()
|
||||
const jsons = []
|
||||
const xvas = new Set()
|
||||
const xvaSums = []
|
||||
const entries = await handler.list(vmDir, {
|
||||
@@ -243,7 +207,7 @@ exports.cleanVm = async function cleanVm(
|
||||
})
|
||||
entries.forEach(path => {
|
||||
if (isMetadataFile(path)) {
|
||||
jsons.add(path)
|
||||
jsons.push(path)
|
||||
} else if (isXvaFile(path)) {
|
||||
xvas.add(path)
|
||||
} else if (isXvaSumFile(path)) {
|
||||
@@ -265,25 +229,22 @@ exports.cleanVm = async function cleanVm(
|
||||
// compile the list of unused XVAs and VHDs, and remove backup metadata which
|
||||
// reference a missing XVA/VHD
|
||||
await asyncMap(jsons, async json => {
|
||||
let metadata
|
||||
try {
|
||||
metadata = JSON.parse(await handler.readFile(json))
|
||||
} catch (error) {
|
||||
onLog(`failed to read metadata file ${json}`, { error })
|
||||
jsons.delete(json)
|
||||
return
|
||||
}
|
||||
|
||||
const metadata = JSON.parse(await handler.readFile(json))
|
||||
const { mode } = metadata
|
||||
let size
|
||||
if (mode === 'full') {
|
||||
const linkedXva = resolve('/', vmDir, metadata.xva)
|
||||
|
||||
if (xvas.has(linkedXva)) {
|
||||
unusedXvas.delete(linkedXva)
|
||||
|
||||
size = await handler.getSize(linkedXva).catch(error => {
|
||||
onLog(`failed to get size of ${json}`, { error })
|
||||
})
|
||||
} else {
|
||||
onLog(`the XVA linked to the metadata ${json} is missing`)
|
||||
if (remove) {
|
||||
onLog(`deleting incomplete backup ${json}`)
|
||||
jsons.delete(json)
|
||||
await handler.unlink(json)
|
||||
}
|
||||
}
|
||||
@@ -293,24 +254,38 @@ exports.cleanVm = async function cleanVm(
|
||||
return Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
|
||||
})()
|
||||
|
||||
const missingVhds = linkedVhds.filter(_ => !vhds.has(_))
|
||||
|
||||
// FIXME: find better approach by keeping as much of the backup as
|
||||
// possible (existing disks) even if one disk is missing
|
||||
if (missingVhds.length === 0) {
|
||||
if (linkedVhds.every(_ => vhds.has(_))) {
|
||||
linkedVhds.forEach(_ => unusedVhds.delete(_))
|
||||
linkedVhds.forEach(path => {
|
||||
vhdsToJSons[path] = json
|
||||
|
||||
size = await asyncMap(linkedVhds, vhd => handler.getSize(vhd)).then(sum, error => {
|
||||
onLog(`failed to get size of ${json}`, { error })
|
||||
})
|
||||
} else {
|
||||
onLog(`Some VHDs linked to the metadata ${json} are missing`, { missingVhds })
|
||||
onLog(`Some VHDs linked to the metadata ${json} are missing`)
|
||||
if (remove) {
|
||||
onLog(`deleting incomplete backup ${json}`)
|
||||
jsons.delete(json)
|
||||
await handler.unlink(json)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const metadataSize = metadata.size
|
||||
if (size !== undefined && metadataSize !== size) {
|
||||
onLog(`incorrect size in metadata: ${metadataSize ?? 'none'} instead of ${size}`)
|
||||
|
||||
// don't update if the the stored size is greater than found files,
|
||||
// it can indicates a problem
|
||||
if (fixMetadata && (metadataSize === undefined || metadataSize < size)) {
|
||||
try {
|
||||
metadata.size = size
|
||||
await handler.writeFile(json, JSON.stringify(metadata), { flags: 'w' })
|
||||
} catch (error) {
|
||||
onLog(`failed to update size in backup metadata ${json}`, { error })
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// TODO: parallelize by vm/job/vdi
|
||||
@@ -349,7 +324,7 @@ exports.cleanVm = async function cleanVm(
|
||||
onLog(`the VHD ${vhd} is unused`)
|
||||
if (remove) {
|
||||
onLog(`deleting unused VHD ${vhd}`)
|
||||
unusedVhdsDeletion.push(VhdAbstract.unlink(handler, vhd))
|
||||
unusedVhdsDeletion.push(handler.unlink(vhd))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -358,9 +333,9 @@ exports.cleanVm = async function cleanVm(
|
||||
})
|
||||
|
||||
// merge interrupted VHDs
|
||||
for (const parent of interruptedVhds.keys()) {
|
||||
vhdsList.interruptedVhds.forEach(parent => {
|
||||
vhdChainsToMerge[parent] = [vhdChildren[parent], parent]
|
||||
}
|
||||
})
|
||||
|
||||
Object.values(vhdChainsToMerge).forEach(chain => {
|
||||
if (chain !== undefined) {
|
||||
@@ -369,15 +344,9 @@ exports.cleanVm = async function cleanVm(
|
||||
})
|
||||
}
|
||||
|
||||
const metadataWithMergedVhd = {}
|
||||
const doMerge = async () => {
|
||||
await asyncMap(toMerge, async chain => {
|
||||
const merged = await limitedMergeVhdChain(chain, { handler, onLog, remove, merge })
|
||||
if (merged !== undefined) {
|
||||
const metadataPath = vhdsToJSons[chain[0]] // all the chain should have the same metada file
|
||||
metadataWithMergedVhd[metadataPath] = true
|
||||
}
|
||||
})
|
||||
const doMerge = () => {
|
||||
const promise = asyncMap(toMerge, async chain => limitedMergeVhdChain(chain, { handler, onLog, remove, merge }))
|
||||
return merge ? promise.then(sizes => ({ size: sum(sizes) })) : promise
|
||||
}
|
||||
|
||||
await Promise.all([
|
||||
@@ -402,52 +371,6 @@ exports.cleanVm = async function cleanVm(
|
||||
}),
|
||||
])
|
||||
|
||||
// update size for delta metadata with merged VHD
|
||||
// check for the other that the size is the same as the real file size
|
||||
|
||||
await asyncMap(jsons, async metadataPath => {
|
||||
const metadata = JSON.parse(await handler.readFile(metadataPath))
|
||||
|
||||
let fileSystemSize
|
||||
const merged = metadataWithMergedVhd[metadataPath] !== undefined
|
||||
|
||||
const { mode, size, vhds, xva } = metadata
|
||||
|
||||
try {
|
||||
if (mode === 'full') {
|
||||
// a full backup : check size
|
||||
const linkedXva = resolve('/', vmDir, xva)
|
||||
fileSystemSize = await handler.getSize(linkedXva)
|
||||
} else if (mode === 'delta') {
|
||||
const linkedVhds = Object.keys(vhds).map(key => resolve('/', vmDir, vhds[key]))
|
||||
fileSystemSize = await computeVhdsSize(handler, linkedVhds)
|
||||
|
||||
// the size is not computed in some cases (e.g. VhdDirectory)
|
||||
if (fileSystemSize === undefined) {
|
||||
return
|
||||
}
|
||||
|
||||
// don't warn if the size has changed after a merge
|
||||
if (!merged && fileSystemSize !== size) {
|
||||
onLog(`incorrect size in metadata: ${size ?? 'none'} instead of ${fileSystemSize}`)
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
onLog(`failed to get size of ${metadataPath}`, { error })
|
||||
return
|
||||
}
|
||||
|
||||
// systematically update size after a merge
|
||||
if ((merged || fixMetadata) && size !== fileSystemSize) {
|
||||
metadata.size = fileSystemSize
|
||||
try {
|
||||
await handler.writeFile(metadataPath, JSON.stringify(metadata), { flags: 'w' })
|
||||
} catch (error) {
|
||||
onLog(`failed to update size in backup metadata ${metadataPath} after merge`, { error })
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
// boolean whether some VHDs were merged (or should be merged)
|
||||
merge: toMerge.length !== 0,
|
||||
|
||||
@@ -1,24 +1,11 @@
|
||||
const assert = require('assert')
|
||||
|
||||
const COMPRESSED_MAGIC_NUMBERS = [
|
||||
const isGzipFile = async (handler, fd) => {
|
||||
// https://tools.ietf.org/html/rfc1952.html#page-5
|
||||
Buffer.from('1F8B', 'hex'),
|
||||
const magicNumber = Buffer.allocUnsafe(2)
|
||||
|
||||
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames
|
||||
Buffer.from('28B52FFD', 'hex'),
|
||||
]
|
||||
const MAGIC_NUMBER_MAX_LENGTH = Math.max(...COMPRESSED_MAGIC_NUMBERS.map(_ => _.length))
|
||||
|
||||
const isCompressedFile = async (handler, fd) => {
|
||||
const header = Buffer.allocUnsafe(MAGIC_NUMBER_MAX_LENGTH)
|
||||
assert.strictEqual((await handler.read(fd, header, 0)).bytesRead, header.length)
|
||||
|
||||
for (const magicNumber of COMPRESSED_MAGIC_NUMBERS) {
|
||||
if (magicNumber.compare(header, 0, magicNumber.length) === 0) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
assert.strictEqual((await handler.read(fd, magicNumber, 0)).bytesRead, magicNumber.length)
|
||||
return magicNumber[0] === 31 && magicNumber[1] === 139
|
||||
}
|
||||
|
||||
// TODO: better check?
|
||||
@@ -56,8 +43,8 @@ async function isValidXva(path) {
|
||||
return false
|
||||
}
|
||||
|
||||
return (await isCompressedFile(handler, fd))
|
||||
? true // compressed files cannot be validated at this time
|
||||
return (await isGzipFile(handler, fd))
|
||||
? true // gzip files cannot be validated at this time
|
||||
: await isValidTar(handler, size, fd)
|
||||
} finally {
|
||||
handler.closeFile(fd).catch(noop)
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
"type": "git",
|
||||
"url": "https://github.com/vatesfr/xen-orchestra.git"
|
||||
},
|
||||
"version": "0.18.3",
|
||||
"version": "0.15.1",
|
||||
"engines": {
|
||||
"node": ">=14.6"
|
||||
},
|
||||
@@ -16,11 +16,11 @@
|
||||
"postversion": "npm publish --access public"
|
||||
},
|
||||
"dependencies": {
|
||||
"@vates/compose": "^2.1.0",
|
||||
"@vates/compose": "^2.0.0",
|
||||
"@vates/disposable": "^0.1.1",
|
||||
"@vates/parse-duration": "^0.1.1",
|
||||
"@xen-orchestra/async-map": "^0.1.2",
|
||||
"@xen-orchestra/fs": "^0.19.3",
|
||||
"@xen-orchestra/fs": "^0.18.0",
|
||||
"@xen-orchestra/log": "^0.3.0",
|
||||
"@xen-orchestra/template": "^0.1.0",
|
||||
"compare-versions": "^4.0.1",
|
||||
@@ -35,12 +35,11 @@
|
||||
"promise-toolbox": "^0.20.0",
|
||||
"proper-lockfile": "^4.1.2",
|
||||
"pump": "^3.0.0",
|
||||
"uuid": "^8.3.2",
|
||||
"vhd-lib": "^3.0.0",
|
||||
"vhd-lib": "^1.3.0",
|
||||
"yazl": "^2.5.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@xen-orchestra/xapi": "^0.8.5"
|
||||
"@xen-orchestra/xapi": "^0.8.0"
|
||||
},
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"author": {
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
// a valid footer of a 2
|
||||
exports.VHDFOOTER = {
|
||||
cookie: 'conectix',
|
||||
features: 2,
|
||||
fileFormatVersion: 65536,
|
||||
dataOffset: 512,
|
||||
timestamp: 0,
|
||||
creatorApplication: 'caml',
|
||||
creatorVersion: 1,
|
||||
creatorHostOs: 0,
|
||||
originalSize: 53687091200,
|
||||
currentSize: 53687091200,
|
||||
diskGeometry: { cylinders: 25700, heads: 16, sectorsPerTrackCylinder: 255 },
|
||||
diskType: 3,
|
||||
checksum: 4294962945,
|
||||
uuid: Buffer.from('d8dbcad85265421e8b298d99c2eec551', 'utf-8'),
|
||||
saved: '',
|
||||
hidden: '',
|
||||
reserved: '',
|
||||
}
|
||||
exports.VHDHEADER = {
|
||||
cookie: 'cxsparse',
|
||||
dataOffset: undefined,
|
||||
tableOffset: 2048,
|
||||
headerVersion: 65536,
|
||||
maxTableEntries: 25600,
|
||||
blockSize: 2097152,
|
||||
checksum: 4294964241,
|
||||
parentUuid: null,
|
||||
parentTimestamp: 0,
|
||||
reserved1: 0,
|
||||
parentUnicodeName: '',
|
||||
parentLocatorEntry: [
|
||||
{
|
||||
platformCode: 0,
|
||||
platformDataSpace: 0,
|
||||
platformDataLength: 0,
|
||||
reserved: 0,
|
||||
platformDataOffset: 0,
|
||||
},
|
||||
{
|
||||
platformCode: 0,
|
||||
platformDataSpace: 0,
|
||||
platformDataLength: 0,
|
||||
reserved: 0,
|
||||
platformDataOffset: 0,
|
||||
},
|
||||
{
|
||||
platformCode: 0,
|
||||
platformDataSpace: 0,
|
||||
platformDataLength: 0,
|
||||
reserved: 0,
|
||||
platformDataOffset: 0,
|
||||
},
|
||||
{
|
||||
platformCode: 0,
|
||||
platformDataSpace: 0,
|
||||
platformDataLength: 0,
|
||||
reserved: 0,
|
||||
platformDataOffset: 0,
|
||||
},
|
||||
{
|
||||
platformCode: 0,
|
||||
platformDataSpace: 0,
|
||||
platformDataLength: 0,
|
||||
reserved: 0,
|
||||
platformDataOffset: 0,
|
||||
},
|
||||
{
|
||||
platformCode: 0,
|
||||
platformDataSpace: 0,
|
||||
platformDataLength: 0,
|
||||
reserved: 0,
|
||||
platformDataOffset: 0,
|
||||
},
|
||||
{
|
||||
platformCode: 0,
|
||||
platformDataSpace: 0,
|
||||
platformDataLength: 0,
|
||||
reserved: 0,
|
||||
platformDataOffset: 0,
|
||||
},
|
||||
{
|
||||
platformCode: 0,
|
||||
platformDataSpace: 0,
|
||||
platformDataLength: 0,
|
||||
reserved: 0,
|
||||
platformDataOffset: 0,
|
||||
},
|
||||
],
|
||||
reserved2: '',
|
||||
}
|
||||
@@ -3,7 +3,7 @@ const map = require('lodash/map.js')
|
||||
const mapValues = require('lodash/mapValues.js')
|
||||
const ignoreErrors = require('promise-toolbox/ignoreErrors.js')
|
||||
const { asyncMap } = require('@xen-orchestra/async-map')
|
||||
const { chainVhd, checkVhdChain, openVhd, VhdAbstract } = require('vhd-lib')
|
||||
const { chainVhd, checkVhdChain, VhdFile } = require('vhd-lib')
|
||||
const { createLogger } = require('@xen-orchestra/log')
|
||||
const { dirname } = require('path')
|
||||
|
||||
@@ -16,7 +16,6 @@ const { MixinBackupWriter } = require('./_MixinBackupWriter.js')
|
||||
const { AbstractDeltaWriter } = require('./_AbstractDeltaWriter.js')
|
||||
const { checkVhd } = require('./_checkVhd.js')
|
||||
const { packUuid } = require('./_packUuid.js')
|
||||
const { Disposable } = require('promise-toolbox')
|
||||
|
||||
const { warn } = createLogger('xo:backups:DeltaBackupWriter')
|
||||
|
||||
@@ -24,7 +23,6 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
|
||||
async checkBaseVdis(baseUuidToSrcVdi) {
|
||||
const { handler } = this._adapter
|
||||
const backup = this._backup
|
||||
const adapter = this._adapter
|
||||
|
||||
const backupDir = getVmBackupDir(backup.vm.uuid)
|
||||
const vdisDir = `${backupDir}/vdis/${backup.job.id}`
|
||||
@@ -36,21 +34,16 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
|
||||
filter: _ => _[0] !== '.' && _.endsWith('.vhd'),
|
||||
prependDir: true,
|
||||
})
|
||||
const packedBaseUuid = packUuid(baseUuid)
|
||||
await asyncMap(vhds, async path => {
|
||||
try {
|
||||
await checkVhdChain(handler, path)
|
||||
// Warning, this should not be written as found = found || await adapter.isMergeableParent(packedBaseUuid, path)
|
||||
//
|
||||
// since all the checks of a path are done in parallel, found would be containing
|
||||
// only the last answer of isMergeableParent which is probably not the right one
|
||||
// this led to the support tickets https://help.vates.fr/#ticket/zoom/4751 , 4729, 4665 and 4300
|
||||
|
||||
const isMergeable = await adapter.isMergeableParent(packedBaseUuid, path)
|
||||
found = found || isMergeable
|
||||
const vhd = new VhdFile(handler, path)
|
||||
await vhd.readHeaderAndFooter()
|
||||
found = found || vhd.footer.uuid.equals(packUuid(baseUuid))
|
||||
} catch (error) {
|
||||
warn('checkBaseVdis', { error })
|
||||
await ignoreErrors.call(VhdAbstract.unlink(handler, path))
|
||||
await ignoreErrors.call(handler.unlink(path))
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
@@ -151,7 +144,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
|
||||
// don't do delta for it
|
||||
vdi.uuid
|
||||
: vdi.$snapshot_of$uuid
|
||||
}/${adapter.getVhdFileName(basename)}`
|
||||
}/${basename}.vhd`
|
||||
)
|
||||
|
||||
const metadataFilename = `${backupDir}/${basename}.json`
|
||||
@@ -195,7 +188,7 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
|
||||
await checkVhd(handler, parentPath)
|
||||
}
|
||||
|
||||
await adapter.writeVhd(path, deltaExport.streams[`${id}.vhd`], {
|
||||
await adapter.outputStream(path, deltaExport.streams[`${id}.vhd`], {
|
||||
// no checksum for VHDs, because they will be invalidated by
|
||||
// merges and chainings
|
||||
checksum: false,
|
||||
@@ -207,11 +200,11 @@ exports.DeltaBackupWriter = class DeltaBackupWriter extends MixinBackupWriter(Ab
|
||||
}
|
||||
|
||||
// set the correct UUID in the VHD
|
||||
await Disposable.use(openVhd(handler, path), async vhd => {
|
||||
vhd.footer.uuid = packUuid(vdi.uuid)
|
||||
await vhd.readBlockAllocationTable() // required by writeFooter()
|
||||
await vhd.writeFooter()
|
||||
})
|
||||
const vhd = new VhdFile(handler, path)
|
||||
await vhd.readHeaderAndFooter()
|
||||
vhd.footer.uuid = packUuid(vdi.uuid)
|
||||
await vhd.readBlockAllocationTable() // required by writeFooter()
|
||||
await vhd.writeFooter()
|
||||
})
|
||||
)
|
||||
return {
|
||||
|
||||
@@ -21,18 +21,10 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
|
||||
this.#vmBackupDir = getVmBackupDir(this._backup.vm.uuid)
|
||||
}
|
||||
|
||||
async _cleanVm(options) {
|
||||
try {
|
||||
return await this._adapter.cleanVm(this.#vmBackupDir, {
|
||||
...options,
|
||||
fixMetadata: true,
|
||||
onLog: warn,
|
||||
lock: false,
|
||||
})
|
||||
} catch (error) {
|
||||
warn(error)
|
||||
return {}
|
||||
}
|
||||
_cleanVm(options) {
|
||||
return this._adapter
|
||||
.cleanVm(this.#vmBackupDir, { ...options, fixMetadata: true, onLog: warn, lock: false })
|
||||
.catch(warn)
|
||||
}
|
||||
|
||||
async beforeBackup() {
|
||||
@@ -51,13 +43,7 @@ exports.MixinBackupWriter = (BaseClass = Object) =>
|
||||
// merge worker only compatible with local remotes
|
||||
const { handler } = this._adapter
|
||||
if (merge && !disableMergeWorker && typeof handler._getRealPath === 'function') {
|
||||
const taskFile =
|
||||
join(MergeWorker.CLEAN_VM_QUEUE, formatFilenameDate(new Date())) +
|
||||
'-' +
|
||||
// add a random suffix to avoid collision in case multiple tasks are created at the same second
|
||||
Math.random().toString(36).slice(2)
|
||||
|
||||
await handler.outputFile(taskFile, this._backup.vm.uuid)
|
||||
await handler.outputFile(join(MergeWorker.CLEAN_VM_QUEUE, formatFilenameDate(new Date())), this._backup.vm.uuid)
|
||||
const remotePath = handler._getRealPath()
|
||||
await MergeWorker.run(remotePath)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
const openVhd = require('vhd-lib').openVhd
|
||||
const Disposable = require('promise-toolbox/Disposable')
|
||||
const Vhd = require('vhd-lib').VhdFile
|
||||
|
||||
exports.checkVhd = async function checkVhd(handler, path) {
|
||||
await Disposable.use(openVhd(handler, path), () => {})
|
||||
await new Vhd(handler, path).readHeaderAndFooter()
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"private": false,
|
||||
"name": "@xen-orchestra/fs",
|
||||
"version": "0.19.3",
|
||||
"version": "0.18.0",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"description": "The File System for Xen Orchestra backups.",
|
||||
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/fs",
|
||||
@@ -33,7 +33,7 @@
|
||||
"proper-lockfile": "^4.1.2",
|
||||
"readable-stream": "^3.0.6",
|
||||
"through2": "^4.0.2",
|
||||
"xo-remote-parser": "^0.8.0"
|
||||
"xo-remote-parser": "^0.7.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/cli": "^7.0.0",
|
||||
|
||||
@@ -76,7 +76,6 @@ export default class RemoteHandlerAbstract {
|
||||
|
||||
const sharedLimit = limitConcurrency(options.maxParallelOperations ?? DEFAULT_MAX_PARALLEL_OPERATIONS)
|
||||
this.closeFile = sharedLimit(this.closeFile)
|
||||
this.copy = sharedLimit(this.copy)
|
||||
this.getInfo = sharedLimit(this.getInfo)
|
||||
this.getSize = sharedLimit(this.getSize)
|
||||
this.list = sharedLimit(this.list)
|
||||
@@ -308,17 +307,6 @@ export default class RemoteHandlerAbstract {
|
||||
return p
|
||||
}
|
||||
|
||||
async copy(oldPath, newPath, { checksum = false } = {}) {
|
||||
oldPath = normalizePath(oldPath)
|
||||
newPath = normalizePath(newPath)
|
||||
|
||||
let p = timeout.call(this._copy(oldPath, newPath), this._timeout)
|
||||
if (checksum) {
|
||||
p = Promise.all([p, this._copy(checksumFile(oldPath), checksumFile(newPath))])
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
async rmdir(dir) {
|
||||
await timeout.call(this._rmdir(normalizePath(dir)).catch(ignoreEnoent), this._timeout)
|
||||
}
|
||||
@@ -531,9 +519,6 @@ export default class RemoteHandlerAbstract {
|
||||
async _rename(oldPath, newPath) {
|
||||
throw new Error('Not implemented')
|
||||
}
|
||||
async _copy(oldPath, newPath) {
|
||||
throw new Error('Not implemented')
|
||||
}
|
||||
|
||||
async _rmdir(dir) {
|
||||
throw new Error('Not implemented')
|
||||
|
||||
@@ -33,10 +33,6 @@ export default class LocalHandler extends RemoteHandlerAbstract {
|
||||
return fs.close(fd)
|
||||
}
|
||||
|
||||
async _copy(oldPath, newPath) {
|
||||
return fs.copy(this._getFilePath(oldPath), this._getFilePath(newPath))
|
||||
}
|
||||
|
||||
async _createReadStream(file, options) {
|
||||
if (typeof file === 'string') {
|
||||
const stream = fs.createReadStream(this._getFilePath(file), options)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import aws from '@sullux/aws-sdk'
|
||||
import assert from 'assert'
|
||||
import http from 'http'
|
||||
import https from 'https'
|
||||
import { parse } from 'xo-remote-parser'
|
||||
|
||||
import RemoteHandlerAbstract from './abstract'
|
||||
@@ -17,7 +16,7 @@ const IDEAL_FRAGMENT_SIZE = Math.ceil(MAX_OBJECT_SIZE / MAX_PARTS_COUNT) // the
|
||||
export default class S3Handler extends RemoteHandlerAbstract {
|
||||
constructor(remote, _opts) {
|
||||
super(remote)
|
||||
const { allowUnauthorized, host, path, username, password, protocol, region } = parse(remote.url)
|
||||
const { host, path, username, password, protocol, region } = parse(remote.url)
|
||||
const params = {
|
||||
accessKeyId: username,
|
||||
apiVersion: '2006-03-01',
|
||||
@@ -30,13 +29,8 @@ export default class S3Handler extends RemoteHandlerAbstract {
|
||||
},
|
||||
}
|
||||
if (protocol === 'http') {
|
||||
params.httpOptions.agent = new http.Agent({ keepAlive: true })
|
||||
params.httpOptions.agent = new http.Agent()
|
||||
params.sslEnabled = false
|
||||
} else if (protocol === 'https') {
|
||||
params.httpOptions.agent = new https.Agent({
|
||||
rejectUnauthorized: !allowUnauthorized,
|
||||
keepAlive: true,
|
||||
})
|
||||
}
|
||||
if (region !== undefined) {
|
||||
params.region = region
|
||||
@@ -57,27 +51,6 @@ export default class S3Handler extends RemoteHandlerAbstract {
|
||||
return { Bucket: this._bucket, Key: this._dir + file }
|
||||
}
|
||||
|
||||
async _copy(oldPath, newPath) {
|
||||
const size = await this._getSize(oldPath)
|
||||
const multipartParams = await this._s3.createMultipartUpload({ ...this._createParams(newPath) })
|
||||
const param2 = { ...multipartParams, CopySource: `/${this._bucket}/${this._dir}${oldPath}` }
|
||||
try {
|
||||
const parts = []
|
||||
let start = 0
|
||||
while (start < size) {
|
||||
const range = `bytes=${start}-${Math.min(start + MAX_PART_SIZE, size) - 1}`
|
||||
const partParams = { ...param2, PartNumber: parts.length + 1, CopySourceRange: range }
|
||||
const upload = await this._s3.uploadPartCopy(partParams)
|
||||
parts.push({ ETag: upload.CopyPartResult.ETag, PartNumber: partParams.PartNumber })
|
||||
start += MAX_PART_SIZE
|
||||
}
|
||||
await this._s3.completeMultipartUpload({ ...multipartParams, MultipartUpload: { Parts: parts } })
|
||||
} catch (e) {
|
||||
await this._s3.abortMultipartUpload(multipartParams)
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
async _isNotEmptyDir(path) {
|
||||
const result = await this._s3.listObjectsV2({
|
||||
Bucket: this._bucket,
|
||||
@@ -152,30 +125,16 @@ export default class S3Handler extends RemoteHandlerAbstract {
|
||||
const splitPrefix = splitPath(prefix)
|
||||
const result = await this._s3.listObjectsV2({
|
||||
Bucket: this._bucket,
|
||||
Prefix: splitPrefix.join('/') + '/', // need slash at the end with the use of delimiters
|
||||
Delimiter: '/', // will only return path until delimiters
|
||||
Prefix: splitPrefix.join('/'),
|
||||
})
|
||||
|
||||
if (result.isTruncated) {
|
||||
const error = new Error('more than 1000 objects, unsupported in this implementation')
|
||||
error.dir = dir
|
||||
throw error
|
||||
}
|
||||
|
||||
const uniq = []
|
||||
|
||||
// sub directories
|
||||
for (const entry of result.CommonPrefixes) {
|
||||
const line = splitPath(entry.Prefix)
|
||||
uniq.push(line[line.length - 1])
|
||||
}
|
||||
// files
|
||||
const uniq = new Set()
|
||||
for (const entry of result.Contents) {
|
||||
const line = splitPath(entry.Key)
|
||||
uniq.push(line[line.length - 1])
|
||||
if (line.length > splitPrefix.length) {
|
||||
uniq.add(line[splitPrefix.length])
|
||||
}
|
||||
}
|
||||
|
||||
return uniq
|
||||
return [...uniq]
|
||||
}
|
||||
|
||||
async _mkdir(path) {
|
||||
@@ -188,9 +147,25 @@ export default class S3Handler extends RemoteHandlerAbstract {
|
||||
// nothing to do, directories do not exist, they are part of the files' path
|
||||
}
|
||||
|
||||
// s3 doesn't have a rename operation, so copy + delete source
|
||||
async _rename(oldPath, newPath) {
|
||||
await this.copy(oldPath, newPath)
|
||||
const size = await this._getSize(oldPath)
|
||||
const multipartParams = await this._s3.createMultipartUpload({ ...this._createParams(newPath) })
|
||||
const param2 = { ...multipartParams, CopySource: `/${this._bucket}/${this._dir}${oldPath}` }
|
||||
try {
|
||||
const parts = []
|
||||
let start = 0
|
||||
while (start < size) {
|
||||
const range = `bytes=${start}-${Math.min(start + MAX_PART_SIZE, size) - 1}`
|
||||
const partParams = { ...param2, PartNumber: parts.length + 1, CopySourceRange: range }
|
||||
const upload = await this._s3.uploadPartCopy(partParams)
|
||||
parts.push({ ETag: upload.CopyPartResult.ETag, PartNumber: partParams.PartNumber })
|
||||
start += MAX_PART_SIZE
|
||||
}
|
||||
await this._s3.completeMultipartUpload({ ...multipartParams, MultipartUpload: { Parts: parts } })
|
||||
} catch (e) {
|
||||
await this._s3.abortMultipartUpload(multipartParams)
|
||||
throw e
|
||||
}
|
||||
await this._s3.deleteObject(this._createParams(oldPath))
|
||||
}
|
||||
|
||||
@@ -236,9 +211,9 @@ export default class S3Handler extends RemoteHandlerAbstract {
|
||||
// nothing to do, directories do not exist, they are part of the files' path
|
||||
}
|
||||
|
||||
// reimplement _rmtree to handle efficiantly path with more than 1000 entries in trees
|
||||
// reimplement _rmTree to handle efficiantly path with more than 1000 entries in trees
|
||||
// @todo : use parallel processing for unlink
|
||||
async _rmtree(path) {
|
||||
async _rmTree(path) {
|
||||
let NextContinuationToken
|
||||
do {
|
||||
const result = await this._s3.listObjectsV2({
|
||||
@@ -246,16 +221,11 @@ export default class S3Handler extends RemoteHandlerAbstract {
|
||||
Prefix: this._dir + path + '/',
|
||||
ContinuationToken: NextContinuationToken,
|
||||
})
|
||||
NextContinuationToken = result.isTruncated ? result.NextContinuationToken : undefined
|
||||
for (const { Key } of result.Contents) {
|
||||
// _unlink will add the prefix, but Key contains everything
|
||||
// also we don't need to check if we delete a directory, since the list only return files
|
||||
await this._s3.deleteObject({
|
||||
Bucket: this._bucket,
|
||||
Key,
|
||||
})
|
||||
NextContinuationToken = result.isTruncated ? null : result.NextContinuationToken
|
||||
for (const path of result.Contents) {
|
||||
await this._unlink(path)
|
||||
}
|
||||
} while (NextContinuationToken !== undefined)
|
||||
} while (NextContinuationToken !== null)
|
||||
}
|
||||
|
||||
async _write(file, buffer, position) {
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
"node": ">=6"
|
||||
},
|
||||
"dependencies": {
|
||||
"bind-property-descriptor": "^2.0.0",
|
||||
"bind-property-descriptor": "^1.0.0",
|
||||
"lodash": "^4.17.21"
|
||||
},
|
||||
"scripts": {
|
||||
|
||||
@@ -13,7 +13,7 @@ module.exports = class Config {
|
||||
const watchers = (this._watchers = new Set())
|
||||
|
||||
app.hooks.on('start', async () => {
|
||||
app.hooks.once(
|
||||
app.hooks.on(
|
||||
'stop',
|
||||
await watch({ appDir, appName, ignoreUnknownFormats: true }, (error, config) => {
|
||||
if (error != null) {
|
||||
@@ -32,7 +32,7 @@ module.exports = class Config {
|
||||
get(path) {
|
||||
const value = get(this._config, path)
|
||||
if (value === undefined) {
|
||||
throw new TypeError('missing config entry: ' + path)
|
||||
throw new TypeError('missing config entry: ' + value)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
"url": "https://vates.fr"
|
||||
},
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"version": "0.1.2",
|
||||
"version": "0.1.1",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
@@ -22,7 +22,7 @@
|
||||
"@vates/parse-duration": "^0.1.1",
|
||||
"@xen-orchestra/emit-async": "^0.1.0",
|
||||
"@xen-orchestra/log": "^0.3.0",
|
||||
"app-conf": "^1.0.0",
|
||||
"app-conf": "^0.9.0",
|
||||
"lodash": "^4.17.21"
|
||||
},
|
||||
"scripts": {
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
"@iarna/toml": "^2.2.0",
|
||||
"@vates/read-chunk": "^0.1.2",
|
||||
"ansi-colors": "^4.1.1",
|
||||
"app-conf": "^1.0.0",
|
||||
"app-conf": "^0.9.0",
|
||||
"content-type": "^1.0.4",
|
||||
"cson-parser": "^4.0.7",
|
||||
"getopts": "^2.2.3",
|
||||
|
||||
@@ -20,7 +20,6 @@ keepAliveInterval = 10e3
|
||||
dirMode = 0o700
|
||||
disableMergeWorker = false
|
||||
snapshotNameLabelTpl = '[XO Backup {job.name}] {vm.name_label}'
|
||||
vhdDirectoryCompression = 'brotli'
|
||||
|
||||
[backups.defaultSettings]
|
||||
reportWhen = 'failure'
|
||||
@@ -88,20 +87,3 @@ ignoreNobakVdis = false
|
||||
|
||||
maxUncoalescedVdis = 1
|
||||
watchEvents = ['network', 'PIF', 'pool', 'SR', 'task', 'VBD', 'VDI', 'VIF', 'VM']
|
||||
|
||||
|
||||
|
||||
#compact mode
|
||||
[reverseProxies]
|
||||
# '/http/' = 'http://localhost:8081/'
|
||||
#The target can have a path ( like `http://target/sub/directory/`),
|
||||
# parameters (`?param=one`) and hash (`#jwt:32154`) that are automatically added to all queries transfered by the proxy.
|
||||
# If a parameter is present in the configuration and in the query, only the config parameter is transferred.
|
||||
# '/another' = http://hiddenServer:8765/path/
|
||||
|
||||
# And use the extended mode when required
|
||||
# The additionnal options of a proxy's configuraiton's section are used to instantiate the `https` Agent(respectively the `http`).
|
||||
# A notable option is `rejectUnauthorized` which allow to connect to a HTTPS backend with an invalid/ self signed certificate
|
||||
#[reverseProxies.'/https/']
|
||||
# target = 'https://localhost:8080/'
|
||||
# rejectUnauthorized = false
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"private": true,
|
||||
"name": "@xen-orchestra/proxy",
|
||||
"version": "0.17.3",
|
||||
"version": "0.15.2",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"description": "XO Proxy used to remotely execute backup jobs",
|
||||
"keywords": [
|
||||
@@ -22,33 +22,32 @@
|
||||
"xo-proxy": "dist/index.mjs"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.18"
|
||||
"node": ">=14.13"
|
||||
},
|
||||
"dependencies": {
|
||||
"@iarna/toml": "^2.2.0",
|
||||
"@koa/router": "^10.0.0",
|
||||
"@vates/compose": "^2.1.0",
|
||||
"@vates/decorate-with": "^1.0.0",
|
||||
"@vates/compose": "^2.0.0",
|
||||
"@vates/decorate-with": "^0.1.0",
|
||||
"@vates/disposable": "^0.1.1",
|
||||
"@xen-orchestra/async-map": "^0.1.2",
|
||||
"@xen-orchestra/backups": "^0.18.3",
|
||||
"@xen-orchestra/fs": "^0.19.3",
|
||||
"@xen-orchestra/backups": "^0.15.1",
|
||||
"@xen-orchestra/fs": "^0.18.0",
|
||||
"@xen-orchestra/log": "^0.3.0",
|
||||
"@xen-orchestra/mixin": "^0.1.0",
|
||||
"@xen-orchestra/mixins": "^0.1.2",
|
||||
"@xen-orchestra/mixins": "^0.1.1",
|
||||
"@xen-orchestra/self-signed": "^0.1.0",
|
||||
"@xen-orchestra/xapi": "^0.8.5",
|
||||
"@xen-orchestra/xapi": "^0.8.0",
|
||||
"ajv": "^8.0.3",
|
||||
"app-conf": "^1.0.0",
|
||||
"app-conf": "^0.9.0",
|
||||
"async-iterator-to-stream": "^1.1.0",
|
||||
"fs-extra": "^10.0.0",
|
||||
"get-stream": "^6.0.0",
|
||||
"getopts": "^2.2.3",
|
||||
"golike-defer": "^0.5.1",
|
||||
"http-server-plus": "^0.11.0",
|
||||
"http2-proxy": "^5.0.53",
|
||||
"json-rpc-protocol": "^0.13.1",
|
||||
"jsonrpc-websocket-client": "^0.7.2",
|
||||
"jsonrpc-websocket-client": "^0.6.0",
|
||||
"koa": "^2.5.1",
|
||||
"koa-compress": "^5.0.1",
|
||||
"koa-helmet": "^5.1.0",
|
||||
@@ -58,7 +57,7 @@
|
||||
"promise-toolbox": "^0.20.0",
|
||||
"source-map-support": "^0.5.16",
|
||||
"stoppable": "^1.0.6",
|
||||
"xdg-basedir": "^5.1.0",
|
||||
"xdg-basedir": "^4.0.0",
|
||||
"xen-api": "^0.35.1",
|
||||
"xo-common": "^0.7.0"
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@ import { createLogger } from '@xen-orchestra/log'
|
||||
|
||||
const { debug, warn } = createLogger('xo:proxy:api')
|
||||
|
||||
const ndJsonStream = asyncIteratorToStream(async function* (responseId, iterable) {
|
||||
const ndJsonStream = asyncIteratorToStream(async function*(responseId, iterable) {
|
||||
try {
|
||||
let cursor, iterator
|
||||
try {
|
||||
@@ -45,14 +45,14 @@ export default class Api {
|
||||
constructor(app, { appVersion, httpServer }) {
|
||||
this._ajv = new Ajv({ allErrors: true })
|
||||
this._methods = { __proto__: null }
|
||||
const PREFIX = '/api/v1'
|
||||
const router = new Router({ prefix: PREFIX }).post('/', async ctx => {
|
||||
|
||||
const router = new Router({ prefix: '/api/v1' }).post('/', async ctx => {
|
||||
// Before Node 13.0 there was an inactivity timeout of 2 mins, which may
|
||||
// not be enough for the API.
|
||||
ctx.req.setTimeout(0)
|
||||
|
||||
const profile = await app.authentication.findProfile({
|
||||
authenticationToken: ctx.cookies.get('authenticationToken'),
|
||||
authenticationToken: ctx.cookies.get('authenticationToken')
|
||||
})
|
||||
if (profile === undefined) {
|
||||
ctx.status = 401
|
||||
@@ -102,7 +102,6 @@ export default class Api {
|
||||
// breaks, send some data every 10s to keep it opened.
|
||||
const stopTimer = clearInterval.bind(
|
||||
undefined,
|
||||
// @to check : can this add space inside binary data ?
|
||||
setInterval(() => stream.push(' '), keepAliveInterval)
|
||||
)
|
||||
stream.on('end', stopTimer).on('error', stopTimer)
|
||||
@@ -119,19 +118,12 @@ export default class Api {
|
||||
.use(router.routes())
|
||||
.use(router.allowedMethods())
|
||||
|
||||
const callback = koa.callback()
|
||||
httpServer.on('request', (req, res) => {
|
||||
// only answers to query to the root url of this mixin
|
||||
// do it before giving the request to Koa to ensure it's not modified
|
||||
if (req.url.startsWith(PREFIX)) {
|
||||
callback(req, res)
|
||||
}
|
||||
})
|
||||
httpServer.on('request', koa.callback())
|
||||
|
||||
this.addMethods({
|
||||
system: {
|
||||
getMethodsInfo: [
|
||||
function* () {
|
||||
function*() {
|
||||
const methods = this._methods
|
||||
for (const name in methods) {
|
||||
const { description, params = {} } = methods[name]
|
||||
@@ -139,25 +131,25 @@ export default class Api {
|
||||
}
|
||||
}.bind(this),
|
||||
{
|
||||
description: 'returns the signatures of all available API methods',
|
||||
},
|
||||
description: 'returns the signatures of all available API methods'
|
||||
}
|
||||
],
|
||||
getServerVersion: [
|
||||
() => appVersion,
|
||||
{
|
||||
description: 'returns the version of xo-server',
|
||||
},
|
||||
description: 'returns the version of xo-server'
|
||||
}
|
||||
],
|
||||
listMethods: [
|
||||
function* () {
|
||||
function*() {
|
||||
const methods = this._methods
|
||||
for (const name in methods) {
|
||||
yield name
|
||||
}
|
||||
}.bind(this),
|
||||
{
|
||||
description: 'returns the name of all available API methods',
|
||||
},
|
||||
description: 'returns the name of all available API methods'
|
||||
}
|
||||
],
|
||||
methodSignature: [
|
||||
({ method: name }) => {
|
||||
@@ -172,14 +164,14 @@ export default class Api {
|
||||
{
|
||||
description: 'returns the signature of an API method',
|
||||
params: {
|
||||
method: { type: 'string' },
|
||||
},
|
||||
},
|
||||
],
|
||||
method: { type: 'string' }
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
test: {
|
||||
range: [
|
||||
function* ({ start = 0, stop, step }) {
|
||||
function*({ start = 0, stop, step }) {
|
||||
if (step === undefined) {
|
||||
step = start > stop ? -1 : 1
|
||||
}
|
||||
@@ -197,11 +189,11 @@ export default class Api {
|
||||
params: {
|
||||
start: { optional: true, type: 'number' },
|
||||
step: { optional: true, type: 'number' },
|
||||
stop: { type: 'number' },
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
stop: { type: 'number' }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -228,7 +220,7 @@ export default class Api {
|
||||
return required
|
||||
}),
|
||||
|
||||
type: 'object',
|
||||
type: 'object'
|
||||
})
|
||||
|
||||
const m = params => {
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import assert from 'assert'
|
||||
import fse from 'fs-extra'
|
||||
import { xdgConfig } from 'xdg-basedir'
|
||||
import xdg from 'xdg-basedir'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { execFileSync } from 'child_process'
|
||||
|
||||
@@ -11,48 +10,33 @@ const { warn } = createLogger('xo:proxy:authentication')
|
||||
const isValidToken = t => typeof t === 'string' && t.length !== 0
|
||||
|
||||
export default class Authentication {
|
||||
#token
|
||||
constructor(_, { appName, config: { authenticationToken: token } }) {
|
||||
if (!isValidToken(token)) {
|
||||
token = JSON.parse(execFileSync('xenstore-read', ['vm-data/xo-proxy-authenticationToken']))
|
||||
|
||||
constructor(app, { appName, config: { authenticationToken: token } }) {
|
||||
const setToken = ({ token }) => {
|
||||
assert(isValidToken(token), 'invalid authentication token: ' + token)
|
||||
|
||||
// save this token in the automatically handled conf file
|
||||
fse.outputFileSync(
|
||||
// this file must take precedence over normal user config
|
||||
`${xdgConfig}/${appName}/config.z-auto.json`,
|
||||
JSON.stringify({ authenticationToken: token }),
|
||||
{ mode: 0o600 }
|
||||
)
|
||||
|
||||
this.#token = token
|
||||
}
|
||||
|
||||
if (isValidToken(token)) {
|
||||
this.#token = token
|
||||
} else {
|
||||
setToken({ token: JSON.parse(execFileSync('xenstore-read', ['vm-data/xo-proxy-authenticationToken'])) })
|
||||
if (!isValidToken(token)) {
|
||||
throw new Error('missing authenticationToken in configuration')
|
||||
}
|
||||
|
||||
try {
|
||||
// save this token in the automatically handled conf file
|
||||
fse.outputFileSync(
|
||||
// this file must take precedence over normal user config
|
||||
`${xdg.config}/${appName}/config.z-auto.json`,
|
||||
JSON.stringify({ authenticationToken: token }),
|
||||
{ mode: 0o600 }
|
||||
)
|
||||
execFileSync('xenstore-rm', ['vm-data/xo-proxy-authenticationToken'])
|
||||
} catch (error) {
|
||||
warn('failed to remove token from XenStore', { error })
|
||||
}
|
||||
}
|
||||
|
||||
app.api.addMethod('authentication.setToken', setToken, {
|
||||
description: 'change the authentication token used by this XO Proxy',
|
||||
params: {
|
||||
token: {
|
||||
type: 'string',
|
||||
minLength: 1,
|
||||
},
|
||||
},
|
||||
})
|
||||
this._token = token
|
||||
}
|
||||
|
||||
async findProfile(credentials) {
|
||||
if (credentials?.authenticationToken === this.#token) {
|
||||
if (credentials?.authenticationToken === this._token) {
|
||||
return new Profile()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -164,17 +164,6 @@ export default class Backups {
|
||||
},
|
||||
},
|
||||
],
|
||||
deleteVmBackups: [
|
||||
({ filenames, remote }) =>
|
||||
Disposable.use(this.getAdapter(remote), adapter => adapter.deleteVmBackups(filenames)),
|
||||
{
|
||||
description: 'delete VM backups',
|
||||
params: {
|
||||
filenames: { type: 'array', items: { type: 'string' } },
|
||||
remote: { type: 'object' },
|
||||
},
|
||||
},
|
||||
],
|
||||
fetchPartitionFiles: [
|
||||
({ disk: diskId, remote, partition: partitionId, paths }) =>
|
||||
Disposable.use(this.getAdapter(remote), adapter => adapter.fetchPartitionFiles(diskId, partitionId, paths)),
|
||||
@@ -414,7 +403,6 @@ export default class Backups {
|
||||
return new RemoteAdapter(yield app.remotes.getHandler(remote), {
|
||||
debounceResource: app.debounceResource.bind(app),
|
||||
dirMode: app.config.get('backups.dirMode'),
|
||||
vhdDirectoryCompression: app.config.get('backups.vhdDirectoryCompression'),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
import { urlToHttpOptions } from 'url'
|
||||
import proxy from 'http2-proxy'
|
||||
|
||||
function removeSlash(str) {
|
||||
return str.replace(/^\/|\/$/g, '')
|
||||
}
|
||||
|
||||
function mergeUrl(relative, base) {
|
||||
const res = new URL(base)
|
||||
const relativeUrl = new URL(relative, base)
|
||||
res.pathname = relativeUrl.pathname
|
||||
relativeUrl.searchParams.forEach((value, name) => {
|
||||
// we do not allow to modify params already specified by config
|
||||
if (!res.searchParams.has(name)) {
|
||||
res.searchParams.append(name, value)
|
||||
}
|
||||
})
|
||||
res.hash = relativeUrl.hash.length > 0 ? relativeUrl.hash : res.hash
|
||||
return res
|
||||
}
|
||||
|
||||
export function backendToLocalPath(basePath, target, backendUrl) {
|
||||
// keep redirect url relative to local server
|
||||
const localPath = `${basePath}/${backendUrl.pathname.substring(target.pathname.length)}${backendUrl.search}${
|
||||
backendUrl.hash
|
||||
}`
|
||||
return localPath
|
||||
}
|
||||
|
||||
export function localToBackendUrl(basePath, target, localPath) {
|
||||
let localPathWithoutBase = removeSlash(localPath).substring(basePath.length)
|
||||
localPathWithoutBase = './' + removeSlash(localPathWithoutBase)
|
||||
const url = mergeUrl(localPathWithoutBase, target)
|
||||
return url
|
||||
}
|
||||
|
||||
export default class ReverseProxy {
|
||||
constructor(app, { httpServer }) {
|
||||
app.config.watch('reverseProxies', proxies => {
|
||||
this._proxies = Object.keys(proxies)
|
||||
.sort((a, b) => b.length - a.length)
|
||||
.map(path => {
|
||||
let config = proxies[path]
|
||||
if (typeof config === 'string') {
|
||||
config = { target: config }
|
||||
}
|
||||
config.path = '/proxy/v1/' + removeSlash(path) + '/'
|
||||
|
||||
return config
|
||||
})
|
||||
})
|
||||
|
||||
httpServer.on('request', (req, res) => this._proxy(req, res))
|
||||
httpServer.on('upgrade', (req, socket, head) => this._upgrade(req, socket, head))
|
||||
}
|
||||
|
||||
_getConfigFromRequest(req) {
|
||||
return this._proxies.find(({ path }) => req.url.startsWith(path))
|
||||
}
|
||||
|
||||
_proxy(req, res) {
|
||||
const config = this._getConfigFromRequest(req)
|
||||
|
||||
if (config === undefined) {
|
||||
res.writeHead(404)
|
||||
res.end('404')
|
||||
return
|
||||
}
|
||||
|
||||
const url = new URL(config.target)
|
||||
const targetUrl = localToBackendUrl(config.path, url, req.originalUrl || req.url)
|
||||
proxy.web(req, res, {
|
||||
...urlToHttpOptions(targetUrl),
|
||||
...config.options,
|
||||
onReq: (req, { headers }) => {
|
||||
headers['x-forwarded-for'] = req.socket.remoteAddress
|
||||
headers['x-forwarded-proto'] = req.socket.encrypted ? 'https' : 'http'
|
||||
if (req.headers.host !== undefined) {
|
||||
headers['x-forwarded-host'] = req.headers.host
|
||||
}
|
||||
},
|
||||
onRes: (req, res, proxyRes) => {
|
||||
// rewrite redirect to pass through this proxy
|
||||
if (proxyRes.statusCode === 301 || proxyRes.statusCode === 302) {
|
||||
// handle relative/ absolute location
|
||||
const redirectTargetLocation = new URL(proxyRes.headers.location, url)
|
||||
|
||||
// this proxy should only allow communication between known hosts. Don't open it too much
|
||||
if (redirectTargetLocation.hostname !== url.hostname || redirectTargetLocation.protocol !== url.protocol) {
|
||||
throw new Error(`Can't redirect from ${url.hostname} to ${redirectTargetLocation.hostname} `)
|
||||
}
|
||||
res.writeHead(proxyRes.statusCode, {
|
||||
...proxyRes.headers,
|
||||
location: backendToLocalPath(config.path, url, redirectTargetLocation),
|
||||
})
|
||||
res.end()
|
||||
return
|
||||
}
|
||||
// pass through the answer of the remote server
|
||||
res.writeHead(proxyRes.statusCode, proxyRes.headers)
|
||||
// pass through content
|
||||
proxyRes.pipe(res)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
_upgrade(req, socket, head) {
|
||||
const config = this._getConfigFromRequest(req)
|
||||
if (config === undefined) {
|
||||
return
|
||||
}
|
||||
|
||||
const { path, target, options } = config
|
||||
const targetUrl = localToBackendUrl(path, target, req.originalUrl || req.url)
|
||||
proxy.ws(req, socket, head, {
|
||||
...urlToHttpOptions(targetUrl),
|
||||
...options,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDETCCAfkCFHXO1U7YJHI61bPNhYDvyBNJYH4LMA0GCSqGSIb3DQEBCwUAMEUx
|
||||
CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
|
||||
cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjIwMTEwMTI0MTU4WhcNNDkwNTI3MTI0
|
||||
MTU4WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE
|
||||
CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC
|
||||
AQ8AMIIBCgKCAQEA1jMLdHuZu2R1fETyB2iRect1alwv76clp/7A8tx4zNaVA9qB
|
||||
BcHbI83mkozuyrXpsEUblTvvcWkheBPAvWD4gj0eWSDSiuf0edcIS6aky+Lr/n0T
|
||||
W/vL5kVNrgPTlsO8OyQcXjDeuUOR1xDWIa8G71Ynd6wtATB7oXe7kaV/Z6b2fENr
|
||||
4wlW0YEDnMHik59c9jXDshhQYDlErwZsSyHuLwkC7xuYO26SUW9fPcHJA3uOfxeG
|
||||
BrCxMuSMOJtdmslRWhLCjbk0PT12OYCCRlvuTvPHa8N57GEQbi4xAu+XATgO1DUm
|
||||
Dq/oCSj0TcWUXXOykN/PAC2cjIyqkU2e7orGaQIDAQABMA0GCSqGSIb3DQEBCwUA
|
||||
A4IBAQCTshhF3V5WVhnpFGHd+tPfeHmUVrUnbC+xW7fSeWpamNmTjHb7XB6uDR0O
|
||||
DGswhEitbbSOsCiwz4/zpfE3/3+X07O8NPbdHTVHCei6D0uyegEeWQ2HoocfZs3X
|
||||
8CORe8TItuvQAevV17D0WkGRoJGVAOiKo+izpjI55QXQ+FjkJ0bfl1iksnUJk0+I
|
||||
ZNmRRNjNyOxo7NAzomSBHfJ5rDE+E440F2uvXIE9OIwHRiq6FGvQmvGijPeeP5J0
|
||||
LzcSK98jfINFSsA/Wn5vWE+gfH9ySD2G3r2cDTS904T77PNiYH+cNSP6ujtmNzvK
|
||||
Bgoa3jXZPRBi82TUOb2jj5DB33bg
|
||||
-----END CERTIFICATE-----
|
||||
@@ -1,27 +0,0 @@
|
||||
-----BEGIN RSA PRIVATE KEY-----
|
||||
MIIEpAIBAAKCAQEA1jMLdHuZu2R1fETyB2iRect1alwv76clp/7A8tx4zNaVA9qB
|
||||
BcHbI83mkozuyrXpsEUblTvvcWkheBPAvWD4gj0eWSDSiuf0edcIS6aky+Lr/n0T
|
||||
W/vL5kVNrgPTlsO8OyQcXjDeuUOR1xDWIa8G71Ynd6wtATB7oXe7kaV/Z6b2fENr
|
||||
4wlW0YEDnMHik59c9jXDshhQYDlErwZsSyHuLwkC7xuYO26SUW9fPcHJA3uOfxeG
|
||||
BrCxMuSMOJtdmslRWhLCjbk0PT12OYCCRlvuTvPHa8N57GEQbi4xAu+XATgO1DUm
|
||||
Dq/oCSj0TcWUXXOykN/PAC2cjIyqkU2e7orGaQIDAQABAoIBAQC65uVq6WLWGa1O
|
||||
FtbdUggGL1svyGrngYChGvB/uZMKoX57U1DbljDCCCrV23WNmbfkYBjWWervmZ1j
|
||||
qlC2roOJGQ1/Fd3A6O7w1YnegPUxFrt3XunijE55iiVi3uHknryDGlpKcfgVzfjW
|
||||
oVFHKPMzKYjcqnbGn+hwlwoq5y7JYFTOa57/dZbyommbodRyy9Dpn0OES0grQqwR
|
||||
VD1amrQ7XJhukcxQgYPuDc/jM3CuowoBsv9f+Q2zsPgr6CpHxxLLIs+kt8NQJT9v
|
||||
neg/pm8ojcwOa9qoILdtu6ue7ee3VE9cFnB1vutxS1+MPeI5wgTJjaYrgPCMxXBM
|
||||
2LdJJEmBAoGBAPA6LpuU1vv5R3x66hzenSk4LS1fj24K0WuBdTwFvzQmCr70oKdo
|
||||
Yywxt+ZkBw5aEtzQlB8GewolHobDJrzxMorU+qEXX3jP2BIPDVQl2orfjr03Yyus
|
||||
s5mYS/Qa6Zf1yObrjulTNm8oTn1WaG3TIvi8c5DyG2OK28N/9oMI1XGRAoGBAORD
|
||||
YKyII/S66gZsJSf45qmrhq1hHuVt1xae5LUPP6lVD+MCCAmuoJnReV8fc9h7Dvgd
|
||||
YPMINkWUTePFr3o4p1mh2ZC7ldczgDn6X4TldY2J3Zg47xJa5hL0L6JL4NiCGRIE
|
||||
FV5rLJxkGh/DDBfmC9hQQ6Yg6cHvyewso5xVnBtZAoGAI+OdWPMIl0ZrrqYyWbPM
|
||||
aP8SiMfRBtCo7tW9bQUyxpi0XEjxw3Dt+AlJfysMftFoJgMnTedK9H4NLHb1T579
|
||||
PQ6KjwyN39+1WSVUiXDKUJsLmSswLrMzdcvx9PscUO6QYCdrB2K+LCcqasFBAr9b
|
||||
ZyvIXCw/eUSihneUnYjxUnECgYAoPgCzKiU8ph9QFozOaUExNH4/3tl1lVHQOR8V
|
||||
FKUik06DtP35xwGlXJrLPF5OEhPnhjZrYk0/IxBAUb/ICmjmknQq4gdes0Ot9QgW
|
||||
A+Yfl+irR45ObBwXx1kGgd4YDYeh93pU9QweXj+Ezfw50mLQNgZXKYJMoJu2uX/2
|
||||
tdkZsQKBgQCTfDcW8qBntI6V+3Gh+sIThz+fjdv5+qT54heO4EHadc98ykEZX0M1
|
||||
sCWJiAQWM/zWXcsTndQDgDsvo23jpoulVPDitSEISp5gSe9FEN2njsVVID9h1OIM
|
||||
f30s5kwcJoiV9kUCya/BFtuS7kbuQfAyPU0v3I+lUey6VCW6A83OTg==
|
||||
-----END RSA PRIVATE KEY-----
|
||||
@@ -1,60 +0,0 @@
|
||||
import { createServer as creatServerHttps } from 'https'
|
||||
import { createServer as creatServerHttp } from 'http'
|
||||
|
||||
import { WebSocketServer } from 'ws'
|
||||
import fs from 'fs'
|
||||
|
||||
const httpsServer = creatServerHttps({
|
||||
key: fs.readFileSync('key.pem'),
|
||||
cert: fs.readFileSync('cert.pem'),
|
||||
})
|
||||
const httpServer = creatServerHttp()
|
||||
|
||||
const wss = new WebSocketServer({ noServer: true, perMessageDeflate: false })
|
||||
|
||||
function upgrade(request, socket, head) {
|
||||
const { pathname } = new URL(request.url)
|
||||
// web socket server only on /foo url
|
||||
if (pathname === '/foo') {
|
||||
wss.handleUpgrade(request, socket, head, function done(ws) {
|
||||
wss.emit('connection', ws, request)
|
||||
ws.on('message', function message(data) {
|
||||
ws.send(data)
|
||||
})
|
||||
})
|
||||
} else {
|
||||
socket.destroy()
|
||||
}
|
||||
}
|
||||
|
||||
function httpHandler(req, res) {
|
||||
switch (req.url) {
|
||||
case '/index.html':
|
||||
res.end('hi')
|
||||
return
|
||||
case '/redirect':
|
||||
res.writeHead(301, {
|
||||
Location: 'index.html',
|
||||
})
|
||||
res.end()
|
||||
return
|
||||
case '/chainRedirect':
|
||||
res.writeHead(301, {
|
||||
Location: '/redirect',
|
||||
})
|
||||
res.end()
|
||||
return
|
||||
default:
|
||||
res.writeHead(404)
|
||||
res.end()
|
||||
}
|
||||
}
|
||||
|
||||
httpsServer.on('upgrade', upgrade)
|
||||
httpServer.on('upgrade', upgrade)
|
||||
|
||||
httpsServer.on('request', httpHandler)
|
||||
httpServer.on('request', httpHandler)
|
||||
|
||||
httpsServer.listen(8080)
|
||||
httpServer.listen(8081)
|
||||
@@ -1,123 +0,0 @@
|
||||
import ReverseProxy, { backendToLocalPath, localToBackendUrl } from '../dist/app/mixins/reverseProxy.mjs'
|
||||
import { deepEqual, strictEqual } from 'assert'
|
||||
|
||||
function makeApp(reverseProxies) {
|
||||
return {
|
||||
config: {
|
||||
get: () => reverseProxies,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const app = makeApp({
|
||||
https: {
|
||||
target: 'https://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
|
||||
oneOption: true,
|
||||
},
|
||||
http: 'http://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
|
||||
})
|
||||
|
||||
// test localToBackendUrl
|
||||
const expectedLocalToRemote = {
|
||||
https: [
|
||||
{
|
||||
local: '/proxy/v1/https/',
|
||||
remote: 'https://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
|
||||
},
|
||||
{
|
||||
local: '/proxy/v1/https/sub',
|
||||
remote: 'https://localhost:8080/remotePath/sub?baseParm=1#one=2&another=3',
|
||||
},
|
||||
{
|
||||
local: '/proxy/v1/https/sub/index.html',
|
||||
remote: 'https://localhost:8080/remotePath/sub/index.html?baseParm=1#one=2&another=3',
|
||||
},
|
||||
{
|
||||
local: '/proxy/v1/https/sub?param=1',
|
||||
remote: 'https://localhost:8080/remotePath/sub?baseParm=1¶m=1#one=2&another=3',
|
||||
},
|
||||
{
|
||||
local: '/proxy/v1/https/sub?baseParm=willbeoverwritten¶m=willstay',
|
||||
remote: 'https://localhost:8080/remotePath/sub?baseParm=1¶m=willstay#one=2&another=3',
|
||||
},
|
||||
{
|
||||
local: '/proxy/v1/https/sub?param=1#another=willoverwrite',
|
||||
remote: 'https://localhost:8080/remotePath/sub?baseParm=1¶m=1#another=willoverwrite',
|
||||
},
|
||||
],
|
||||
}
|
||||
const proxy = new ReverseProxy(app, { httpServer: { on: () => {} } })
|
||||
for (const proxyId in expectedLocalToRemote) {
|
||||
for (const { local, remote } of expectedLocalToRemote[proxyId]) {
|
||||
const config = proxy._getConfigFromRequest({ url: local })
|
||||
const url = new URL(config.target)
|
||||
strictEqual(localToBackendUrl(config.path, url, local).href, remote, 'error converting to backend')
|
||||
}
|
||||
}
|
||||
|
||||
// test backendToLocalPath
|
||||
const expectedRemoteToLocal = {
|
||||
https: [
|
||||
{
|
||||
local: '/proxy/v1/https/',
|
||||
remote: 'https://localhost:8080/remotePath/',
|
||||
},
|
||||
{
|
||||
local: '/proxy/v1/https/sub/index.html',
|
||||
remote: '/remotePath/sub/index.html',
|
||||
},
|
||||
{
|
||||
local: '/proxy/v1/https/?baseParm=1#one=2&another=3',
|
||||
remote: '?baseParm=1#one=2&another=3',
|
||||
},
|
||||
{
|
||||
local: '/proxy/v1/https/sub?baseParm=1#one=2&another=3',
|
||||
remote: 'https://localhost:8080/remotePath/sub?baseParm=1#one=2&another=3',
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
for (const proxyId in expectedRemoteToLocal) {
|
||||
for (const { local, remote } of expectedRemoteToLocal[proxyId]) {
|
||||
const config = proxy._getConfigFromRequest({ url: local })
|
||||
const targetUrl = new URL('https://localhost:8080/remotePath/?baseParm=1#one=2&another=3')
|
||||
const remoteUrl = new URL(remote, targetUrl)
|
||||
strictEqual(backendToLocalPath(config.path, targetUrl, remoteUrl), local, 'error converting to local')
|
||||
}
|
||||
}
|
||||
|
||||
// test _getConfigFromRequest
|
||||
|
||||
const expectedConfig = [
|
||||
{
|
||||
local: '/proxy/v1/http/other',
|
||||
config: {
|
||||
target: 'http://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
|
||||
options: {},
|
||||
path: '/proxy/v1/http',
|
||||
},
|
||||
},
|
||||
{
|
||||
local: '/proxy/v1/http',
|
||||
config: undefined,
|
||||
},
|
||||
|
||||
{
|
||||
local: '/proxy/v1/other',
|
||||
config: undefined,
|
||||
},
|
||||
{
|
||||
local: '/proxy/v1/https/',
|
||||
config: {
|
||||
target: 'https://localhost:8080/remotePath/?baseParm=1#one=2&another=3',
|
||||
options: {
|
||||
oneOption: true,
|
||||
},
|
||||
path: '/proxy/v1/https',
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
for (const { local, config } of expectedConfig) {
|
||||
deepEqual(proxy._getConfigFromRequest({ url: local }), config)
|
||||
}
|
||||
@@ -44,8 +44,8 @@
|
||||
"pw": "^0.0.4",
|
||||
"strip-indent": "^3.0.0",
|
||||
"xdg-basedir": "^4.0.0",
|
||||
"xo-lib": "^0.11.1",
|
||||
"xo-vmdk-to-vhd": "^2.0.3"
|
||||
"xo-lib": "^0.10.1",
|
||||
"xo-vmdk-to-vhd": "^2.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/cli": "^7.0.0",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@xen-orchestra/xapi",
|
||||
"version": "0.8.5",
|
||||
"version": "0.8.0",
|
||||
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/xapi",
|
||||
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
|
||||
"repository": {
|
||||
@@ -38,7 +38,7 @@
|
||||
"prepublishOnly": "yarn run build"
|
||||
},
|
||||
"dependencies": {
|
||||
"@vates/decorate-with": "^1.0.0",
|
||||
"@vates/decorate-with": "^0.1.0",
|
||||
"@xen-orchestra/async-map": "^0.1.2",
|
||||
"@xen-orchestra/log": "^0.3.0",
|
||||
"d3-time-format": "^3.0.0",
|
||||
|
||||
@@ -15,18 +15,16 @@ exports.VDI_FORMAT_VHD = 'vhd'
|
||||
// xapi.call('host.get_servertime', host.$ref) for example
|
||||
exports.formatDateTime = utcFormat('%Y%m%dT%H:%M:%SZ')
|
||||
|
||||
const dateTimeParsers = ['%Y%m%dT%H:%M:%SZ', '%Y-%m-%dT%H:%M:%S.%LZ'].map(utcParse)
|
||||
const parseDateTimeHelper = utcParse('%Y%m%dT%H:%M:%SZ')
|
||||
exports.parseDateTime = function (str, defaultValue) {
|
||||
for (const parser of dateTimeParsers) {
|
||||
const date = parser(str)
|
||||
if (date !== null) {
|
||||
return date.getTime()
|
||||
const date = parseDateTimeHelper(str)
|
||||
if (date === null) {
|
||||
if (arguments.length > 1) {
|
||||
return defaultValue
|
||||
}
|
||||
throw new RangeError(`unable to parse XAPI datetime ${JSON.stringify(str)}`)
|
||||
}
|
||||
if (arguments.length > 1) {
|
||||
return defaultValue
|
||||
}
|
||||
throw new RangeError(`unable to parse XAPI datetime ${JSON.stringify(str)}`)
|
||||
return date.getTime()
|
||||
}
|
||||
|
||||
const hasProps = o => {
|
||||
@@ -100,16 +98,18 @@ function removeWatcher(predicate, cb) {
|
||||
|
||||
class Xapi extends Base {
|
||||
constructor({
|
||||
callRetryWhenTooManyPendingTasks = { delay: 5e3, tries: 10 },
|
||||
callRetryWhenTooManyPendingTasks,
|
||||
ignoreNobakVdis,
|
||||
maxUncoalescedVdis,
|
||||
vdiDestroyRetryWhenInUse = { delay: 5e3, tries: 10 },
|
||||
vdiDestroyRetryWhenInUse,
|
||||
...opts
|
||||
}) {
|
||||
assert.notStrictEqual(ignoreNobakVdis, undefined)
|
||||
|
||||
super(opts)
|
||||
this._callRetryWhenTooManyPendingTasks = {
|
||||
delay: 5e3,
|
||||
tries: 10,
|
||||
...callRetryWhenTooManyPendingTasks,
|
||||
onRetry,
|
||||
when: { code: 'TOO_MANY_PENDING_TASKS' },
|
||||
@@ -117,6 +117,8 @@ class Xapi extends Base {
|
||||
this._ignoreNobakVdis = ignoreNobakVdis
|
||||
this._maxUncoalescedVdis = maxUncoalescedVdis
|
||||
this._vdiDestroyRetryWhenInUse = {
|
||||
delay: 5e3,
|
||||
retries: 10,
|
||||
...vdiDestroyRetryWhenInUse,
|
||||
onRetry,
|
||||
when: { code: 'VDI_IN_USE' },
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
/* eslint-env jest */
|
||||
|
||||
const { parseDateTime } = require('./')
|
||||
|
||||
describe('parseDateTime()', () => {
|
||||
it('parses legacy XAPI format', () => {
|
||||
expect(parseDateTime('20220106T21:25:15Z')).toBe(1641504315000)
|
||||
})
|
||||
|
||||
it('parses ISO 8601 with millisconds format', () => {
|
||||
expect(parseDateTime('2022-01-06T17:32:35.000Z')).toBe(1641490355000)
|
||||
})
|
||||
|
||||
it('throws when value cannot be parsed', () => {
|
||||
expect(() => parseDateTime('foo bar')).toThrow('unable to parse XAPI datetime "foo bar"')
|
||||
})
|
||||
})
|
||||
@@ -60,7 +60,7 @@ module.exports = class Vm {
|
||||
try {
|
||||
vdi = await this[vdiRefOrUuid.startsWith('OpaqueRef:') ? 'getRecord' : 'getRecordByUuid']('VDI', vdiRefOrUuid)
|
||||
} catch (error) {
|
||||
warn('_assertHealthyVdiChain, could not fetch VDI', { error })
|
||||
warn(error)
|
||||
return
|
||||
}
|
||||
cache[vdi.$ref] = vdi
|
||||
@@ -81,7 +81,7 @@ module.exports = class Vm {
|
||||
try {
|
||||
vdi = await this.getRecord('VDI', vdiRef)
|
||||
} catch (error) {
|
||||
warn('_assertHealthyVdiChain, could not fetch VDI', { error })
|
||||
warn(error)
|
||||
return
|
||||
}
|
||||
cache[vdiRef] = vdi
|
||||
@@ -99,7 +99,6 @@ module.exports = class Vm {
|
||||
// should coalesce
|
||||
const children = childrenMap[vdi.uuid]
|
||||
if (
|
||||
children !== undefined && // unused unmanaged VDI, will be GC-ed
|
||||
children.length === 1 &&
|
||||
!children[0].managed && // some SRs do not coalesce the leaf
|
||||
tolerance-- <= 0
|
||||
@@ -167,7 +166,7 @@ module.exports = class Vm {
|
||||
memory_static_min,
|
||||
name_description,
|
||||
name_label,
|
||||
NVRAM,
|
||||
// NVRAM, // experimental
|
||||
order,
|
||||
other_config = {},
|
||||
PCI_bus = '',
|
||||
@@ -256,7 +255,6 @@ module.exports = class Vm {
|
||||
is_vmss_snapshot,
|
||||
name_description,
|
||||
name_label,
|
||||
NVRAM,
|
||||
order,
|
||||
reference_label,
|
||||
shutdown_delay,
|
||||
|
||||
159
CHANGELOG.md
159
CHANGELOG.md
@@ -1,152 +1,9 @@
|
||||
## **5.64.0** (2021-10-29)
|
||||
|
||||
# ChangeLog
|
||||
|
||||
## **5.66.2** (2022-01-05)
|
||||
|
||||
<img id="latest" src="https://badgen.net/badge/channel/latest/yellow" alt="Channel: latest" />
|
||||
|
||||
### Bug fixes
|
||||
|
||||
- [Import/Disk] Fix `JSON.parse` and `createReadableSparseStream is not a function` errors [#6068](https://github.com/vatesfr/xen-orchestra/issues/6068)
|
||||
- [Backup] Fix delta backup are almost always full backup instead of differentials [Forum#5256](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it/69) [Forum#5371](https://xcp-ng.org/forum/topic/5371/delta-backup-changes-in-5-66) (PR [#6075](https://github.com/vatesfr/xen-orchestra/pull/6075))
|
||||
|
||||
### Released packages
|
||||
|
||||
- vhd-lib 3.0.0
|
||||
- xo-vmdk-to-vhd 2.0.3
|
||||
- @xen-orchestra/backups 0.18.3
|
||||
- @xen-orchestra/proxy 0.17.3
|
||||
- xo-server 5.86.3
|
||||
- xo-web 5.91.2
|
||||
|
||||
## **5.66.1** (2021-12-23)
|
||||
|
||||
### Bug fixes
|
||||
|
||||
- [Dashboard/Health] Fix `error has occured` when a pool has no default SR
|
||||
- [Delta Backup] Fix unnecessary full backup when not using S3 [Forum #5371](https://xcp-ng.org/forum/topic/5371/delta-backup-changes-in-5-66)d (PR [#6070](https://github.com/vatesfr/xen-orchestra/pull/6070))
|
||||
- [Backup] Fix incorrect warnings `incorrect size [...] instead of undefined`
|
||||
|
||||
### Released packages
|
||||
|
||||
- @xen-orchestra/backups 0.18.2
|
||||
- @xen-orchestra/proxy 0.17.2
|
||||
- xo-server 5.86.2
|
||||
- xo-web 5.91.1
|
||||
|
||||
## **5.66.0** (2021-12-21)
|
||||
|
||||
### Enhancements
|
||||
|
||||
- [About] Show commit instead of version numbers for source users (PR [#6045](https://github.com/vatesfr/xen-orchestra/pull/6045))
|
||||
- [Health] Display default SRs that aren't shared [#5871](https://github.com/vatesfr/xen-orchestra/issues/5871) (PR [#6033](https://github.com/vatesfr/xen-orchestra/pull/6033))
|
||||
- [Pool,VM/advanced] Ability to change the suspend SR [#4163](https://github.com/vatesfr/xen-orchestra/issues/4163) (PR [#6044](https://github.com/vatesfr/xen-orchestra/pull/6044))
|
||||
- [Home/VMs/Backup filter] Filter out VMs in disabled backup jobs (PR [#6037](https://github.com/vatesfr/xen-orchestra/pull/6037))
|
||||
- [Rolling Pool Update] Automatically disable High Availability during the update [#5711](https://github.com/vatesfr/xen-orchestra/issues/5711) (PR [#6057](https://github.com/vatesfr/xen-orchestra/pull/6057))
|
||||
- [Delta Backup on S3] Compress blocks by default ([Brotli](https://en.wikipedia.org/wiki/Brotli)) which reduces remote usage and increase backup speed (PR [#5932](https://github.com/vatesfr/xen-orchestra/pull/5932))
|
||||
|
||||
### Bug fixes
|
||||
|
||||
- [Tables/actions] Fix collapsed actions being clickable despite being disabled (PR [#6023](https://github.com/vatesfr/xen-orchestra/pull/6023))
|
||||
- [Backup] Remove incorrect size warning following a merge [Forum #5727](https://xcp-ng.org/forum/topic/4769/warnings-showing-in-system-logs-following-each-backup-job/4) (PR [#6010](https://github.com/vatesfr/xen-orchestra/pull/6010))
|
||||
- [Delta Backup] Preserve UEFI boot parameters [#6054](https://github.com/vatesfr/xen-orchestra/issues/6054) [Forum #5319](https://xcp-ng.org/forum/topic/5319/bug-uefi-boot-parameters-not-preserved-with-delta-backups)
|
||||
|
||||
### Released packages
|
||||
|
||||
- @xen-orchestra/mixins 0.1.2
|
||||
- @xen-orchestra/xapi 0.8.5
|
||||
- vhd-lib 2.1.0
|
||||
- xo-vmdk-to-vhd 2.0.2
|
||||
- @xen-orchestra/backups 0.18.1
|
||||
- @xen-orchestra/proxy 0.17.1
|
||||
- xo-server 5.86.1
|
||||
- xo-web 5.91.0
|
||||
|
||||
## **5.65.3** (2021-12-20)
|
||||
|
||||
<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
|
||||
|
||||
### Bug fixes
|
||||
|
||||
- [Continuous Replication] Fix `could not find the base VM`
|
||||
- [Backup/Smart mode] Always ignore replicated VMs created by the current job
|
||||
- [Backup] Fix `Unexpected end of JSON input` during merge step
|
||||
- [Backup] Fix stuck jobs when using S3 remotes (PR [#6067](https://github.com/vatesfr/xen-orchestra/pull/6067))
|
||||
|
||||
### Released packages
|
||||
|
||||
- @xen-orchestra/fs 0.19.3
|
||||
- vhd-lib 2.0.4
|
||||
- @xen-orchestra/backups 0.17.1
|
||||
- xo-server 5.85.1
|
||||
|
||||
## **5.65.2** (2021-12-10)
|
||||
|
||||
### Bug fixes
|
||||
|
||||
- [Backup] Fix `handler.rmTree` is not a function (Forum [5256](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it/29) PR [#6041](https://github.com/vatesfr/xen-orchestra/pull/6041) )
|
||||
- [Backup] Fix `EEXIST` in logs when multiple merge tasks are created at the same time ([Forum #5301](https://xcp-ng.org/forum/topic/5301/warnings-errors-in-journalctl))
|
||||
- [Backup] Fix missing backup on restore (Forum [5256](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it/29) (PR [#6048](https://github.com/vatesfr/xen-orchestra/pull/6048))
|
||||
|
||||
### Released packages
|
||||
|
||||
- @xen-orchestra/fs 0.19.2
|
||||
- vhd-lib 2.0.3
|
||||
- @xen-orchestra/backups 0.16.2
|
||||
- xo-server 5.84.3
|
||||
- @xen-orchestra/proxy 0.15.5
|
||||
|
||||
## **5.65.1** (2021-12-03)
|
||||
|
||||
### Bug fixes
|
||||
|
||||
- [Delta Backup Restoration] Fix assertion error [Forum #5257](https://xcp-ng.org/forum/topic/5257/problems-building-from-source/16)
|
||||
- [Delta Backup Restoration] `TypeError: this disposable has already been disposed` [Forum #5257](https://xcp-ng.org/forum/topic/5257/problems-building-from-source/20)
|
||||
- [Backups] Fix: `Error: Chaining alias is forbidden xo-vm-backups/..alias.vhd to xo-vm-backups/....alias.vhd` when backuping a file to s3 [Forum #5226](https://xcp-ng.org/forum/topic/5256/s3-backup-try-it)
|
||||
- [Delta Backup Restoration] `VDI_IO_ERROR(Device I/O errors)` [Forum #5727](https://xcp-ng.org/forum/topic/5257/problems-building-from-source/4) (PR [#6031](https://github.com/vatesfr/xen-orchestra/pull/6031))
|
||||
- [Delta Backup] Fix `Cannot read property 'uuid' of undefined` when a VDI has been removed from a backed up VM (PR [#6034](https://github.com/vatesfr/xen-orchestra/pull/6034))
|
||||
|
||||
### Released packages
|
||||
|
||||
- @vates/compose 2.1.0
|
||||
- vhd-lib 2.0.2
|
||||
- xo-vmdk-to-vhd 2.0.1
|
||||
- @xen-orchestra/backups 0.16.1
|
||||
- @xen-orchestra/proxy 0.15.4
|
||||
- xo-server 5.84.2
|
||||
|
||||
## **5.65.0** (2021-11-30)
|
||||
|
||||
### Highlights
|
||||
|
||||
- [VM] Ability to export a snapshot's memory (PR [#6015](https://github.com/vatesfr/xen-orchestra/pull/6015))
|
||||
- [Cloud config] Ability to create a network cloud config template and reuse it in the VM creation [#5931](https://github.com/vatesfr/xen-orchestra/issues/5931) (PR [#5979](https://github.com/vatesfr/xen-orchestra/pull/5979))
|
||||
- [Backup/logs] identify XAPI errors (PR [#6001](https://github.com/vatesfr/xen-orchestra/pull/6001))
|
||||
- [lite] Highlight selected VM (PR [#5939](https://github.com/vatesfr/xen-orchestra/pull/5939))
|
||||
|
||||
### Enhancements
|
||||
|
||||
- [S3] Ability to authorize self signed certificates for S3 remote (PR [#5961](https://github.com/vatesfr/xen-orchestra/pull/5961))
|
||||
|
||||
### Bug fixes
|
||||
|
||||
- [Import/VM] Fix the import of OVA files (PR [#5976](https://github.com/vatesfr/xen-orchestra/pull/5976))
|
||||
|
||||
### Released packages
|
||||
|
||||
- @vates/async-each 0.1.0
|
||||
- xo-remote-parser 0.8.4
|
||||
- @xen-orchestra/fs 0.19.0
|
||||
- @xen-orchestra/xapi patch
|
||||
- vhd-lib 2.0.1
|
||||
- @xen-orchestra/backups 0.16.0
|
||||
- xo-lib 0.11.1
|
||||
- @xen-orchestra/proxy 0.15.3
|
||||
- xo-server 5.84.1
|
||||
- vhd-cli 0.6.0
|
||||
- xo-web 5.90.0
|
||||
|
||||
## **5.64.0** (2021-10-29)
|
||||
|
||||
## Highlights
|
||||
|
||||
- [Netbox] Support older versions of Netbox and prevent "active is not a valid choice" error [#5898](https://github.com/vatesfr/xen-orchestra/issues/5898) (PR [#5946](https://github.com/vatesfr/xen-orchestra/pull/5946))
|
||||
@@ -154,8 +11,8 @@
|
||||
- [Host] Handle evacuation failure during host shutdown (PR [#5966](https://github.com/vatesfr/xen-orchestra/pull/#5966))
|
||||
- [Menu] Notify user when proxies need to be upgraded (PR [#5930](https://github.com/vatesfr/xen-orchestra/pull/5930))
|
||||
- [Servers] Ability to use an HTTP proxy between XO and a server (PR [#5958](https://github.com/vatesfr/xen-orchestra/pull/5958))
|
||||
- [VM/export] Ability to copy the export URL (PR [#5948](https://github.com/vatesfr/xen-orchestra/pull/5948))
|
||||
- [Pool/advanced] Ability to define network for importing/exporting VMs/VDIs (PR [#5957](https://github.com/vatesfr/xen-orchestra/pull/5957))
|
||||
- [VM/export] Ability to copy the export URL (PR [#5948](https://github.com/vatesfr/xen-orchestra/pull/5948))
|
||||
- [Pool/advanced] Ability to define network for importing/exporting VMs/VDIs (PR [#5957](https://github.com/vatesfr/xen-orchestra/pull/5957))
|
||||
- [Host/advanced] Add button to enable/disable the host (PR [#5952](https://github.com/vatesfr/xen-orchestra/pull/5952))
|
||||
- [Backups] Enable merge worker by default
|
||||
|
||||
@@ -188,11 +45,13 @@
|
||||
|
||||
## **5.63.0** (2021-09-30)
|
||||
|
||||
<img id="stable" src="https://badgen.net/badge/channel/stable/green" alt="Channel: stable" />
|
||||
|
||||
### Highlights
|
||||
|
||||
- [Backup] Go back to previous page instead of going to the overview after editing a job: keeps current filters and page (PR [#5913](https://github.com/vatesfr/xen-orchestra/pull/5913))
|
||||
- [Health] Do not take into consideration duplicated MAC addresses from CR VMs (PR [#5916](https://github.com/vatesfr/xen-orchestra/pull/5916))
|
||||
- [Health] Ability to filter duplicated MAC addresses by running VMs (PR [#5917](https://github.com/vatesfr/xen-orchestra/pull/5917))
|
||||
- [Health] Do not take into consideration duplicated MAC addresses from CR VMs (PR [#5916](https://github.com/vatesfr/xen-orchestra/pull/5916))
|
||||
- [Health] Ability to filter duplicated MAC addresses by running VMs (PR [#5917](https://github.com/vatesfr/xen-orchestra/pull/5917))
|
||||
- [Tables] Move the search bar and pagination to the top of the table (PR [#5914](https://github.com/vatesfr/xen-orchestra/pull/5914))
|
||||
- [Netbox] Handle nested prefixes by always assigning an IP to the smallest prefix it matches (PR [#5908](https://github.com/vatesfr/xen-orchestra/pull/5908))
|
||||
|
||||
@@ -459,7 +318,7 @@
|
||||
- [Proxy] _Redeploy_ now works when the bound VM is missing
|
||||
- [VM template] Fix confirmation modal doesn't appear on deleting a default template (PR [#5644](https://github.com/vatesfr/xen-orchestra/pull/5644))
|
||||
- [OVA VM Import] Fix imported VMs all having the same MAC addresses
|
||||
- [Disk import] Fix `an error has occurred` when importing wrong format or corrupted files [#5663](https://github.com/vatesfr/xen-orchestra/issues/5663) (PR [#5683](https://github.com/vatesfr/xen-orchestra/pull/5683))
|
||||
- [Disk import] Fix `an error has occurred` when importing wrong format or corrupted files [#5663](https://github.com/vatesfr/xen-orchestra/issues/5663) (PR [#5683](https://github.com/vatesfr/xen-orchestra/pull/5683))
|
||||
|
||||
### Released packages
|
||||
|
||||
|
||||
@@ -7,15 +7,11 @@
|
||||
|
||||
> Users must be able to say: “Nice enhancement, I'm eager to test it”
|
||||
|
||||
- Limit number of concurrent VM migrations per pool to `3` [#6065](https://github.com/vatesfr/xen-orchestra/issues/6065) (PR [#6076](https://github.com/vatesfr/xen-orchestra/pull/6076))
|
||||
Can be changed in `xo-server`'s configuration file: `xapiOptions.vmMigrationConcurrency`
|
||||
- [Proxy] Now ships a reverse proxy [PR#6072](https://github.com/vatesfr/xen-orchestra/pull/6072)
|
||||
|
||||
### Bug fixes
|
||||
|
||||
> Users must be able to say: “I had this issue, happy to know it's fixed”
|
||||
|
||||
- [Backup] Detect and clear orphan merge states, fix `ENOENT` errors (PR [#6087](https://github.com/vatesfr/xen-orchestra/pull/6087))
|
||||
[Import/VM] Fix the import of OVA files (PR [#5976](https://github.com/vatesfr/xen-orchestra/pull/5976))
|
||||
|
||||
### Packages to release
|
||||
|
||||
@@ -34,7 +30,7 @@
|
||||
>
|
||||
> In case of conflict, the highest (lowest in previous list) `$version` wins.
|
||||
|
||||
- @xen-orchestra/backups minor
|
||||
- @xen-orchestra/backups-cli minor
|
||||
- @xen-orchestra/proxy minor
|
||||
- xo-server minor
|
||||
- @xen-orchestra/fs minor
|
||||
- vhd-lib minor
|
||||
- xo-server patch
|
||||
- vhd-cli minor
|
||||
|
||||
@@ -341,16 +341,6 @@ XO will try to find the right prefix for each IP address. If it can't find a pre
|
||||
- Generate a token:
|
||||
- Go to Admin > Tokens > Add token
|
||||
- Create a token with "Write enabled"
|
||||
- The owner of the token must have at least the following permissions:
|
||||
- View permissions on:
|
||||
- extras > custom-fields
|
||||
- ipam > prefixes
|
||||
- All permissions on:
|
||||
- ipam > ip-addresses
|
||||
- virtualization > cluster-types
|
||||
- virtualization > clusters
|
||||
- virtualization > interfaces
|
||||
- virtualization > virtual-machines
|
||||
- Add a UUID custom field (for **Netbox 2.x**):
|
||||
- Got to Admin > Custom fields > Add custom field
|
||||
- Create a custom field called "uuid" (lower case!)
|
||||
|
||||
@@ -26,12 +26,6 @@ Each backups' job execution is identified by a `runId`. You can find this `runId
|
||||
|
||||

|
||||
|
||||
## Exclude disks
|
||||
|
||||
During a backup job, you can avoid saving all disks of the VM. To do that is trivial: just edit the VM disk name and add `[NOBAK]` before the current name, eg: `data-disk` will become `[NOBAK] data-disk` (with a space or not, doesn't matter).
|
||||
|
||||
The disks marked with `[NOBAK]` will be now ignored in all following backups.
|
||||
|
||||
## Schedule
|
||||
|
||||
:::tip
|
||||
|
||||
@@ -43,6 +43,12 @@ Just go into your "Backup" view, and select Delta Backup. Then, it's the same as
|
||||
|
||||
Unlike other types of backup jobs which delete the associated snapshot when the job is done and it has been exported, delta backups always keep a snapshot of every VM in the backup job, and uses it for the delta. Do not delete these snapshots!
|
||||
|
||||
## Exclude disks
|
||||
|
||||
During a delta backup job, you can avoid saving all disks of the VM. To do that is trivial: just edit the VM disk name and add `[NOBAK]` before the current name, eg: `data-disk` will become `[NOBAK] data-disk` (with a space or not, doesn't matter).
|
||||
|
||||
The disks marked with `[NOBAK]` will be now ignored in all following backups.
|
||||
|
||||
## Delta backup initial seed
|
||||
|
||||
If you don't want to do an initial full directly toward the destination, you can create a local delta backup first, then transfer the files to your destination.
|
||||
|
||||
@@ -103,9 +103,9 @@ In that case, you already set the password for `xoa` user. If you forgot it, see
|
||||
|
||||
### Manually deployed
|
||||
|
||||
If you connect via SSH or console for the first time without using our [web deploy form](https://xen-orchestra.com/#!/xoa), be aware **there is NO default password set for security reasons**. To set it, you need to connect to your host to find the XOA VM UUID (eg via `xe vm-list`).
|
||||
If you connect via SSH or console for the first time without using our [web deploy form](https://xen-orchestra.com/#!/xoa), be aware **there's NO default password set for security reasons**. To set it, you need to connect to your host to find the XOA VM UUID (eg via `xe vm-list`).
|
||||
|
||||
Next, you can replace `<UUID>` with the UUID you found previously, and `<password>` with your password:
|
||||
Then replace `<UUID>` with the previously find UUID, and `<password>` with your password:
|
||||
|
||||
```
|
||||
xe vm-param-set uuid=<UUID> xenstore-data:vm-data/system-account-xoa-password=<password>
|
||||
@@ -115,9 +115,7 @@ xe vm-param-set uuid=<UUID> xenstore-data:vm-data/system-account-xoa-password=<p
|
||||
Don't forget to use quotes for your password, eg: `xenstore-data:vm-data/system-account-xoa-password='MyPassW0rd!'`
|
||||
:::
|
||||
|
||||
Finally, you must reboot the VM to implement the changes.
|
||||
|
||||
You can now connect with the `xoa` username and password you defined in the previous command, eg with `ssh xoa@<XOA IP ADDRESS>`.
|
||||
Then, you could connect with `xoa` username and the password you defined in the previous command, eg with `ssh xoa@<XOA IP ADDRESS>`.
|
||||
|
||||
### Using sudo
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
"handlebars": "^4.7.6",
|
||||
"husky": "^4.2.5",
|
||||
"jest": "^27.3.1",
|
||||
"lint-staged": "^12.0.3",
|
||||
"lint-staged": "^11.1.2",
|
||||
"lodash": "^4.17.4",
|
||||
"prettier": "^2.0.5",
|
||||
"promise-toolbox": "^0.20.0",
|
||||
@@ -46,6 +46,7 @@
|
||||
],
|
||||
"^(value-matcher)$": "$1/src",
|
||||
"^(vhd-cli)$": "$1/src",
|
||||
"^(vhd-lib)$": "$1/src",
|
||||
"^(xo-[^/]+)$": [
|
||||
"$1/src",
|
||||
"$1"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"private": false,
|
||||
"name": "vhd-cli",
|
||||
"version": "0.6.0",
|
||||
"version": "0.5.0",
|
||||
"license": "ISC",
|
||||
"description": "Tools to read/create and merge VHD files",
|
||||
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-cli",
|
||||
@@ -24,12 +24,11 @@
|
||||
"node": ">=8.10"
|
||||
},
|
||||
"dependencies": {
|
||||
"@xen-orchestra/fs": "^0.19.3",
|
||||
"@xen-orchestra/fs": "^0.18.0",
|
||||
"cli-progress": "^3.1.0",
|
||||
"exec-promise": "^0.7.0",
|
||||
"getopts": "^2.2.3",
|
||||
"human-format": "^0.11.0",
|
||||
"vhd-lib": "^3.0.0"
|
||||
"vhd-lib": "^1.3.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@babel/cli": "^7.0.0",
|
||||
|
||||
@@ -1,34 +1,6 @@
|
||||
import { Constants, VhdFile } from 'vhd-lib'
|
||||
import { VhdFile } from 'vhd-lib'
|
||||
import { getHandler } from '@xen-orchestra/fs'
|
||||
import { resolve } from 'path'
|
||||
import humanFormat from 'human-format'
|
||||
import invert from 'lodash/invert.js'
|
||||
|
||||
const { PLATFORMS } = Constants
|
||||
|
||||
const DISK_TYPES_MAP = invert(Constants.DISK_TYPES)
|
||||
const PLATFORMS_MAP = invert(PLATFORMS)
|
||||
|
||||
const MAPPERS = {
|
||||
bytes: humanFormat.bytes,
|
||||
date: _ => (_ !== 0 ? new Date(_) : 0),
|
||||
diskType: _ => DISK_TYPES_MAP[_],
|
||||
platform: _ => PLATFORMS_MAP[_],
|
||||
}
|
||||
function mapProperties(object, mapping) {
|
||||
const result = { ...object }
|
||||
for (const prop of Object.keys(mapping)) {
|
||||
const value = object[prop]
|
||||
if (value !== undefined) {
|
||||
let mapper = mapping[prop]
|
||||
if (typeof mapper === 'string') {
|
||||
mapper = MAPPERS[mapper]
|
||||
}
|
||||
result[prop] = mapper(value)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export default async args => {
|
||||
const vhd = new VhdFile(getHandler({ url: 'file:///' }), resolve(args[0]))
|
||||
@@ -40,26 +12,6 @@ export default async args => {
|
||||
await vhd.readHeaderAndFooter(false)
|
||||
}
|
||||
|
||||
console.log(
|
||||
mapProperties(vhd.footer, {
|
||||
currentSize: 'bytes',
|
||||
diskType: 'diskType',
|
||||
originalSize: 'bytes',
|
||||
timestamp: 'date',
|
||||
})
|
||||
)
|
||||
|
||||
console.log(
|
||||
mapProperties(vhd.header, {
|
||||
blockSize: 'bytes',
|
||||
parentTimestamp: 'date',
|
||||
parentLocatorEntry: _ =>
|
||||
_.filter(_ => _.platformCode !== PLATFORMS.NONE) // hide empty
|
||||
.map(_ =>
|
||||
mapProperties(_, {
|
||||
platformCode: 'platform',
|
||||
})
|
||||
),
|
||||
})
|
||||
)
|
||||
console.log(vhd.header)
|
||||
console.log(vhd.footer)
|
||||
}
|
||||
|
||||
@@ -1,18 +1,13 @@
|
||||
import { openVhd } from 'vhd-lib'
|
||||
import { getSyncedHandler } from '@xen-orchestra/fs'
|
||||
import { createContentStream } from 'vhd-lib'
|
||||
import { getHandler } from '@xen-orchestra/fs'
|
||||
import { resolve } from 'path'
|
||||
|
||||
import { writeStream } from '../_utils'
|
||||
import { Disposable } from 'promise-toolbox'
|
||||
|
||||
export default async args => {
|
||||
if (args.length < 2 || args.some(_ => _ === '-h' || _ === '--help')) {
|
||||
return `Usage: ${this.command} <input VHD> [<output raw>]`
|
||||
}
|
||||
|
||||
await Disposable.use(async function* () {
|
||||
const handler = getSyncedHandler({ url: 'file:///' })
|
||||
const vhd = openVhd(handler, resolve(args[0]))
|
||||
await writeStream(vhd.rawContent())
|
||||
})
|
||||
await writeStream(createContentStream(getHandler({ url: 'file:///' }), resolve(args[0])), args[1])
|
||||
}
|
||||
|
||||
1
packages/vhd-lib/.babelrc.js
Normal file
1
packages/vhd-lib/.babelrc.js
Normal file
@@ -0,0 +1 @@
|
||||
module.exports = require('../../@xen-orchestra/babel-config')(require('./package.json'))
|
||||
@@ -1,275 +0,0 @@
|
||||
/* eslint-env jest */
|
||||
|
||||
const rimraf = require('rimraf')
|
||||
const tmp = require('tmp')
|
||||
const fs = require('fs-extra')
|
||||
const { getSyncedHandler } = require('@xen-orchestra/fs')
|
||||
const { Disposable, pFromCallback } = require('promise-toolbox')
|
||||
|
||||
const { openVhd } = require('../index')
|
||||
const { checkFile, createRandomFile, convertFromRawToVhd, createRandomVhdDirectory } = require('../tests/utils')
|
||||
const { VhdAbstract } = require('./VhdAbstract')
|
||||
const { BLOCK_UNUSED, FOOTER_SIZE, HEADER_SIZE, PLATFORMS, SECTOR_SIZE } = require('../_constants')
|
||||
const { unpackHeader, unpackFooter } = require('./_utils')
|
||||
|
||||
let tempDir
|
||||
|
||||
jest.setTimeout(60000)
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await pFromCallback(cb => tmp.dir(cb))
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await pFromCallback(cb => rimraf(tempDir, cb))
|
||||
})
|
||||
|
||||
const streamToBuffer = stream => {
|
||||
let buffer = Buffer.alloc(0)
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', data => (buffer = Buffer.concat([buffer, data])))
|
||||
stream.on('end', () => resolve(buffer))
|
||||
})
|
||||
}
|
||||
|
||||
test('It creates an alias', async () => {
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: 'file://' + tempDir })
|
||||
const aliasPath = `alias/alias.alias.vhd`
|
||||
const aliasFsPath = `${tempDir}/${aliasPath}`
|
||||
await fs.mkdirp(`${tempDir}/alias`)
|
||||
|
||||
const testOneCombination = async ({ targetPath, targetContent }) => {
|
||||
await VhdAbstract.createAlias(handler, aliasPath, targetPath)
|
||||
// alias file is created
|
||||
expect(await fs.exists(aliasFsPath)).toEqual(true)
|
||||
// content is the target path relative to the alias location
|
||||
const content = await fs.readFile(aliasFsPath, 'utf-8')
|
||||
expect(content).toEqual(targetContent)
|
||||
// create alias fails if alias already exists, remove it before next loop step
|
||||
await fs.unlink(aliasFsPath)
|
||||
}
|
||||
|
||||
const combinations = [
|
||||
{ targetPath: `targets.vhd`, targetContent: `../targets.vhd` },
|
||||
{ targetPath: `alias/targets.vhd`, targetContent: `targets.vhd` },
|
||||
{ targetPath: `alias/sub/targets.vhd`, targetContent: `sub/targets.vhd` },
|
||||
{ targetPath: `sibling/targets.vhd`, targetContent: `../sibling/targets.vhd` },
|
||||
]
|
||||
|
||||
for (const { targetPath, targetContent } of combinations) {
|
||||
await testOneCombination({ targetPath, targetContent })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
test('alias must have *.alias.vhd extension', async () => {
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: 'file:///' })
|
||||
const aliasPath = `${tempDir}/invalidalias.vhd`
|
||||
const targetPath = `${tempDir}/targets.vhd`
|
||||
expect(async () => await VhdAbstract.createAlias(handler, aliasPath, targetPath)).rejects.toThrow()
|
||||
|
||||
expect(await fs.exists(aliasPath)).toEqual(false)
|
||||
})
|
||||
})
|
||||
|
||||
test('alias must not be chained', async () => {
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: 'file:///' })
|
||||
const aliasPath = `${tempDir}/valid.alias.vhd`
|
||||
const targetPath = `${tempDir}/an.other.valid.alias.vhd`
|
||||
expect(async () => await VhdAbstract.createAlias(handler, aliasPath, targetPath)).rejects.toThrow()
|
||||
expect(await fs.exists(aliasPath)).toEqual(false)
|
||||
})
|
||||
})
|
||||
|
||||
test('It rename and unlink a VHDFile', async () => {
|
||||
const initalSize = 4
|
||||
const rawFileName = `${tempDir}/randomfile`
|
||||
await createRandomFile(rawFileName, initalSize)
|
||||
const vhdFileName = `${tempDir}/randomfile.vhd`
|
||||
await convertFromRawToVhd(rawFileName, vhdFileName)
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: 'file:///' })
|
||||
const { size } = await fs.stat(vhdFileName)
|
||||
const targetFileName = `${tempDir}/renamed.vhd`
|
||||
|
||||
await VhdAbstract.rename(handler, vhdFileName, targetFileName)
|
||||
expect(await fs.exists(vhdFileName)).toEqual(false)
|
||||
const { size: renamedSize } = await fs.stat(targetFileName)
|
||||
expect(size).toEqual(renamedSize)
|
||||
await VhdAbstract.unlink(handler, targetFileName)
|
||||
expect(await fs.exists(targetFileName)).toEqual(false)
|
||||
})
|
||||
})
|
||||
|
||||
test('It rename and unlink a VhdDirectory', async () => {
|
||||
const initalSize = 4
|
||||
const vhdDirectory = `${tempDir}/randomfile.dir`
|
||||
await createRandomVhdDirectory(vhdDirectory, initalSize)
|
||||
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: 'file:///' })
|
||||
const vhd = yield openVhd(handler, vhdDirectory)
|
||||
expect(vhd.header.cookie).toEqual('cxsparse')
|
||||
expect(vhd.footer.cookie).toEqual('conectix')
|
||||
|
||||
const targetFileName = `${tempDir}/renamed.vhd`
|
||||
// it should clean an existing directory
|
||||
await fs.mkdir(targetFileName)
|
||||
await fs.writeFile(`${targetFileName}/dummy`, 'I exists')
|
||||
await VhdAbstract.rename(handler, vhdDirectory, targetFileName)
|
||||
expect(await fs.exists(vhdDirectory)).toEqual(false)
|
||||
expect(await fs.exists(targetFileName)).toEqual(true)
|
||||
expect(await fs.exists(`${targetFileName}/dummy`)).toEqual(false)
|
||||
await VhdAbstract.unlink(handler, targetFileName)
|
||||
expect(await fs.exists(targetFileName)).toEqual(false)
|
||||
})
|
||||
})
|
||||
|
||||
test('It create , rename and unlink alias', async () => {
|
||||
const initalSize = 4
|
||||
const rawFileName = `${tempDir}/randomfile`
|
||||
await createRandomFile(rawFileName, initalSize)
|
||||
const vhdFileName = `${tempDir}/randomfile.vhd`
|
||||
await convertFromRawToVhd(rawFileName, vhdFileName)
|
||||
const aliasFileName = `${tempDir}/aliasFileName.alias.vhd`
|
||||
const aliasFileNameRenamed = `${tempDir}/aliasFileNameRenamed.alias.vhd`
|
||||
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: 'file:///' })
|
||||
await VhdAbstract.createAlias(handler, aliasFileName, vhdFileName)
|
||||
expect(await fs.exists(aliasFileName)).toEqual(true)
|
||||
expect(await fs.exists(vhdFileName)).toEqual(true)
|
||||
|
||||
await VhdAbstract.rename(handler, aliasFileName, aliasFileNameRenamed)
|
||||
expect(await fs.exists(aliasFileName)).toEqual(false)
|
||||
expect(await fs.exists(vhdFileName)).toEqual(true)
|
||||
expect(await fs.exists(aliasFileNameRenamed)).toEqual(true)
|
||||
|
||||
await VhdAbstract.unlink(handler, aliasFileNameRenamed)
|
||||
expect(await fs.exists(aliasFileName)).toEqual(false)
|
||||
expect(await fs.exists(vhdFileName)).toEqual(false)
|
||||
expect(await fs.exists(aliasFileNameRenamed)).toEqual(false)
|
||||
})
|
||||
})
|
||||
|
||||
test('it can create a vhd stream', async () => {
|
||||
const initialNbBlocks = 3
|
||||
const initalSize = initialNbBlocks * 2
|
||||
const rawFileName = `${tempDir}/randomfile`
|
||||
await createRandomFile(rawFileName, initalSize)
|
||||
const vhdFileName = `${tempDir}/vhd.vhd`
|
||||
await convertFromRawToVhd(rawFileName, vhdFileName)
|
||||
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: 'file://' + tempDir })
|
||||
|
||||
const vhd = yield openVhd(handler, 'vhd.vhd')
|
||||
await vhd.readBlockAllocationTable()
|
||||
|
||||
const parentLocatorBase = Buffer.from('a file path, not aligned', 'utf16le')
|
||||
const aligned = Buffer.alloc(SECTOR_SIZE, 0)
|
||||
parentLocatorBase.copy(aligned)
|
||||
await vhd.writeParentLocator({
|
||||
id: 0,
|
||||
platformCode: PLATFORMS.W2KU,
|
||||
data: parentLocatorBase,
|
||||
})
|
||||
await vhd.writeFooter()
|
||||
const stream = vhd.stream()
|
||||
|
||||
// read all the stream into a buffer
|
||||
|
||||
const buffer = await streamToBuffer(stream)
|
||||
const length = buffer.length
|
||||
const bufFooter = buffer.slice(0, FOOTER_SIZE)
|
||||
|
||||
// footer is still valid
|
||||
expect(() => unpackFooter(bufFooter)).not.toThrow()
|
||||
const footer = unpackFooter(bufFooter)
|
||||
|
||||
// header is still valid
|
||||
const bufHeader = buffer.slice(FOOTER_SIZE, HEADER_SIZE + FOOTER_SIZE)
|
||||
expect(() => unpackHeader(bufHeader, footer)).not.toThrow()
|
||||
|
||||
// 1 deleted block should be in ouput
|
||||
let start = FOOTER_SIZE + HEADER_SIZE + vhd.batSize
|
||||
|
||||
const parentLocatorData = buffer.slice(start, start + SECTOR_SIZE)
|
||||
expect(parentLocatorData.equals(aligned)).toEqual(true)
|
||||
start += SECTOR_SIZE // parent locator
|
||||
expect(length).toEqual(start + initialNbBlocks * vhd.fullBlockSize + FOOTER_SIZE)
|
||||
expect(stream.length).toEqual(buffer.length)
|
||||
// blocks
|
||||
const blockBuf = Buffer.alloc(vhd.sectorsPerBlock * SECTOR_SIZE, 0)
|
||||
for (let i = 0; i < initialNbBlocks; i++) {
|
||||
const blockDataStart = start + i * vhd.fullBlockSize + 512 /* block bitmap */
|
||||
const blockDataEnd = blockDataStart + vhd.sectorsPerBlock * SECTOR_SIZE
|
||||
const content = buffer.slice(blockDataStart, blockDataEnd)
|
||||
await handler.read('randomfile', blockBuf, i * vhd.sectorsPerBlock * SECTOR_SIZE)
|
||||
expect(content.equals(blockBuf)).toEqual(true)
|
||||
}
|
||||
// footer
|
||||
const endFooter = buffer.slice(length - FOOTER_SIZE)
|
||||
expect(bufFooter).toEqual(endFooter)
|
||||
|
||||
await handler.writeFile('out.vhd', buffer)
|
||||
// check that the vhd is still valid
|
||||
await checkFile(`${tempDir}/out.vhd`)
|
||||
})
|
||||
})
|
||||
|
||||
it('can stream content', async () => {
|
||||
const initalSizeMb = 5 // 2 block and an half
|
||||
const initialNbBlocks = Math.ceil(initalSizeMb / 2)
|
||||
const initialByteSize = initalSizeMb * 1024 * 1024
|
||||
const rawFileName = `${tempDir}/randomfile`
|
||||
await createRandomFile(rawFileName, initalSizeMb)
|
||||
const vhdFileName = `${tempDir}/vhd.vhd`
|
||||
await convertFromRawToVhd(rawFileName, vhdFileName)
|
||||
const bat = Buffer.alloc(512)
|
||||
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: 'file://' + tempDir })
|
||||
|
||||
const vhd = yield openVhd(handler, 'vhd.vhd')
|
||||
// mark first block as unused
|
||||
await handler.read('vhd.vhd', bat, vhd.header.tableOffset)
|
||||
bat.writeUInt32BE(BLOCK_UNUSED, 0)
|
||||
await handler.write('vhd.vhd', bat, vhd.header.tableOffset)
|
||||
|
||||
// read our modified block allocation table
|
||||
await vhd.readBlockAllocationTable()
|
||||
const stream = vhd.rawContent()
|
||||
const buffer = await streamToBuffer(stream)
|
||||
|
||||
// qemu can modify size, to align it to geometry
|
||||
|
||||
// check that data didn't change
|
||||
const blockDataLength = vhd.sectorsPerBlock * SECTOR_SIZE
|
||||
|
||||
// first block should be empty
|
||||
const EMPTY = Buffer.alloc(blockDataLength, 0)
|
||||
const firstBlock = buffer.slice(0, blockDataLength)
|
||||
// using buffer1 toEquals buffer2 make jest crash trying to stringify it on failure
|
||||
expect(firstBlock.equals(EMPTY)).toEqual(true)
|
||||
|
||||
let remainingLength = initialByteSize - blockDataLength // already checked the first block
|
||||
for (let i = 1; i < initialNbBlocks; i++) {
|
||||
// last block will be truncated
|
||||
const blockSize = Math.min(blockDataLength, remainingLength - blockDataLength)
|
||||
const blockDataStart = i * blockDataLength // first block have been deleted
|
||||
const blockDataEnd = blockDataStart + blockSize
|
||||
const content = buffer.slice(blockDataStart, blockDataEnd)
|
||||
|
||||
const blockBuf = Buffer.alloc(blockSize, 0)
|
||||
|
||||
await handler.read('randomfile', blockBuf, i * blockDataLength)
|
||||
expect(content.equals(blockBuf)).toEqual(true)
|
||||
remainingLength -= blockSize
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -1,114 +0,0 @@
|
||||
/* eslint-env jest */
|
||||
|
||||
const rimraf = require('rimraf')
|
||||
const tmp = require('tmp')
|
||||
const fs = require('fs-extra')
|
||||
const { getHandler, getSyncedHandler } = require('@xen-orchestra/fs')
|
||||
const { Disposable, pFromCallback } = require('promise-toolbox')
|
||||
|
||||
const { openVhd, VhdDirectory } = require('../')
|
||||
const { createRandomFile, convertFromRawToVhd, convertToVhdDirectory } = require('../tests/utils')
|
||||
|
||||
let tempDir = null
|
||||
|
||||
jest.setTimeout(60000)
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await pFromCallback(cb => tmp.dir(cb))
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await pFromCallback(cb => rimraf(tempDir, cb))
|
||||
})
|
||||
|
||||
test('Can coalesce block', async () => {
|
||||
const initalSize = 4
|
||||
const parentrawFileName = `${tempDir}/randomfile`
|
||||
const parentFileName = `${tempDir}/parent.vhd`
|
||||
const parentDirectoryName = `${tempDir}/parent.dir.vhd`
|
||||
|
||||
await createRandomFile(parentrawFileName, initalSize)
|
||||
await convertFromRawToVhd(parentrawFileName, parentFileName)
|
||||
await convertToVhdDirectory(parentrawFileName, parentFileName, parentDirectoryName)
|
||||
|
||||
const childrawFileName = `${tempDir}/randomfile`
|
||||
const childFileName = `${tempDir}/childFile.vhd`
|
||||
await createRandomFile(childrawFileName, initalSize)
|
||||
await convertFromRawToVhd(childrawFileName, childFileName)
|
||||
const childRawDirectoryName = `${tempDir}/randomFile2.vhd`
|
||||
const childDirectoryFileName = `${tempDir}/childDirFile.vhd`
|
||||
const childDirectoryName = `${tempDir}/childDir.vhd`
|
||||
await createRandomFile(childRawDirectoryName, initalSize)
|
||||
await convertFromRawToVhd(childRawDirectoryName, childDirectoryFileName)
|
||||
await convertToVhdDirectory(childRawDirectoryName, childDirectoryFileName, childDirectoryName)
|
||||
|
||||
await Disposable.use(async function* () {
|
||||
const handler = getHandler({ url: 'file://' })
|
||||
const parentVhd = yield openVhd(handler, parentDirectoryName, { flags: 'w' })
|
||||
await parentVhd.readBlockAllocationTable()
|
||||
const childFileVhd = yield openVhd(handler, childFileName)
|
||||
await childFileVhd.readBlockAllocationTable()
|
||||
const childDirectoryVhd = yield openVhd(handler, childDirectoryName)
|
||||
await childDirectoryVhd.readBlockAllocationTable()
|
||||
|
||||
await parentVhd.coalesceBlock(childFileVhd, 0)
|
||||
await parentVhd.writeFooter()
|
||||
await parentVhd.writeBlockAllocationTable()
|
||||
let parentBlockData = (await parentVhd.readBlock(0)).data
|
||||
let childBlockData = (await childFileVhd.readBlock(0)).data
|
||||
expect(parentBlockData.equals(childBlockData)).toEqual(true)
|
||||
|
||||
await parentVhd.coalesceBlock(childDirectoryVhd, 0)
|
||||
await parentVhd.writeFooter()
|
||||
await parentVhd.writeBlockAllocationTable()
|
||||
parentBlockData = (await parentVhd.readBlock(0)).data
|
||||
childBlockData = (await childDirectoryVhd.readBlock(0)).data
|
||||
expect(parentBlockData).toEqual(childBlockData)
|
||||
})
|
||||
})
|
||||
|
||||
test('compressed blocks and metadata works', async () => {
|
||||
const initalSize = 4
|
||||
const rawFileName = `${tempDir}/randomfile`
|
||||
const vhdName = `${tempDir}/parent.vhd`
|
||||
|
||||
await createRandomFile(rawFileName, initalSize)
|
||||
await convertFromRawToVhd(rawFileName, vhdName)
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
|
||||
const vhd = yield openVhd(handler, 'parent.vhd')
|
||||
await vhd.readBlockAllocationTable()
|
||||
const compressedVhd = yield VhdDirectory.create(handler, 'compressed.vhd', { compression: 'gzip' })
|
||||
compressedVhd.header = vhd.header
|
||||
compressedVhd.footer = vhd.footer
|
||||
for await (const block of vhd.blocks()) {
|
||||
await compressedVhd.writeEntireBlock(block)
|
||||
}
|
||||
await Promise
|
||||
.all[(await compressedVhd.writeHeader(), await compressedVhd.writeFooter(), await compressedVhd.writeBlockAllocationTable())]
|
||||
|
||||
// compressed vhd have a metadata file
|
||||
expect(await fs.exists(`${tempDir}/compressed.vhd/metadata.json`)).toEqual(true)
|
||||
const metada = JSON.parse(await handler.readFile('compressed.vhd/metadata.json'))
|
||||
expect(metada.compression.type).toEqual('gzip')
|
||||
expect(metada.compression.options.level).toEqual(1)
|
||||
|
||||
// compressed vhd should not be broken
|
||||
await compressedVhd.readHeaderAndFooter()
|
||||
await compressedVhd.readBlockAllocationTable()
|
||||
|
||||
// check that footer and header are not modified
|
||||
expect(compressedVhd.footer).toEqual(vhd.footer)
|
||||
expect(compressedVhd.header).toEqual(vhd.header)
|
||||
|
||||
// their block content should not have changed
|
||||
let counter = 0
|
||||
for await (const block of compressedVhd.blocks()) {
|
||||
const source = await vhd.readBlock(block.id)
|
||||
expect(source.data.equals(block.data)).toEqual(true)
|
||||
counter++
|
||||
}
|
||||
// neither the number of blocks
|
||||
expect(counter).toEqual(2)
|
||||
})
|
||||
})
|
||||
@@ -1,83 +0,0 @@
|
||||
/* eslint-env jest */
|
||||
|
||||
const rimraf = require('rimraf')
|
||||
const tmp = require('tmp')
|
||||
const { Disposable, pFromCallback } = require('promise-toolbox')
|
||||
const { getSyncedHandler } = require('@xen-orchestra/fs')
|
||||
|
||||
const { SECTOR_SIZE, PLATFORMS } = require('../_constants')
|
||||
const { createRandomFile, convertFromRawToVhd } = require('../tests/utils')
|
||||
const { openVhd, chainVhd } = require('..')
|
||||
const { VhdSynthetic } = require('./VhdSynthetic')
|
||||
|
||||
let tempDir = null
|
||||
|
||||
jest.setTimeout(60000)
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await pFromCallback(cb => tmp.dir(cb))
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await pFromCallback(cb => rimraf(tempDir, cb))
|
||||
})
|
||||
|
||||
test('It can read block and parent locator from a synthetic vhd', async () => {
|
||||
const bigRawFileName = `/bigrandomfile`
|
||||
await createRandomFile(`${tempDir}/${bigRawFileName}`, 8)
|
||||
const bigVhdFileName = `/bigrandomfile.vhd`
|
||||
await convertFromRawToVhd(`${tempDir}/${bigRawFileName}`, `${tempDir}/${bigVhdFileName}`)
|
||||
|
||||
const smallRawFileName = `/smallrandomfile`
|
||||
await createRandomFile(`${tempDir}/${smallRawFileName}`, 4)
|
||||
const smallVhdFileName = `/smallrandomfile.vhd`
|
||||
await convertFromRawToVhd(`${tempDir}/${smallRawFileName}`, `${tempDir}/${smallVhdFileName}`)
|
||||
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
|
||||
// ensure the two VHD are linked, with the child of type DISK_TYPES.DIFFERENCING
|
||||
await chainVhd(handler, bigVhdFileName, handler, smallVhdFileName, true)
|
||||
|
||||
const [smallVhd, bigVhd] = yield Disposable.all([
|
||||
openVhd(handler, smallVhdFileName),
|
||||
openVhd(handler, bigVhdFileName),
|
||||
])
|
||||
// add parent locato
|
||||
// this will also scramble the block inside the vhd files
|
||||
await bigVhd.writeParentLocator({
|
||||
id: 0,
|
||||
platformCode: PLATFORMS.W2KU,
|
||||
data: Buffer.from('I am in the big one'),
|
||||
})
|
||||
const syntheticVhd = new VhdSynthetic([smallVhd, bigVhd])
|
||||
await syntheticVhd.readBlockAllocationTable()
|
||||
|
||||
expect(syntheticVhd.header.diskType).toEqual(bigVhd.header.diskType)
|
||||
expect(syntheticVhd.header.parentTimestamp).toEqual(bigVhd.header.parentTimestamp)
|
||||
|
||||
// first two block should be from small
|
||||
const buf = Buffer.alloc(syntheticVhd.sectorsPerBlock * SECTOR_SIZE, 0)
|
||||
let content = (await syntheticVhd.readBlock(0)).data
|
||||
await handler.read(smallRawFileName, buf, 0)
|
||||
expect(content).toEqual(buf)
|
||||
|
||||
content = (await syntheticVhd.readBlock(1)).data
|
||||
await handler.read(smallRawFileName, buf, buf.length)
|
||||
expect(content).toEqual(buf)
|
||||
|
||||
// the next one from big
|
||||
|
||||
content = (await syntheticVhd.readBlock(2)).data
|
||||
await handler.read(bigRawFileName, buf, buf.length * 2)
|
||||
expect(content).toEqual(buf)
|
||||
|
||||
content = (await syntheticVhd.readBlock(3)).data
|
||||
await handler.read(bigRawFileName, buf, buf.length * 3)
|
||||
expect(content).toEqual(buf)
|
||||
|
||||
// the parent locator should the one of the root vhd
|
||||
const parentLocator = await syntheticVhd.readParentLocator(0)
|
||||
expect(parentLocator.platformCode).toEqual(PLATFORMS.W2KU)
|
||||
expect(Buffer.from(parentLocator.data, 'utf-8').toString()).toEqual('I am in the big one')
|
||||
})
|
||||
})
|
||||
@@ -1,88 +0,0 @@
|
||||
const UUID = require('uuid')
|
||||
const cloneDeep = require('lodash/cloneDeep.js')
|
||||
const { asyncMap } = require('@xen-orchestra/async-map')
|
||||
const { VhdAbstract } = require('./VhdAbstract')
|
||||
const { DISK_TYPES, FOOTER_SIZE, HEADER_SIZE } = require('../_constants')
|
||||
|
||||
const assert = require('assert')
|
||||
|
||||
exports.VhdSynthetic = class VhdSynthetic extends VhdAbstract {
|
||||
#vhds = []
|
||||
|
||||
get header() {
|
||||
// this the VHD we want to synthetize
|
||||
const vhd = this.#vhds[0]
|
||||
|
||||
// this is the root VHD
|
||||
const rootVhd = this.#vhds[this.#vhds.length - 1]
|
||||
|
||||
// data of our synthetic VHD
|
||||
// TODO: set parentLocatorEntry-s in header
|
||||
return {
|
||||
...vhd.header,
|
||||
parentLocatorEntry: cloneDeep(rootVhd.header.parentLocatorEntry),
|
||||
tableOffset: FOOTER_SIZE + HEADER_SIZE,
|
||||
parentTimestamp: rootVhd.header.parentTimestamp,
|
||||
parentUnicodeName: rootVhd.header.parentUnicodeName,
|
||||
parentUuid: rootVhd.header.parentUuid,
|
||||
}
|
||||
}
|
||||
|
||||
get footer() {
|
||||
// this is the root VHD
|
||||
const rootVhd = this.#vhds[this.#vhds.length - 1]
|
||||
return {
|
||||
...this.#vhds[0].footer,
|
||||
dataOffset: FOOTER_SIZE,
|
||||
diskType: rootVhd.footer.diskType,
|
||||
}
|
||||
}
|
||||
|
||||
static async open(vhds) {
|
||||
const vhd = new VhdSynthetic(vhds)
|
||||
return {
|
||||
dispose: () => {},
|
||||
value: vhd,
|
||||
}
|
||||
}
|
||||
/**
|
||||
* @param {Array<VhdAbstract>} vhds the chain of Vhds used to compute this Vhd, from the deepest child (in position 0), to the root (in the last position)
|
||||
* only the last one can have any type. Other must have type DISK_TYPES.DIFFERENCING (delta)
|
||||
*/
|
||||
constructor(vhds) {
|
||||
assert(vhds.length > 0)
|
||||
super()
|
||||
this.#vhds = vhds
|
||||
}
|
||||
|
||||
async readBlockAllocationTable() {
|
||||
await asyncMap(this.#vhds, vhd => vhd.readBlockAllocationTable())
|
||||
}
|
||||
|
||||
containsBlock(blockId) {
|
||||
return this.#vhds.some(vhd => vhd.containsBlock(blockId))
|
||||
}
|
||||
|
||||
async readHeaderAndFooter() {
|
||||
const vhds = this.#vhds
|
||||
|
||||
await asyncMap(vhds, vhd => vhd.readHeaderAndFooter())
|
||||
|
||||
for (let i = 0, n = vhds.length - 1; i < n; ++i) {
|
||||
const child = vhds[i]
|
||||
const parent = vhds[i + 1]
|
||||
assert.strictEqual(child.footer.diskType, DISK_TYPES.DIFFERENCING)
|
||||
assert.strictEqual(UUID.stringify(child.header.parentUuid), UUID.stringify(parent.footer.uuid))
|
||||
}
|
||||
}
|
||||
|
||||
async readBlock(blockId, onlyBitmap = false) {
|
||||
const index = this.#vhds.findIndex(vhd => vhd.containsBlock(blockId))
|
||||
// only read the content of the first vhd containing this block
|
||||
return await this.#vhds[index].readBlock(blockId, onlyBitmap)
|
||||
}
|
||||
|
||||
_readParentLocatorData(id) {
|
||||
return this.#vhds[this.#vhds.length - 1]._readParentLocatorData(id)
|
||||
}
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
const assert = require('assert')
|
||||
const { BLOCK_UNUSED, SECTOR_SIZE } = require('../_constants')
|
||||
const { fuFooter, fuHeader, checksumStruct, unpackField } = require('../_structs')
|
||||
const checkFooter = require('../checkFooter')
|
||||
const checkHeader = require('../_checkHeader')
|
||||
|
||||
const computeBatSize = entries => sectorsToBytes(sectorsRoundUpNoZero(entries * 4))
|
||||
exports.computeBatSize = computeBatSize
|
||||
|
||||
const computeSectorsPerBlock = blockSize => blockSize / SECTOR_SIZE
|
||||
exports.computeSectorsPerBlock = computeSectorsPerBlock
|
||||
// one bit per sector
|
||||
const computeBlockBitmapSize = blockSize => computeSectorsPerBlock(blockSize) >>> 3
|
||||
exports.computeBlockBitmapSize = computeBlockBitmapSize
|
||||
const computeFullBlockSize = blockSize => blockSize + SECTOR_SIZE * computeSectorOfBitmap(blockSize)
|
||||
exports.computeFullBlockSize = computeFullBlockSize
|
||||
const computeSectorOfBitmap = blockSize => sectorsRoundUpNoZero(computeBlockBitmapSize(blockSize))
|
||||
exports.computeSectorOfBitmap = computeSectorOfBitmap
|
||||
|
||||
// Sectors conversions.
|
||||
const sectorsRoundUpNoZero = bytes => Math.ceil(bytes / SECTOR_SIZE) || 1
|
||||
exports.sectorsRoundUpNoZero = sectorsRoundUpNoZero
|
||||
const sectorsToBytes = sectors => sectors * SECTOR_SIZE
|
||||
exports.sectorsToBytes = sectorsToBytes
|
||||
|
||||
const assertChecksum = (name, buf, struct) => {
|
||||
const actual = unpackField(struct.fields.checksum, buf)
|
||||
const expected = checksumStruct(buf, struct)
|
||||
assert.strictEqual(actual, expected, `invalid ${name} checksum ${actual}, expected ${expected}`)
|
||||
}
|
||||
exports.assertChecksum = assertChecksum
|
||||
|
||||
// unused block as buffer containing a uint32BE
|
||||
const BUF_BLOCK_UNUSED = Buffer.allocUnsafe(4)
|
||||
BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
|
||||
exports.BUF_BLOCK_UNUSED = BUF_BLOCK_UNUSED
|
||||
|
||||
/**
|
||||
* Check and parse the header buffer to build an header object
|
||||
*
|
||||
* @param {Buffer} bufHeader
|
||||
* @param {Object} footer
|
||||
* @returns {Object} the parsed header
|
||||
*/
|
||||
exports.unpackHeader = (bufHeader, footer) => {
|
||||
assertChecksum('header', bufHeader, fuHeader)
|
||||
|
||||
const header = fuHeader.unpack(bufHeader)
|
||||
checkHeader(header, footer)
|
||||
return header
|
||||
}
|
||||
|
||||
/**
|
||||
* Check and parse the footer buffer to build a footer object
|
||||
*
|
||||
* @param {Buffer} bufHeader
|
||||
* @param {Object} footer
|
||||
* @returns {Object} the parsed footer
|
||||
*/
|
||||
|
||||
exports.unpackFooter = bufFooter => {
|
||||
assertChecksum('footer', bufFooter, fuFooter)
|
||||
|
||||
const footer = fuFooter.unpack(bufFooter)
|
||||
checkFooter(footer)
|
||||
return footer
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
const MASK = 0x80
|
||||
|
||||
exports.set = (map, bit) => {
|
||||
map[bit >> 3] |= MASK >> (bit & 7)
|
||||
}
|
||||
|
||||
exports.test = (map, bit) => ((map[bit >> 3] << (bit & 7)) & MASK) !== 0
|
||||
@@ -1,40 +0,0 @@
|
||||
exports.BLOCK_UNUSED = 0xffffffff
|
||||
|
||||
// This lib has been extracted from the Xen Orchestra project.
|
||||
exports.CREATOR_APPLICATION = 'xo '
|
||||
|
||||
// Sizes in bytes.
|
||||
exports.FOOTER_SIZE = 512
|
||||
exports.HEADER_SIZE = 1024
|
||||
exports.SECTOR_SIZE = 512
|
||||
exports.DEFAULT_BLOCK_SIZE = 0x00200000 // from the spec
|
||||
|
||||
exports.FOOTER_COOKIE = 'conectix'
|
||||
exports.HEADER_COOKIE = 'cxsparse'
|
||||
|
||||
exports.DISK_TYPES = {
|
||||
__proto__: null,
|
||||
|
||||
FIXED: 2,
|
||||
DYNAMIC: 3,
|
||||
DIFFERENCING: 4,
|
||||
}
|
||||
|
||||
exports.PARENT_LOCATOR_ENTRIES = 8
|
||||
|
||||
exports.PLATFORMS = {
|
||||
__proto__: null,
|
||||
|
||||
NONE: 0,
|
||||
WI2R: 0x57693272,
|
||||
WI2K: 0x5769326b,
|
||||
W2RU: 0x57327275,
|
||||
W2KU: 0x57326b75,
|
||||
MAC: 0x4d616320,
|
||||
MACX: 0x4d616358,
|
||||
}
|
||||
|
||||
exports.FILE_FORMAT_VERSION = 1 << 16
|
||||
exports.HEADER_VERSION = 1 << 16
|
||||
|
||||
exports.ALIAS_MAX_PATH_LENGTH = 1024
|
||||
@@ -1 +0,0 @@
|
||||
module.exports = Function.prototype
|
||||
@@ -1,64 +0,0 @@
|
||||
/* eslint-env jest */
|
||||
|
||||
const rimraf = require('rimraf')
|
||||
const tmp = require('tmp')
|
||||
const { getSyncedHandler } = require('@xen-orchestra/fs')
|
||||
const { Disposable, pFromCallback } = require('promise-toolbox')
|
||||
|
||||
const { isVhdAlias, resolveAlias } = require('./_resolveAlias')
|
||||
const { ALIAS_MAX_PATH_LENGTH } = require('./_constants')
|
||||
|
||||
let tempDir
|
||||
|
||||
jest.setTimeout(60000)
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await pFromCallback(cb => tmp.dir(cb))
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await pFromCallback(cb => rimraf(tempDir, cb))
|
||||
})
|
||||
|
||||
test('is vhd alias recognize only *.alias.vhd files', () => {
|
||||
expect(isVhdAlias('filename.alias.vhd')).toEqual(true)
|
||||
expect(isVhdAlias('alias.vhd')).toEqual(false)
|
||||
expect(isVhdAlias('filename.vhd')).toEqual(false)
|
||||
expect(isVhdAlias('filename.alias.vhd.other')).toEqual(false)
|
||||
})
|
||||
|
||||
test('resolve return the path in argument for a non alias file ', async () => {
|
||||
expect(await resolveAlias(null, 'filename.vhd')).toEqual('filename.vhd')
|
||||
})
|
||||
test('resolve get the path of the target file for an alias', async () => {
|
||||
await Disposable.use(async function* () {
|
||||
// same directory
|
||||
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
|
||||
const alias = `alias.alias.vhd`
|
||||
await handler.writeFile(alias, 'target.vhd')
|
||||
await expect(await resolveAlias(handler, alias)).toEqual(`target.vhd`)
|
||||
|
||||
// different directory
|
||||
await handler.mkdir(`sub`)
|
||||
await handler.writeFile(alias, 'sub/target.vhd', { flags: 'w' })
|
||||
await expect(await resolveAlias(handler, alias)).toEqual(`sub/target.vhd`)
|
||||
})
|
||||
})
|
||||
|
||||
test('resolve throws an error an alias to an alias', async () => {
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
|
||||
const alias = `alias.alias.vhd`
|
||||
const target = `target.alias.vhd`
|
||||
await handler.writeFile(alias, target)
|
||||
await expect(async () => await resolveAlias(handler, alias)).rejects.toThrow(Error)
|
||||
})
|
||||
})
|
||||
|
||||
test('resolve throws an error on a file too big ', async () => {
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
|
||||
await handler.writeFile('toobig.alias.vhd', Buffer.alloc(ALIAS_MAX_PATH_LENGTH + 1, 0))
|
||||
await expect(async () => await resolveAlias(handler, 'toobig.alias.vhd')).rejects.toThrow(Error)
|
||||
})
|
||||
})
|
||||
@@ -1,25 +0,0 @@
|
||||
const { ALIAS_MAX_PATH_LENGTH } = require('./_constants')
|
||||
const resolveRelativeFromFile = require('./_resolveRelativeFromFile')
|
||||
|
||||
function isVhdAlias(filename) {
|
||||
return filename.endsWith('.alias.vhd')
|
||||
}
|
||||
exports.isVhdAlias = isVhdAlias
|
||||
|
||||
exports.resolveAlias = async function resolveAlias(handler, filename) {
|
||||
if (!isVhdAlias(filename)) {
|
||||
return filename
|
||||
}
|
||||
const size = await handler.getSize(filename)
|
||||
if (size > ALIAS_MAX_PATH_LENGTH) {
|
||||
// seems reasonnable for a relative path
|
||||
throw new Error(`The alias file ${filename} is too big (${size} bytes)`)
|
||||
}
|
||||
const aliasContent = (await handler.readFile(filename)).toString().trim()
|
||||
// also handle circular references and unreasonnably long chains
|
||||
if (isVhdAlias(aliasContent)) {
|
||||
throw new Error(`Chaining alias is forbidden ${filename} to ${aliasContent}`)
|
||||
}
|
||||
// the target is relative to the alias location
|
||||
return resolveRelativeFromFile(filename, aliasContent)
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
const { dirname, relative } = require('path')
|
||||
|
||||
const { openVhd } = require('./openVhd')
|
||||
const { DISK_TYPES } = require('./_constants')
|
||||
const { Disposable } = require('promise-toolbox')
|
||||
|
||||
module.exports = async function chain(parentHandler, parentPath, childHandler, childPath, force = false) {
|
||||
await Disposable.use(
|
||||
[openVhd(parentHandler, parentPath), openVhd(childHandler, childPath)],
|
||||
async ([parentVhd, childVhd]) => {
|
||||
await childVhd.readHeaderAndFooter()
|
||||
const { header, footer } = childVhd
|
||||
|
||||
if (footer.diskType !== DISK_TYPES.DIFFERENCING) {
|
||||
if (!force) {
|
||||
throw new Error('cannot chain disk of type ' + footer.diskType)
|
||||
}
|
||||
footer.diskType = DISK_TYPES.DIFFERENCING
|
||||
}
|
||||
await childVhd.readBlockAllocationTable()
|
||||
|
||||
const parentName = relative(dirname(childPath), parentPath)
|
||||
header.parentUuid = parentVhd.footer.uuid
|
||||
header.parentUnicodeName = parentName
|
||||
await childVhd.setUniqueParentLocator(parentName)
|
||||
await childVhd.writeHeader()
|
||||
await childVhd.writeFooter()
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
const { openVhd } = require('./openVhd')
|
||||
const resolveRelativeFromFile = require('./_resolveRelativeFromFile')
|
||||
const { DISK_TYPES } = require('./_constants')
|
||||
const { Disposable } = require('promise-toolbox')
|
||||
|
||||
module.exports = async function checkChain(handler, path) {
|
||||
await Disposable.use(function* () {
|
||||
let vhd
|
||||
do {
|
||||
vhd = yield openVhd(handler, path)
|
||||
path = resolveRelativeFromFile(path, vhd.header.parentUnicodeName)
|
||||
} while (vhd.footer.diskType !== DISK_TYPES.DYNAMIC)
|
||||
})
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
const assert = require('assert')
|
||||
|
||||
const { DISK_TYPES, FILE_FORMAT_VERSION, FOOTER_COOKIE, FOOTER_SIZE } = require('./_constants')
|
||||
|
||||
module.exports = footer => {
|
||||
assert.strictEqual(footer.cookie, FOOTER_COOKIE)
|
||||
assert.strictEqual(footer.dataOffset, FOOTER_SIZE)
|
||||
assert.strictEqual(footer.fileFormatVersion, FILE_FORMAT_VERSION)
|
||||
assert(footer.originalSize <= footer.currentSize)
|
||||
assert(footer.diskType === DISK_TYPES.DIFFERENCING || footer.diskType === DISK_TYPES.DYNAMIC)
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
const { parseVhdStream } = require('./parseVhdStream.js')
|
||||
const { VhdDirectory } = require('./Vhd/VhdDirectory.js')
|
||||
const { Disposable } = require('promise-toolbox')
|
||||
const { asyncEach } = require('@vates/async-each')
|
||||
|
||||
const buildVhd = Disposable.wrap(async function* (handler, path, inputStream, { concurrency, compression }) {
|
||||
const vhd = yield VhdDirectory.create(handler, path, { compression })
|
||||
await asyncEach(
|
||||
parseVhdStream(inputStream),
|
||||
async function (item) {
|
||||
switch (item.type) {
|
||||
case 'footer':
|
||||
vhd.footer = item.footer
|
||||
break
|
||||
case 'header':
|
||||
vhd.header = item.header
|
||||
break
|
||||
case 'parentLocator':
|
||||
await vhd.writeParentLocator({ ...item, data: item.buffer })
|
||||
break
|
||||
case 'block':
|
||||
await vhd.writeEntireBlock(item)
|
||||
break
|
||||
case 'bat':
|
||||
// it exists but I don't care
|
||||
break
|
||||
default:
|
||||
throw new Error(`unhandled type of block generated by parser : ${item.type} while generating ${path}`)
|
||||
}
|
||||
},
|
||||
{
|
||||
concurrency,
|
||||
}
|
||||
)
|
||||
await Promise.all([vhd.writeFooter(), vhd.writeHeader(), vhd.writeBlockAllocationTable()])
|
||||
})
|
||||
|
||||
exports.createVhdDirectoryFromStream = async function createVhdDirectoryFromStream(
|
||||
handler,
|
||||
path,
|
||||
inputStream,
|
||||
{ validator, concurrency = 16, compression } = {}
|
||||
) {
|
||||
try {
|
||||
await buildVhd(handler, path, inputStream, { concurrency, compression })
|
||||
if (validator !== undefined) {
|
||||
await validator.call(this, path)
|
||||
}
|
||||
} catch (error) {
|
||||
// cleanup on error
|
||||
await handler.rmtree(path)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
exports.chainVhd = require('./chain')
|
||||
exports.checkFooter = require('./checkFooter')
|
||||
exports.checkVhdChain = require('./checkChain')
|
||||
exports.createReadableRawStream = require('./createReadableRawStream')
|
||||
exports.createReadableSparseStream = require('./createReadableSparseStream')
|
||||
exports.createVhdStreamWithLength = require('./createVhdStreamWithLength')
|
||||
exports.createVhdDirectoryFromStream = require('./createVhdDirectoryFromStream').createVhdDirectoryFromStream
|
||||
exports.mergeVhd = require('./merge')
|
||||
exports.peekFooterFromVhdStream = require('./peekFooterFromVhdStream')
|
||||
exports.openVhd = require('./openVhd').openVhd
|
||||
exports.parseVhdStream = require('./parseVhdStream').parseVhdStream
|
||||
exports.VhdAbstract = require('./Vhd/VhdAbstract').VhdAbstract
|
||||
exports.VhdDirectory = require('./Vhd/VhdDirectory').VhdDirectory
|
||||
exports.VhdFile = require('./Vhd/VhdFile').VhdFile
|
||||
exports.VhdSynthetic = require('./Vhd/VhdSynthetic').VhdSynthetic
|
||||
exports.Constants = require('./_constants')
|
||||
@@ -1,157 +0,0 @@
|
||||
/* eslint-env jest */
|
||||
|
||||
const fs = require('fs-extra')
|
||||
const rimraf = require('rimraf')
|
||||
const tmp = require('tmp')
|
||||
const { getHandler } = require('@xen-orchestra/fs')
|
||||
const { pFromCallback } = require('promise-toolbox')
|
||||
|
||||
const { VhdFile, chainVhd, mergeVhd: vhdMerge } = require('./index')
|
||||
|
||||
const { checkFile, createRandomFile, convertFromRawToVhd } = require('./tests/utils')
|
||||
|
||||
let tempDir = null
|
||||
|
||||
jest.setTimeout(60000)
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await pFromCallback(cb => tmp.dir(cb))
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await pFromCallback(cb => rimraf(tempDir, cb))
|
||||
})
|
||||
|
||||
test('merge works in normal cases', async () => {
|
||||
const mbOfFather = 8
|
||||
const mbOfChildren = 4
|
||||
const parentRandomFileName = `${tempDir}/randomfile`
|
||||
const childRandomFileName = `${tempDir}/small_randomfile`
|
||||
const parentFileName = `${tempDir}/parent.vhd`
|
||||
const child1FileName = `${tempDir}/child1.vhd`
|
||||
const handler = getHandler({ url: 'file://' })
|
||||
|
||||
await createRandomFile(parentRandomFileName, mbOfFather)
|
||||
await convertFromRawToVhd(parentRandomFileName, parentFileName)
|
||||
|
||||
await createRandomFile(childRandomFileName, mbOfChildren)
|
||||
await convertFromRawToVhd(childRandomFileName, child1FileName)
|
||||
await chainVhd(handler, parentFileName, handler, child1FileName, true)
|
||||
|
||||
// merge
|
||||
await vhdMerge(handler, parentFileName, handler, child1FileName)
|
||||
|
||||
// check that vhd is still valid
|
||||
await checkFile(parentFileName)
|
||||
|
||||
const parentVhd = new VhdFile(handler, parentFileName)
|
||||
await parentVhd.readHeaderAndFooter()
|
||||
await parentVhd.readBlockAllocationTable()
|
||||
|
||||
let offset = 0
|
||||
// check that the data are the same as source
|
||||
for await (const block of parentVhd.blocks()) {
|
||||
const blockContent = block.data
|
||||
const file = offset < mbOfChildren * 1024 * 1024 ? childRandomFileName : parentRandomFileName
|
||||
const buffer = Buffer.alloc(blockContent.length)
|
||||
const fd = await fs.open(file, 'r')
|
||||
await fs.read(fd, buffer, 0, buffer.length, offset)
|
||||
|
||||
expect(buffer.equals(blockContent)).toEqual(true)
|
||||
offset += parentVhd.header.blockSize
|
||||
}
|
||||
})
|
||||
|
||||
test('it can resume a merge ', async () => {
|
||||
const mbOfFather = 8
|
||||
const mbOfChildren = 4
|
||||
const parentRandomFileName = `${tempDir}/randomfile`
|
||||
const childRandomFileName = `${tempDir}/small_randomfile`
|
||||
const handler = getHandler({ url: `file://${tempDir}` })
|
||||
|
||||
await createRandomFile(`${tempDir}/randomfile`, mbOfFather)
|
||||
await convertFromRawToVhd(`${tempDir}/randomfile`, `${tempDir}/parent.vhd`)
|
||||
const parentVhd = new VhdFile(handler, 'parent.vhd')
|
||||
await parentVhd.readHeaderAndFooter()
|
||||
|
||||
await createRandomFile(`${tempDir}/small_randomfile`, mbOfChildren)
|
||||
await convertFromRawToVhd(`${tempDir}/small_randomfile`, `${tempDir}/child1.vhd`)
|
||||
await chainVhd(handler, 'parent.vhd', handler, 'child1.vhd', true)
|
||||
const childVhd = new VhdFile(handler, 'child1.vhd')
|
||||
await childVhd.readHeaderAndFooter()
|
||||
|
||||
await handler.writeFile(
|
||||
'.parent.vhd.merge.json',
|
||||
JSON.stringify({
|
||||
parent: {
|
||||
header: parentVhd.header.checksum,
|
||||
},
|
||||
child: {
|
||||
header: 'NOT CHILD HEADER ',
|
||||
},
|
||||
})
|
||||
)
|
||||
// expect merge to fail since child header is not ok
|
||||
await expect(async () => await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd')).rejects.toThrow()
|
||||
|
||||
await handler.unlink('.parent.vhd.merge.json')
|
||||
await handler.writeFile(
|
||||
'.parent.vhd.merge.json',
|
||||
JSON.stringify({
|
||||
parent: {
|
||||
header: 'NOT PARENT HEADER',
|
||||
},
|
||||
child: {
|
||||
header: childVhd.header.checksum,
|
||||
},
|
||||
})
|
||||
)
|
||||
// expect merge to fail since parent header is not ok
|
||||
await expect(async () => await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd')).rejects.toThrow()
|
||||
|
||||
// break the end footer of parent
|
||||
const size = await handler.getSize('parent.vhd')
|
||||
const fd = await handler.openFile('parent.vhd', 'r+')
|
||||
const buffer = Buffer.alloc(512, 0)
|
||||
// add a fake footer at the end
|
||||
handler.write(fd, buffer, size)
|
||||
await handler.closeFile(fd)
|
||||
// check vhd should fail
|
||||
await expect(async () => await parentVhd.readHeaderAndFooter()).rejects.toThrow()
|
||||
|
||||
await handler.unlink('.parent.vhd.merge.json')
|
||||
await handler.writeFile(
|
||||
'.parent.vhd.merge.json',
|
||||
JSON.stringify({
|
||||
parent: {
|
||||
header: parentVhd.header.checksum,
|
||||
},
|
||||
child: {
|
||||
header: childVhd.header.checksum,
|
||||
},
|
||||
currentBlock: 1,
|
||||
})
|
||||
)
|
||||
|
||||
// really merge
|
||||
await vhdMerge(handler, 'parent.vhd', handler, 'child1.vhd')
|
||||
|
||||
// reload header footer and block allocation table , they should succed
|
||||
await parentVhd.readHeaderAndFooter()
|
||||
await parentVhd.readBlockAllocationTable()
|
||||
let offset = 0
|
||||
// check that the data are the same as source
|
||||
for await (const block of parentVhd.blocks()) {
|
||||
const blockContent = block.data
|
||||
// first block is marked as already merged, should not be modified
|
||||
// second block should come from children
|
||||
// then two block only in parent
|
||||
const file = block.id === 1 ? childRandomFileName : parentRandomFileName
|
||||
const buffer = Buffer.alloc(blockContent.length)
|
||||
const fd = await fs.open(file, 'r')
|
||||
await fs.read(fd, buffer, 0, buffer.length, offset)
|
||||
|
||||
expect(buffer.equals(blockContent)).toEqual(true)
|
||||
offset += parentVhd.header.blockSize
|
||||
}
|
||||
})
|
||||
@@ -1,127 +0,0 @@
|
||||
// TODO: remove once completely merged in vhd.js
|
||||
|
||||
const assert = require('assert')
|
||||
const noop = require('./_noop')
|
||||
const { createLogger } = require('@xen-orchestra/log')
|
||||
const { limitConcurrency } = require('limit-concurrency-decorator')
|
||||
|
||||
const { openVhd } = require('./openVhd')
|
||||
const { basename, dirname } = require('path')
|
||||
const { DISK_TYPES } = require('./_constants')
|
||||
const { Disposable } = require('promise-toolbox')
|
||||
|
||||
const { warn } = createLogger('vhd-lib:merge')
|
||||
|
||||
// Merge vhd child into vhd parent.
|
||||
//
|
||||
// TODO: rename the VHD file during the merge
|
||||
module.exports = limitConcurrency(2)(async function merge(
|
||||
parentHandler,
|
||||
parentPath,
|
||||
childHandler,
|
||||
childPath,
|
||||
{ onProgress = noop } = {}
|
||||
) {
|
||||
const mergeStatePath = dirname(parentPath) + '/' + '.' + basename(parentPath) + '.merge.json'
|
||||
|
||||
return await Disposable.use(async function* () {
|
||||
let mergeState = await parentHandler
|
||||
.readFile(mergeStatePath)
|
||||
.then(content => {
|
||||
const state = JSON.parse(content)
|
||||
|
||||
// ensure the correct merge will be continued
|
||||
assert.strictEqual(parentVhd.header.checksum, state.parent.header)
|
||||
assert.strictEqual(childVhd.header.checksum, state.child.header)
|
||||
|
||||
return state
|
||||
})
|
||||
.catch(error => {
|
||||
if (error.code !== 'ENOENT') {
|
||||
warn('problem while checking the merge state', { error })
|
||||
}
|
||||
})
|
||||
|
||||
// during merging, the end footer of the parent can be overwritten by new blocks
|
||||
// we should use it as a way to check vhd health
|
||||
const parentVhd = yield openVhd(parentHandler, parentPath, {
|
||||
flags: 'r+',
|
||||
checkSecondFooter: mergeState === undefined,
|
||||
})
|
||||
const childVhd = yield openVhd(childHandler, childPath)
|
||||
if (mergeState === undefined) {
|
||||
assert.strictEqual(childVhd.header.blockSize, parentVhd.header.blockSize)
|
||||
|
||||
const parentDiskType = parentVhd.footer.diskType
|
||||
assert(parentDiskType === DISK_TYPES.DIFFERENCING || parentDiskType === DISK_TYPES.DYNAMIC)
|
||||
assert.strictEqual(childVhd.footer.diskType, DISK_TYPES.DIFFERENCING)
|
||||
}
|
||||
|
||||
// Read allocation table of child/parent.
|
||||
await Promise.all([parentVhd.readBlockAllocationTable(), childVhd.readBlockAllocationTable()])
|
||||
|
||||
const { maxTableEntries } = childVhd.header
|
||||
|
||||
if (mergeState === undefined) {
|
||||
await parentVhd.ensureBatSize(childVhd.header.maxTableEntries)
|
||||
|
||||
mergeState = {
|
||||
child: { header: childVhd.header.checksum },
|
||||
parent: { header: parentVhd.header.checksum },
|
||||
currentBlock: 0,
|
||||
mergedDataSize: 0,
|
||||
}
|
||||
|
||||
// finds first allocated block for the 2 following loops
|
||||
while (mergeState.currentBlock < maxTableEntries && !childVhd.containsBlock(mergeState.currentBlock)) {
|
||||
++mergeState.currentBlock
|
||||
}
|
||||
}
|
||||
|
||||
// counts number of allocated blocks
|
||||
let nBlocks = 0
|
||||
for (let block = mergeState.currentBlock; block < maxTableEntries; block++) {
|
||||
if (childVhd.containsBlock(block)) {
|
||||
nBlocks += 1
|
||||
}
|
||||
}
|
||||
|
||||
onProgress({ total: nBlocks, done: 0 })
|
||||
|
||||
// merges blocks
|
||||
for (let i = 0; i < nBlocks; ++i, ++mergeState.currentBlock) {
|
||||
while (!childVhd.containsBlock(mergeState.currentBlock)) {
|
||||
++mergeState.currentBlock
|
||||
}
|
||||
|
||||
await parentHandler.writeFile(mergeStatePath, JSON.stringify(mergeState), { flags: 'w' }).catch(warn)
|
||||
|
||||
mergeState.mergedDataSize += await parentVhd.coalesceBlock(childVhd, mergeState.currentBlock)
|
||||
onProgress({
|
||||
total: nBlocks,
|
||||
done: i + 1,
|
||||
})
|
||||
}
|
||||
|
||||
// some blocks could have been created or moved in parent : write bat
|
||||
await parentVhd.writeBlockAllocationTable()
|
||||
|
||||
const cFooter = childVhd.footer
|
||||
const pFooter = parentVhd.footer
|
||||
|
||||
pFooter.currentSize = cFooter.currentSize
|
||||
pFooter.diskGeometry = { ...cFooter.diskGeometry }
|
||||
pFooter.originalSize = cFooter.originalSize
|
||||
pFooter.timestamp = cFooter.timestamp
|
||||
pFooter.uuid = cFooter.uuid
|
||||
|
||||
// necessary to update values and to recreate the footer after block
|
||||
// creation
|
||||
await parentVhd.writeFooter()
|
||||
|
||||
// should be a disposable
|
||||
parentHandler.unlink(mergeStatePath).catch(warn)
|
||||
|
||||
return mergeState.mergedDataSize
|
||||
})
|
||||
})
|
||||
@@ -1,15 +0,0 @@
|
||||
const { resolveAlias } = require('./_resolveAlias')
|
||||
const { VhdDirectory } = require('./Vhd/VhdDirectory.js')
|
||||
const { VhdFile } = require('./Vhd/VhdFile.js')
|
||||
|
||||
exports.openVhd = async function openVhd(handler, path, opts) {
|
||||
const resolved = await resolveAlias(handler, path)
|
||||
try {
|
||||
return await VhdFile.open(handler, resolved, opts)
|
||||
} catch (e) {
|
||||
if (e.code !== 'EISDIR') {
|
||||
throw e
|
||||
}
|
||||
return await VhdDirectory.open(handler, resolved, opts)
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"private": false,
|
||||
"name": "vhd-lib",
|
||||
"version": "3.0.0",
|
||||
"version": "1.3.0",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"description": "Primitives for VHD file handling",
|
||||
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/vhd-lib",
|
||||
@@ -11,11 +11,11 @@
|
||||
"type": "git",
|
||||
"url": "https://github.com/vatesfr/xen-orchestra.git"
|
||||
},
|
||||
"main": "dist/",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
"node": ">=10"
|
||||
},
|
||||
"dependencies": {
|
||||
"@vates/async-each": "^0.1.0",
|
||||
"@vates/read-chunk": "^0.1.2",
|
||||
"@xen-orchestra/log": "^0.3.0",
|
||||
"async-iterator-to-stream": "^1.0.2",
|
||||
@@ -27,12 +27,25 @@
|
||||
"uuid": "^8.3.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@xen-orchestra/fs": "^0.19.3",
|
||||
"@babel/cli": "^7.0.0",
|
||||
"@babel/core": "^7.0.0",
|
||||
"@babel/preset-env": "^7.0.0",
|
||||
"@xen-orchestra/fs": "^0.18.0",
|
||||
"babel-plugin-lodash": "^3.3.2",
|
||||
"cross-env": "^7.0.2",
|
||||
"execa": "^5.0.0",
|
||||
"get-stream": "^6.0.0",
|
||||
"readable-stream": "^3.0.6",
|
||||
"rimraf": "^3.0.0",
|
||||
"tmp": "^0.2.1"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
|
||||
"clean": "rimraf dist/",
|
||||
"dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
|
||||
"prebuild": "yarn run clean",
|
||||
"predev": "yarn run clean",
|
||||
"prepublishOnly": "yarn run build",
|
||||
"postversion": "npm publish"
|
||||
},
|
||||
"author": {
|
||||
|
||||
@@ -1,134 +0,0 @@
|
||||
const { BLOCK_UNUSED, FOOTER_SIZE, HEADER_SIZE, SECTOR_SIZE } = require('./_constants')
|
||||
const { readChunk } = require('@vates/read-chunk')
|
||||
const assert = require('assert')
|
||||
const { unpackFooter, unpackHeader, computeFullBlockSize } = require('./Vhd/_utils')
|
||||
|
||||
const cappedBufferConcat = (buffers, maxSize) => {
|
||||
let buffer = Buffer.concat(buffers)
|
||||
if (buffer.length > maxSize) {
|
||||
buffer = buffer.slice(buffer.length - maxSize)
|
||||
}
|
||||
return buffer
|
||||
}
|
||||
|
||||
exports.parseVhdStream = async function* parseVhdStream(stream) {
|
||||
let bytesRead = 0
|
||||
|
||||
// handle empty space between elements
|
||||
// ensure we read stream in order
|
||||
async function read(offset, size) {
|
||||
assert(bytesRead <= offset, `offset is ${offset} but we already read ${bytesRead} bytes`)
|
||||
if (bytesRead < offset) {
|
||||
// empty spaces
|
||||
await read(bytesRead, offset - bytesRead)
|
||||
}
|
||||
const buf = await readChunk(stream, size)
|
||||
assert.strictEqual(buf.length, size, `read ${buf.length} instead of ${size}`)
|
||||
bytesRead += size
|
||||
return buf
|
||||
}
|
||||
|
||||
const bufFooter = await read(0, FOOTER_SIZE)
|
||||
|
||||
const footer = unpackFooter(bufFooter)
|
||||
yield { type: 'footer', footer, offset: 0 }
|
||||
|
||||
const bufHeader = await read(FOOTER_SIZE, HEADER_SIZE)
|
||||
const header = unpackHeader(bufHeader, footer)
|
||||
|
||||
yield { type: 'header', header, offset: SECTOR_SIZE }
|
||||
const blockSize = header.blockSize
|
||||
assert.strictEqual(blockSize % SECTOR_SIZE, 0)
|
||||
|
||||
const fullBlockSize = computeFullBlockSize(blockSize)
|
||||
|
||||
const bitmapSize = fullBlockSize - blockSize
|
||||
|
||||
const index = []
|
||||
|
||||
for (const parentLocatorId in header.parentLocatorEntry) {
|
||||
const parentLocatorEntry = header.parentLocatorEntry[parentLocatorId]
|
||||
// empty parent locator entry, does not exist in the content
|
||||
if (parentLocatorEntry.platformDataSpace === 0) {
|
||||
continue
|
||||
}
|
||||
index.push({
|
||||
...parentLocatorEntry,
|
||||
type: 'parentLocator',
|
||||
offset: parentLocatorEntry.platformDataOffset,
|
||||
size: parentLocatorEntry.platformDataLength,
|
||||
id: parentLocatorId,
|
||||
})
|
||||
}
|
||||
|
||||
const batOffset = header.tableOffset
|
||||
const batSize = Math.max(1, Math.ceil((header.maxTableEntries * 4) / SECTOR_SIZE)) * SECTOR_SIZE
|
||||
|
||||
index.push({
|
||||
type: 'bat',
|
||||
offset: batOffset,
|
||||
size: batSize,
|
||||
})
|
||||
|
||||
// sometimes some parent locator are before the BAT
|
||||
index.sort((a, b) => a.offset - b.offset)
|
||||
while (index.length > 0) {
|
||||
const item = index.shift()
|
||||
const buffer = await read(item.offset, item.size)
|
||||
item.buffer = buffer
|
||||
|
||||
const { type } = item
|
||||
if (type === 'bat') {
|
||||
// found the BAT : read it and add block to index
|
||||
|
||||
let blockCount = 0
|
||||
for (let blockCounter = 0; blockCounter < header.maxTableEntries; blockCounter++) {
|
||||
const batEntrySector = buffer.readUInt32BE(blockCounter * 4)
|
||||
// unallocated block, no need to export it
|
||||
if (batEntrySector !== BLOCK_UNUSED) {
|
||||
const batEntryBytes = batEntrySector * SECTOR_SIZE
|
||||
// ensure the block is not before the bat
|
||||
assert.ok(batEntryBytes >= batOffset + batSize)
|
||||
index.push({
|
||||
type: 'block',
|
||||
id: blockCounter,
|
||||
offset: batEntryBytes,
|
||||
size: fullBlockSize,
|
||||
})
|
||||
blockCount++
|
||||
}
|
||||
}
|
||||
// sort again index to ensure block and parent locator are in the right order
|
||||
index.sort((a, b) => a.offset - b.offset)
|
||||
item.blockCount = blockCount
|
||||
} else if (type === 'block') {
|
||||
item.bitmap = buffer.slice(0, bitmapSize)
|
||||
item.data = buffer.slice(bitmapSize)
|
||||
}
|
||||
|
||||
yield item
|
||||
}
|
||||
|
||||
/**
|
||||
* the second footer is at filesize - 512 , there can be empty spaces between last block
|
||||
* and the start of the footer
|
||||
*
|
||||
* we read till the end of the stream, and use the last 512 bytes as the footer
|
||||
*/
|
||||
const bufFooterEnd = await readLastSector(stream)
|
||||
assert(bufFooter.equals(bufFooterEnd), 'footer1 !== footer2')
|
||||
}
|
||||
|
||||
function readLastSector(stream) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let bufFooterEnd = Buffer.alloc(0)
|
||||
stream.on('data', chunk => {
|
||||
if (chunk.length > 0) {
|
||||
bufFooterEnd = cappedBufferConcat([bufFooterEnd, chunk], SECTOR_SIZE)
|
||||
}
|
||||
})
|
||||
|
||||
stream.on('end', () => resolve(bufFooterEnd))
|
||||
stream.on('error', reject)
|
||||
})
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
const { readChunk } = require('@vates/read-chunk')
|
||||
|
||||
const { FOOTER_SIZE } = require('./_constants')
|
||||
const { fuFooter } = require('./_structs')
|
||||
|
||||
module.exports = async function peekFooterFromStream(stream) {
|
||||
const footerBuffer = await readChunk(stream, FOOTER_SIZE)
|
||||
const footer = fuFooter.unpack(footerBuffer)
|
||||
stream.unshift(footerBuffer)
|
||||
return footer
|
||||
}
|
||||
138
packages/vhd-lib/src/Vhd/VhdAbstract.integ.spec.js
Normal file
138
packages/vhd-lib/src/Vhd/VhdAbstract.integ.spec.js
Normal file
@@ -0,0 +1,138 @@
|
||||
/* eslint-env jest */
|
||||
|
||||
import rimraf from 'rimraf'
|
||||
import tmp from 'tmp'
|
||||
import fs from 'fs-extra'
|
||||
import { getSyncedHandler } from '@xen-orchestra/fs'
|
||||
import { Disposable, pFromCallback } from 'promise-toolbox'
|
||||
|
||||
import { openVhd } from '../index'
|
||||
import { createRandomFile, convertFromRawToVhd, createRandomVhdDirectory } from '../tests/utils'
|
||||
import { VhdAbstract } from './VhdAbstract'
|
||||
|
||||
let tempDir
|
||||
|
||||
jest.setTimeout(60000)
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await pFromCallback(cb => tmp.dir(cb))
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await pFromCallback(cb => rimraf(tempDir, cb))
|
||||
})
|
||||
|
||||
test('It creates an alias', async () => {
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: 'file://' + tempDir })
|
||||
const aliasPath = `alias/alias.alias.vhd`
|
||||
const aliasFsPath = `${tempDir}/${aliasPath}`
|
||||
await fs.mkdirp(`${tempDir}/alias`)
|
||||
|
||||
const testOneCombination = async ({ targetPath, targetContent }) => {
|
||||
await VhdAbstract.createAlias(handler, aliasPath, targetPath)
|
||||
// alias file is created
|
||||
expect(await fs.exists(aliasFsPath)).toEqual(true)
|
||||
// content is the target path relative to the alias location
|
||||
const content = await fs.readFile(aliasFsPath, 'utf-8')
|
||||
expect(content).toEqual(targetContent)
|
||||
// create alias fails if alias already exists, remove it before next loop step
|
||||
await fs.unlink(aliasFsPath)
|
||||
}
|
||||
|
||||
const combinations = [
|
||||
{ targetPath: `targets.vhd`, targetContent: `../targets.vhd` },
|
||||
{ targetPath: `alias/targets.vhd`, targetContent: `targets.vhd` },
|
||||
{ targetPath: `alias/sub/targets.vhd`, targetContent: `sub/targets.vhd` },
|
||||
{ targetPath: `sibling/targets.vhd`, targetContent: `../sibling/targets.vhd` },
|
||||
]
|
||||
|
||||
for (const { targetPath, targetContent } of combinations) {
|
||||
await testOneCombination({ targetPath, targetContent })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
test('alias must have *.alias.vhd extension', async () => {
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
|
||||
const aliasPath = 'invalidalias.vhd'
|
||||
const targetPath = 'targets.vhd'
|
||||
expect(async () => await VhdAbstract.createAlias(handler, aliasPath, targetPath)).rejects.toThrow()
|
||||
|
||||
expect(await fs.exists(aliasPath)).toEqual(false)
|
||||
})
|
||||
})
|
||||
|
||||
test('alias must not be chained', async () => {
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
|
||||
const aliasPath = 'valid.alias.vhd'
|
||||
const targetPath = 'an.other.valid.alias.vhd'
|
||||
expect(async () => await VhdAbstract.createAlias(handler, aliasPath, targetPath)).rejects.toThrow()
|
||||
expect(await fs.exists(aliasPath)).toEqual(false)
|
||||
})
|
||||
})
|
||||
|
||||
test('It rename and unlink a VHDFile', async () => {
|
||||
const initalSize = 4
|
||||
const rawFileName = `${tempDir}/randomfile`
|
||||
await createRandomFile(rawFileName, initalSize)
|
||||
await convertFromRawToVhd(rawFileName, `${tempDir}/randomfile.vhd`)
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
|
||||
const { size } = await fs.stat(`${tempDir}/randomfile.vhd`)
|
||||
|
||||
await VhdAbstract.rename(handler, 'randomfile.vhd', 'renamed.vhd')
|
||||
expect(await fs.exists(`${tempDir}/randomfile.vhd`)).toEqual(false)
|
||||
const { size: renamedSize } = await fs.stat(`${tempDir}/renamed.vhd`)
|
||||
expect(size).toEqual(renamedSize)
|
||||
await VhdAbstract.unlink(handler, 'renamed.vhd')
|
||||
expect(await fs.exists(`${tempDir}/renamed.vhd`)).toEqual(false)
|
||||
})
|
||||
})
|
||||
|
||||
test('It rename and unlink a VhdDirectory', async () => {
|
||||
const initalSize = 4
|
||||
const vhdDirectory = `${tempDir}/randomfile.dir`
|
||||
await createRandomVhdDirectory(vhdDirectory, initalSize)
|
||||
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
|
||||
const vhd = yield openVhd(handler, 'randomfile.dir')
|
||||
expect(vhd.header.cookie).toEqual('cxsparse')
|
||||
expect(vhd.footer.cookie).toEqual('conectix')
|
||||
|
||||
await VhdAbstract.rename(handler, 'randomfile.dir', 'renamed.vhd')
|
||||
expect(await fs.exists(`${tempDir}/randomfile.dir`)).toEqual(false)
|
||||
await VhdAbstract.unlink(handler, `renamed.vhd`)
|
||||
expect(await fs.exists(`${tempDir}/renamed.vhd`)).toEqual(false)
|
||||
})
|
||||
})
|
||||
|
||||
test('It create , rename and unlink alias', async () => {
|
||||
const initalSize = 4
|
||||
const rawFileName = `${tempDir}/randomfile`
|
||||
await createRandomFile(rawFileName, initalSize)
|
||||
const vhdFileName = `${tempDir}/randomfile.vhd`
|
||||
await convertFromRawToVhd(rawFileName, vhdFileName)
|
||||
const aliasFileName = `${tempDir}/aliasFileName.alias.vhd`
|
||||
const aliasFileNameRenamed = `${tempDir}/aliasFileNameRenamed.alias.vhd`
|
||||
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
|
||||
await VhdAbstract.createAlias(handler, 'aliasFileName.alias.vhd', 'randomfile.vhd')
|
||||
expect(await fs.exists(aliasFileName)).toEqual(true)
|
||||
expect(await fs.exists(vhdFileName)).toEqual(true)
|
||||
|
||||
await VhdAbstract.rename(handler, 'aliasFileName.alias.vhd', 'aliasFileNameRenamed.alias.vhd')
|
||||
expect(await fs.exists(aliasFileName)).toEqual(false)
|
||||
expect(await fs.exists(vhdFileName)).toEqual(true)
|
||||
expect(await fs.exists(aliasFileNameRenamed)).toEqual(true)
|
||||
|
||||
await VhdAbstract.unlink(handler, 'aliasFileNameRenamed.alias.vhd')
|
||||
expect(await fs.exists(aliasFileName)).toEqual(false)
|
||||
expect(await fs.exists(vhdFileName)).toEqual(false)
|
||||
expect(await fs.exists(aliasFileNameRenamed)).toEqual(false)
|
||||
})
|
||||
})
|
||||
@@ -1,48 +1,29 @@
|
||||
const {
|
||||
computeBatSize,
|
||||
computeFullBlockSize,
|
||||
computeSectorOfBitmap,
|
||||
computeSectorsPerBlock,
|
||||
sectorsToBytes,
|
||||
} = require('./_utils')
|
||||
const {
|
||||
ALIAS_MAX_PATH_LENGTH,
|
||||
PLATFORMS,
|
||||
SECTOR_SIZE,
|
||||
PARENT_LOCATOR_ENTRIES,
|
||||
FOOTER_SIZE,
|
||||
HEADER_SIZE,
|
||||
BLOCK_UNUSED,
|
||||
} = require('../_constants')
|
||||
const assert = require('assert')
|
||||
const path = require('path')
|
||||
const asyncIteratorToStream = require('async-iterator-to-stream')
|
||||
const { checksumStruct, fuFooter, fuHeader } = require('../_structs')
|
||||
const { isVhdAlias, resolveAlias } = require('../_resolveAlias')
|
||||
import { computeBatSize, sectorsRoundUpNoZero, sectorsToBytes } from './_utils'
|
||||
import { PLATFORM_NONE, SECTOR_SIZE, PLATFORM_W2KU, PARENT_LOCATOR_ENTRIES } from '../_constants'
|
||||
import { resolveAlias, isVhdAlias } from '../_resolveAlias'
|
||||
|
||||
exports.VhdAbstract = class VhdAbstract {
|
||||
get bitmapSize() {
|
||||
return sectorsToBytes(this.sectorsOfBitmap)
|
||||
}
|
||||
import assert from 'assert'
|
||||
import path from 'path'
|
||||
|
||||
get fullBlockSize() {
|
||||
return computeFullBlockSize(this.header.blockSize)
|
||||
}
|
||||
|
||||
get sectorsOfBitmap() {
|
||||
return computeSectorOfBitmap(this.header.blockSize)
|
||||
}
|
||||
|
||||
get sectorsPerBlock() {
|
||||
return computeSectorsPerBlock(this.header.blockSize)
|
||||
}
|
||||
export class VhdAbstract {
|
||||
#header
|
||||
bitmapSize
|
||||
footer
|
||||
fullBlockSize
|
||||
sectorsOfBitmap
|
||||
sectorsPerBlock
|
||||
|
||||
get header() {
|
||||
throw new Error('get header is not implemented')
|
||||
assert.notStrictEqual(this.#header, undefined, `header must be read before it's used`)
|
||||
return this.#header
|
||||
}
|
||||
|
||||
get footer() {
|
||||
throw new Error('get footer not implemented')
|
||||
set header(header) {
|
||||
this.#header = header
|
||||
this.sectorsPerBlock = header.blockSize / SECTOR_SIZE
|
||||
this.sectorsOfBitmap = sectorsRoundUpNoZero(this.sectorsPerBlock >> 3)
|
||||
this.fullBlockSize = sectorsToBytes(this.sectorsOfBitmap + this.sectorsPerBlock)
|
||||
this.bitmapSize = sectorsToBytes(this.sectorsOfBitmap)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -102,10 +83,8 @@ exports.VhdAbstract = class VhdAbstract {
|
||||
*
|
||||
* @returns {number} the merged data size
|
||||
*/
|
||||
async coalesceBlock(child, blockId) {
|
||||
const block = await child.readBlock(blockId)
|
||||
await this.writeEntireBlock(block)
|
||||
return block.data.length
|
||||
coalesceBlock(child, blockId) {
|
||||
throw new Error(`coalescing the block ${blockId} from ${child} is not implemented`)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -138,7 +117,7 @@ exports.VhdAbstract = class VhdAbstract {
|
||||
return computeBatSize(this.header.maxTableEntries)
|
||||
}
|
||||
|
||||
async writeParentLocator({ id, platformCode = PLATFORMS.NONE, data = Buffer.alloc(0) }) {
|
||||
async writeParentLocator({ id, platformCode = PLATFORM_NONE, data = Buffer.alloc(0) }) {
|
||||
assert(id >= 0, 'parent Locator id must be a positive number')
|
||||
assert(id < PARENT_LOCATOR_ENTRIES, `parent Locator id must be less than ${PARENT_LOCATOR_ENTRIES}`)
|
||||
|
||||
@@ -147,7 +126,7 @@ exports.VhdAbstract = class VhdAbstract {
|
||||
const entry = this.header.parentLocatorEntry[id]
|
||||
const dataSpaceSectors = Math.ceil(data.length / SECTOR_SIZE)
|
||||
entry.platformCode = platformCode
|
||||
entry.platformDataSpace = dataSpaceSectors
|
||||
entry.platformDataSpace = dataSpaceSectors * SECTOR_SIZE
|
||||
entry.platformDataLength = data.length
|
||||
}
|
||||
|
||||
@@ -167,14 +146,14 @@ exports.VhdAbstract = class VhdAbstract {
|
||||
async setUniqueParentLocator(fileNameString) {
|
||||
await this.writeParentLocator({
|
||||
id: 0,
|
||||
platformCode: PLATFORMS.W2KU,
|
||||
platformCode: PLATFORM_W2KU,
|
||||
data: Buffer.from(fileNameString, 'utf16le'),
|
||||
})
|
||||
|
||||
for (let i = 1; i < PARENT_LOCATOR_ENTRIES; i++) {
|
||||
await this.writeParentLocator({
|
||||
id: i,
|
||||
platformCode: PLATFORMS.NONE,
|
||||
platformCode: PLATFORM_NONE,
|
||||
data: Buffer.alloc(0),
|
||||
})
|
||||
}
|
||||
@@ -190,10 +169,6 @@ exports.VhdAbstract = class VhdAbstract {
|
||||
}
|
||||
|
||||
static async rename(handler, sourcePath, targetPath) {
|
||||
try {
|
||||
// delete target if it already exists
|
||||
await VhdAbstract.unlink(handler, targetPath)
|
||||
} catch (e) {}
|
||||
await handler.rename(sourcePath, targetPath)
|
||||
}
|
||||
|
||||
@@ -227,109 +202,6 @@ exports.VhdAbstract = class VhdAbstract {
|
||||
const aliasDir = path.dirname(path.resolve('/', aliasPath))
|
||||
// only store the relative path from alias to target
|
||||
const relativePathToTarget = path.relative(aliasDir, path.resolve('/', targetPath))
|
||||
|
||||
if (relativePathToTarget.length > ALIAS_MAX_PATH_LENGTH) {
|
||||
throw new Error(
|
||||
`Alias relative path ${relativePathToTarget} is too long : ${relativePathToTarget.length} chars, max is ${ALIAS_MAX_PATH_LENGTH}`
|
||||
)
|
||||
}
|
||||
await handler.writeFile(aliasPath, relativePathToTarget)
|
||||
}
|
||||
|
||||
stream() {
|
||||
const { footer, batSize } = this
|
||||
const { ...header } = this.header // copy since we don't ant to modifiy the current header
|
||||
const rawFooter = fuFooter.pack(footer)
|
||||
checksumStruct(rawFooter, fuFooter)
|
||||
|
||||
// update them in header
|
||||
// update checksum in header
|
||||
|
||||
let offset = FOOTER_SIZE + HEADER_SIZE + batSize
|
||||
|
||||
const rawHeader = fuHeader.pack(header)
|
||||
checksumStruct(rawHeader, fuHeader)
|
||||
|
||||
// add parentlocator size
|
||||
for (let i = 0; i < PARENT_LOCATOR_ENTRIES; i++) {
|
||||
header.parentLocatorEntry[i] = {
|
||||
...header.parentLocatorEntry[i],
|
||||
platformDataOffset: offset,
|
||||
}
|
||||
offset += header.parentLocatorEntry[i].platformDataSpace * SECTOR_SIZE
|
||||
}
|
||||
|
||||
assert.strictEqual(offset % SECTOR_SIZE, 0)
|
||||
|
||||
const bat = Buffer.allocUnsafe(batSize)
|
||||
let offsetSector = offset / SECTOR_SIZE
|
||||
const blockSizeInSectors = this.fullBlockSize / SECTOR_SIZE
|
||||
let fileSize = offsetSector * SECTOR_SIZE + FOOTER_SIZE /* the footer at the end */
|
||||
// compute BAT , blocks starts after parent locator entries
|
||||
for (let i = 0; i < header.maxTableEntries; i++) {
|
||||
if (this.containsBlock(i)) {
|
||||
bat.writeUInt32BE(offsetSector, i * 4)
|
||||
offsetSector += blockSizeInSectors
|
||||
fileSize += this.fullBlockSize
|
||||
} else {
|
||||
bat.writeUInt32BE(BLOCK_UNUSED, i * 4)
|
||||
}
|
||||
}
|
||||
|
||||
const self = this
|
||||
async function* iterator() {
|
||||
yield rawFooter
|
||||
yield rawHeader
|
||||
yield bat
|
||||
|
||||
// yield parent locator
|
||||
|
||||
for (let i = 0; i < PARENT_LOCATOR_ENTRIES; i++) {
|
||||
const space = header.parentLocatorEntry[i].platformDataSpace * SECTOR_SIZE
|
||||
if (space > 0) {
|
||||
const data = (await self.readParentLocator(i)).data
|
||||
// align data to a sector
|
||||
const buffer = Buffer.alloc(space, 0)
|
||||
data.copy(buffer)
|
||||
yield buffer
|
||||
}
|
||||
}
|
||||
|
||||
// yield all blocks
|
||||
// since contains() can be costly for synthetic vhd, use the computed bat
|
||||
for (let i = 0; i < header.maxTableEntries; i++) {
|
||||
if (bat.readUInt32BE(i * 4) !== BLOCK_UNUSED) {
|
||||
const block = await self.readBlock(i)
|
||||
yield block.buffer
|
||||
}
|
||||
}
|
||||
// yield footer again
|
||||
yield rawFooter
|
||||
}
|
||||
|
||||
const stream = asyncIteratorToStream(iterator())
|
||||
stream.length = fileSize
|
||||
return stream
|
||||
}
|
||||
|
||||
rawContent() {
|
||||
const { header, footer } = this
|
||||
const { blockSize } = header
|
||||
const self = this
|
||||
async function* iterator() {
|
||||
const nBlocks = header.maxTableEntries
|
||||
let remainingSize = footer.currentSize
|
||||
const EMPTY = Buffer.alloc(blockSize, 0)
|
||||
for (let blockId = 0; blockId < nBlocks; ++blockId) {
|
||||
let buffer = self.containsBlock(blockId) ? (await self.readBlock(blockId)).data : EMPTY
|
||||
// the last block can be truncated since raw size is not a multiple of blockSize
|
||||
buffer = remainingSize < blockSize ? buffer.slice(0, remainingSize) : buffer
|
||||
remainingSize -= blockSize
|
||||
yield buffer
|
||||
}
|
||||
}
|
||||
const stream = asyncIteratorToStream(iterator())
|
||||
stream.length = footer.currentSize
|
||||
return stream
|
||||
}
|
||||
}
|
||||
@@ -1,67 +1,15 @@
|
||||
const { unpackHeader, unpackFooter, sectorsToBytes } = require('./_utils')
|
||||
const { createLogger } = require('@xen-orchestra/log')
|
||||
const { fuFooter, fuHeader, checksumStruct } = require('../_structs')
|
||||
const { test, set: setBitmap } = require('../_bitmap')
|
||||
const { VhdAbstract } = require('./VhdAbstract')
|
||||
const assert = require('assert')
|
||||
const promisify = require('promise-toolbox/promisify')
|
||||
const zlib = require('zlib')
|
||||
import { buildHeader, buildFooter } from './_utils'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { fuFooter, fuHeader, checksumStruct } from '../_structs'
|
||||
import { test, set as setBitmap } from '../_bitmap'
|
||||
import { VhdAbstract } from './VhdAbstract'
|
||||
import assert from 'assert'
|
||||
|
||||
const { debug } = createLogger('vhd-lib:VhdDirectory')
|
||||
|
||||
const NULL_COMPRESSOR = {
|
||||
compress: buffer => buffer,
|
||||
decompress: buffer => buffer,
|
||||
baseOptions: {},
|
||||
}
|
||||
|
||||
const COMPRESSORS = {
|
||||
gzip: {
|
||||
compress: (
|
||||
gzip => buffer =>
|
||||
gzip(buffer, { level: zlib.constants.Z_BEST_SPEED })
|
||||
)(promisify(zlib.gzip)),
|
||||
decompress: promisify(zlib.gunzip),
|
||||
},
|
||||
brotli: {
|
||||
compress: (
|
||||
brotliCompress => buffer =>
|
||||
brotliCompress(buffer, {
|
||||
params: {
|
||||
[zlib.constants.BROTLI_PARAM_QUALITY]: zlib.constants.BROTLI_MIN_QUALITY,
|
||||
},
|
||||
})
|
||||
)(promisify(zlib.brotliCompress)),
|
||||
decompress: promisify(zlib.brotliDecompress),
|
||||
},
|
||||
}
|
||||
|
||||
// inject identifiers
|
||||
for (const id of Object.keys(COMPRESSORS)) {
|
||||
COMPRESSORS[id].id = id
|
||||
}
|
||||
|
||||
function getCompressor(compressorType) {
|
||||
if (compressorType === undefined) {
|
||||
return NULL_COMPRESSOR
|
||||
}
|
||||
|
||||
const compressor = COMPRESSORS[compressorType]
|
||||
|
||||
if (compressor === undefined) {
|
||||
throw new Error(`Compression type ${compressorType} is not supported`)
|
||||
}
|
||||
|
||||
return compressor
|
||||
}
|
||||
|
||||
// ===================================================================
|
||||
// Directory format
|
||||
// <path>
|
||||
// ├─ chunk-filters.json
|
||||
// │ Ordered array of filters that have been applied before writing chunks.
|
||||
// │ These filters needs to be applied in reverse order to read them.
|
||||
// │
|
||||
// ├─ header // raw content of the header
|
||||
// ├─ footer // raw content of the footer
|
||||
// ├─ bat // bit array. A zero bit indicates at a position that this block is not present
|
||||
@@ -70,24 +18,16 @@ function getCompressor(compressorType) {
|
||||
// └─ <the first to {blockId.length -3} numbers of blockId >
|
||||
// └─ <the three last numbers of blockID > // block content.
|
||||
|
||||
exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
|
||||
export class VhdDirectory extends VhdAbstract {
|
||||
#uncheckedBlockTable
|
||||
#header
|
||||
footer
|
||||
#compressor
|
||||
|
||||
get compressionType() {
|
||||
return this.#compressor.id
|
||||
}
|
||||
|
||||
set header(header) {
|
||||
this.#header = header
|
||||
super.header = header
|
||||
this.#blockTable = Buffer.alloc(header.maxTableEntries)
|
||||
}
|
||||
|
||||
get header() {
|
||||
assert.notStrictEqual(this.#header, undefined, `header must be read before it's used`)
|
||||
return this.#header
|
||||
return super.header
|
||||
}
|
||||
|
||||
get #blockTable() {
|
||||
@@ -99,8 +39,8 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
|
||||
this.#uncheckedBlockTable = blockTable
|
||||
}
|
||||
|
||||
static async open(handler, path, { flags = 'r+' } = {}) {
|
||||
const vhd = new VhdDirectory(handler, path, { flags })
|
||||
static async open(handler, path) {
|
||||
const vhd = new VhdDirectory(handler, path)
|
||||
|
||||
// openning a file for reading does not trigger EISDIR as long as we don't really read from it :
|
||||
// https://man7.org/linux/man-pages/man2/open.2.html
|
||||
@@ -114,21 +54,19 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
|
||||
}
|
||||
}
|
||||
|
||||
static async create(handler, path, { flags = 'wx+', compression } = {}) {
|
||||
static async create(handler, path) {
|
||||
await handler.mkdir(path)
|
||||
const vhd = new VhdDirectory(handler, path, { flags, compression })
|
||||
const vhd = new VhdDirectory(handler, path)
|
||||
return {
|
||||
dispose: () => {},
|
||||
value: vhd,
|
||||
}
|
||||
}
|
||||
|
||||
constructor(handler, path, opts) {
|
||||
constructor(handler, path) {
|
||||
super()
|
||||
this._handler = handler
|
||||
this._path = path
|
||||
this._opts = opts
|
||||
this.#compressor = getCompressor(opts?.compression)
|
||||
}
|
||||
|
||||
async readBlockAllocationTable() {
|
||||
@@ -140,29 +78,34 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
|
||||
return test(this.#blockTable, blockId)
|
||||
}
|
||||
|
||||
_getChunkPath(partName) {
|
||||
getChunkPath(partName) {
|
||||
return this._path + '/' + partName
|
||||
}
|
||||
|
||||
async _readChunk(partName) {
|
||||
// here we can implement compression and / or crypto
|
||||
const buffer = await this._handler.readFile(this._getChunkPath(partName))
|
||||
const buffer = await this._handler.readFile(this.getChunkPath(partName))
|
||||
|
||||
const uncompressed = await this.#compressor.decompress(buffer)
|
||||
return {
|
||||
buffer: uncompressed,
|
||||
buffer: Buffer.from(buffer),
|
||||
}
|
||||
}
|
||||
|
||||
async _writeChunk(partName, buffer) {
|
||||
assert.notStrictEqual(
|
||||
this._opts?.flags,
|
||||
'r',
|
||||
`Can't write a chunk ${partName} in ${this._path} with read permission`
|
||||
)
|
||||
assert(Buffer.isBuffer(buffer))
|
||||
// here we can implement compression and / or crypto
|
||||
|
||||
const compressed = await this.#compressor.compress(buffer)
|
||||
return this._handler.outputFile(this._getChunkPath(partName), compressed, this._opts)
|
||||
// chunks can be in sub directories : create direcotries if necessary
|
||||
const pathParts = partName.split('/')
|
||||
let currentPath = this._path
|
||||
|
||||
// the last one is the file name
|
||||
for (let i = 0; i < pathParts.length - 1; i++) {
|
||||
currentPath += '/' + pathParts[i]
|
||||
await this._handler.mkdir(currentPath)
|
||||
}
|
||||
|
||||
return this._handler.writeFile(this.getChunkPath(partName), buffer)
|
||||
}
|
||||
|
||||
// put block in subdirectories to limit impact when doing directory listing
|
||||
@@ -173,12 +116,10 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
|
||||
}
|
||||
|
||||
async readHeaderAndFooter() {
|
||||
// we need to know if thre is compression before reading headers
|
||||
await this.#readChunkFilters()
|
||||
const { buffer: bufHeader } = await this._readChunk('header')
|
||||
const { buffer: bufFooter } = await this._readChunk('footer')
|
||||
const footer = unpackFooter(bufFooter)
|
||||
const header = unpackHeader(bufHeader, footer)
|
||||
const footer = buildFooter(bufFooter)
|
||||
const header = buildHeader(bufHeader, footer)
|
||||
|
||||
this.footer = footer
|
||||
this.header = header
|
||||
@@ -211,13 +152,12 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
|
||||
await this._writeChunk('footer', rawFooter)
|
||||
}
|
||||
|
||||
async writeHeader() {
|
||||
writeHeader() {
|
||||
const { header } = this
|
||||
const rawHeader = fuHeader.pack(header)
|
||||
header.checksum = checksumStruct(rawHeader, fuHeader)
|
||||
debug(`Write header (checksum=${header.checksum}). (data=${rawHeader.toString('hex')})`)
|
||||
await this._writeChunk('header', rawHeader)
|
||||
await this.#writeChunkFilters()
|
||||
return this._writeChunk('header', rawHeader)
|
||||
}
|
||||
|
||||
writeBlockAllocationTable() {
|
||||
@@ -227,22 +167,11 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
|
||||
return this._writeChunk('bat', this.#blockTable)
|
||||
}
|
||||
|
||||
// only works if data are in the same handler
|
||||
// only works if data are in the same bucket
|
||||
// and if the full block is modified in child ( which is the case whit xcp)
|
||||
// and if the compression type is same on both sides
|
||||
async coalesceBlock(child, blockId) {
|
||||
if (
|
||||
!(child instanceof VhdDirectory) ||
|
||||
this._handler !== child._handler ||
|
||||
child.compressionType !== this.compressionType
|
||||
) {
|
||||
return super.coalesceBlock(child, blockId)
|
||||
}
|
||||
await this._handler.copy(
|
||||
child._getChunkPath(child._getBlockPath(blockId)),
|
||||
this._getChunkPath(this._getBlockPath(blockId))
|
||||
)
|
||||
return sectorsToBytes(this.sectorsPerBlock)
|
||||
|
||||
coalesceBlock(child, blockId) {
|
||||
this._handler.copy(child.getChunkPath(blockId), this.getChunkPath(blockId))
|
||||
}
|
||||
|
||||
async writeEntireBlock(block) {
|
||||
@@ -258,24 +187,4 @@ exports.VhdDirectory = class VhdDirectory extends VhdAbstract {
|
||||
await this._writeChunk('parentLocatorEntry' + id, data)
|
||||
this.header.parentLocatorEntry[id].platformDataOffset = 0
|
||||
}
|
||||
|
||||
async #writeChunkFilters() {
|
||||
const compressionType = this.compressionType
|
||||
const path = this._path + '/chunk-filters.json'
|
||||
if (compressionType === undefined) {
|
||||
await this._handler.unlink(path)
|
||||
} else {
|
||||
await this._handler.writeFile(path, JSON.stringify([compressionType]))
|
||||
}
|
||||
}
|
||||
|
||||
async #readChunkFilters() {
|
||||
const chunkFilters = await this._handler.readFile(this._path + '/chunk-filters.json').then(JSON.parse, error => {
|
||||
if (error.code === 'ENOENT') {
|
||||
return []
|
||||
}
|
||||
throw error
|
||||
})
|
||||
this.#compressor = getCompressor(chunkFilters[0])
|
||||
}
|
||||
}
|
||||
@@ -1,25 +1,18 @@
|
||||
/* eslint-env jest */
|
||||
|
||||
const execa = require('execa')
|
||||
const fs = require('fs-extra')
|
||||
const getStream = require('get-stream')
|
||||
const rimraf = require('rimraf')
|
||||
const tmp = require('tmp')
|
||||
const { getHandler } = require('@xen-orchestra/fs')
|
||||
const { Disposable, pFromCallback } = require('promise-toolbox')
|
||||
const { randomBytes } = require('crypto')
|
||||
import execa from 'execa'
|
||||
import fs from 'fs-extra'
|
||||
import getStream from 'get-stream'
|
||||
import rimraf from 'rimraf'
|
||||
import tmp from 'tmp'
|
||||
import { getHandler } from '@xen-orchestra/fs'
|
||||
import { pFromCallback } from 'promise-toolbox'
|
||||
import { randomBytes } from 'crypto'
|
||||
|
||||
const { VhdFile } = require('./VhdFile')
|
||||
const { openVhd } = require('../openVhd')
|
||||
import { VhdFile } from './VhdFile'
|
||||
|
||||
const { SECTOR_SIZE } = require('../_constants')
|
||||
const {
|
||||
checkFile,
|
||||
createRandomFile,
|
||||
convertFromRawToVhd,
|
||||
convertToVhdDirectory,
|
||||
recoverRawContent,
|
||||
} = require('../tests/utils')
|
||||
import { SECTOR_SIZE } from '../_constants'
|
||||
import { checkFile, createRandomFile, convertFromRawToVhd, recoverRawContent } from '../tests/utils'
|
||||
|
||||
let tempDir = null
|
||||
|
||||
@@ -33,29 +26,6 @@ afterEach(async () => {
|
||||
await pFromCallback(cb => rimraf(tempDir, cb))
|
||||
})
|
||||
|
||||
test('respect the checkSecondFooter flag', async () => {
|
||||
const initalSize = 0
|
||||
const rawFileName = `${tempDir}/randomfile`
|
||||
await createRandomFile(rawFileName, initalSize)
|
||||
const vhdFileName = `${tempDir}/randomfile.vhd`
|
||||
await convertFromRawToVhd(rawFileName, vhdFileName)
|
||||
|
||||
const handler = getHandler({ url: `file://${tempDir}` })
|
||||
|
||||
const size = await handler.getSize('randomfile.vhd')
|
||||
const fd = await handler.openFile('randomfile.vhd', 'r+')
|
||||
const buffer = Buffer.alloc(512, 0)
|
||||
// add a fake footer at the end
|
||||
handler.write(fd, buffer, size)
|
||||
await handler.closeFile(fd)
|
||||
// not using openVhd to be able to call readHeaderAndFooter separatly
|
||||
const vhd = new VhdFile(handler, 'randomfile.vhd')
|
||||
|
||||
await expect(async () => await vhd.readHeaderAndFooter()).rejects.toThrow()
|
||||
await expect(async () => await vhd.readHeaderAndFooter(true)).rejects.toThrow()
|
||||
await expect(await vhd.readHeaderAndFooter(false)).toEqual(undefined)
|
||||
})
|
||||
|
||||
test('blocks can be moved', async () => {
|
||||
const initalSize = 4
|
||||
const rawFileName = `${tempDir}/randomfile`
|
||||
@@ -88,7 +58,6 @@ test('the BAT MSB is not used for sign', async () => {
|
||||
// here we are moving the first sector very far in the VHD to prove the BAT doesn't use signed int32
|
||||
const hugePositionBytes = hugeWritePositionSectors * SECTOR_SIZE
|
||||
await vhd._freeFirstBlockSpace(hugePositionBytes)
|
||||
await vhd.writeFooter()
|
||||
|
||||
// we recover the data manually for speed reasons.
|
||||
// fs.write() with offset is way faster than qemu-img when there is a 1.5To
|
||||
@@ -193,45 +162,3 @@ test('BAT can be extended and blocks moved', async () => {
|
||||
await recoverRawContent(vhdFileName, recoveredFileName, originalSize)
|
||||
expect(await fs.readFile(recoveredFileName)).toEqual(await fs.readFile(rawFileName))
|
||||
})
|
||||
|
||||
test('Can coalesce block', async () => {
|
||||
const initalSize = 4
|
||||
const parentrawFileName = `${tempDir}/randomfile`
|
||||
const parentFileName = `${tempDir}/parent.vhd`
|
||||
await createRandomFile(parentrawFileName, initalSize)
|
||||
await convertFromRawToVhd(parentrawFileName, parentFileName)
|
||||
const childrawFileName = `${tempDir}/randomfile`
|
||||
const childFileName = `${tempDir}/childFile.vhd`
|
||||
await createRandomFile(childrawFileName, initalSize)
|
||||
await convertFromRawToVhd(childrawFileName, childFileName)
|
||||
const childRawDirectoryName = `${tempDir}/randomFile2.vhd`
|
||||
const childDirectoryFileName = `${tempDir}/childDirFile.vhd`
|
||||
const childDirectoryName = `${tempDir}/childDir.vhd`
|
||||
await createRandomFile(childRawDirectoryName, initalSize)
|
||||
await convertFromRawToVhd(childRawDirectoryName, childDirectoryFileName)
|
||||
await convertToVhdDirectory(childRawDirectoryName, childDirectoryFileName, childDirectoryName)
|
||||
|
||||
await Disposable.use(async function* () {
|
||||
const handler = getHandler({ url: 'file://' })
|
||||
const parentVhd = yield openVhd(handler, parentFileName, { flags: 'r+' })
|
||||
await parentVhd.readBlockAllocationTable()
|
||||
const childFileVhd = yield openVhd(handler, childFileName)
|
||||
await childFileVhd.readBlockAllocationTable()
|
||||
const childDirectoryVhd = yield openVhd(handler, childDirectoryName)
|
||||
await childDirectoryVhd.readBlockAllocationTable()
|
||||
|
||||
await parentVhd.coalesceBlock(childFileVhd, 0)
|
||||
await parentVhd.writeFooter()
|
||||
await parentVhd.writeBlockAllocationTable()
|
||||
let parentBlockData = (await parentVhd.readBlock(0)).data
|
||||
let childBlockData = (await childFileVhd.readBlock(0)).data
|
||||
expect(parentBlockData).toEqual(childBlockData)
|
||||
|
||||
await parentVhd.coalesceBlock(childDirectoryVhd, 0)
|
||||
await parentVhd.writeFooter()
|
||||
await parentVhd.writeBlockAllocationTable()
|
||||
parentBlockData = (await parentVhd.readBlock(0)).data
|
||||
childBlockData = (await childDirectoryVhd.readBlock(0)).data
|
||||
expect(parentBlockData).toEqual(childBlockData)
|
||||
})
|
||||
})
|
||||
@@ -1,18 +1,18 @@
|
||||
const {
|
||||
import {
|
||||
BLOCK_UNUSED,
|
||||
FOOTER_SIZE,
|
||||
HEADER_SIZE,
|
||||
PLATFORMS,
|
||||
PLATFORM_NONE,
|
||||
SECTOR_SIZE,
|
||||
PARENT_LOCATOR_ENTRIES,
|
||||
} = require('../_constants')
|
||||
const { computeBatSize, sectorsToBytes, unpackHeader, unpackFooter, BUF_BLOCK_UNUSED } = require('./_utils')
|
||||
const { createLogger } = require('@xen-orchestra/log')
|
||||
const { fuFooter, fuHeader, checksumStruct } = require('../_structs')
|
||||
const { set: mapSetBit } = require('../_bitmap')
|
||||
const { VhdAbstract } = require('./VhdAbstract')
|
||||
const assert = require('assert')
|
||||
const getFirstAndLastBlocks = require('../_getFirstAndLastBlocks')
|
||||
} from '../_constants'
|
||||
import { computeBatSize, sectorsToBytes, buildHeader, buildFooter, BUF_BLOCK_UNUSED } from './_utils'
|
||||
import { createLogger } from '@xen-orchestra/log'
|
||||
import { fuFooter, fuHeader, checksumStruct } from '../_structs'
|
||||
import { set as mapSetBit, test as mapTestBit } from '../_bitmap'
|
||||
import { VhdAbstract } from './VhdAbstract'
|
||||
import assert from 'assert'
|
||||
import getFirstAndLastBlocks from '../_getFirstAndLastBlocks'
|
||||
|
||||
const { debug } = createLogger('vhd-lib:VhdFile')
|
||||
|
||||
@@ -50,10 +50,8 @@ const { debug } = createLogger('vhd-lib:VhdFile')
|
||||
// - parentLocatorSize(i) = header.parentLocatorEntry[i].platformDataSpace * sectorSize
|
||||
// - sectorSize = 512
|
||||
|
||||
exports.VhdFile = class VhdFile extends VhdAbstract {
|
||||
export class VhdFile extends VhdAbstract {
|
||||
#uncheckedBlockTable
|
||||
#header
|
||||
footer
|
||||
|
||||
get #blockTable() {
|
||||
assert.notStrictEqual(this.#uncheckedBlockTable, undefined, 'Block table must be initialized before access')
|
||||
@@ -69,7 +67,7 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
|
||||
}
|
||||
|
||||
set header(header) {
|
||||
this.#header = header
|
||||
super.header = header
|
||||
const size = this.batSize
|
||||
this.#blockTable = Buffer.alloc(size)
|
||||
for (let i = 0; i < this.header.maxTableEntries; i++) {
|
||||
@@ -77,26 +75,26 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
|
||||
}
|
||||
}
|
||||
get header() {
|
||||
return this.#header
|
||||
return super.header
|
||||
}
|
||||
|
||||
static async open(handler, path, { flags, checkSecondFooter = true } = {}) {
|
||||
const fd = await handler.openFile(path, flags ?? 'r+')
|
||||
static async open(handler, path) {
|
||||
const fd = await handler.openFile(path, 'r+')
|
||||
const vhd = new VhdFile(handler, fd)
|
||||
// openning a file for reading does not trigger EISDIR as long as we don't really read from it :
|
||||
// https://man7.org/linux/man-pages/man2/open.2.html
|
||||
// EISDIR pathname refers to a directory and the access requested
|
||||
// involved writing (that is, O_WRONLY or O_RDWR is set).
|
||||
// reading the header ensure we have a well formed file immediatly
|
||||
await vhd.readHeaderAndFooter(checkSecondFooter)
|
||||
await vhd.readHeaderAndFooter()
|
||||
return {
|
||||
dispose: () => handler.closeFile(fd),
|
||||
value: vhd,
|
||||
}
|
||||
}
|
||||
|
||||
static async create(handler, path, { flags } = {}) {
|
||||
const fd = await handler.openFile(path, flags ?? 'wx')
|
||||
static async create(handler, path) {
|
||||
const fd = await handler.openFile(path, 'wx')
|
||||
const vhd = new VhdFile(handler, fd)
|
||||
return {
|
||||
dispose: () => handler.closeFile(fd),
|
||||
@@ -131,7 +129,7 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
|
||||
for (let i = 0; i < PARENT_LOCATOR_ENTRIES; i++) {
|
||||
const entry = header.parentLocatorEntry[i]
|
||||
|
||||
if (entry.platformCode !== PLATFORMS.NONE) {
|
||||
if (entry.platformCode !== PLATFORM_NONE) {
|
||||
end = Math.max(end, entry.platformDataOffset + sectorsToBytes(entry.platformDataSpace))
|
||||
}
|
||||
}
|
||||
@@ -179,8 +177,8 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
|
||||
const bufFooter = buf.slice(0, FOOTER_SIZE)
|
||||
const bufHeader = buf.slice(FOOTER_SIZE)
|
||||
|
||||
const footer = unpackFooter(bufFooter)
|
||||
const header = unpackHeader(bufHeader, footer)
|
||||
const footer = buildFooter(bufFooter)
|
||||
const header = buildHeader(bufHeader, footer)
|
||||
|
||||
if (checkSecondFooter) {
|
||||
const size = await this._handler.getSize(this._path)
|
||||
@@ -345,6 +343,47 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
|
||||
)
|
||||
}
|
||||
|
||||
async coalesceBlock(child, blockId) {
|
||||
const block = await child.readBlock(blockId)
|
||||
const { bitmap, data } = block
|
||||
|
||||
debug(`coalesceBlock block=${blockId}`)
|
||||
|
||||
// For each sector of block data...
|
||||
const { sectorsPerBlock } = child
|
||||
let parentBitmap = null
|
||||
for (let i = 0; i < sectorsPerBlock; i++) {
|
||||
// If no changes on one sector, skip.
|
||||
if (!mapTestBit(bitmap, i)) {
|
||||
continue
|
||||
}
|
||||
let endSector = i + 1
|
||||
|
||||
// Count changed sectors.
|
||||
while (endSector < sectorsPerBlock && mapTestBit(bitmap, endSector)) {
|
||||
++endSector
|
||||
}
|
||||
|
||||
// Write n sectors into parent.
|
||||
debug(`coalesceBlock: write sectors=${i}...${endSector}`)
|
||||
|
||||
const isFullBlock = i === 0 && endSector === sectorsPerBlock
|
||||
if (isFullBlock) {
|
||||
await this.writeEntireBlock(block)
|
||||
} else {
|
||||
if (parentBitmap === null) {
|
||||
parentBitmap = (await this.readBlock(blockId, true)).bitmap
|
||||
}
|
||||
await this._writeBlockSectors(block, i, endSector, parentBitmap)
|
||||
}
|
||||
|
||||
i = endSector
|
||||
}
|
||||
|
||||
// Return the merged data size
|
||||
return data.length
|
||||
}
|
||||
|
||||
// Write a context footer. (At the end and beginning of a vhd file.)
|
||||
async writeFooter(onlyEndFooter = false) {
|
||||
const { footer } = this
|
||||
@@ -423,7 +462,7 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
|
||||
async _readParentLocatorData(parentLocatorId) {
|
||||
const { platformDataOffset, platformDataLength } = this.header.parentLocatorEntry[parentLocatorId]
|
||||
if (platformDataLength > 0) {
|
||||
return await this._read(platformDataOffset, platformDataLength)
|
||||
return (await this._read(platformDataOffset, platformDataLength)).buffer
|
||||
}
|
||||
return Buffer.alloc(0)
|
||||
}
|
||||
@@ -435,8 +474,7 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
|
||||
// reset offset if data is empty
|
||||
header.parentLocatorEntry[parentLocatorId].platformDataOffset = 0
|
||||
} else {
|
||||
const space = header.parentLocatorEntry[parentLocatorId].platformDataSpace * SECTOR_SIZE
|
||||
if (data.length <= space) {
|
||||
if (data.length <= header.parentLocatorEntry[parentLocatorId].platformDataSpace) {
|
||||
// new parent locator length is smaller than available space : keep it in place
|
||||
position = header.parentLocatorEntry[parentLocatorId].platformDataOffset
|
||||
} else {
|
||||
@@ -451,7 +489,7 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
|
||||
// move the first(s) block(s) at the end of the data
|
||||
// move the parent locator to the precedent position of the first block
|
||||
const { firstSector } = firstAndLastBlocks
|
||||
await this._freeFirstBlockSpace(space)
|
||||
await this._freeFirstBlockSpace(header.parentLocatorEntry[parentLocatorId].platformDataSpace)
|
||||
position = sectorsToBytes(firstSector)
|
||||
}
|
||||
}
|
||||
@@ -459,8 +497,4 @@ exports.VhdFile = class VhdFile extends VhdAbstract {
|
||||
header.parentLocatorEntry[parentLocatorId].platformDataOffset = position
|
||||
}
|
||||
}
|
||||
|
||||
async getSize() {
|
||||
return await this._handler.getSize(this._path)
|
||||
}
|
||||
}
|
||||
52
packages/vhd-lib/src/Vhd/_utils.js
Normal file
52
packages/vhd-lib/src/Vhd/_utils.js
Normal file
@@ -0,0 +1,52 @@
|
||||
import assert from 'assert'
|
||||
import { BLOCK_UNUSED, SECTOR_SIZE } from '../_constants'
|
||||
import { fuFooter, fuHeader, checksumStruct, unpackField } from '../_structs'
|
||||
import checkFooter from '../checkFooter'
|
||||
import checkHeader from '../_checkHeader'
|
||||
|
||||
export const computeBatSize = entries => sectorsToBytes(sectorsRoundUpNoZero(entries * 4))
|
||||
|
||||
// Sectors conversions.
|
||||
export const sectorsRoundUpNoZero = bytes => Math.ceil(bytes / SECTOR_SIZE) || 1
|
||||
export const sectorsToBytes = sectors => sectors * SECTOR_SIZE
|
||||
|
||||
export const assertChecksum = (name, buf, struct) => {
|
||||
const actual = unpackField(struct.fields.checksum, buf)
|
||||
const expected = checksumStruct(buf, struct)
|
||||
assert.strictEqual(actual, expected, `invalid ${name} checksum ${actual}, expected ${expected}`)
|
||||
}
|
||||
|
||||
// unused block as buffer containing a uint32BE
|
||||
export const BUF_BLOCK_UNUSED = Buffer.allocUnsafe(4)
|
||||
BUF_BLOCK_UNUSED.writeUInt32BE(BLOCK_UNUSED, 0)
|
||||
|
||||
/**
|
||||
* Check and parse the header buffer to build an header object
|
||||
*
|
||||
* @param {Buffer} bufHeader
|
||||
* @param {Object} footer
|
||||
* @returns {Object} the parsed header
|
||||
*/
|
||||
export const buildHeader = (bufHeader, footer) => {
|
||||
assertChecksum('header', bufHeader, fuHeader)
|
||||
|
||||
const header = fuHeader.unpack(bufHeader)
|
||||
checkHeader(header, footer)
|
||||
return header
|
||||
}
|
||||
|
||||
/**
|
||||
* Check and parse the footer buffer to build a footer object
|
||||
*
|
||||
* @param {Buffer} bufHeader
|
||||
* @param {Object} footer
|
||||
* @returns {Object} the parsed footer
|
||||
*/
|
||||
|
||||
export const buildFooter = bufFooter => {
|
||||
assertChecksum('footer', bufFooter, fuFooter)
|
||||
|
||||
const footer = fuFooter.unpack(bufFooter)
|
||||
checkFooter(footer)
|
||||
return footer
|
||||
}
|
||||
7
packages/vhd-lib/src/_bitmap.js
Normal file
7
packages/vhd-lib/src/_bitmap.js
Normal file
@@ -0,0 +1,7 @@
|
||||
const MASK = 0x80
|
||||
|
||||
export const set = (map, bit) => {
|
||||
map[bit >> 3] |= MASK >> (bit & 7)
|
||||
}
|
||||
|
||||
export const test = (map, bit) => ((map[bit >> 3] << (bit & 7)) & MASK) !== 0
|
||||
@@ -1,8 +1,8 @@
|
||||
const assert = require('assert')
|
||||
import assert from 'assert'
|
||||
|
||||
const { HEADER_COOKIE, HEADER_VERSION, SECTOR_SIZE } = require('./_constants')
|
||||
import { HEADER_COOKIE, HEADER_VERSION, SECTOR_SIZE } from './_constants'
|
||||
|
||||
module.exports = (header, footer) => {
|
||||
export default (header, footer) => {
|
||||
assert.strictEqual(header.cookie, HEADER_COOKIE)
|
||||
assert.strictEqual(header.dataOffset, undefined)
|
||||
assert.strictEqual(header.headerVersion, HEADER_VERSION)
|
||||
@@ -1,6 +1,6 @@
|
||||
const { SECTOR_SIZE } = require('./_constants')
|
||||
import { SECTOR_SIZE } from './_constants'
|
||||
|
||||
module.exports = function computeGeometryForSize(size) {
|
||||
export default function computeGeometryForSize(size) {
|
||||
const totalSectors = Math.min(Math.ceil(size / 512), 65535 * 16 * 255)
|
||||
let sectorsPerTrackCylinder
|
||||
let heads
|
||||
30
packages/vhd-lib/src/_constants.js
Normal file
30
packages/vhd-lib/src/_constants.js
Normal file
@@ -0,0 +1,30 @@
|
||||
export const BLOCK_UNUSED = 0xffffffff
|
||||
|
||||
// This lib has been extracted from the Xen Orchestra project.
|
||||
export const CREATOR_APPLICATION = 'xo '
|
||||
|
||||
// Sizes in bytes.
|
||||
export const FOOTER_SIZE = 512
|
||||
export const HEADER_SIZE = 1024
|
||||
export const SECTOR_SIZE = 512
|
||||
export const DEFAULT_BLOCK_SIZE = 0x00200000 // from the spec
|
||||
|
||||
export const FOOTER_COOKIE = 'conectix'
|
||||
export const HEADER_COOKIE = 'cxsparse'
|
||||
|
||||
export const DISK_TYPE_FIXED = 2
|
||||
export const DISK_TYPE_DYNAMIC = 3
|
||||
export const DISK_TYPE_DIFFERENCING = 4
|
||||
|
||||
export const PARENT_LOCATOR_ENTRIES = 8
|
||||
|
||||
export const PLATFORM_NONE = 0
|
||||
export const PLATFORM_WI2R = 0x57693272
|
||||
export const PLATFORM_WI2K = 0x5769326b
|
||||
export const PLATFORM_W2RU = 0x57327275
|
||||
export const PLATFORM_W2KU = 0x57326b75
|
||||
export const PLATFORM_MAC = 0x4d616320
|
||||
export const PLATFORM_MACX = 0x4d616358
|
||||
|
||||
export const FILE_FORMAT_VERSION = 1 << 16
|
||||
export const HEADER_VERSION = 1 << 16
|
||||
@@ -1,5 +1,5 @@
|
||||
/* eslint-env jest */
|
||||
const { createFooter } = require('./_createFooterHeader')
|
||||
import { createFooter } from './_createFooterHeader'
|
||||
|
||||
test('createFooter() does not crash', () => {
|
||||
createFooter(104448, Math.floor(Date.now() / 1000), {
|
||||
@@ -1,20 +1,20 @@
|
||||
const { v4: generateUuid } = require('uuid')
|
||||
import { v4 as generateUuid } from 'uuid'
|
||||
|
||||
const { checksumStruct, fuFooter, fuHeader } = require('./_structs')
|
||||
const {
|
||||
import { checksumStruct, fuFooter, fuHeader } from './_structs'
|
||||
import {
|
||||
CREATOR_APPLICATION,
|
||||
DEFAULT_BLOCK_SIZE: VHD_BLOCK_SIZE_BYTES,
|
||||
DISK_TYPES,
|
||||
DEFAULT_BLOCK_SIZE as VHD_BLOCK_SIZE_BYTES,
|
||||
DISK_TYPE_FIXED,
|
||||
FILE_FORMAT_VERSION,
|
||||
FOOTER_COOKIE,
|
||||
FOOTER_SIZE,
|
||||
HEADER_COOKIE,
|
||||
HEADER_SIZE,
|
||||
HEADER_VERSION,
|
||||
PLATFORMS,
|
||||
} = require('./_constants')
|
||||
PLATFORM_WI2K,
|
||||
} from './_constants'
|
||||
|
||||
exports.createFooter = function createFooter(size, timestamp, geometry, dataOffset, diskType = DISK_TYPES.FIXED) {
|
||||
export function createFooter(size, timestamp, geometry, dataOffset, diskType = DISK_TYPE_FIXED) {
|
||||
const footer = fuFooter.pack({
|
||||
cookie: FOOTER_COOKIE,
|
||||
features: 2,
|
||||
@@ -22,7 +22,7 @@ exports.createFooter = function createFooter(size, timestamp, geometry, dataOffs
|
||||
dataOffset,
|
||||
timestamp,
|
||||
creatorApplication: CREATOR_APPLICATION,
|
||||
creatorHostOs: PLATFORMS.WI2K, // it looks like everybody is using Wi2k
|
||||
creatorHostOs: PLATFORM_WI2K, // it looks like everybody is using Wi2k
|
||||
originalSize: size,
|
||||
currentSize: size,
|
||||
diskGeometry: geometry,
|
||||
@@ -33,7 +33,7 @@ exports.createFooter = function createFooter(size, timestamp, geometry, dataOffs
|
||||
return footer
|
||||
}
|
||||
|
||||
exports.createHeader = function createHeader(
|
||||
export function createHeader(
|
||||
maxTableEntries,
|
||||
tableOffset = HEADER_SIZE + FOOTER_SIZE,
|
||||
blockSize = VHD_BLOCK_SIZE_BYTES
|
||||
@@ -1,10 +1,10 @@
|
||||
const assert = require('assert')
|
||||
import assert from 'assert'
|
||||
|
||||
const { BLOCK_UNUSED } = require('./_constants')
|
||||
import { BLOCK_UNUSED } from './_constants'
|
||||
|
||||
// get the identifiers and first sectors of the first and last block
|
||||
// in the file
|
||||
module.exports = bat => {
|
||||
export default bat => {
|
||||
const n = bat.length
|
||||
if (n === 0) {
|
||||
return
|
||||
1
packages/vhd-lib/src/_noop.js
Normal file
1
packages/vhd-lib/src/_noop.js
Normal file
@@ -0,0 +1 @@
|
||||
export default Function.prototype
|
||||
66
packages/vhd-lib/src/_resolveAlias.integ.spec.js
Normal file
66
packages/vhd-lib/src/_resolveAlias.integ.spec.js
Normal file
@@ -0,0 +1,66 @@
|
||||
/* eslint-env jest */
|
||||
|
||||
import rimraf from 'rimraf'
|
||||
import tmp from 'tmp'
|
||||
import { getSyncedHandler } from '@xen-orchestra/fs'
|
||||
import { Disposable, pFromCallback } from 'promise-toolbox'
|
||||
|
||||
import { isVhdAlias, resolveAlias } from './_resolveAlias'
|
||||
|
||||
let tempDir
|
||||
|
||||
jest.setTimeout(60000)
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await pFromCallback(cb => tmp.dir(cb))
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await pFromCallback(cb => rimraf(tempDir, cb))
|
||||
})
|
||||
|
||||
test('is vhd alias recognize only *.alias.vhd files', () => {
|
||||
expect(isVhdAlias('filename.alias.vhd')).toEqual(true)
|
||||
expect(isVhdAlias('alias.vhd')).toEqual(false)
|
||||
expect(isVhdAlias('filename.vhd')).toEqual(false)
|
||||
expect(isVhdAlias('filename.alias.vhd.other')).toEqual(false)
|
||||
})
|
||||
|
||||
test('resolve return the path in argument for a non alias file ', async () => {
|
||||
expect(await resolveAlias(null, 'filename.vhd')).toEqual('filename.vhd')
|
||||
})
|
||||
test('resolve get the path of the target file for an alias', async () => {
|
||||
await Disposable.use(async function* () {
|
||||
// same directory
|
||||
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
|
||||
await handler.mkdir(`alias`)
|
||||
const aliasPath = 'alias/alias.alias.vhd'
|
||||
const testOneCombination = async ({ targetPath, targetContent }) => {
|
||||
await handler.writeFile(aliasPath, targetPath, { flags: 'w' })
|
||||
const resolved = await resolveAlias(handler, aliasPath)
|
||||
expect(resolved).toEqual(targetContent)
|
||||
await handler.unlink(aliasPath)
|
||||
}
|
||||
// the alias contain the relative path to the file. The resolved values is the full path from the root of the remote
|
||||
const combinations = [
|
||||
{ targetPath: `../targets.vhd`, targetContent: `targets.vhd` },
|
||||
{ targetPath: `targets.vhd`, targetContent: `alias/targets.vhd` },
|
||||
{ targetPath: `sub/targets.vhd`, targetContent: `alias/sub/targets.vhd` },
|
||||
{ targetPath: `../sibling/targets.vhd`, targetContent: `sibling/targets.vhd` },
|
||||
]
|
||||
|
||||
for (const { targetPath, targetContent } of combinations) {
|
||||
await testOneCombination({ targetPath, targetContent })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
test('resolve throws an error an alias to an alias', async () => {
|
||||
await Disposable.use(async function* () {
|
||||
const handler = yield getSyncedHandler({ url: `file://${tempDir}` })
|
||||
const alias = 'alias.alias.vhd'
|
||||
const target = 'target.alias.vhd'
|
||||
await handler.writeFile(alias, target)
|
||||
expect(async () => await resolveAlias(handler, alias)).rejects.toThrow(Error)
|
||||
})
|
||||
})
|
||||
18
packages/vhd-lib/src/_resolveAlias.js
Normal file
18
packages/vhd-lib/src/_resolveAlias.js
Normal file
@@ -0,0 +1,18 @@
|
||||
import resolveRelativeFromFile from './_resolveRelativeFromFile'
|
||||
|
||||
export function isVhdAlias(filename) {
|
||||
return filename.endsWith('.alias.vhd')
|
||||
}
|
||||
|
||||
export async function resolveAlias(handler, filename) {
|
||||
if (!isVhdAlias(filename)) {
|
||||
return filename
|
||||
}
|
||||
const aliasContent = (await handler.readFile(filename)).toString().trim()
|
||||
// also handle circular references and unreasonnably long chains
|
||||
if (isVhdAlias(aliasContent)) {
|
||||
throw new Error(`Chaining alias is forbidden ${filename} to ${aliasContent}`)
|
||||
}
|
||||
// the target is relative to the alias location
|
||||
return resolveRelativeFromFile(filename, aliasContent)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user