Compare commits

..

2 Commits

Author SHA1 Message Date
Florent BEAUCHAMP
6fdf0a97e4 Simplify @xen-orchestra/backups/_runners/_writers/IncrementalRemoteWriter.mjs
Co-authored-by: Julien Fontanet <julien.fontanet@isonoe.net>
2024-02-14 07:46:39 +01:00
Florent Beauchamp
3a4bdd3b1e fix(backups): ensure vhd are chained with older vhd 2024-02-13 12:31:22 +00:00
25 changed files with 91 additions and 256 deletions

View File

@@ -65,11 +65,10 @@ module.exports = {
typescript: true,
'eslint-import-resolver-custom-alias': {
alias: {
'@core': '../web-core/lib',
'@': './src',
},
extensions: ['.ts'],
packages: ['@xen-orchestra/lite', '@xen-orchestra/web'],
packages: ['@xen-orchestra/lite'],
},
},
},

View File

@@ -160,10 +160,10 @@ export class ImportVmBackup {
// update the stream with the negative vhd stream
stream = await negativeVhd.stream()
vdis[vdiRef].baseVdi = snapshotCandidate
} catch (error) {
} catch (err) {
// can be a broken VHD chain, a vhd chain with a key backup, ....
// not an irrecuperable error, don't dispose parentVhd, and fallback to full restore
warn(`can't use differential restore`, { error })
warn(`can't use differential restore`, err)
disposableDescendants?.dispose()
}
}

View File

@@ -7,7 +7,7 @@ import { chainVhd, checkVhdChain, openVhd, VhdAbstract } from 'vhd-lib'
import { createLogger } from '@xen-orchestra/log'
import { decorateClass } from '@vates/decorate-with'
import { defer } from 'golike-defer'
import { dirname } from 'node:path'
import { dirname, basename as pathBasename } from 'node:path'
import { formatFilenameDate } from '../../_filenameDate.mjs'
import { getOldEntries } from '../../_getOldEntries.mjs'
@@ -143,10 +143,8 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
let metadataContent = await this._isAlreadyTransferred(timestamp)
if (metadataContent !== undefined) {
// skip backup while being vigilant to not stuck the forked stream
// @todo : should skip backup while being vigilant to not stuck the forked stream
Task.info('This backup has already been transfered')
Object.values(deltaExport.streams).forEach(stream => stream.destroy())
return { size: 0 }
}
const basename = formatFilenameDate(timestamp)
@@ -201,7 +199,10 @@ export class IncrementalRemoteWriter extends MixinRemoteWriter(AbstractIncrement
undefined,
`missing parent of ${id} in ${dirname(path)}, looking for ${vdi.other_config[TAG_BASE_DELTA]}`
)
assert.ok(
pathBasename(parentPath) < pathBasename(path),
`vhd must be sorted to be chained`
)
parentPath = parentPath.slice(1) // remove leading slash
// TODO remove when this has been done before the export

View File

@@ -113,13 +113,13 @@ export const MixinRemoteWriter = (BaseClass = Object) =>
)
}
async _isAlreadyTransferred(timestamp) {
_isAlreadyTransferred(timestamp) {
const vmUuid = this._vmUuid
const adapter = this._adapter
const backupDir = getVmBackupDir(vmUuid)
try {
const actualMetadata = JSON.parse(
await adapter._handler.readFile(`${backupDir}/${formatFilenameDate(timestamp)}.json`)
adapter._handler.readFile(`${backupDir}/${formatFilenameDate(timestamp)}.json`)
)
return actualMetadata
} catch (error) {}

View File

@@ -20,7 +20,5 @@ export function split(path) {
return parts
}
// paths are made absolute otherwise fs.relative() would resolve them against working directory
export const relativeFromFile = (file, path) => relative(dirname(normalize(file)), normalize(path))
export const relativeFromFile = (file, path) => relative(dirname(file), path)
export const resolveFromFile = (file, path) => resolve('/', dirname(file), path).slice(1)

View File

@@ -1,17 +0,0 @@
import { describe, it } from 'test'
import { strict as assert } from 'assert'
import { relativeFromFile } from './path.js'
describe('relativeFromFile()', function () {
for (const [title, args] of Object.entries({
'file absolute and path absolute': ['/foo/bar/file.vhd', '/foo/baz/path.vhd'],
'file relative and path absolute': ['foo/bar/file.vhd', '/foo/baz/path.vhd'],
'file absolute and path relative': ['/foo/bar/file.vhd', 'foo/baz/path.vhd'],
'file relative and path relative': ['foo/bar/file.vhd', 'foo/baz/path.vhd'],
})) {
it('works with ' + title, function () {
assert.equal(relativeFromFile(...args), '../baz/path.vhd')
})
}
})

View File

@@ -54,10 +54,10 @@ async function handleExistingFile(root, indexPath, path) {
await indexFile(fullPath, indexPath)
}
}
} catch (error) {
if (error.code !== 'EEXIST') {
} catch (err) {
if (err.code !== 'EEXIST') {
// there can be a symbolic link in the tree
warn('handleExistingFile', { error })
warn('handleExistingFile', err)
}
}
}
@@ -106,7 +106,7 @@ export async function watchRemote(remoteId, { root, immutabilityDuration, rebuil
await File.liftImmutability(settingPath)
} catch (error) {
// file may not exists, and it's not really a problem
info('lifting immutability on current settings', { error })
info('lifting immutability on current settings', error)
}
await fs.writeFile(
settingPath,

View File

@@ -1,15 +1,13 @@
{
"extends": "@vue/tsconfig/tsconfig.dom.json",
"include": ["env.d.ts", "src/**/*", "src/**/*.vue", "../web-core/lib/**/*", "../web-core/lib/**/*.vue"],
"include": ["env.d.ts", "src/**/*", "src/**/*.vue"],
"exclude": ["src/**/__tests__/*"],
"compilerOptions": {
"composite": true,
"noEmit": true,
"baseUrl": ".",
"rootDir": "..",
"paths": {
"@/*": ["./src/*"],
"@core/*": ["../web-core/lib/*"]
"@/*": ["./src/*"]
}
}
}

View File

@@ -23,7 +23,6 @@ export default defineConfig({
resolve: {
alias: {
'@': fileURLToPath(new URL('./src', import.meta.url)),
'@core': fileURLToPath(new URL('../web-core/lib', import.meta.url)),
},
},

View File

@@ -27,16 +27,6 @@ log.error('could not join server', {
})
```
A logging method has the following signature:
```ts
interface LoggingMethod {
(error): void
(message: string, data?: { error?: Error; [property: string]: any }): void
}
```
### Consumer
Then, at application level, configure the logs are handled:

View File

@@ -45,16 +45,6 @@ log.error('could not join server', {
})
```
A logging method has the following signature:
```ts
interface LoggingMethod {
(error): void
(message: string, data?: { error?: Error; [property: string]: any }): void
}
```
### Consumer
Then, at application level, configure the logs are handled:

View File

@@ -10,8 +10,7 @@
}
},
"devDependencies": {
"vue": "^3.4.13",
"@vue/tsconfig": "^0.5.1"
"vue": "^3.4.13"
},
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/web-core",
"bugs": "https://github.com/vatesfr/xen-orchestra/issues",
@@ -26,6 +25,6 @@
},
"license": "AGPL-3.0-or-later",
"engines": {
"node": ">=18"
"node": ">=8.10"
}
}

View File

@@ -1,12 +0,0 @@
{
"extends": "@vue/tsconfig/tsconfig.dom.json",
"include": ["env.d.ts", "lib/**/*", "lib/**/*.vue"],
"exclude": ["lib/**/__tests__/*"],
"compilerOptions": {
"noEmit": true,
"baseUrl": ".",
"paths": {
"@core/*": ["./lib/*"]
}
}
}

View File

@@ -1,22 +1,13 @@
{
"extends": "@vue/tsconfig/tsconfig.dom.json",
"include": [
"env.d.ts",
"typed-router.d.ts",
"src/**/*",
"src/**/*.vue",
"../web-core/lib/**/*",
"../web-core/lib/**/*.vue"
],
"include": ["env.d.ts", "typed-router.d.ts", "src/**/*", "src/**/*.vue"],
"exclude": ["src/**/__tests__/*"],
"compilerOptions": {
"composite": true,
"noEmit": true,
"baseUrl": ".",
"rootDir": "..",
"paths": {
"@/*": ["./src/*"],
"@core/*": ["../web-core/lib/*"]
"@/*": ["./src/*"]
}
}
}

View File

@@ -11,7 +11,6 @@ export default defineConfig({
resolve: {
alias: {
'@': fileURLToPath(new URL('./src', import.meta.url)),
'@core': fileURLToPath(new URL('../web-core/lib', import.meta.url)),
},
},
})

View File

@@ -21,23 +21,12 @@ export default class Vif {
MAC = '',
} = {}
) {
if (device === undefined) {
const allowedDevices = await this.call('VM.get_allowed_VIF_devices', VM)
if (allowedDevices.length === 0) {
const error = new Error('could not find an allowed VIF device')
error.poolUuid = this.pool.uuid
error.vmRef = VM
throw error
}
device = allowedDevices[0]
}
const [powerState, ...rest] = await Promise.all([
this.getField('VM', VM, 'power_state'),
MTU ?? this.getField('network', network, 'MTU'),
device ?? (await this.call('VM.get_allowed_VIF_devices', VM))[0],
MTU ?? (await this.getField('network', network, 'MTU')),
])
;[MTU] = rest
;[device, MTU] = rest
const vifRef = await this.call('VIF.create', {
currently_attached: powerState === 'Suspended' ? currently_attached : undefined,

View File

@@ -8,8 +8,6 @@
> Users must be able to say: “Nice enhancement, I'm eager to test it”
- Disable search engine indexing via a `robots.txt`
- [Stats] Support format used by XAPI 23.31
- [Storage/Disks] Handle link to VM for suspended VDIs (PR [#7391](https://github.com/vatesfr/xen-orchestra/pull/7391))
### Bug fixes
@@ -36,11 +34,8 @@
<!--packages-start-->
- @xen-orchestra/backups patch
- @xen-orchestra/fs patch
- @xen-orchestra/xapi patch
- vhd-lib patch
- xo-server minor
- xo-server patch
- xo-server-audit patch
- xo-web minor
- xo-web patch
<!--packages-end-->

View File

@@ -93,21 +93,6 @@ Follow the instructions:
You can also download XOA from xen-orchestra.com in an XVA file. Once you've got the XVA file, you can import it with `xe vm-import filename=xoa_unified.xva` or via XenCenter.
If you want to use static IP address for your appliance:
```sh
xe vm-param-set uuid="$uuid" \
xenstore-data:vm-data/ip="$ip" \
xenstore-data:vm-data/netmask="$netmask" \
xenstore-data:vm-data/gateway="$gateway"
```
If you want to replace the default DNS server:
```sh
xe vm-param-set uuid="$uuid" xenstore-data:vm-data/dns="$dns"
```
After the VM is imported, you just need to start it with `xe vm-start vm="XOA"` or with XenCenter.
## First console connection

View File

@@ -1,6 +1,6 @@
'use strict'
const { relativeFromFile } = require('@xen-orchestra/fs/path')
const { dirname, relative } = require('path')
const { openVhd } = require('./openVhd')
const { DISK_TYPES } = require('./_constants')
@@ -21,7 +21,7 @@ module.exports = async function chain(parentHandler, parentPath, childHandler, c
}
await childVhd.readBlockAllocationTable()
const parentName = relativeFromFile(childPath, parentPath)
const parentName = relative(dirname(childPath), parentPath)
header.parentUuid = parentVhd.footer.uuid
header.parentUnicodeName = parentName
await childVhd.setUniqueParentLocator(parentName)

View File

@@ -27,7 +27,7 @@ async function sendToNagios(app, jobName, vmBackupInfo) {
jobName
)
} catch (error) {
warn('sendToNagios:', { error })
warn('sendToNagios:', error)
}
}

View File

@@ -413,7 +413,6 @@ const TRANSFORMS = {
startTime: metrics && toTimestamp(metrics.start_time),
secureBoot: obj.platform.secureboot === 'true',
suspendSr: link(obj, 'suspend_SR'),
suspendVdi: link(obj, 'suspend_VDI'),
tags: obj.tags,
VIFs: link(obj, 'VIFs'),
VTPMs: link(obj, 'VTPMs'),
@@ -450,6 +449,7 @@ const TRANSFORMS = {
vm.snapshot_time = toTimestamp(obj.snapshot_time)
vm.$snapshot_of = link(obj, 'snapshot_of')
vm.suspendVdi = link(obj, 'suspend_VDI')
} else if (obj.is_a_template) {
const defaultTemplate = isDefaultTemplate(obj)
vm.type += '-template'

View File

@@ -45,17 +45,7 @@ const RRD_POINTS_PER_STEP = {
// Utils
// -------------------------------------------------------------------
function parseNumber(value) {
// Starting from XAPI 23.31, numbers in the JSON payload are encoded as
// strings to support NaN, Infinity and -Infinity
if (typeof value === 'string') {
const asNumber = +value
if (isNaN(asNumber) && value !== 'NaN') {
throw new Error('cannot parse number: ' + value)
}
value = asNumber
}
function convertNanToNull(value) {
return isNaN(value) ? null : value
}
@@ -68,7 +58,7 @@ async function getServerTimestamp(xapi, hostRef) {
// -------------------------------------------------------------------
const computeValues = (dataRow, legendIndex, transformValue = identity) =>
map(dataRow, ({ values }) => transformValue(parseNumber(values[legendIndex])))
map(dataRow, ({ values }) => transformValue(convertNanToNull(values[legendIndex])))
const combineStats = (stats, path, combineValues) => zipWith(...map(stats, path), (...values) => combineValues(values))
@@ -255,15 +245,7 @@ export default class XapiStats {
start: timestamp,
},
})
.then(response => response.text())
.then(data => {
try {
// starting from XAPI 23.31, the response is valid JSON
return JSON.parse(data)
} catch (_) {
return JSON5.parse(data)
}
})
.then(response => response.text().then(JSON5.parse))
.catch(err => {
delete this.#hostCache[hostUuid][step]
throw err
@@ -317,7 +299,7 @@ export default class XapiStats {
// To avoid crossing over the boundary, we ask for one less step
const optimumTimestamp = currentTimeStamp - maxDuration + step
const json = await this._getJson(xapi, host, optimumTimestamp, step)
const actualStep = parseNumber(json.meta.step)
const actualStep = json.meta.step
if (actualStep !== step) {
throw new FaultyGranularity(`Unable to get the true granularity: ${actualStep}`)
@@ -344,10 +326,9 @@ export default class XapiStats {
return
}
const endTimestamp = parseNumber(json.meta.end)
if (stepStats === undefined || stepStats.endTimestamp !== endTimestamp) {
if (stepStats === undefined || stepStats.endTimestamp !== json.meta.end) {
stepStats = {
endTimestamp,
endTimestamp: json.meta.end,
interval: actualStep,
stats: {},
}

View File

@@ -1,6 +1,5 @@
import asyncMapSettled from '@xen-orchestra/async-map/legacy.js'
import { basename } from 'path'
import { createLogger } from '@xen-orchestra/log'
import { format, parse } from 'xo-remote-parser'
import {
DEFAULT_ENCRYPTION_ALGORITHM,
@@ -18,35 +17,17 @@ import { Remotes } from '../models/remote.mjs'
// ===================================================================
const { warn } = createLogger('xo:mixins:remotes')
const obfuscateRemote = ({ url, ...remote }) => {
const parsedUrl = parse(url)
remote.url = format(sensitiveValues.obfuscate(parsedUrl))
return remote
}
// these properties should be defined on the remote object itself and not as
// part of the remote URL
//
// there is a bug somewhere that keep putting them into the URL, this list
// is here to help track it
const INVALID_URL_PARAMS = ['benchmarks', 'id', 'info', 'name', 'proxy', 'enabled', 'error', 'url']
function validateUrl(url) {
const parsedUrl = parse(url)
const { path } = parsedUrl
function validatePath(url) {
const { path } = parse(url)
if (path !== undefined && basename(path) === 'xo-vm-backups') {
throw invalidParameters('remote url should not end with xo-vm-backups')
}
for (const param of INVALID_URL_PARAMS) {
if (Object.hasOwn(parsedUrl, param)) {
// log with stack trace
warn(new Error('invalid remote URL param ' + param))
}
}
}
export default class {
@@ -201,22 +182,6 @@ export default class {
if (remote === undefined) {
throw noSuchObject(id, 'remote')
}
const parsedUrl = parse(remote.url)
let fixed = false
for (const param of INVALID_URL_PARAMS) {
if (Object.hasOwn(parsedUrl, param)) {
// delete the value to trace its real origin when it's added back
// with `updateRemote()`
delete parsedUrl[param]
fixed = true
}
}
if (fixed) {
remote.url = format(parsedUrl)
this._remotes.update(remote).catch(warn)
}
return remote
}
@@ -237,7 +202,7 @@ export default class {
}
async createRemote({ name, options, proxy, url }) {
validateUrl(url)
validatePath(url)
const params = {
enabled: false,
@@ -254,10 +219,6 @@ export default class {
}
updateRemote(id, { enabled, name, options, proxy, url }) {
if (url !== undefined) {
validateUrl(url)
}
const handlers = this._handlers
const handler = handlers[id]
if (handler !== undefined) {
@@ -277,7 +238,7 @@ export default class {
@synchronized()
async _updateRemote(id, { url, ...props }) {
if (url !== undefined) {
validateUrl(url)
validatePath(url)
}
const remote = await this._getRemote(id)

View File

@@ -75,7 +75,7 @@ export const reportOnSupportPanel = async ({ files = [], formatMessage = identit
ADDITIONAL_FILES.map(({ fetch, name }) =>
timeout.call(fetch(), ADDITIONAL_FILES_FETCH_TIMEOUT).then(
file => formData.append('attachments', createBlobFromString(file), name),
error => logger.warn(`cannot get ${name}`, { error })
error => logger.warn(`cannot get ${name}`, error)
)
)
)

View File

@@ -12,7 +12,7 @@ import PropTypes from 'prop-types'
import React from 'react'
import SortedTable from 'sorted-table'
import TabButton from 'tab-button'
import renderXoItem, { Vdi, Vm } from 'render-xo-item'
import renderXoItem, { Vdi } from 'render-xo-item'
import { confirm } from 'modal'
import { injectIntl } from 'react-intl'
import { Text } from 'editable'
@@ -123,76 +123,67 @@ const COLUMNS = [
vms: getAllVms(state, props),
vbds: getVbds(state, props),
})
})(({ item: vdi, vbds, vms, userData: { vmsSnapshotsBySuspendVdi } }) => {
const vmSnapshot = vmsSnapshotsBySuspendVdi[vdi.uuid]?.[0]
})(({ vbds, vms }) => {
if (isEmpty(vms)) {
return null
}
return (
<Container>
{vbds.length > 0 ? (
map(vbds, (vbd, index) => {
const vm = vms[vbd.VM]
{map(vbds, (vbd, index) => {
const vm = vms[vbd.VM]
if (vm === undefined) {
return null
}
if (vm === undefined) {
return null
}
const type = vm.type
let link
if (type === 'VM') {
link = `/vms/${vm.id}`
} else if (type === 'VM-template') {
link = `/home?s=${vm.id}&t=VM-template`
} else {
link = vm.$snapshot_of === undefined ? '/dashboard/health' : `/vms/${vm.$snapshot_of}/snapshots`
}
const type = vm.type
let link
if (type === 'VM') {
link = `/vms/${vm.id}`
} else if (type === 'VM-template') {
link = `/home?s=${vm.id}&t=VM-template`
} else {
link = vm.$snapshot_of === undefined ? '/dashboard/health' : `/vms/${vm.$snapshot_of}/snapshots`
}
return (
<Row className={index > 0 && 'mt-1'}>
<Col mediumSize={8}>
<Link to={link}>{renderXoItem(vm)}</Link>
</Col>
<Col mediumSize={4}>
<ButtonGroup>
{vbd.attached ? (
<ActionRowButton
btnStyle='danger'
handler={disconnectVbd}
handlerParam={vbd}
icon='disconnect'
tooltip={_('vbdDisconnect')}
/>
) : (
<ActionRowButton
btnStyle='primary'
disabled={some(vbds, 'attached') || !isVmRunning(vm)}
handler={connectVbd}
handlerParam={vbd}
icon='connect'
tooltip={_('vbdConnect')}
/>
)}
return (
<Row className={index > 0 && 'mt-1'}>
<Col mediumSize={8}>
<Link to={link}>{renderXoItem(vm)}</Link>
</Col>
<Col mediumSize={4}>
<ButtonGroup>
{vbd.attached ? (
<ActionRowButton
btnStyle='danger'
handler={deleteVbd}
handler={disconnectVbd}
handlerParam={vbd}
icon='vdi-forget'
tooltip={_('vdiForget')}
icon='disconnect'
tooltip={_('vbdDisconnect')}
/>
</ButtonGroup>
</Col>
</Row>
)
})
) : (
<Col mediumSize={8}>
<Link to={`/vms/${vmSnapshot.$snapshot_of}/snapshots`}>
<Vm id={vmSnapshot.$snapshot_of} />
</Link>
</Col>
)}
) : (
<ActionRowButton
btnStyle='primary'
disabled={some(vbds, 'attached') || !isVmRunning(vm)}
handler={connectVbd}
handlerParam={vbd}
icon='connect'
tooltip={_('vbdConnect')}
/>
)}
<ActionRowButton
btnStyle='danger'
handler={deleteVbd}
handlerParam={vbd}
icon='vdi-forget'
tooltip={_('vdiForget')}
/>
</ButtonGroup>
</Col>
</Row>
)
})}
</Container>
)
}),
@@ -313,7 +304,6 @@ class NewDisk extends Component {
@connectStore(() => ({
checkPermissions: getCheckPermissions,
vbds: createGetObjectsOfType('VBD'),
vmsSnapshotsBySuspendVdi: createGetObjectsOfType('VM-snapshot').groupBy('suspendVdi'),
}))
export default class SrDisks extends Component {
_closeNewDiskForm = () => this.setState({ newDisk: false })
@@ -444,7 +434,6 @@ export default class SrDisks extends Component {
columns={COLUMNS}
data-isVdiAttached={this._getIsVdiAttached()}
data-vdisByBaseCopy={this._getVdisByBaseCopy()}
data-vmsSnapshotsBySuspendVdi={this.props.vmsSnapshotsBySuspendVdi}
defaultFilter='filterOnlyManaged'
filters={FILTERS}
groupedActions={GROUPED_ACTIONS}