Compare commits

..

10 Commits

Author SHA1 Message Date
Florent Beauchamp
26c9338f54 feat: data_destroy 2024-02-21 16:54:46 +00:00
Florent Beauchamp
4ae0e3912f fixes : backup runs 2024-02-21 15:45:43 +00:00
Florent Beauchamp
c3571325c5 feat(backups): use CBT if selected 2024-02-21 14:54:41 +00:00
Florent Beauchamp
9d56ab2a00 feat:add ux 2024-02-21 14:54:41 +00:00
Florent Beauchamp
b3e163d090 remove broken import 2024-02-21 14:54:41 +00:00
Florent Beauchamp
1e0c411d5f feat(backups): destroy data of cbt enabled snapshots 2024-02-21 14:54:41 +00:00
Florent Beauchamp
d30b5950fc feat(backup): use cbt in exports incremental vm 2024-02-21 14:54:41 +00:00
Florent Beauchamp
98ec3f4c5e feat(xapi,vhd-lib): implement cbt for reading changed data 2024-02-21 14:54:41 +00:00
Florent Beauchamp
ff57bc2a0b feat(xapi): implement Change Block Tracking function 2024-02-21 14:54:39 +00:00
Florent Beauchamp
ee0fd9ab8e feat(nbd-client): implement buffer passthrough in read block 2024-02-21 14:54:12 +00:00
24 changed files with 156 additions and 1031 deletions

View File

@@ -61,22 +61,23 @@ export default class MultiNbdClient {
async *readBlocks(indexGenerator) {
// default : read all blocks
const readAhead = []
const makeReadBlockPromise = (index, size) => {
const promise = this.readBlock(index, size)
const makeReadBlockPromise = (index, buffer, size) => {
// pass through any pre loaded buffer
const promise = buffer ? Promise.resolve(buffer) : this.readBlock(index, size)
// error is handled during unshift
promise.catch(() => {})
return promise
}
// read all blocks, but try to keep readAheadMaxLength promise waiting ahead
for (const { index, size } of indexGenerator()) {
for (const { index, buffer, size } of indexGenerator()) {
// stack readAheadMaxLength promises before starting to handle the results
if (readAhead.length === this.#readAhead) {
// any error will stop reading blocks
yield readAhead.shift()
}
readAhead.push(makeReadBlockPromise(index, size))
readAhead.push(makeReadBlockPromise(index, buffer, size))
}
while (readAhead.length > 0) {
yield readAhead.shift()

View File

@@ -79,9 +79,16 @@ export async function exportIncrementalVm(
$SR$uuid: vdi.$SR.uuid,
}
let changedBlocks
console.log('CBT ? ', vdi.cbt_enabled,vdiRef,baseVdi?.$ref)
if (vdi.cbt_enabled && baseVdi?.$ref) {
// @todo log errors and fallback to default mode
changedBlocks = await vdi.$listChangedBlock(baseVdi?.$ref)
}
streams[`${vdiRef}.vhd`] = await vdi.$exportContent({
baseRef: baseVdi?.$ref,
cancelToken,
changedBlocks,
format: 'vhd',
nbdConcurrency,
preferNbd,

View File

@@ -193,6 +193,17 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
const allSettings = this.job.settings
const baseSettings = this._baseSettings
const baseVmRef = this._baseVm?.$ref
if (this._settings.deltaComputeMode === 'CBT' && this._exportedVm?.$ref && this._exportedVm?.$ref != this._vm.$ref) {
console.log('WILL PURGE',this._exportedVm?.$ref)
const xapi = this._xapi
const vdiRefs = await this._xapi.VM_getDisks(this._exportedVm?.$ref)
await xapi.call('VM.destroy',this._exportedVm.$ref)
// @todo: ensure it is really the snapshot
for (const vdiRef of vdiRefs) {
// @todo handle error
await xapi.VDI_dataDestroy(vdiRef)
}
}
const snapshotsPerSchedule = groupBy(this._jobSnapshots, _ => _.other_config['xo:backup:schedule'])
const xapi = this._xapi
@@ -208,6 +219,8 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
}
})
})
}
async copy() {
@@ -226,6 +239,22 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
throw new Error('Not implemented')
}
async enableCbt() {
// for each disk of the VM , enable CBT
if (this._settings.deltaComputeMode !== 'CBT') {
return
}
const vm = this._vm
const xapi = this._xapi
console.log(vm.VBDs)
const vdiRefs = await vm.$getDisks(vm.VBDs)
for (const vdiRef of vdiRefs) {
// @todo handle error
await xapi.VDI_enableChangeBlockTracking(vdiRef)
}
// @todo : when do we disable CBT ?
}
async run($defer) {
const settings = this._settings
assert(
@@ -246,7 +275,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
await this._cleanMetadata()
await this._removeUnusedSnapshots()
await this.enableCbt()
const vm = this._vm
const isRunning = vm.power_state === 'Running'
const startAfter = isRunning && (settings.offlineBackup ? 'backup' : settings.offlineSnapshot && 'snapshot')
@@ -267,6 +296,7 @@ export const AbstractXapi = class AbstractXapiVmBackupRunner extends Abstract {
await this._exportedVm.update_blocked_operations({ pool_migrate: reason, migrate_send: reason })
try {
await this._copy()
// @todo if CBT is enabled : should call vdi.datadestroy on snapshot here
} finally {
await this._exportedVm.update_blocked_operations({ pool_migrate, migrate_send })
}

View File

@@ -1,61 +0,0 @@
# `useCollectionNavigation`
The `useCollectionNavigation` composable handles the navigation across a collection (i.e., changing the active item).
It is mainly used to navigate between items in a collection with a keyboard.
## Usage
```ts
const definition = defineCollection(/* ... */)
const { items, activeItem } = useCollection(definitions)
const { moveUp, moveDown, moveLeft, moveRight, handleKeydown } = useCollectionNavigation(items, activeItem)
```
## `moveUp`
The `moveUp` function set the `activeItem` to the previous one, if any, or `undefined` otherwise.
If the previous item is a `Group` and it is expanded, the `moveUp` function will set the `activeItem` to the last item
of that group.
## `moveDown`
The `moveDown` function set the `activeItem` to the next one, if any, or `undefined` otherwise.
If the current `activeItem` is a `Group` and it is expanded, the `moveDown` function will set the `activeItem` to the
first item of that group.
## `moveLeft`
If the current `activeItem` is a `Group` and it is expanded, the `moveLeft` function will collapse the group.
In any other case, the `moveLeft` function will set the `activeItem` to the parent `Group`, if any, or will do nothing
otherwise.
## `moveRight`
If the current `activeItem` is a `Group` and it is collapsed, the `moveRight` function will expand the group.
In any other case, the `moveRight` function will act the same as `moveDown`.
## `handleKeydown`
The `handleKeydown` function is a helper function that can be used with the `@keydown` event binding.
```html
<div @keydown="handleKeydown" tabindex="0">...</div>
<!-- Is equivalent to -->
<div
@keydown.left.prevent="moveLeft"
@keydown.right.prevent="moveRight"
@keydown.up.prevent="moveUp"
@keydown.down.prevent="moveDown"
tabindex="0"
>
...
</div>
```

View File

@@ -1,492 +0,0 @@
# `useCollection` composable
The `useCollection` composable handles a collection of items (called `Leaf` and `Group`) in a tree structure.
`Leaf` and `Group` can be _selected_, _activated_, and/or _filtered_.
Additionally, `Group` can be _expanded_ and contains `Leaf` and/or `Group` children.
Multiple items can be selected at the same time (if `allowMultiSelect` is `true`). But only one item can be activated at
a time.
## Usage
The `useCollection` composable takes an array of definitions (called `LeafDefinition` and `GroupDefinition`) as first
argument, and an optional object of options as second argument.
```ts
useCollection(definitions)
useCollection(definitions, options)
```
| | Required | Type | Default | |
| -------------------------- | :------: | --------------------------------------- | ------- | ----------------------------------------------------- |
| `definitions` | ✓ | `(LeafDefinition \| GroupDefinition)[]` | | The definitions of the items in the collection |
| `options.allowMultiSelect` | | `boolean` | `false` | Whether more than one item can be selected at a time. |
| `options.expanded` | | `boolean` | `true` | Whether all groups are initially expanded. |
## `useCollection` return values
| | Type | |
| --------------- | ----------------------------------------- | ------------------------------------------------------------------------- |
| `items` | `(Leaf \| Group)[]` | Array of visible `Leaf` and `Group` instances (See Item Visibility below) |
| `activeItem` | `ComputedRef<Leaf \| Group \| undefined>` | The active item instance |
| `selectedItems` | `ComputedRef<(Leaf \| Group)[]>` | Array of selected item instances |
| `expandedItems` | `ComputedRef<Group[]>` | Array of expanded group instances |
## `LeafDefinition`
```ts
new LeafDefinition(id, data)
new LeafDefinition(id, data, options)
```
| | Required | Type | Default | |
| ----------------------- | :------: | ----------------------------------- | ----------- | -------------------------------------------------------------------------------------- |
| `id` | ✓ | `string` | | unique identifier across the whole collection of leafs and groups |
| `data` | ✓ | `T` | | data to be stored in the item |
| `options.discriminator` | | `string` | `undefined` | discriminator for the item when you mix different data types (see Discriminator below) |
| `options.passesFilter` | | `(data: T) => boolean \| undefined` | `undefined` | filter function (see Filtering below) |
### Example
```ts
const definition = new LeafDefinition('jd-1', { name: 'John Doe', age: 30 })
```
## `GroupDefinition`
A `GroupDefinition` is very similar to a `LeafDefinition`, but it contains a collection of children definitions.
```ts
new GroupDefinition(id, data, children)
new GroupDefinition(id, data, options, children)
```
| | | Type | Default | |
| ----------------------- | --- | --------------------------------------- | ----------- | -------------------------------------------------------------------------------------- |
| `id` | ✓ | `string` | | unique identifier across the whole collection of leafs and groups |
| `data` | ✓ | `any` | | data to be stored in the item |
| `options.discriminator` | | `string` | `undefined` | discriminator for the item when you mix different data types (see Discriminator below) |
| `options.passesFilter` | | `(data) => boolean \| undefined` | `undefined` | filter function (see Filtering below) |
| `children` | ✓ | `(LeafDefinition \| GroupDefinition)[]` | | array of items that are contained in this group |
### Example
```ts
const definition = new GroupDefinition('smithes', { name: 'The Smithes' }, [
new ItemDefinition('jd-1', { name: 'John Smith', age: 30 }),
new ItemDefinition('jd-2', { name: 'Jane Smith', age: 28 }),
])
```
## Discriminator
The `discriminator` is a string used to differentiate between different types of items. This is useful when you want to
mix different types of items at the same collection depth.
### Mixed data without discriminator
```ts
const definitions = [
new LeafDefinition('jd-1', { name: 'John Doe', age: 30 }),
new LeafDefinition('rx-1', { name: 'Rex', breed: 'Golden Retriever' }),
]
const { items } = useCollection(definitions)
items.value.forEach(item => {
// item.data.<cursor> neither 'age' nor 'breed' are available here because we can't know the type of the item
})
```
### Using the discriminator
```ts
const definitions = [
new LeafDefinition('jd-1', { name: 'John Doe', age: 30 }, { discriminator: 'person' }),
new LeafDefinition('rx-1', { name: 'Rex', breed: 'Golden Retriever' }, { discriminator: 'animal' }),
]
const { items } = useCollection(definitions)
items.value.forEach(item => {
if (item.discriminator === 'person') {
// item.data.<cursor> `name` and `age` are available here
} else {
// item.data.<cursor> `name` and `breed` are available here
}
})
```
### Mixing `GroupDefinition` and `LeafDefinition` (of same types each)
If you mix `LeafDefinition` and `GroupDefinition` (of same types each), you don't need to use the discriminator because
the `isGroup` property will serve the same purpose.
```ts
const definitions = [
new LeafDefinition('jd-1', { name: 'John Doe', age: 30 }),
new GroupDefinition('dogs', { name: 'Dogs', legs: 4 }, [
/* ... */
]),
]
const { items } = useCollection(definitions)
items.value.forEach(item => {
if (item.isGroup) {
// item.data.<cursor> `name` and `legs` are available here
} else {
// item.data.<cursor> `name` and `age` are available here
}
})
```
## Filtering
The optional `passesFilter` function is used to filter the item across the collection and can affect its visibility (see
Item Visibility below).
It takes the `data` as first argument and will return:
- `true` if the item explicitly passes the filter
- `false` if the item explicitly doesn't pass the filter
- `undefined` if the filter is ignored
## `defineCollection` helper
The `defineCollection` helper creates a collection of definitions in a more convenient way.
```ts
defineCollection(entries)
defineCollection(entries, options)
defineCollection(entries, getChildren)
defineCollection(entries, options, getChildren)
```
| | Required | Type | Default | |
| ----------------------- | :------: | -------------------------------- | ----------- | ------------------------------------------------------------------------------ |
| `entries` | ✓ | `T[]` | | array of items to be stored in the collection |
| `options.idField` | | `keyof T` | `id` | field to be used as the unique identifier for the items. |
| `options.discriminator` | | `string` | `undefined` | discriminator for the item when you mix different data types |
| `options.passesFilter` | | `(data) => boolean \| undefined` | `undefined` | filter function that takes the data as first argument |
| `getChildren` | ✓ | `(data: T) => Definition[]` | | function that returns an array of definitions that are contained in this group |
Let's take this `families` example:
```ts
const families = [
{
id: 'does',
name: 'The Does',
members: [
{
id: 'jd-1',
name: 'John Doe',
age: 30,
animals: [
{
id: 'jd-1-dog',
name: 'Rex',
},
],
},
{
id: 'jd-2',
name: 'Jane Doe',
age: 28,
animals: [],
},
],
},
{
id: 'smiths',
name: 'The Smiths',
members: [
{
id: 'js-1',
name: 'John Smith',
age: 35,
animals: [
{
id: 'js-1-cat',
name: 'Whiskers',
},
{
id: 'js-1-dog',
name: 'Fido',
},
],
},
{
id: 'js-2',
name: 'Jane Smith',
age: 33,
animals: [
{
id: 'js-2-cat',
name: 'Mittens',
},
],
},
],
},
]
```
You can use the `defineCollection` helper:
```ts
const definitions = defineCollection(families, family =>
defineCollection(family.members, person => defineCollection(person.animals))
)
```
This is the equivalent of the following code:
```ts
const definitions = families.map(
family =>
new GroupDefinition(
family.id,
family,
family.members.map(
person =>
new GroupDefinition(
person.id,
person,
person.animals.map(animal => new ItemDefinition(animal.id, animal))
)
)
)
)
```
## `Leaf` and `Group` instances
`Leaf` and `Group` instances have the following properties:
| | | |
| --------------- | --------------------------- | ----------------------------------------------------------------- |
| `id` | `string` | unique identifier across the whole collection of leafs and groups |
| `isGroup` | `boolean` | `true`for `Group` instances, `false` for `Leaf` instances |
| `discriminator` | `string` \| `undefined` | discriminator for the item when you mix different data types |
| `data` | `T` | data stored in the item |
| `depth` | `number` | depth of the item in the collection |
| `isSelected` | `boolean` | whether the item is selected |
| `isActive` | `boolean` | whether the item is active |
| `isVisible` | `boolean` | whether the item is visible (see Item Visibility below) |
| `activate` | `() => void` | function to activate the item |
| `toggleSelect` | `(force?: boolean) => void` | function to toggle the selection of the item |
| `labelClasses` | `{ [name]: boolean }` | object of classes to be used in the template (see below) |
### `labelClasses`
The `labelClasses` properties are classes to be used in the template `:class`.
For a `Leaf` instance, it contains the following properties:
- `selected`: whether the leaf is selected
- `active`: whether the leaf is active
- `matches`: whether the leaf matches the filter
## `Group` instances
Additionally, `Group` instances have the following properties:
| | | |
| ------------------------------ | --------- | ----------------------------------------------- |
| `isExpanded` | `boolean` | whether the item is expanded |
| `areChildrenFullySelected` | `boolean` | whether all children are selected |
| `areChildrenPartiallySelected` | `boolean` | whether some children are selected |
| `rawChildren` | `Item[]` | array of all children instances |
| `children` | `Item[]` | array of visible children instances (see below) |
### `labelClasses`
For a `Group` instance, it contains the following properties:
- `selected`: whether the group is selected
- `selected-partial`: whether the group is partially selected (i.e., some children are selected)
- `selected-full`: whether the group is fully selected (i.e., all children are selected)
- `expanded`: whether the group is expanded
- `active`: whether the group is active
- `matches`: whether the group matches the filter
## Item Visibility
Here are the rules to determine whether an item is visible or not.
**Note**: Only the first matching rule determines an item's visibility. Subsequent rules are not evaluated.
1. If `passesFilter` returns `true` => _visible_
2. If any of its ancestors `passesFilter` returns `true` => _visible_
3. _(`Group` only)_ If any of its descendants `passesFilter` returns `true` => _visible_
4. If `passesFilter` returns `false` => _**not** visible_
5. If it doesn't have a parent => _visible_
6. If the parent's `isExpanded` is `true` => _visible_
7. If the parent's `isExpanded` is `false` => _**not** visible_
## Example 1: Tree View
```html
<template>
<ul>
<li v-for="family in items" :key="family.id">
<div class="label" @click="family.toggleExpand()">{{ family.isExpanded ? '↓' : '→' }} {{ family.data.name }}</div>
<ul v-if="family.isExpanded" class="persons">
<li v-for="person in family.children" :key="person.id">
<div class="label" @click="person.toggleExpand()">
{{ person.isExpanded ? '↓' : '→' }} {{ person.data.name }} ({{ person.data.age }})
</div>
<ul v-if="person.isExpanded" class="animals">
<li v-for="animal in person.children" :key="animal.id">{{ animal.data.name }}</li>
</ul>
</li>
</ul>
</li>
</ul>
</template>
<script lang="ts" setup>
const definitions = defineCollection(families, ({ members }) =>
defineCollection(members, ({ animals }) => defineCollection(animals))
)
const { items } = useCollection(definitions)
</script>
<style lang="postcss" scoped>
.persons,
.animals {
padding-left: 20px;
}
.animals li {
padding-left: 10px;
}
.label {
cursor: pointer;
}
</style>
```
## Example 2: Multi-select
```html
<template>
<ul>
<li v-for="family in items" :key="family.id">
<div
class="label family"
:class="family.labelClasses"
@mouseenter="family.activate()"
@click="family.toggleChildrenSelect()"
>
{{ family.data.name }}
</div>
<ul class="persons">
<li v-for="person in family.children" :key="person.id">
<div
class="label person"
:class="person.labelClasses"
@mouseenter="person.activate()"
@click="person.toggleSelect()"
>
{{ person.data.name }} ({{ person.data.age }})
</div>
</li>
</ul>
</li>
</ul>
</template>
<script lang="ts" setup>
const definitions = defineCollection(families, ({ members }) => defineCollection(members))
const { items } = useCollection(definitions, { allowMultiSelect: true })
</script>
<style lang="postcss" scoped>
.persons {
padding-left: 20px;
}
.family {
background-color: #eaeaea;
&.selected-full {
background-color: #add8e6;
}
&.active {
filter: brightness(1.1);
}
}
.person {
background-color: #f5f5f5;
&.selected {
background-color: #b5e2f1;
}
&.active {
filter: brightness(1.07);
}
}
</style>
```
### Example 3: Filtering
```html
<template>
<div>
<input v-model="filter" placeholder="Filter" />
</div>
<ul>
<li v-for="family in items" :key="family.id">
<div :class="family.labelClasses">{{ family.data.name }}</div>
<ul class="sub">
<li v-for="person in family.children" :key="person.id">
<div :class="person.labelClasses">{{ person.data.name }} ({{ person.data.age }})</div>
<ul class="sub">
<li v-for="animal in person.children" :key="animal.id">
<div :class="animal.labelClasses">{{ animal.data.name }}</div>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</template>
<script lang="ts" setup>
const filter = ref<string>()
const predicate = ({ name }: { name: string }) => {
const filterValue = filter.value?.trim().toLocaleLowerCase() ?? false
return !filterValue ? undefined : name.toLocaleLowerCase().includes(filterValue)
}
const definitions = defineCollection(families, { predicate }, ({ members }) =>
defineCollection(members, { predicate }, ({ animals }) => defineCollection(animals, { predicate }))
)
const { items } = useCollection(definitions, { expand: false })
</script>
<style lang="postcss" scoped>
.sub {
padding-left: 20px;
}
.matches {
font-weight: bold;
}
</style>
```

View File

@@ -1,73 +0,0 @@
import { Group } from '@core/composables/collection/group'
import type { Item } from '@core/composables/collection/types'
import { computed, type ComputedRef } from 'vue'
export function useCollectionNavigation<TItem extends Item>(
items: ComputedRef<TItem[]>,
activeItem: ComputedRef<TItem | undefined>
) {
const flatItems = computed(() => {
const result = [] as any[]
function add(item: Item) {
result.push(item)
if (item instanceof Group) {
item.children.forEach(child => add(child))
}
}
items.value.forEach(item => add(item))
return result
}) as ComputedRef<TItem[]>
const activeIndex = computed(() => {
const id = activeItem.value?.id
return id === undefined ? -1 : flatItems.value.findIndex(item => item.id === id)
})
const moveDown = () => {
flatItems.value[activeIndex.value === -1 ? 0 : activeIndex.value + 1]?.activate()
}
const moveUp = () => {
flatItems.value[activeIndex.value - 1]?.activate()
}
const moveLeft = () => {
if (activeItem.value instanceof Group && activeItem.value.isExpanded) {
return activeItem.value.toggleExpand(false, true)
}
activeItem.value?.parent?.activate()
}
const moveRight = () => {
if (activeItem.value instanceof Group && !activeItem.value.isExpanded) {
return activeItem.value.toggleExpand(true)
}
moveDown()
}
const handleKeydown = (event: KeyboardEvent) => {
if (['ArrowDown', 'ArrowUp', 'ArrowLeft', 'ArrowRight'].includes(event.key)) {
event.preventDefault()
}
switch (event.key) {
case 'ArrowDown':
return moveDown()
case 'ArrowUp':
return moveUp()
case 'ArrowLeft':
return moveLeft()
case 'ArrowRight':
return moveRight()
}
}
return { moveUp, moveDown, moveLeft, moveRight, handleKeydown }
}

View File

@@ -1,33 +0,0 @@
import { buildCollection } from '@core/composables/collection/build-collection'
import type { CollectionContext, Definition, Item } from '@core/composables/collection/types'
import { computed, type MaybeRefOrGetter, reactive, type Ref, ref, toValue } from 'vue'
export function useCollection<TDefinition extends Definition>(
definitions: MaybeRefOrGetter<TDefinition[]>,
options?: { allowMultiSelect?: boolean; expand?: boolean }
) {
const selected = ref(new Map()) as Ref<Map<string, Item>>
const expanded = ref(new Map()) as Ref<Map<string, Item>>
const active = ref() as Ref<Item | undefined>
const context = reactive({
allowMultiSelect: options?.allowMultiSelect ?? false,
selected,
expanded,
active,
}) as CollectionContext
const rawItems = computed(() => buildCollection(toValue(definitions), context))
const items = computed(() => rawItems.value.filter(item => item.isVisible))
if (options?.expand !== false) {
items.value.forEach(item => item.isGroup && item.toggleExpand(true, true))
}
return {
items,
activeItem: computed(() => context.active),
selectedItems: computed(() => Array.from(context.selected.values())),
expandedItems: computed(() => Array.from(context.expanded.values())),
}
}

View File

@@ -1,69 +0,0 @@
import type { Group } from '@core/composables/collection/group'
import type { CollectionContext, ItemOptions } from '@core/composables/collection/types'
export abstract class Base<T = any, TDiscriminator = any> {
abstract readonly isGroup: boolean
abstract passesFilterDownwards: boolean
abstract isVisible: boolean
abstract labelClasses: Record<string, boolean>
readonly id: string
readonly data: T
readonly depth: number
readonly discriminator: TDiscriminator | undefined
readonly parent: Group | undefined
readonly context: CollectionContext
readonly predicate: undefined | ((data: T) => boolean | undefined)
constructor(
id: string,
data: T,
parent: Group | undefined,
context: CollectionContext,
depth: number,
options?: ItemOptions<T, TDiscriminator>
) {
this.id = id
this.data = data
this.parent = parent
this.context = context
this.depth = depth
this.discriminator = options?.discriminator
this.predicate = options?.predicate
}
get passesFilter() {
return this.predicate?.(this.data)
}
get isSelected() {
return this.context.selected.has(this.id)
}
get isActive() {
return this.context.active?.id === this.id
}
get passesFilterUpwards(): boolean {
return this.passesFilter || (this.parent?.passesFilterUpwards ?? false)
}
activate() {
this.context.active = this
}
toggleSelect(force?: boolean) {
const shouldSelect = force ?? !this.isSelected
if (shouldSelect) {
if (!this.context.allowMultiSelect) {
this.context.selected.clear()
}
this.context.selected.set(this.id, this)
} else {
this.context.selected.delete(this.id)
}
}
}

View File

@@ -1,21 +0,0 @@
import { Group } from '@core/composables/collection/group'
import { GroupDefinition } from '@core/composables/collection/group-definition'
import { Leaf } from '@core/composables/collection/leaf'
import type { CollectionContext, Definition, DefinitionToItem, Item } from '@core/composables/collection/types'
export function buildCollection<TDefinition extends Definition>(
definitions: TDefinition[],
context: CollectionContext
): DefinitionToItem<TDefinition>[] {
function create(definitions: Definition[], parent: Group | undefined, depth: number): Item[] {
return definitions.map(definition =>
definition instanceof GroupDefinition
? new Group(definition.id, definition.data, parent, context, depth, definition.options, thisGroup =>
create(definition.children, thisGroup, depth + 1)
)
: new Leaf(definition.id, definition.data, parent, context, depth, definition.options)
)
}
return create(definitions, undefined, 0) as DefinitionToItem<TDefinition>[]
}

View File

@@ -1,43 +0,0 @@
import { GroupDefinition } from '@core/composables/collection/group-definition'
import { LeafDefinition } from '@core/composables/collection/leaf-definition'
import type { DefineCollectionOptions, Definition } from '@core/composables/collection/types'
// Overload 1: Leaf with no options
export function defineCollection<T, const TDiscriminator>(entries: T[]): LeafDefinition<T, TDiscriminator>[]
// Overload 2: Leaf with options
export function defineCollection<T, const TDiscriminator>(
entries: T[],
options: DefineCollectionOptions<T, TDiscriminator>
): LeafDefinition<T, TDiscriminator>[]
// Overload 3: Group with no options
export function defineCollection<T, TChildDefinition extends Definition, const TDiscriminator>(
entries: T[],
getChildren: (data: T) => TChildDefinition[]
): GroupDefinition<T, TChildDefinition, TDiscriminator>[]
// Overload 4: Group with options
export function defineCollection<T, TChildDefinition extends Definition, const TDiscriminator>(
entries: T[],
options: DefineCollectionOptions<T, TDiscriminator>,
getChildren: (data: T) => TChildDefinition[]
): GroupDefinition<T, TChildDefinition, TDiscriminator>[]
// Implementation
export function defineCollection<T, TChildDefinition extends Definition, const TDiscriminator>(
entries: T[],
optionsOrGetChildren?: DefineCollectionOptions<T, TDiscriminator> | ((data: T) => TChildDefinition[]),
getChildren?: (data: T) => TChildDefinition[]
) {
const options = typeof optionsOrGetChildren === 'function' ? {} : optionsOrGetChildren ?? {}
const getChildrenFn = typeof optionsOrGetChildren === 'function' ? optionsOrGetChildren : getChildren
const { idField = 'id' as keyof T, ...otherOptions } = options
if (getChildrenFn !== undefined) {
return entries.map(data => new GroupDefinition(data[idField] as string, data, otherOptions, getChildrenFn(data)))
}
return entries.map(data => new LeafDefinition(data[idField] as string, data, options))
}

View File

@@ -1,13 +0,0 @@
import type { ItemOptions } from '@core/composables/collection/types'
export abstract class DefinitionBase<T, TDiscriminator> {
id: string
data: T
options: ItemOptions<T, TDiscriminator>
constructor(id: string, data: T, options: ItemOptions<T, TDiscriminator> = {}) {
this.data = data
this.options = options
this.id = id
}
}

View File

@@ -1,23 +0,0 @@
import { DefinitionBase } from '@core/composables/collection/definition-base'
import type { Definition, ItemOptions } from '@core/composables/collection/types'
export class GroupDefinition<
T = any,
TChildDefinition extends Definition = Definition,
const TDiscriminator = any,
> extends DefinitionBase<T, TDiscriminator> {
children: TChildDefinition[]
constructor(id: string, data: T, children: TChildDefinition[])
constructor(id: string, data: T, options: ItemOptions<T, TDiscriminator>, children: TChildDefinition[])
constructor(
id: string,
data: T,
optionsOrChildren: ItemOptions<T, TDiscriminator> | TChildDefinition[],
children?: TChildDefinition[]
) {
super(id, data, Array.isArray(optionsOrChildren) ? {} : optionsOrChildren)
this.children = Array.isArray(optionsOrChildren) ? optionsOrChildren : children!
}
}

View File

@@ -1,106 +0,0 @@
import { Base } from '@core/composables/collection/base'
import type { CollectionContext, Item, ItemOptions } from '@core/composables/collection/types'
export class Group<T = any, TChild extends Item = Item, const TDiscriminator = any> extends Base<T, TDiscriminator> {
readonly isGroup = true
readonly rawChildren: TChild[]
constructor(
id: string,
data: T,
parent: Group | undefined,
context: CollectionContext,
depth: number,
options: ItemOptions<T, TDiscriminator> | undefined,
getChildren: (thisGroup: Group<T, TChild, TDiscriminator>) => TChild[]
) {
super(id, data, parent, context, depth, options)
this.rawChildren = getChildren(this)
}
get children() {
return this.rawChildren.filter(child => child.isVisible)
}
get passesFilterDownwards(): boolean {
return this.passesFilter || this.rawChildren.some(child => child.passesFilterDownwards)
}
get isVisible() {
if (this.passesFilterUpwards || this.passesFilterDownwards) {
return true
}
if (this.passesFilter === false) {
return false
}
return this.parent?.isExpanded ?? true
}
get isExpanded() {
return this.context.expanded.has(this.id) || this.passesFilterDownwards || this.passesFilterUpwards
}
get areChildrenFullySelected(): boolean {
if (!this.context.allowMultiSelect) {
throw new Error('allowMultiSelect must be enabled to use areChildrenFullySelected')
}
return this.rawChildren.every(child => (child.isGroup ? child.areChildrenFullySelected : child.isSelected))
}
get areChildrenPartiallySelected(): boolean {
if (!this.context.allowMultiSelect) {
throw new Error('allowMultiSelect must be enabled to use areChildrenPartiallySelected')
}
if (this.areChildrenFullySelected) {
return false
}
return this.rawChildren.some(child => (child.isGroup ? child.areChildrenPartiallySelected : child.isSelected))
}
get labelClasses() {
return {
active: this.isActive,
selected: this.isSelected,
matches: this.passesFilter === true,
'selected-partial': this.context.allowMultiSelect && this.areChildrenPartiallySelected,
'selected-full': this.context.allowMultiSelect && this.areChildrenFullySelected,
expanded: this.isExpanded,
}
}
toggleExpand(force?: boolean, recursive?: boolean) {
const shouldExpand = force ?? !this.isExpanded
if (shouldExpand) {
this.context.expanded.set(this.id, this)
} else {
this.context.expanded.delete(this.id)
}
const shouldPropagate = recursive ?? !shouldExpand
if (shouldPropagate) {
this.rawChildren.forEach(child => {
if (child.isGroup) {
child.toggleExpand(shouldExpand, recursive)
}
})
}
}
toggleChildrenSelect(force?: boolean) {
if (!this.context.allowMultiSelect) {
throw new Error('allowMultiSelect must be enabled to use toggleChildrenSelect')
}
const shouldSelect = force ?? !this.areChildrenFullySelected
this.rawChildren.forEach(child => {
child instanceof Group ? child.toggleChildrenSelect(shouldSelect) : child.toggleSelect(shouldSelect)
})
}
}

View File

@@ -1,3 +0,0 @@
import { DefinitionBase } from '@core/composables/collection/definition-base'
export class LeafDefinition<T = any, const TDiscriminator = any> extends DefinitionBase<T, TDiscriminator> {}

View File

@@ -1,29 +0,0 @@
import { Base } from '@core/composables/collection/base'
export class Leaf<T = any, const TDiscriminator = any> extends Base<T, TDiscriminator> {
readonly isGroup = false
get passesFilterDownwards(): boolean {
return this.passesFilter ?? false
}
get isVisible() {
if (this.passesFilterUpwards) {
return true
}
if (this.passesFilter === false) {
return false
}
return this.parent?.isExpanded ?? true
}
get labelClasses() {
return {
active: this.isActive,
selected: this.isSelected,
matches: this.passesFilter === true,
}
}
}

View File

@@ -1,32 +0,0 @@
import type { Base } from '@core/composables/collection/base'
import type { Group } from '@core/composables/collection/group'
import type { GroupDefinition } from '@core/composables/collection/group-definition'
import type { Leaf } from '@core/composables/collection/leaf'
import type { LeafDefinition } from '@core/composables/collection/leaf-definition'
export type ItemOptions<T, TDiscriminator> = {
discriminator?: TDiscriminator
predicate?: (data: T) => boolean | undefined
}
export type DefineCollectionOptions<T, TDiscriminator> = ItemOptions<T, TDiscriminator> & {
idField?: keyof T
}
export type Definition = LeafDefinition | GroupDefinition
export type CollectionContext = {
allowMultiSelect: boolean
selected: Map<string, Base>
expanded: Map<string, Base>
active: Base | undefined
}
export type DefinitionToItem<TDefinition> =
TDefinition extends GroupDefinition<infer T, infer TChildDefinition, infer TDiscriminator>
? Group<T, DefinitionToItem<TChildDefinition>, TDiscriminator>
: TDefinition extends LeafDefinition<infer T, infer TDiscriminator>
? Leaf<T, TDiscriminator>
: never
export type Item = Leaf | Group

View File

@@ -83,9 +83,31 @@ class Vdi {
}
}
// return an buffer with 0/1 bit, showing if the 64KB block corresponding
// in the raw vdi has changed
async listChangedBlock(ref, baseRef){
console.log('listchanged blocks', ref, baseRef)
const encoded = await this.call('VDI.list_changed_blocks', baseRef, ref)
console.log({encoded})
const buf = Buffer.from(encoded, 'base64')
console.log({buf})
return buf
}
async enableChangeBlockTracking(ref){
return this.call('VDI.enable_cbt', ref)
}
async disableChangeBlockTracking(ref){
return this.call('VDI.disable_cbt', ref)
}
async dataDestroy(ref){
return this.call('VDI.data_destroy', ref)
}
async exportContent(
ref,
{ baseRef, cancelToken = CancelToken.none, format, nbdConcurrency = 1, preferNbd = this._preferNbd }
{ baseRef, cancelToken = CancelToken.none, changedBlocks, format, nbdConcurrency = 1, preferNbd = this._preferNbd }
) {
const query = {
format,
@@ -114,7 +136,7 @@ class Vdi {
})
if (nbdClient !== undefined && format === VDI_FORMAT_VHD) {
const taskRef = await this.task_create(`Exporting content of VDI ${vdiName} using NBD`)
stream = await createNbdVhdStream(nbdClient, stream)
stream = await createNbdVhdStream(nbdClient, stream, {changedBlocks})
stream.on('progress', progress => this.call('task.set_progress', taskRef, progress))
finished(stream, () => this.task_destroy(taskRef))
}

View File

@@ -42,7 +42,8 @@
- @xen-orchestra/backups patch
- @xen-orchestra/fs patch
- @xen-orchestra/xapi patch
- @xen-orchestra/xapi minor
- @vates/nbd-client minor
- vhd-lib patch
- xo-server minor
- xo-server-audit patch

View File

@@ -14,6 +14,7 @@ const {
const { fuHeader, checksumStruct } = require('./_structs')
const assert = require('node:assert')
const NBD_DEFAULT_BLOCK_SIZE = 64 * 1024
const MAX_DURATION_BETWEEN_PROGRESS_EMIT = 5e3
const MIN_TRESHOLD_PERCENT_BETWEEN_PROGRESS_EMIT = 1
@@ -34,10 +35,42 @@ exports.createNbdRawStream = function createRawStream(nbdClient) {
return stream
}
function batContainsBlock(bat, blockId) {
const entry = bat.readUInt32BE(blockId * 4)
if (entry !== BLOCK_UNUSED) {
return [{ blockId, size: DEFAULT_BLOCK_SIZE }]
}
}
// one 2MB VHD block is in 32 blocks of 64KB
// 32 bits are written in 8 4bytes uint32
const EMPTY_NBD_BUFFER = Buffer.alloc(NBD_DEFAULT_BLOCK_SIZE, 0)
function cbtContainsBlock(cbt, blockId) {
const subBlocks = []
let hasOne = false
for (let i = 0; i < 32; i++) {
const position = blockId * 32 + i
const bitOffset = position & 7 // in byte
const byteIndex = position >> 3 // in buffer
const bit = (cbt[byteIndex] >> bitOffset) & 1
if (bit === 1) {
console.log('CBT contains block', blockId)
console.log({position,bitOffset,byteIndex, cbt:cbt[byteIndex],bit})
subBlocks.push({ blockId: position, size: NBD_DEFAULT_BLOCK_SIZE })
hasOne = true
} else {
// don't read empty blocks
subBlocks.push({ buffer: EMPTY_NBD_BUFFER })
}
}
if (hasOne) {
return subBlocks
}
}
exports.createNbdVhdStream = async function createVhdStream(
nbdClient,
sourceStream,
{
changedBlocks,
maxDurationBetweenProgressEmit = MAX_DURATION_BETWEEN_PROGRESS_EMIT,
minTresholdPercentBetweenProgressEmit = MIN_TRESHOLD_PERCENT_BETWEEN_PROGRESS_EMIT,
} = {}
@@ -51,7 +84,10 @@ exports.createNbdVhdStream = async function createVhdStream(
await skipStrict(sourceStream, header.tableOffset - (FOOTER_SIZE + HEADER_SIZE))
// new table offset
header.tableOffset = FOOTER_SIZE + HEADER_SIZE
const streamBat = await readChunkStrict(sourceStream, batSize)
let streamBat
if (changedBlocks === undefined) {
streamBat = await readChunkStrict(sourceStream, batSize)
}
let offset = FOOTER_SIZE + HEADER_SIZE + batSize
// check if parentlocator are ordered
let precLocator = 0
@@ -79,14 +115,14 @@ exports.createNbdVhdStream = async function createVhdStream(
// compute a BAT with the position that the block will have in the resulting stream
// blocks starts directly after parent locator entries
const entries = []
for (let i = 0; i < header.maxTableEntries; i++) {
const entry = streamBat.readUInt32BE(i * 4)
if (entry !== BLOCK_UNUSED) {
bat.writeUInt32BE(offsetSector, i * 4)
entries.push(i)
for (let blockId = 0; blockId < header.maxTableEntries; blockId++) {
const subBlocks = changedBlocks ? cbtContainsBlock(changedBlocks, blockId) : batContainsBlock(streamBat, blockId)
if (subBlocks !== undefined) {
bat.writeUInt32BE(offsetSector, blockId * 4)
entries.push({ blockId, subBlocks })
offsetSector += blockSizeInSectors
} else {
bat.writeUInt32BE(BLOCK_UNUSED, i * 4)
bat.writeUInt32BE(BLOCK_UNUSED, blockId * 4)
}
}
@@ -137,8 +173,10 @@ exports.createNbdVhdStream = async function createVhdStream(
// yield blocks from nbd
const nbdIterator = nbdClient.readBlocks(function* () {
for (const entry of entries) {
yield { index: entry, size: DEFAULT_BLOCK_SIZE }
for (const { subBlocks } of entries) {
for (const { blockId, buffer, size } of subBlocks) {
yield { index: blockId, buffer, size }
}
}
})
const bitmap = Buffer.alloc(SECTOR_SIZE, 255)

View File

@@ -615,7 +615,7 @@ const TRANSFORMS = {
vdi(obj) {
const vdi = {
type: 'VDI',
cbt_enabled: obj.cbt_enabled,
missing: obj.missing,
name_description: obj.name_description,
name_label: obj.name_label,

View File

@@ -590,6 +590,9 @@ const messages = {
preferNbd: 'Use NBD protocol to transfer disk if available',
preferNbdInformation: 'A network accessible by XO or the proxy must have NBD enabled',
nbdConcurrency: 'Number of NBD connexion per disk',
deltaComputationMode: 'Delta computation mode',
deltaComputationModeSnapshot: 'Snapshot comparison',
deltaComputationModeCbt: 'Change Block Tracking',
// ------ New Remote -----
newRemote: 'New file system remote',

View File

@@ -54,9 +54,13 @@ export default class IsoDevice extends Component {
() => this.props.vm.$pool,
() => this.props.vm.$container,
(vmPool, vmContainer) => sr => {
const vmRunning = vmContainer !== vmPool
const sameHost = vmContainer === sr.$container
const samePool = vmPool === sr.$pool
return (
vmPool === sr.$pool &&
(sr.shared || vmContainer === sr.$container) &&
samePool &&
(vmRunning ? sr.shared || sameHost : true) &&
(sr.SR_type === 'iso' || (sr.SR_type === 'udev' && sr.size))
)
}

View File

@@ -45,6 +45,7 @@ import { RemoteProxy, RemoteProxyWarning } from './_remoteProxy'
import getSettingsWithNonDefaultValue from '../_getSettingsWithNonDefaultValue'
import { canDeltaBackup, constructPattern, destructPattern, FormFeedback, FormGroup, Input, Li, Ul } from './../utils'
import Select from '../../../common/form/select'
export NewMetadataBackup from './metadata'
export NewMirrorBackup from './mirror'
@@ -635,11 +636,18 @@ const New = decorate([
nbdConcurrency,
})
},
setDeltaComputationMode({ setGlobalSettings }, deltaComputeMode) {
console.log({deltaComputeMode})
setGlobalSettings({
deltaComputeMode: deltaComputeMode.value,
})
},
},
computed: {
compressionId: generateId,
formId: generateId,
inputConcurrencyId: generateId,
inputDeltaComputationMode: generateId,
inputFullIntervalId: generateId,
inputMaxExportRate: generateId,
inputPreferNbd: generateId,
@@ -753,6 +761,7 @@ const New = decorate([
const {
checkpointSnapshot,
concurrency,
deltaComputationMode = 'AGAINST_PREVIOUS_SNAPSHOT',
fullInterval,
maxExportRate,
nbdConcurrency = 1,
@@ -1107,6 +1116,24 @@ const New = decorate([
offlineSnapshot={offlineSnapshot}
setGlobalSettings={effects.setGlobalSettings}
/>
{state.isDelta && (
<FormGroup>
<label htmlFor={state.inputDeltaComputationMode}>
<strong>{_('deltaComputationMode')}</strong>
</label>
<Select
id={state.inputDeltaComputationMode}
onChange={effects.setDeltaComputationMode}
value={deltaComputationMode}
disabled={!state.inputPreferNbd}
options={[
{ label: _('deltaComputationModeSnapshot'), value: 'AGAINST_PREVIOUS_SNAPSHOT' },
{ label: _('deltaComputationModeCbt'), value: 'CBT' },
]}
/>
</FormGroup>
)}
</div>
)}
</CardBlock>

View File

@@ -3,6 +3,7 @@ import _ from 'intl'
import Copiable from 'copiable'
import decorate from 'apply-decorators'
import Icon from 'icon'
import map from 'lodash/map'
import React from 'react'
import store from 'store'
import HomeTags from 'home-tags'
@@ -23,21 +24,10 @@ export default decorate([
provideState({
computed: {
areHostsVersionsEqual: ({ areHostsVersionsEqualByPool }, { host }) => areHostsVersionsEqualByPool[host.$pool],
inMemoryVms: (_, { vms }) => {
const result = []
for (const key of Object.keys(vms)) {
const vm = vms[key]
const { power_state } = vm
if (power_state === 'Running' || power_state === 'Paused') {
result.push(vm)
}
}
return result
},
},
}),
injectState,
({ statsOverview, host, nVms, vmController, state: { areHostsVersionsEqual, inMemoryVms } }) => {
({ statsOverview, host, nVms, vmController, vms, state: { areHostsVersionsEqual } }) => {
const pool = getObject(store.getState(), host.$pool)
const vmsFilter = encodeURIComponent(new CM.Property('$container', new CM.String(host.id)).toString())
return (
@@ -130,7 +120,7 @@ export default decorate([
tooltip={`${host.productBrand} (${formatSize(vmController.memory.size)})`}
value={vmController.memory.size}
/>
{inMemoryVms.map(vm => (
{map(vms, vm => (
<UsageElement
tooltip={`${vm.name_label} (${formatSize(vm.memory.size)})`}
key={vm.id}