usb nbd for al exports

Reported by @benjamreis

@vates/fuse-vhd/index.mjs

@@ -1,9 +1,7 @@
-'use strict'
-
-const LRU = require('lru-cache')
-const Fuse = require('fuse-native')
-const { VhdSynthetic } = require('vhd-lib')
-const { Disposable, fromCallback } = require('promise-toolbox')
+import LRU from 'lru-cache'
+import Fuse from 'fuse-native'
+import { VhdSynthetic } from 'vhd-lib'
+import { Disposable, fromCallback } from 'promise-toolbox'
 
 // build a s stat object from https://github.com/fuse-friends/fuse-native/blob/master/test/fixtures/stat.js
 const stat = st => ({
@@ -16,7 +14,7 @@ const stat = st => ({
   gid: st.gid !== undefined ? st.gid : process.getgid(),
 })
 
-exports.mount = Disposable.factory(async function* mount(handler, diskPath, mountDir) {
+export const mount = Disposable.factory(async function* mount(handler, diskPath, mountDir) {
   const vhd = yield VhdSynthetic.fromVhdChain(handler, diskPath)
 
   const cache = new LRU({

@vates/fuse-vhd/package.json

@@ -15,8 +15,9 @@
     "url": "https://vates.fr"
   },
   "engines": {
-    "node": ">=10.0"
+    "node": ">=14"
   },
+  "main": "./index.mjs",
   "dependencies": {
     "fuse-native": "^2.2.6",
     "lru-cache": "^7.14.0",

@vates/nbd-client/bench.mjs

@@ -0,0 +1,32 @@
+
+import NbdClient from "./index.mjs";
+
+
+
+async function bench(){
+    const client = new NbdClient({
+        address:'172.16.210.14',
+        port: 8077,
+        exportname: 'bench_export'
+    })
+    await client.connect()
+    console.log('connected', client.exportSize)
+
+    for(let chunk_size=16*1024; chunk_size < 16*1024*1024; chunk_size *=2){
+
+
+    let i=0
+    const start = + new Date()
+    for await(const block of client.readBlocks(chunk_size) ){
+        i++
+        if((i*chunk_size) % (16*1024*1024) ===0){
+                process.stdout.write('.')
+        }
+        if(i*chunk_size > 1024*1024*1024) break
+    }
+    console.log(chunk_size,Math.round( (i*chunk_size/1024/1024*1000)/ (new Date() - start)))
+
+    }
+}
+
+bench()

@vates/nbd-client/constants.js

@@ -1,42 +0,0 @@
-'use strict'
-exports.INIT_PASSWD = Buffer.from('NBDMAGIC') // "NBDMAGIC" ensure we're connected to a nbd server
-exports.OPTS_MAGIC = Buffer.from('IHAVEOPT') // "IHAVEOPT" start an option block
-exports.NBD_OPT_REPLY_MAGIC = 1100100111001001n // magic received during negociation
-exports.NBD_OPT_EXPORT_NAME = 1
-exports.NBD_OPT_ABORT = 2
-exports.NBD_OPT_LIST = 3
-exports.NBD_OPT_STARTTLS = 5
-exports.NBD_OPT_INFO = 6
-exports.NBD_OPT_GO = 7
-
-exports.NBD_FLAG_HAS_FLAGS = 1 << 0
-exports.NBD_FLAG_READ_ONLY = 1 << 1
-exports.NBD_FLAG_SEND_FLUSH = 1 << 2
-exports.NBD_FLAG_SEND_FUA = 1 << 3
-exports.NBD_FLAG_ROTATIONAL = 1 << 4
-exports.NBD_FLAG_SEND_TRIM = 1 << 5
-
-exports.NBD_FLAG_FIXED_NEWSTYLE = 1 << 0
-
-exports.NBD_CMD_FLAG_FUA = 1 << 0
-exports.NBD_CMD_FLAG_NO_HOLE = 1 << 1
-exports.NBD_CMD_FLAG_DF = 1 << 2
-exports.NBD_CMD_FLAG_REQ_ONE = 1 << 3
-exports.NBD_CMD_FLAG_FAST_ZERO = 1 << 4
-
-exports.NBD_CMD_READ = 0
-exports.NBD_CMD_WRITE = 1
-exports.NBD_CMD_DISC = 2
-exports.NBD_CMD_FLUSH = 3
-exports.NBD_CMD_TRIM = 4
-exports.NBD_CMD_CACHE = 5
-exports.NBD_CMD_WRITE_ZEROES = 6
-exports.NBD_CMD_BLOCK_STATUS = 7
-exports.NBD_CMD_RESIZE = 8
-
-exports.NBD_REQUEST_MAGIC = 0x25609513 // magic number to create a new NBD request to send to the server
-exports.NBD_REPLY_MAGIC = 0x67446698 // magic number received from the server when reading response to a nbd request
-exports.NBD_REPLY_ACK = 1
-
-exports.NBD_DEFAULT_PORT = 10809
-exports.NBD_DEFAULT_BLOCK_SIZE = 64 * 1024

@vates/nbd-client/constants.mjs

@@ -0,0 +1,41 @@
+export const INIT_PASSWD = Buffer.from('NBDMAGIC') // "NBDMAGIC" ensure we're connected to a nbd server
+export const OPTS_MAGIC = Buffer.from('IHAVEOPT') // "IHAVEOPT" start an option block
+export const NBD_OPT_REPLY_MAGIC = 1100100111001001n // magic received during negociation
+export const NBD_OPT_EXPORT_NAME = 1
+export const NBD_OPT_ABORT = 2
+export const NBD_OPT_LIST = 3
+export const NBD_OPT_STARTTLS = 5
+export const NBD_OPT_INFO = 6
+export const NBD_OPT_GO = 7
+
+export const NBD_FLAG_HAS_FLAGS = 1 << 0
+export const NBD_FLAG_READ_ONLY = 1 << 1
+export const NBD_FLAG_SEND_FLUSH = 1 << 2
+export const NBD_FLAG_SEND_FUA = 1 << 3
+export const NBD_FLAG_ROTATIONAL = 1 << 4
+export const NBD_FLAG_SEND_TRIM = 1 << 5
+
+export const NBD_FLAG_FIXED_NEWSTYLE = 1 << 0
+
+export const NBD_CMD_FLAG_FUA = 1 << 0
+export const NBD_CMD_FLAG_NO_HOLE = 1 << 1
+export const NBD_CMD_FLAG_DF = 1 << 2
+export const NBD_CMD_FLAG_REQ_ONE = 1 << 3
+export const NBD_CMD_FLAG_FAST_ZERO = 1 << 4
+
+export const NBD_CMD_READ = 0
+export const NBD_CMD_WRITE = 1
+export const NBD_CMD_DISC = 2
+export const NBD_CMD_FLUSH = 3
+export const NBD_CMD_TRIM = 4
+export const NBD_CMD_CACHE = 5
+export const NBD_CMD_WRITE_ZEROES = 6
+export const NBD_CMD_BLOCK_STATUS = 7
+export const NBD_CMD_RESIZE = 8
+
+export const NBD_REQUEST_MAGIC = 0x25609513 // magic number to create a new NBD request to send to the server
+export const NBD_REPLY_MAGIC = 0x67446698 // magic number received from the server when reading response to a nbd request
+export const NBD_REPLY_ACK = 1
+
+export const NBD_DEFAULT_PORT = 10809
+export const NBD_DEFAULT_BLOCK_SIZE = 64 * 1024

@vates/nbd-client/index.mjs

@@ -1,8 +1,11 @@
-'use strict'
-const assert = require('node:assert')
-const { Socket } = require('node:net')
-const { connect } = require('node:tls')
-const {
+import assert from 'node:assert'
+import { Socket } from 'node:net'
+import { connect } from 'node:tls'
+import { fromCallback, pRetry, pDelay, pTimeout } from 'promise-toolbox'
+import { readChunkStrict } from '@vates/read-chunk'
+import { createLogger } from '@xen-orchestra/log'
+
+import {
   INIT_PASSWD,
   NBD_CMD_READ,
   NBD_DEFAULT_BLOCK_SIZE,
@@ -17,16 +20,13 @@ const {
   NBD_REQUEST_MAGIC,
   OPTS_MAGIC,
   NBD_CMD_DISC,
-} = require('./constants.js')
-const { fromCallback, pRetry, pDelay, pTimeout } = require('promise-toolbox')
-const { readChunkStrict } = require('@vates/read-chunk')
-const { createLogger } = require('@xen-orchestra/log')
+} from './constants.mjs'
 
 const { warn } = createLogger('vates:nbd-client')
 
 // documentation is here : https://github.com/NetworkBlockDevice/nbd/blob/master/doc/proto.md
 
-module.exports = class NbdClient {
+export default class NbdClient {
   #serverAddress
   #serverCert
   #serverPort
@@ -307,11 +307,11 @@ module.exports = class NbdClient {
     })
   }
 
-  async *readBlocks(indexGenerator) {
+  async *readBlocks(indexGenerator = 2*1024*1024) {
     // default : read all blocks
-    if (indexGenerator === undefined) {
+    if (typeof indexGenerator === 'number') {
       const exportSize = this.#exportSize
-      const chunkSize = 2 * 1024 * 1024
+      const chunkSize =  indexGenerator
       indexGenerator = function* () {
         const nbBlocks = Math.ceil(Number(exportSize / BigInt(chunkSize)))
         for (let index = 0; BigInt(index) < nbBlocks; index++) {
@@ -319,6 +319,7 @@ module.exports = class NbdClient {
         }
       }
     }
+
     const readAhead = []
     const readAheadMaxLength = this.#readAhead
     const makeReadBlockPromise = (index, size) => {

@vates/nbd-client/package.json

@@ -17,6 +17,7 @@
   "engines": {
     "node": ">=14.0"
   },
+  "main": "./index.mjs",
   "dependencies": {
     "@vates/async-each": "^1.0.0",
     "@vates/read-chunk": "^1.1.1",
@@ -31,6 +32,6 @@
   },
   "scripts": {
     "postversion": "npm publish --access public",
-    "test-integration": "tap --lines 97 --functions 95 --branches 74 --statements 97 tests/*.integ.js"
+    "test-integration": "tap --lines 97 --functions 95 --branches 74 --statements 97 tests/*.integ.mjs"
   }
 }

@vates/nbd-client/tests/nbdclient.integ.mjs

@@ -1,13 +1,12 @@
-'use strict'
-const NbdClient = require('../index.js')
-const { spawn, exec } = require('node:child_process')
-const fs = require('node:fs/promises')
-const { test } = require('tap')
-const tmp = require('tmp')
-const { pFromCallback } = require('promise-toolbox')
-const { Socket } = require('node:net')
-const { NBD_DEFAULT_PORT } = require('../constants.js')
-const assert = require('node:assert')
+import NbdClient from '../index.mjs'
+import { spawn, exec } from 'node:child_process'
+import fs from 'node:fs/promises'
+import { test } from 'tap'
+import tmp from 'tmp'
+import { pFromCallback } from 'promise-toolbox'
+import { Socket } from 'node:net'
+import { NBD_DEFAULT_PORT } from '../constants.mjs'
+import assert from 'node:assert'
 
 const FILE_SIZE = 10 * 1024 * 1024
 

@vates/node-vsphere-soap/lib/client.mjs

@@ -1,4 +1,3 @@
-'use strict'
 /*
 
   node-vsphere-soap
@@ -12,17 +11,18 @@
 
 */
 
-const EventEmitter = require('events').EventEmitter
-const axios = require('axios')
-const https = require('node:https')
-const util = require('util')
-const soap = require('soap')
-const Cookie = require('soap-cookie') // required for session persistence
+import { EventEmitter } from 'events'
+import axios from 'axios'
+import https from 'node:https'
+import util from 'util'
+import soap from 'soap'
+import Cookie from 'soap-cookie' // required for session persistence
+
 // Client class
 // inherits from EventEmitter
 // possible events: connect, error, ready
 
-function Client(vCenterHostname, username, password, sslVerify) {
+export function Client(vCenterHostname, username, password, sslVerify) {
   this.status = 'disconnected'
   this.reconnectCount = 0
 
@@ -228,4 +228,3 @@ function _soapErrorHandler(self, emitter, command, args, err) {
 }
 
 // end
-exports.Client = Client

@vates/node-vsphere-soap/package.json

@@ -2,7 +2,7 @@
   "name": "@vates/node-vsphere-soap",
   "version": "1.0.0",
   "description": "interface to vSphere SOAP/WSDL from node for interfacing with vCenter or ESXi, forked from node-vsphere-soap",
-  "main": "lib/client.js",
+  "main": "lib/client.mjs",
   "author": "reedog117",
   "repository": {
     "directory": "@vates/node-vsphere-soap",
@@ -30,7 +30,7 @@
   "private": false,
   "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@vates/node-vsphere-soap",
   "engines": {
-    "node": ">=8.10"
+    "node": ">=14"
   },
   "scripts": {
     "postversion": "npm publish --access public"

@vates/node-vsphere-soap/test/config-test.stub.mjs

@@ -1,15 +1,11 @@
-'use strict'
-
 // place your own credentials here for a vCenter or ESXi server
 // this information will be used for connecting to a vCenter instance
 // for module testing
 // name the file config-test.js
 
-const vCenterTestCreds = {
+export const vCenterTestCreds = {
   vCenterIP: 'vcsa',
   vCenterUser: 'vcuser',
   vCenterPassword: 'vcpw',
   vCenter: true,
 }
-
-exports.vCenterTestCreds = vCenterTestCreds

@vates/node-vsphere-soap/test/vsphere-soap.test.mjs

@@ -1,18 +1,16 @@
-'use strict'
-
 /*
   vsphere-soap.test.js
 
   tests for the vCenterConnectionInstance class
 */
 
-const assert = require('assert')
-const { describe, it } = require('test')
+import assert from 'assert'
+import { describe, it } from 'test'
 
-const vc = require('../lib/client')
+import * as vc from '../lib/client.mjs'
 
-// eslint-disable-next-line n/no-missing-require
-const TestCreds = require('../config-test.js').vCenterTestCreds
+// eslint-disable-next-line n/no-missing-import
+import { vCenterTestCreds as TestCreds } from '../config-test.mjs'
 
 const VItest = new vc.Client(TestCreds.vCenterIP, TestCreds.vCenterUser, TestCreds.vCenterPassword, false)
 

@vates/read-chunk/index.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const assert = require('assert')
+const isUtf8 = require('isutf8')
 
 /**
  * Read a chunk of data from a stream.
@@ -81,6 +82,13 @@ exports.readChunkStrict = async function readChunkStrict(stream, size) {
 
   if (size !== undefined && chunk.length !== size) {
     const error = new Error(`stream has ended with not enough data (actual: ${chunk.length}, expected: ${size})`)
+
+    // Buffer.isUtf8 is too recent for now
+    // @todo : replace external package by Buffer.isUtf8 when the supported version of node reach 18
+
+    if (chunk.length < 1024 && isUtf8(chunk)) {
+      error.text = chunk.toString('utf8')
+    }
     Object.defineProperties(error, {
       chunk: {
         value: chunk,

@vates/read-chunk/index.test.js

@@ -102,12 +102,37 @@ describe('readChunkStrict', function () {
     assert.strictEqual(error.chunk, undefined)
   })
 
-  it('throws if stream ends with not enough data', async () => {
+  it('throws if stream ends with not enough data, utf8', async () => {
     const error = await rejectionOf(readChunkStrict(makeStream(['foo', 'bar']), 10))
     assert(error instanceof Error)
     assert.strictEqual(error.message, 'stream has ended with not enough data (actual: 6, expected: 10)')
+    assert.strictEqual(error.text, 'foobar')
     assert.deepEqual(error.chunk, Buffer.from('foobar'))
   })
+
+  it('throws if stream ends with not enough data, non utf8 ', async () => {
+    const source = [Buffer.alloc(10, 128), Buffer.alloc(10, 128)]
+    const error = await rejectionOf(readChunkStrict(makeStream(source), 30))
+    assert(error instanceof Error)
+    assert.strictEqual(error.message, 'stream has ended with not enough data (actual: 20, expected: 30)')
+    assert.strictEqual(error.text, undefined)
+    assert.deepEqual(error.chunk, Buffer.concat(source))
+  })
+
+  it('throws if stream ends with not enough data,  utf8 , long data', async () => {
+    const source = Buffer.from('a'.repeat(1500))
+    const error = await rejectionOf(readChunkStrict(makeStream([source]), 2000))
+    assert(error instanceof Error)
+    assert.strictEqual(error.message, `stream has ended with not enough data (actual: 1500, expected: 2000)`)
+    assert.strictEqual(error.text, undefined)
+    assert.deepEqual(error.chunk, source)
+  })
+
+  it('succeed', async () => {
+    const source = Buffer.from('a'.repeat(20))
+    const chunk = await readChunkStrict(makeStream([source]), 10)
+    assert.deepEqual(source.subarray(10), chunk)
+  })
 })
 
 describe('skip', function () {
@@ -134,6 +159,16 @@ describe('skip', function () {
   it('returns less size if stream ends', async () => {
     assert.deepEqual(await skip(makeStream('foo bar'), 10), 7)
   })
+
+  it('put back if it read too much', async () => {
+    let source = makeStream(['foo', 'bar'])
+    await skip(source, 1) // read part of data chunk
+    const chunk = (await readChunkStrict(source, 2)).toString('utf-8')
+    assert.strictEqual(chunk, 'oo')
+
+    source = makeStream(['foo', 'bar'])
+    assert.strictEqual(await skip(source, 3), 3) // read aligned with data chunk
+  })
 })
 
 describe('skipStrict', function () {
@@ -144,4 +179,9 @@ describe('skipStrict', function () {
     assert.strictEqual(error.message, 'stream has ended with not enough data (actual: 7, expected: 10)')
     assert.deepEqual(error.bytesSkipped, 7)
   })
+  it('succeed', async () => {
+    const source = makeStream(['foo', 'bar', 'baz'])
+    const res = await skipStrict(source, 4)
+    assert.strictEqual(res, undefined)
+  })
 })

@vates/read-chunk/package.json

@@ -33,5 +33,8 @@
   },
   "devDependencies": {
     "test": "^3.2.1"
+  },
+  "dependencies": {
+    "isutf8": "^4.0.0"
   }
 }

@xen-orchestra/fs/src/abstract.test.js

@@ -209,7 +209,7 @@ describe('encryption', () => {
     // encrypt with a non default algorithm
     const encryptor = _getEncryptor('aes-256-cbc', '73c1838d7d8a6088ca2317fb5f29cd91')
 
-    await fs.writeFile(`${dir}/encryption.json`, `{"algorithm": "aes-256-gmc"}`)
+    await fs.writeFile(`${dir}/encryption.json`, `{"algorithm": "aes-256-gcm"}`)
     await fs.writeFile(`${dir}/metadata.json`, encryptor.encryptData(`{"random": "NOTSORANDOM"}`))
 
     // remote is now non empty : can't modify key anymore

@xen-orchestra/lite/CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## **next**
 
+- Ability to export selected VMs as CSV file (PR [#6915](https://github.com/vatesfr/xen-orchestra/pull/6915))
+- [Pool/VMs] Ability to export selected VMs as JSON file (PR [#6911](https://github.com/vatesfr/xen-orchestra/pull/6911))
+
 ## **0.1.1** (2023-07-03)
 
 - Invalidate sessionId token after logout (PR [#6480](https://github.com/vatesfr/xen-orchestra/pull/6480))

@xen-orchestra/lite/package.json

@@ -17,6 +17,7 @@
     "@fortawesome/vue-fontawesome": "^3.0.1",
     "@novnc/novnc": "^1.3.0",
     "@types/d3-time-format": "^4.0.0",
+    "@types/file-saver": "^2.0.5",
     "@types/lodash-es": "^4.17.6",
     "@types/marked": "^4.0.8",
     "@vueuse/core": "^10.1.2",
@@ -25,6 +26,7 @@
     "d3-time-format": "^4.1.0",
     "decorator-synchronized": "^0.6.0",
     "echarts": "^5.3.3",
+    "file-saver": "^2.0.5",
     "highlight.js": "^11.6.0",
     "human-format": "^1.1.0",
     "iterable-backoff": "^0.1.0",

@xen-orchestra/lite/src/components/pool/dashboard/PoolDashboardCpuUsage.vue

@@ -1,21 +1,48 @@
 <template>
   <UiCard :color="hasError ? 'error' : undefined">
-    <UiCardTitle>{{ $t("cpu-usage") }}</UiCardTitle>
+    <UiCardTitle>
+      {{ $t("cpu-usage") }}
+      <template v-if="vmStatsCanBeExpired || hostStatsCanBeExpired" #right>
+        <UiSpinner v-tooltip="$t('fetching-fresh-data')" />
+      </template>
+    </UiCardTitle>
     <HostsCpuUsage />
     <VmsCpuUsage />
   </UiCard>
 </template>
 <script lang="ts" setup>
+import { vTooltip } from "@/directives/tooltip.directive";
 import HostsCpuUsage from "@/components/pool/dashboard/cpuUsage/HostsCpuUsage.vue";
 import VmsCpuUsage from "@/components/pool/dashboard/cpuUsage/VmsCpuUsage.vue";
 import UiCard from "@/components/ui/UiCard.vue";
 import UiCardTitle from "@/components/ui/UiCardTitle.vue";
 import { useHostStore } from "@/stores/host.store";
 import { useVmStore } from "@/stores/vm.store";
-import { computed } from "vue";
+import { computed, inject, type ComputedRef } from "vue";
+import type { Stat } from "@/composables/fetch-stats.composable";
+import type { HostStats, VmStats } from "@/libs/xapi-stats";
+import UiSpinner from "@/components/ui/UiSpinner.vue";
 
 const { hasError: hasVmError } = useVmStore().subscribe();
 const { hasError: hasHostError } = useHostStore().subscribe();
 
+const vmStats = inject<ComputedRef<Stat<VmStats>[]>>(
+  "vmStats",
+  computed(() => [])
+);
+
+const hostStats = inject<ComputedRef<Stat<HostStats>[]>>(
+  "hostStats",
+  computed(() => [])
+);
+
+const vmStatsCanBeExpired = computed(() =>
+  vmStats.value.some((stat) => stat.canBeExpired)
+);
+
+const hostStatsCanBeExpired = computed(() =>
+  hostStats.value.some((stat) => stat.canBeExpired)
+);
+
 const hasError = computed(() => hasVmError.value || hasHostError.value);
 </script>

@xen-orchestra/lite/src/components/pool/dashboard/PoolDashboardNetworkChart.vue

@@ -1,5 +1,6 @@
 <template>
   <!-- TODO: add a loader when data is not fully loaded or undefined -->
+  <!-- TODO: add small loader with tooltips when stats can be expired -->
   <!-- TODO: display the NoData component in case of a data recovery error -->
   <LinearChart
     :data="data"

@xen-orchestra/lite/src/components/pool/dashboard/PoolDashboardRamUsage.vue

@@ -1,22 +1,50 @@
 <template>
   <UiCard :color="hasError ? 'error' : undefined">
-    <UiCardTitle>{{ $t("ram-usage") }}</UiCardTitle>
+    <UiCardTitle>
+      {{ $t("ram-usage") }}
+      <template v-if="vmStatsCanBeExpired || hostStatsCanBeExpired" #right>
+        <UiSpinner v-tooltip="$t('fetching-fresh-data')" />
+      </template>
+    </UiCardTitle>
     <HostsRamUsage />
     <VmsRamUsage />
   </UiCard>
 </template>
 
 <script lang="ts" setup>
+import { vTooltip } from "@/directives/tooltip.directive";
 import HostsRamUsage from "@/components/pool/dashboard/ramUsage/HostsRamUsage.vue";
 import VmsRamUsage from "@/components/pool/dashboard/ramUsage/VmsRamUsage.vue";
 import UiCard from "@/components/ui/UiCard.vue";
 import UiCardTitle from "@/components/ui/UiCardTitle.vue";
 import { useHostStore } from "@/stores/host.store";
 import { useVmStore } from "@/stores/vm.store";
-import { computed } from "vue";
+import { computed, inject } from "vue";
+import type { ComputedRef } from "vue";
+import type { HostStats, VmStats } from "@/libs/xapi-stats";
+import type { Stat } from "@/composables/fetch-stats.composable";
+import UiSpinner from "@/components/ui/UiSpinner.vue";
 
 const { hasError: hasVmError } = useVmStore().subscribe();
 const { hasError: hasHostError } = useHostStore().subscribe();
 
+const vmStats = inject<ComputedRef<Stat<VmStats>[]>>(
+  "vmStats",
+  computed(() => [])
+);
+
+const hostStats = inject<ComputedRef<Stat<HostStats>[]>>(
+  "hostStats",
+  computed(() => [])
+);
+
+const vmStatsCanBeExpired = computed(() =>
+  vmStats.value.some((stat) => stat.canBeExpired)
+);
+
+const hostStatsCanBeExpired = computed(() =>
+  hostStats.value.some((stat) => stat.canBeExpired)
+);
+
 const hasError = computed(() => hasVmError.value || hasHostError.value);
 </script>

@xen-orchestra/lite/src/components/pool/dashboard/cpuUsage/PoolCpuUsageChart.vue

@@ -1,5 +1,6 @@
 <template>
   <!-- TODO: add a loader when data is not fully loaded or undefined -->
+  <!-- TODO: add small loader with tooltips when stats can be expired -->
   <!-- TODO: Display the NoDataError component in case of a data recovery error -->
   <LinearChart
     :data="data"

@xen-orchestra/lite/src/components/pool/dashboard/ramUsage/PoolRamUsage.vue

@@ -1,5 +1,6 @@
 <template>
   <!-- TODO: add a loader when data is not fully loaded or undefined -->
+  <!-- TODO: add small loader with tooltips when stats can be expired -->
   <!-- TODO: display the NoDataError component in case of a data recovery error -->
   <LinearChart
     :data="data"

@xen-orchestra/lite/src/components/vm/VmActionItems/VmActionExportItem.vue

@@ -0,0 +1,51 @@
+<template>
+  <MenuItem :icon="faFileExport">
+    {{ $t("export") }}
+    <template #submenu>
+      <MenuItem
+        v-tooltip="{ content: $t('coming-soon'), placement: 'left' }"
+        :icon="faDisplay"
+      >
+        {{ $t("export-vms") }}
+      </MenuItem>
+      <MenuItem
+        :icon="faCode"
+        @click="
+          exportVmsAsJsonFile(vms, `vms_${new Date().toISOString()}.json`)
+        "
+      >
+        {{ $t("export-table-to", { type: ".json" }) }}
+      </MenuItem>
+      <MenuItem
+        :icon="faFileCsv"
+        @click="exportVmsAsCsvFile(vms, `vms_${new Date().toISOString()}.csv`)"
+      >
+        {{ $t("export-table-to", { type: ".csv" }) }}
+      </MenuItem>
+    </template>
+  </MenuItem>
+</template>
+
+<script lang="ts" setup>
+import { computed } from "vue";
+import { exportVmsAsCsvFile, exportVmsAsJsonFile } from "@/libs/vm";
+import MenuItem from "@/components/menu/MenuItem.vue";
+import {
+  faCode,
+  faDisplay,
+  faFileCsv,
+  faFileExport,
+} from "@fortawesome/free-solid-svg-icons";
+import { useVmStore } from "@/stores/vm.store";
+import { vTooltip } from "@/directives/tooltip.directive";
+import type { XenApiVm } from "@/libs/xen-api";
+
+const props = defineProps<{
+  vmRefs: XenApiVm["$ref"][];
+}>();
+
+const { getByOpaqueRef: getVm } = useVmStore().subscribe();
+const vms = computed(() =>
+  props.vmRefs.map(getVm).filter((vm): vm is XenApiVm => vm !== undefined)
+);
+</script>

@xen-orchestra/lite/src/components/vm/VmsActionsBar.vue

@@ -25,30 +25,8 @@
     <MenuItem v-tooltip="$t('coming-soon')" :icon="faCamera">
       {{ $t("snapshot") }}
     </MenuItem>
+    <VmActionExportItem :vm-refs="selectedRefs" />
     <VmActionDeleteItem :vm-refs="selectedRefs" />
-    <MenuItem :icon="faFileExport">
-      {{ $t("export") }}
-      <template #submenu>
-        <MenuItem
-          v-tooltip="{ content: $t('coming-soon'), placement: 'left' }"
-          :icon="faDisplay"
-        >
-          {{ $t("export-vms") }}
-        </MenuItem>
-        <MenuItem
-          v-tooltip="{ content: $t('coming-soon'), placement: 'left' }"
-          :icon="faCode"
-        >
-          {{ $t("export-table-to", { type: ".json" }) }}
-        </MenuItem>
-        <MenuItem
-          v-tooltip="{ content: $t('coming-soon'), placement: 'left' }"
-          :icon="faFileCsv"
-        >
-          {{ $t("export-table-to", { type: ".csv" }) }}
-        </MenuItem>
-      </template>
-    </MenuItem>
   </AppMenu>
 </template>
 
@@ -57,6 +35,7 @@ import AppMenu from "@/components/menu/AppMenu.vue";
 import MenuItem from "@/components/menu/MenuItem.vue";
 import UiButton from "@/components/ui/UiButton.vue";
 import VmActionCopyItem from "@/components/vm/VmActionItems/VmActionCopyItem.vue";
+import VmActionExportItem from "@/components/vm/VmActionItems/VmActionExportItem.vue";
 import VmActionDeleteItem from "@/components/vm/VmActionItems/VmActionDeleteItem.vue";
 import VmActionPowerStateItems from "@/components/vm/VmActionItems/VmActionPowerStateItems.vue";
 import { vTooltip } from "@/directives/tooltip.directive";
@@ -64,12 +43,8 @@ import type { XenApiVm } from "@/libs/xen-api";
 import { useUiStore } from "@/stores/ui.store";
 import {
   faCamera,
-  faCode,
-  faDisplay,
   faEdit,
   faEllipsis,
-  faFileCsv,
-  faFileExport,
   faPowerOff,
   faRoute,
 } from "@fortawesome/free-solid-svg-icons";

@xen-orchestra/lite/src/composables/fetch-stats.composable.ts

@@ -10,6 +10,7 @@ import { type Pausable, promiseTimeout, useTimeoutPoll } from "@vueuse/core";
 import { computed, type ComputedRef, onUnmounted, ref } from "vue";
 
 export type Stat<T> = {
+  canBeExpired: boolean;
   id: string;
   name: string;
   stats: T | undefined;
@@ -21,8 +22,10 @@ type GetStats<
   S extends HostStats | VmStats
 > = (
   uuid: T["uuid"],
-  granularity: GRANULARITY
-) => Promise<XapiStatsResponse<S>> | undefined;
+  granularity: GRANULARITY,
+  ignoreExpired: boolean,
+  opts: { abortSignal?: AbortSignal }
+) => Promise<XapiStatsResponse<S> | undefined> | undefined;
 
 export type FetchedStats<
   T extends XenApiHost | XenApiVm,
@@ -41,6 +44,7 @@ export default function useFetchStats<
 >(getStats: GetStats<T, S>, granularity: GRANULARITY): FetchedStats<T, S> {
   const stats = ref<Map<string, Stat<S>>>(new Map());
   const timestamp = ref<number[]>([0, 0]);
+  const abortController = new AbortController();
 
   const register = (object: T) => {
     const mapKey = `${object.uuid}-${granularity}`;
@@ -49,13 +53,18 @@ export default function useFetchStats<
       return;
     }
 
+    const ignoreExpired = computed(() => !stats.value.has(mapKey));
+
     const pausable = useTimeoutPoll(
       async () => {
-        if (!stats.value.has(mapKey)) {
-          return;
-        }
-
-        const newStats = await getStats(object.uuid, granularity);
+        const newStats = (await getStats(
+          object.uuid,
+          granularity,
+          ignoreExpired.value,
+          {
+            abortSignal: abortController.signal,
+          }
+        )) as XapiStatsResponse<S>;
 
         if (newStats === undefined) {
           return;
@@ -69,6 +78,7 @@ export default function useFetchStats<
         ];
 
         stats.value.get(mapKey)!.stats = newStats.stats;
+        stats.value.get(mapKey)!.canBeExpired = newStats.canBeExpired;
         await promiseTimeout(newStats.interval * 1000);
       },
       0,
@@ -76,6 +86,7 @@ export default function useFetchStats<
     );
 
     stats.value.set(mapKey, {
+      canBeExpired: false,
       id: object.uuid,
       name: object.name_label,
       stats: undefined,
@@ -90,6 +101,7 @@ export default function useFetchStats<
   };
 
   onUnmounted(() => {
+    abortController.abort();
     stats.value.forEach((stat) => stat.pausable.pause());
   });
 

@xen-orchestra/lite/src/libs/vm.ts

@@ -0,0 +1,41 @@
+import { saveAs } from "file-saver";
+import type { XenApiVm } from "@/libs/xen-api";
+
+function stringifyCsvValue(value: any) {
+  let res = "";
+  if (Array.isArray(value)) {
+    res = value.join(";");
+  } else if (typeof value === "object") {
+    res = JSON.stringify(value);
+  } else {
+    res = String(value);
+  }
+  return `"${res.replace(/"/g, '""')}"`;
+}
+
+export function exportVmsAsCsvFile(vms: XenApiVm[], fileName: string) {
+  const csvHeaders = Object.keys(vms[0]);
+
+  const csvRows = vms.map((vm) =>
+    csvHeaders.map((header) => stringifyCsvValue(vm[header as keyof XenApiVm]))
+  );
+
+  saveAs(
+    new Blob(
+      [[csvHeaders, ...csvRows].map((row) => row.join(",")).join("\n")],
+      {
+        type: "text/csv;charset=utf-8",
+      }
+    ),
+    fileName
+  );
+}
+
+export function exportVmsAsJsonFile(vms: XenApiVm[], fileName: string) {
+  saveAs(
+    new Blob([JSON.stringify(vms, null, 2)], {
+      type: "application/json",
+    }),
+    fileName
+  );
+}

@xen-orchestra/lite/src/libs/xapi-stats.ts

@@ -295,18 +295,22 @@ export type HostStats = {
 };
 
 export type XapiStatsResponse<T> = {
+  canBeExpired: boolean;
   endTimestamp: number;
   interval: number;
   stats: T;
 };
 
+type StatsByObject = {
+  [uuid: string]: {
+    [step: string]: XapiStatsResponse<HostStats | VmStats>;
+  };
+};
+
 export default class XapiStats {
   #xapi;
-  #statsByObject: {
-    [uuid: string]: {
-      [step: string]: XapiStatsResponse<HostStats | any>;
-    };
-  } = {};
+  #statsByObject: StatsByObject = {};
+  #cachedStatsByObject: StatsByObject = {};
   constructor(xapi: XenApi) {
     this.#xapi = xapi;
   }
@@ -314,7 +318,12 @@ export default class XapiStats {
   // Execute one http request on a XenServer for get stats
   // Return stats (Json format) or throws got exception
   @limitConcurrency(3)
-  async _getJson(host: XenApiHost, timestamp: any, step: any) {
+  async _getJson(
+    host: XenApiHost,
+    timestamp: number,
+    step: RRD_STEP,
+    { abortSignal }: { abortSignal?: AbortSignal } = {}
+  ) {
     const resp = await this.#xapi.getResource("/rrd_updates", {
       host,
       query: {
@@ -324,13 +333,23 @@ export default class XapiStats {
         json: "true",
         start: timestamp,
       },
+      abortSignal,
     });
     return JSON5.parse(await resp.text());
   }
 
   // To avoid multiple requests, we keep a cache for the stats and
   // only return it if we not exceed a step
-  #getCachedStats(uuid: any, step: any, currentTimeStamp: any) {
+  #getCachedStats(
+    uuid: string,
+    step: RRD_STEP,
+    currentTimeStamp: number,
+    ignoreExpired = false
+  ) {
+    if (ignoreExpired) {
+      return this.#cachedStatsByObject[uuid]?.[step];
+    }
+
     const statsByObject = this.#statsByObject;
 
     const stats = statsByObject[uuid]?.[step];
@@ -347,12 +366,16 @@ export default class XapiStats {
   }
 
   @synchronized.withKey(({ host }: { host: XenApiHost }) => host.uuid)
-  async _getAndUpdateStats({
+  async _getAndUpdateStats<T extends VmStats | HostStats>({
+    abortSignal,
     host,
+    ignoreExpired = false,
     uuid,
     granularity,
   }: {
+    abortSignal?: AbortSignal;
     host: XenApiHost;
+    ignoreExpired?: boolean;
     uuid: any;
     granularity: GRANULARITY;
   }) {
@@ -367,7 +390,13 @@ export default class XapiStats {
     }
     const currentTimeStamp = Math.floor(new Date().getTime() / 1000);
 
-    const stats = this.#getCachedStats(uuid, step, currentTimeStamp);
+    const stats = this.#getCachedStats(
+      uuid,
+      step,
+      currentTimeStamp,
+      ignoreExpired
+    ) as XapiStatsResponse<T>;
+
     if (stats !== undefined) {
       return stats;
     }
@@ -376,75 +405,113 @@ export default class XapiStats {
 
     // To avoid crossing over the boundary, we ask for one less step
     const optimumTimestamp = currentTimeStamp - maxDuration + step;
-    const json = await this._getJson(host, optimumTimestamp, step);
 
-    const actualStep = json.meta.step as number;
+    try {
+      const json = await this._getJson(host, optimumTimestamp, step, {
+        abortSignal,
+      });
 
-    if (json.data.length > 0) {
-      // fetched data is organized from the newest to the oldest
-      // but this implementation requires it in the other direction
-      json.data.reverse();
-      json.meta.legend.forEach((legend: any, index: number) => {
-        const [, type, uuid, metricType] = /^AVERAGE:([^:]+):(.+):(.+)$/.exec(
-          legend
-        ) as any;
+      const actualStep = json.meta.step as number;
 
-        const metrics = STATS[type] as any;
-        if (metrics === undefined) {
-          return;
-        }
+      if (json.data.length > 0) {
+        // fetched data is organized from the newest to the oldest
+        // but this implementation requires it in the other direction
+        json.data.reverse();
+        json.meta.legend.forEach((legend: any, index: number) => {
+          const [, type, uuid, metricType] = /^AVERAGE:([^:]+):(.+):(.+)$/.exec(
+            legend
+          ) as any;
 
-        const { metric, testResult } = findMetric(metrics, metricType) as any;
-        if (metric === undefined) {
-          return;
-        }
-
-        const xoObjectStats = createGetProperty(this.#statsByObject, uuid, {});
-        let stepStats = xoObjectStats[actualStep];
-        if (
-          stepStats === undefined ||
-          stepStats.endTimestamp !== json.meta.end
-        ) {
-          stepStats = xoObjectStats[actualStep] = {
-            endTimestamp: json.meta.end,
-            interval: actualStep,
-          };
-        }
-
-        const path =
-          metric.getPath !== undefined
-            ? metric.getPath(testResult)
-            : [findKey(metrics, metric)];
-
-        const lastKey = path.length - 1;
-        let metricStats = createGetProperty(stepStats, "stats", {});
-        path.forEach((property: any, key: number) => {
-          if (key === lastKey) {
-            metricStats[property] = computeValues(
-              json.data,
-              index,
-              metric.transformValue
-            );
+          const metrics = STATS[type] as any;
+          if (metrics === undefined) {
             return;
           }
 
-          metricStats = createGetProperty(metricStats, property, {});
+          const { metric, testResult } = findMetric(metrics, metricType) as any;
+          if (metric === undefined) {
+            return;
+          }
+
+          const xoObjectStats = createGetProperty(
+            this.#statsByObject,
+            uuid,
+            {}
+          );
+          const cacheXoObjectStats = createGetProperty(
+            this.#cachedStatsByObject,
+            uuid,
+            {}
+          );
+
+          let stepStats = xoObjectStats[actualStep];
+          let cacheStepStats = cacheXoObjectStats[actualStep];
+          if (
+            stepStats === undefined ||
+            stepStats.endTimestamp !== json.meta.end
+          ) {
+            stepStats = xoObjectStats[actualStep] = {
+              endTimestamp: json.meta.end,
+              interval: actualStep,
+              canBeExpired: false,
+            };
+            cacheStepStats = cacheXoObjectStats[actualStep] = {
+              endTimestamp: json.meta.end,
+              interval: actualStep,
+              canBeExpired: true,
+            };
+          }
+
+          const path =
+            metric.getPath !== undefined
+              ? metric.getPath(testResult)
+              : [findKey(metrics, metric)];
+
+          const lastKey = path.length - 1;
+          let metricStats = createGetProperty(stepStats, "stats", {});
+          let cacheMetricStats = createGetProperty(cacheStepStats, "stats", {});
+
+          path.forEach((property: any, key: number) => {
+            if (key === lastKey) {
+              metricStats[property] = computeValues(
+                json.data,
+                index,
+                metric.transformValue
+              );
+              cacheMetricStats[property] = computeValues(
+                json.data,
+                index,
+                metric.transformValue
+              );
+              return;
+            }
+
+            metricStats = createGetProperty(metricStats, property, {});
+            cacheMetricStats = createGetProperty(
+              cacheMetricStats,
+              property,
+              {}
+            );
+          });
         });
-      });
-    }
-
-    if (actualStep !== step) {
-      throw new FaultyGranularity(
-        `Unable to get the true granularity: ${actualStep}`
-      );
-    }
-
-    return (
-      this.#statsByObject[uuid]?.[step] ?? {
-        endTimestamp: currentTimeStamp,
-        interval: step,
-        stats: {},
       }
-    );
+
+      if (actualStep !== step) {
+        throw new FaultyGranularity(
+          `Unable to get the true granularity: ${actualStep}`
+        );
+      }
+    } catch (error) {
+      if (error instanceof Error && error.name === "AbortError") {
+        return;
+      }
+
+      throw error;
+    }
+
+    return (this.#statsByObject[uuid]?.[step] ?? {
+      endTimestamp: currentTimeStamp,
+      interval: step,
+      stats: {},
+    }) as XapiStatsResponse<T>;
   }
 }

@xen-orchestra/lite/src/libs/xen-api.ts

@@ -266,7 +266,11 @@ export default class XenApi {
 
   async getResource(
     pathname: string,
-    { host, query }: { host: XenApiHost; query: any }
+    {
+      abortSignal,
+      host,
+      query,
+    }: { abortSignal?: AbortSignal; host: XenApiHost; query: any }
   ) {
     const url = new URL("http://localhost");
     url.protocol = window.location.protocol;
@@ -277,7 +281,7 @@ export default class XenApi {
       session_id: this.#sessionId,
     }).toString();
 
-    return fetch(url);
+    return fetch(url, { signal: abortSignal });
   }
 
   async loadRecords<T extends XenApiRecord<string>>(

@xen-orchestra/lite/src/locales/en.json

@@ -36,6 +36,7 @@
   "export": "Export",
   "export-table-to": "Export table to {type}",
   "export-vms": "Export VMs",
+  "fetching-fresh-data": "Fetching fresh data",
   "filter": {
     "comparison": {
       "contains": "Contains",

@xen-orchestra/lite/src/locales/fr.json

@@ -36,6 +36,7 @@
   "export": "Exporter",
   "export-table-to": "Exporter le tableau en {type}",
   "export-vms": "Exporter les VMs",
+  "fetching-fresh-data": "Récupération de données à jour",
   "filter": {
     "comparison": {
       "contains": "Contient",

@xen-orchestra/lite/src/stores/host.store.ts

@@ -1,5 +1,9 @@
 import { isHostRunning, sortRecordsByNameLabel } from "@/libs/utils";
-import type { GRANULARITY, XapiStatsResponse } from "@/libs/xapi-stats";
+import type {
+  GRANULARITY,
+  HostStats,
+  XapiStatsResponse,
+} from "@/libs/xapi-stats";
 import type { XenApiHost, XenApiHostMetrics } from "@/libs/xen-api";
 import { useXapiCollectionStore } from "@/stores/xapi-collection.store";
 import { useXenApiStore } from "@/stores/xen-api.store";
@@ -8,11 +12,15 @@ import { createSubscribe } from "@/types/xapi-collection";
 import { defineStore } from "pinia";
 import { computed, type ComputedRef } from "vue";
 
+type GetStats = (
+  hostUuid: XenApiHost["uuid"],
+  granularity: GRANULARITY,
+  ignoreExpired: boolean,
+  opts: { abortSignal?: AbortSignal }
+) => Promise<XapiStatsResponse<HostStats> | undefined> | undefined;
+
 type GetStatsExtension = {
-  getStats: (
-    hostUuid: XenApiHost["uuid"],
-    granularity: GRANULARITY
-  ) => Promise<XapiStatsResponse<any>> | undefined;
+  getStats: GetStats;
 };
 
 type RunningHostsExtension = [
@@ -31,9 +39,11 @@ export const useHostStore = defineStore("host", () => {
   const subscribe = createSubscribe<XenApiHost, Extensions>((options) => {
     const originalSubscription = hostCollection.subscribe(options);
 
-    const getStats = (
-      hostUuid: XenApiHost["uuid"],
-      granularity: GRANULARITY
+    const getStats: GetStats = (
+      hostUuid,
+      granularity,
+      ignoreExpired = false,
+      { abortSignal }
     ) => {
       const host = originalSubscription.getByUuid(hostUuid);
 
@@ -45,8 +55,10 @@ export const useHostStore = defineStore("host", () => {
         ? xenApiStore.getXapiStats()
         : undefined;
 
-      return xapiStats?._getAndUpdateStats({
+      return xapiStats?._getAndUpdateStats<HostStats>({
+        abortSignal,
         host,
+        ignoreExpired,
         uuid: host.uuid,
         granularity,
       });

@xen-orchestra/lite/src/stores/vm.store.ts

@@ -1,5 +1,9 @@
 import { sortRecordsByNameLabel } from "@/libs/utils";
-import type { GRANULARITY, XapiStatsResponse } from "@/libs/xapi-stats";
+import type {
+  GRANULARITY,
+  VmStats,
+  XapiStatsResponse,
+} from "@/libs/xapi-stats";
 import type { XenApiHost, XenApiVm } from "@/libs/xen-api";
 import { POWER_STATE } from "@/libs/xen-api";
 import { useXapiCollectionStore } from "@/stores/xapi-collection.store";
@@ -8,6 +12,12 @@ import { createSubscribe, type Subscription } from "@/types/xapi-collection";
 import { defineStore } from "pinia";
 import { computed, type ComputedRef } from "vue";
 
+type GetStats = (
+  id: XenApiVm["uuid"],
+  granularity: GRANULARITY,
+  ignoreExpired: boolean,
+  opts: { abortSignal?: AbortSignal }
+) => Promise<XapiStatsResponse<VmStats> | undefined> | undefined;
 type DefaultExtension = {
   recordsByHostRef: ComputedRef<Map<XenApiHost["$ref"], XenApiVm[]>>;
   runningVms: ComputedRef<XenApiVm[]>;
@@ -15,10 +25,7 @@ type DefaultExtension = {
 
 type GetStatsExtension = [
   {
-    getStats: (
-      id: XenApiVm["uuid"],
-      granularity: GRANULARITY
-    ) => Promise<XapiStatsResponse<any>>;
+    getStats: GetStats;
   },
   { hostSubscription: Subscription<XenApiHost, object> }
 ];
@@ -60,33 +67,49 @@ export const useVmStore = defineStore("vm", () => {
 
     const hostSubscription = options?.hostSubscription;
 
-    const getStatsSubscription = hostSubscription !== undefined && {
-      getStats: (vmUuid: XenApiVm["uuid"], granularity: GRANULARITY) => {
-        const xenApiStore = useXenApiStore();
-
-        if (!xenApiStore.isConnected) {
-          return undefined;
+    const getStatsSubscription:
+      | {
+          getStats: GetStats;
         }
+      | undefined =
+      hostSubscription !== undefined
+        ? {
+            getStats: (
+              id,
+              granularity,
+              ignoreExpired = false,
+              { abortSignal }
+            ) => {
+              const xenApiStore = useXenApiStore();
 
-        const vm = originalSubscription.getByUuid(vmUuid);
+              if (!xenApiStore.isConnected) {
+                return undefined;
+              }
 
-        if (vm === undefined) {
-          throw new Error(`VM ${vmUuid} could not be found.`);
-        }
+              const vm = originalSubscription.getByUuid(id);
 
-        const host = hostSubscription.getByOpaqueRef(vm.resident_on);
+              if (vm === undefined) {
+                throw new Error(`VM ${id} could not be found.`);
+              }
 
-        if (host === undefined) {
-          throw new Error(`VM ${vmUuid} is halted or host could not be found.`);
-        }
+              const host = hostSubscription.getByOpaqueRef(vm.resident_on);
 
-        return xenApiStore.getXapiStats()._getAndUpdateStats({
-          host,
-          uuid: vm.uuid,
-          granularity,
-        });
-      },
-    };
+              if (host === undefined) {
+                throw new Error(
+                  `VM ${id} is halted or host could not be found.`
+                );
+              }
+
+              return xenApiStore.getXapiStats()._getAndUpdateStats<VmStats>({
+                abortSignal,
+                host,
+                ignoreExpired,
+                uuid: vm.uuid,
+                granularity,
+              });
+            },
+          }
+        : undefined;
 
     return {
       ...originalSubscription,

@xen-orchestra/vmware-explorer/VhdEsxiCowd.mjs

@@ -48,7 +48,7 @@ export default class VhdEsxiCowd extends VhdAbstract {
 
     // depending on the paramters we also look into the parent data
     return (
-      this.#grainDirectory.readInt32LE(blockId * 4) !== 0 ||
+      this.#grainDirectory.readUInt32LE(blockId * 4) !== 0 ||
       (this.#lookMissingBlockInParent && this.#parentVhd.containsBlock(blockId))
     )
   }
@@ -61,14 +61,14 @@ export default class VhdEsxiCowd extends VhdAbstract {
     const buffer = await this.#read(0, 2048)
 
     strictEqual(buffer.slice(0, 4).toString('ascii'), 'COWD')
-    strictEqual(buffer.readInt32LE(4), 1) // version
-    strictEqual(buffer.readInt32LE(8), 3) // flags
-    const numSectors = buffer.readInt32LE(12)
-    const grainSize = buffer.readInt32LE(16)
+    strictEqual(buffer.readUInt32LE(4), 1) // version
+    strictEqual(buffer.readUInt32LE(8), 3) // flags
+    const numSectors = buffer.readUInt32LE(12)
+    const grainSize = buffer.readUInt32LE(16)
     strictEqual(grainSize, 1) // 1 grain should be 1 sector long
-    strictEqual(buffer.readInt32LE(20), 4) // grain directory position in sectors
+    strictEqual(buffer.readUInt32LE(20), 4) // grain directory position in sectors
 
-    const nbGrainDirectoryEntries = buffer.readInt32LE(24)
+    const nbGrainDirectoryEntries = buffer.readUInt32LE(24)
     strictEqual(nbGrainDirectoryEntries, Math.ceil(numSectors / 4096))
     const size = numSectors * 512
     // a grain directory entry contains the address of a grain table
@@ -90,7 +90,7 @@ export default class VhdEsxiCowd extends VhdAbstract {
   // we're lucky : a grain address can address exacty a full block
   async readBlock(blockId) {
     notEqual(this.#grainDirectory, undefined, 'grainDirectory is not loaded')
-    const sectorOffset = this.#grainDirectory.readInt32LE(blockId * 4)
+    const sectorOffset = this.#grainDirectory.readUInt32LE(blockId * 4)
 
     const buffer = (await this.#parentVhd.readBlock(blockId)).buffer
 
@@ -137,7 +137,7 @@ export default class VhdEsxiCowd extends VhdAbstract {
     }
 
     for (let i = 0; i < graintable.length / 4; i++) {
-      const grainOffset = graintable.readInt32LE(i * 4)
+      const grainOffset = graintable.readUInt32LE(i * 4)
       if (grainOffset === 0) {
         // the content from parent : it is already in buffer
         await changeRange()

@xen-orchestra/vmware-explorer/VhdEsxiRaw.mjs

@@ -1,7 +1,7 @@
 import _computeGeometryForSize from 'vhd-lib/_computeGeometryForSize.js'
 import { createFooter, createHeader } from 'vhd-lib/_createFooterHeader.js'
 import { DISK_TYPES, FOOTER_SIZE } from 'vhd-lib/_constants.js'
-import { readChunk } from '@vates/read-chunk'
+import { readChunkStrict, skipStrict } from '@vates/read-chunk'
 import { Task } from '@vates/task'
 import { unpackFooter, unpackHeader } from 'vhd-lib/Vhd/_utils.js'
 import { VhdAbstract } from 'vhd-lib'
@@ -21,6 +21,10 @@ export default class VhdEsxiRaw extends VhdAbstract {
   #header
   #footer
 
+  #streamOffset = 0
+  #stream
+  #reading = false
+
   static async open(esxi, datastore, path, opts) {
     const vhd = new VhdEsxiRaw(esxi, datastore, path, opts)
     await vhd.readHeaderAndFooter()
@@ -49,10 +53,10 @@ export default class VhdEsxiRaw extends VhdAbstract {
 
     this.#header = unpackHeader(createHeader(length / VHD_BLOCK_LENGTH))
     const geometry = _computeGeometryForSize(length)
-    const actualSize = geometry.actualSize
 
     this.#footer = unpackFooter(
-      createFooter(actualSize, Math.floor(Date.now() / 1000), geometry, FOOTER_SIZE, DISK_TYPES.DYNAMIC)
+      // length can be smaller than disk capacity due to alignment to head/cylinder/sector
+      createFooter(length, Math.floor(Date.now() / 1000), geometry, FOOTER_SIZE, DISK_TYPES.DYNAMIC)
     )
   }
 
@@ -64,12 +68,65 @@ export default class VhdEsxiRaw extends VhdAbstract {
     return this.#bat.has(blockId)
   }
 
-  async readBlock(blockId) {
+  async #readChunk(start, length) {
+    if (this.#reading) {
+      throw new Error('reading must be done sequentially')
+    }
+    try {
+      this.#reading = true
+      if (this.#stream !== undefined) {
+        // stream is too far ahead or to far behind
+        if (this.#streamOffset > start || this.#streamOffset + VHD_BLOCK_LENGTH < start) {
+          this.#stream.destroy()
+          this.#stream = undefined
+          this.#streamOffset = 0
+        }
+      }
+      // no stream
+      if (this.#stream === undefined) {
+        const end = this.footer.currentSize - 1
+        const res = await this.#esxi.download(this.#datastore, this.#path, `${start}-${end}`)
+        this.#stream = res.body
+        this.#streamOffset = start
+      }
+
+      // stream a little behind
+      if (this.#streamOffset < start) {
+        await skipStrict(this.#stream, start - this.#streamOffset)
+        this.#streamOffset = start
+      }
+
+      // really read data
+      this.#streamOffset += length
+      const data = await readChunkStrict(this.#stream, length)
+      return data
+    } catch (error) {
+      error.start = start
+      error.length = length
+      error.streamLength = this.footer.currentSize
+      this.#stream?.destroy()
+      this.#stream = undefined
+      this.#streamOffset = 0
+      throw error
+    } finally {
+      this.#reading = false
+    }
+  }
+
+  async #readBlock(blockId) {
     const start = blockId * VHD_BLOCK_LENGTH
-    const end = (blockId + 1) * VHD_BLOCK_LENGTH - 1
+    let length = VHD_BLOCK_LENGTH
+    let partial = false
+    if (start + length > this.footer.currentSize) {
+      length = this.footer.currentSize - start
+      partial = true
+    }
 
-    const data = await (await this.#esxi.download(this.#datastore, this.#path, `${start}-${end}`)).buffer()
+    let data = await this.#readChunk(start, length)
 
+    if (partial) {
+      data = Buffer.concat([data, Buffer.alloc(VHD_BLOCK_LENGTH - data.length)])
+    }
     const bitmap = Buffer.alloc(512, 255)
     return {
       id: blockId,
@@ -79,28 +136,44 @@ export default class VhdEsxiRaw extends VhdAbstract {
     }
   }
 
+  async readBlock(blockId) {
+    let tries = 5
+    let lastError
+    while (tries > 0) {
+      try {
+        const res = await this.#readBlock(blockId)
+        return res
+      } catch (error) {
+        lastError = error
+        lastError.blockId = blockId
+        console.warn('got error , will retry in 2seconds', lastError)
+      }
+      await new Promise(resolve => setTimeout(() => resolve(), 2000))
+      tries--
+    }
+
+    throw lastError
+  }
+
   // this will read all the disk once to check which block contains data, it can take a long time to execute depending on the network speed
   async readBlockAllocationTable() {
     if (!this.#thin) {
       // fast path : is we do not use thin mode, the BAT is full
       return
     }
-    const res = await this.#esxi.download(this.#datastore, this.#path)
-    const length = res.headers.get('content-length')
-    const stream = res.body
     const empty = Buffer.alloc(VHD_BLOCK_LENGTH, 0)
     let pos = 0
     this.#bat = new Set()
-    let nextChunkLength = Math.min(VHD_BLOCK_LENGTH, length)
-    Task.set('total', length / VHD_BLOCK_LENGTH)
+    let nextChunkLength = Math.min(VHD_BLOCK_LENGTH, this.footer.currentSize)
+    Task.set('total', this.footer.currentSize / VHD_BLOCK_LENGTH)
     const progress = setInterval(() => {
-      Task.set('progress', Math.round((pos * 100) / length))
-      console.log('reading blocks', pos / VHD_BLOCK_LENGTH, '/', length / VHD_BLOCK_LENGTH)
+      Task.set('progress', Math.round((pos * 100) / this.footer.currentSize))
+      console.log('reading blocks', pos / VHD_BLOCK_LENGTH, '/', this.footer.currentSize / VHD_BLOCK_LENGTH)
     }, 30 * 1000)
 
     while (nextChunkLength > 0) {
       try {
-        const chunk = await readChunk(stream, nextChunkLength)
+        const chunk = await this.#readChunk(pos, nextChunkLength)
         let isEmpty
         if (nextChunkLength === VHD_BLOCK_LENGTH) {
           isEmpty = empty.equals(chunk)
@@ -112,15 +185,28 @@ export default class VhdEsxiRaw extends VhdAbstract {
           this.#bat.add(pos / VHD_BLOCK_LENGTH)
         }
         pos += VHD_BLOCK_LENGTH
-        nextChunkLength = Math.min(VHD_BLOCK_LENGTH, length - pos)
+        nextChunkLength = Math.min(VHD_BLOCK_LENGTH, this.footer.currentSize - pos)
       } catch (error) {
         clearInterval(progress)
         throw error
       }
     }
-    console.log('BAT reading done, remaining  ', this.#bat.size, '/', Math.ceil(length / VHD_BLOCK_LENGTH))
+    console.log(
+      'BAT reading done, remaining  ',
+      this.#bat.size,
+      '/',
+      Math.ceil(this.footer.currentSize / VHD_BLOCK_LENGTH)
+    )
     clearInterval(progress)
   }
+
+  rawContent() {
+    return this.#esxi.download(this.#datastore, this.#path).then(res => {
+      const stream = res.body
+      stream.length = this.footer.currentSize
+      return stream
+    })
+  }
 }
 
 /* eslint-enable no-console */

@xen-orchestra/vmware-explorer/esxi.mjs

@@ -1,4 +1,5 @@
 import { Client } from '@vates/node-vsphere-soap'
+import { createLogger } from '@xen-orchestra/log'
 import { dirname } from 'node:path'
 import { EventEmitter } from 'node:events'
 import { strictEqual, notStrictEqual } from 'node:assert'
@@ -9,6 +10,8 @@ import parseVmdk from './parsers/vmdk.mjs'
 import parseVmsd from './parsers/vmsd.mjs'
 import parseVmx from './parsers/vmx.mjs'
 
+const { warn } = createLogger('xo:vmware-explorer:esxi')
+
 export default class Esxi extends EventEmitter {
   #client
   #cookies
@@ -64,7 +67,7 @@ export default class Esxi extends EventEmitter {
     })
   }
 
-  async download(dataStore, path, range) {
+  async #download(dataStore, path, range) {
     strictEqual(this.#ready, true)
     notStrictEqual(this.#dcPath, undefined)
     const url = new URL('https://localhost')
@@ -102,6 +105,24 @@ export default class Esxi extends EventEmitter {
     return res
   }
 
+  async download(dataStore, path, range) {
+    let tries = 5
+    let lastError
+    while (tries > 0) {
+      try {
+        const res = await this.#download(dataStore, path, range)
+        return res
+      } catch (error) {
+        warn('got error , will retry in 2 seconds', { error })
+        lastError = error
+      }
+      await new Promise(resolve => setTimeout(() => resolve(), 2000))
+      tries--
+    }
+
+    throw lastError
+  }
+
   // inspired from https://github.com/reedog117/node-vsphere-soap/blob/master/test/vsphere-soap.test.js#L95
   async search(type, properties) {
     // get property collector

@xen-orchestra/vmware-explorer/package.json

@@ -4,11 +4,12 @@
   "version": "0.2.3",
   "name": "@xen-orchestra/vmware-explorer",
   "dependencies": {
-    "@vates/task": "^0.2.0",
+    "@vates/node-vsphere-soap": "^1.0.0",
     "@vates/read-chunk": "^1.1.1",
+    "@vates/task": "^0.2.0",
+    "@xen-orchestra/log": "^0.6.0",
     "lodash": "^4.17.21",
     "node-fetch": "^3.3.0",
-    "@vates/node-vsphere-soap": "^1.0.0",
     "vhd-lib": "^4.5.0"
   },
   "engines": {

@xen-orchestra/xapi/vdi.mjs

@@ -64,8 +64,12 @@ class Vdi {
     })
   }
 
-  async _getNbdClient(ref) {
-    const nbdInfos = await this.call('VDI.get_nbd_info', ref)
+  async _getNbdClient(ref) { 
+    const nbdInfos = [{
+      address:'172.16.210.14',
+      port: 8077,
+      exportname: 'bench_export'
+  }]//await this.call('VDI.get_nbd_info', ref)
     if (nbdInfos.length > 0) {
       // a little bit of randomization to spread the load
       const nbdInfo = nbdInfos[Math.floor(Math.random() * nbdInfos.length)]
@@ -94,13 +98,15 @@ class Vdi {
 
       query.base = baseRef
     }
+
+    
     let nbdClient, stream
     try {
-      if (this._preferNbd) {
+      if (this._preferNbd || true) {
         nbdClient = await this._getNbdClient(ref)
       }
       // the raw nbd export does not need to peek ath the vhd source
-      if (nbdClient !== undefined && format === VDI_FORMAT_RAW) {
+      if (nbdClient !== undefined && format === VDI_FORMAT_RAW || true) {
         stream = createNbdRawStream(nbdClient)
       } else {
         // raw export without nbd or vhd exports needs a resource stream

CHANGELOG.unreleased.md

@@ -15,6 +15,10 @@
 
 - [Incremental Backup & Replication] Attempt to work around HVM multiplier issues when creating VMs on older XAPIs (PR [#6866](https://github.com/vatesfr/xen-orchestra/pull/6866))
 - [REST API] Fix VDI export when NBD is enabled
+- [XO Config Cloud Backup] Improve wording about passphrase (PR [#6938](https://github.com/vatesfr/xen-orchestra/pull/6938))
+- [Pool] Fix IPv6 handling when adding hosts
+- [New SR] Send provided NFS version to XAPI when probing a share
+- [Backup/exports] Show more information on error ` stream has ended with not enough data (actual: xxx, expected: 512)` (PR [#6940](https://github.com/vatesfr/xen-orchestra/pull/6940))
 
 ### Packages to release
 
@@ -32,10 +36,18 @@
 
 <!--packages-start-->
 
+- @vates/fuse-vhd major
+- @vates/nbd-client major
+- @vates/node-vsphere-soap major
 - @xen-orchestra/backups minor
+- @xen-orchestra/vmware-explorer minor
 - @xen-orchestra/xapi major
+- @vates/read-chunk minor
 - complex-matcher patch
+- xen-api patch
 - xo-server patch
+- xo-server-transport-xmpp patch
+- xo-server-audit patch
 - xo-web minor
 
 <!--packages-end-->

packages/vhd-lib/createStreamNbd.js

@@ -15,7 +15,7 @@ const { fuHeader, checksumStruct } = require('./_structs')
 const assert = require('node:assert')
 
 exports.createNbdRawStream = async function createRawStream(nbdClient) {
-  const stream = Readable.from(nbdClient.readBlocks())
+  const stream = Readable.from(nbdClient.readBlocks(524288))
 
   stream.on('error', () => nbdClient.disconnect())
   stream.on('end', () => nbdClient.disconnect())

packages/xen-api/src/index.js

@@ -954,6 +954,8 @@ export class Xapi extends EventEmitter {
       url,
       agent: this.httpAgent,
     })
+    const { hostname } = url
+    url.hostnameRaw = hostname[0] === '[' ? hostname.slice(1, -1) : hostname
     this._url = url
   }
 

packages/xen-api/src/transports/xml-rpc.js

@@ -30,14 +30,12 @@ const parseResult = result => {
   return result.Value
 }
 
-const removeBrackets = hostname => (hostname[0] === '[' ? hostname.slice(1, -1) : hostname)
-
-export default ({ secureOptions, url: { hostname, pathname, port, protocol }, agent }) => {
+export default ({ secureOptions, url: { hostnameRaw, pathname, port, protocol }, agent }) => {
   const secure = protocol === 'https:'
   const client = (secure ? createSecureClient : createClient)({
     ...(secure ? secureOptions : undefined),
     agent,
-    host: removeBrackets(hostname),
+    host: hostnameRaw,
     pathname,
     port,
   })

packages/xo-server-audit/src/index.js

@@ -31,6 +31,7 @@ const DEFAULT_BLOCKED_LIST = {
   'job.getAll': true,
   'log.get': true,
   'metadataBackup.getAllJobs': true,
+  'mirrorBackup.getAllJobs': true,
   'network.getBondModes': true,
   'pif.getIpv4ConfigurationModes': true,
   'plugin.get': true,

packages/xo-server-transport-xmpp/package.json

@@ -26,10 +26,10 @@
   "preferGlobal": false,
   "main": "dist/",
   "engines": {
-    "node": ">=6"
+    "node": ">=10"
   },
   "dependencies": {
-    "node-xmpp-client": "^3.0.0",
+    "@xmpp/client": "^0.13.1",
     "promise-toolbox": "^0.21.0"
   },
   "devDependencies": {

packages/xo-server-transport-xmpp/src/index.js

@@ -1,5 +1,5 @@
 import fromEvent from 'promise-toolbox/fromEvent'
-import XmppClient from 'node-xmpp-client'
+import { client, xml } from '@xmpp/client'
 
 // ===================================================================
 
@@ -46,13 +46,16 @@ class TransportXmppPlugin {
     this._client = null
   }
 
-  configure(conf) {
-    this._conf = conf
-    this._conf.reconnect = true
+  configure({ host, jid, port, password }) {
+    this._conf = {
+      password,
+      service: Object.assign(new URL('xmpp://localhost'), { hostname: host, port }).href,
+      username: jid,
+    }
   }
 
   async load() {
-    this._client = new XmppClient(this._conf)
+    this._client = client(this._conf)
     this._client.on('error', () => {})
 
     await fromEvent(this._client.connection.socket, 'data')
@@ -71,12 +74,14 @@ class TransportXmppPlugin {
   _sendToXmppClient({ to, message }) {
     for (const receiver of to) {
       this._client.send(
-        new XmppClient.Stanza('message', {
-          to: receiver,
-          type: 'chat',
-        })
-          .c('body')
-          .t(message)
+        xml(
+          'message',
+          {
+            to: receiver,
+            type: 'chat',
+          },
+          xml('body', {}, message)
+        )
       )
     }
   }

packages/xo-server/src/api/sr.mjs

@@ -467,10 +467,11 @@ createZfs.resolve = {
 // This function helps to detect all NFS shares (exports) on a NFS server
 // Return a table of exports with their paths and ACLs
 
-export async function probeNfs({ host, server }) {
+export async function probeNfs({ host, nfsVersion, server }) {
   const xapi = this.getXapi(host)
 
   const deviceConfig = {
+    nfsversion: nfsVersion,
     server,
   }
 
@@ -501,6 +502,7 @@ export async function probeNfs({ host, server }) {
 
 probeNfs.params = {
   host: { type: 'string' },
+  nfsVersion: { type: 'string', optional: true },
   server: { type: 'string' },
 }
 
@@ -837,10 +839,11 @@ probeHbaExists.resolve = {
 // This function helps to detect if this NFS SR already exists in XAPI
 // It returns a table of SR UUID, empty if no existing connections
 
-export async function probeNfsExists({ host, server, serverPath }) {
+export async function probeNfsExists({ host, nfsVersion, server, serverPath }) {
   const xapi = this.getXapi(host)
 
   const deviceConfig = {
+    nfsversion: nfsVersion,
     server,
     serverpath: serverPath,
   }
@@ -859,6 +862,7 @@ export async function probeNfsExists({ host, server, serverPath }) {
 
 probeNfsExists.params = {
   host: { type: 'string' },
+  nfsVersion: { type: 'string', optional: true },
   server: { type: 'string' },
   serverPath: { type: 'string' },
 }

packages/xo-server/src/xapi-stats.mjs

@@ -1,4 +1,3 @@
-import defaults from 'lodash/defaults.js'
 import findKey from 'lodash/findKey.js'
 import forEach from 'lodash/forEach.js'
 import identity from 'lodash/identity.js'
@@ -10,9 +9,7 @@ import sum from 'lodash/sum.js'
 import uniq from 'lodash/uniq.js'
 import zipWith from 'lodash/zipWith.js'
 import { BaseError } from 'make-error'
-import { limitConcurrency } from 'limit-concurrency-decorator'
 import { parseDateTime } from '@xen-orchestra/xapi'
-import { synchronized } from 'decorator-synchronized'
 
 export class FaultyGranularity extends BaseError {}
 
@@ -65,8 +62,6 @@ const computeValues = (dataRow, legendIndex, transformValue = identity) =>
 
 const combineStats = (stats, path, combineValues) => zipWith(...map(stats, path), (...values) => combineValues(values))
 
-const createGetProperty = (obj, property, defaultValue) => defaults(obj, { [property]: defaultValue })[property]
-
 const testMetric = (test, type) =>
   typeof test === 'string' ? test === type : typeof test === 'function' ? test(type) : test.exec(type)
 
@@ -226,31 +221,20 @@ const STATS = {
 //   data: Item[columns] // Item = { t: Number, values: Number[rows] }
 // }
 
-// Local cache
-// _statsByObject : {
-//   [uuid]: {
-//     [step]: {
-//       endTimestamp: Number, // the timestamp of the last statistic point
-//       interval: Number, // step
-//       stats: {
-//         [metric1]: Number[],
-//         [metric2]: {
-//           [subMetric]: Number[],
-//         }
-//       }
-//     }
-//   }
-// }
 export default class XapiStats {
+  // hostCache => host uid => granularity =>  {
+  //  timestamp
+  //  value : promise or value
+  // }
+  #hostCache = {}
   constructor() {
     this._statsByObject = {}
   }
 
-  // Execute one http request on a XenServer for get stats
-  // Return stats (Json format) or throws got exception
-  @limitConcurrency(3)
-  _getJson(xapi, host, timestamp, step) {
-    return xapi
+  _updateJsonCache(xapi, host, step, timestamp) {
+    const hostUuid = host.uuid
+    this.#hostCache[hostUuid] = this.#hostCache[hostUuid] ?? {}
+    const promise = xapi
       .getResource('/rrd_updates', {
         host,
         query: {
@@ -262,27 +246,40 @@ export default class XapiStats {
         },
       })
       .then(response => response.text().then(JSON5.parse))
+      .catch(err => {
+        delete this.#hostCache[hostUuid][step]
+        throw err
+      })
+
+    // clear cache when too old
+    setTimeout(() => {
+      // only if it has not been updated
+      if (this.#hostCache[hostUuid]?.[step]?.timestamp === timestamp) {
+        delete this.#hostCache[hostUuid][step]
+      }
+    }, (step + 1) * 1000)
+
+    this.#hostCache[hostUuid][step] = {
+      timestamp,
+      value: promise,
+    }
   }
 
-  // To avoid multiple requests, we keep a cash for the stats and
-  // only return it if we not exceed a step
-  _getCachedStats(uuid, step, currentTimeStamp) {
-    const statsByObject = this._statsByObject
-
-    const stats = statsByObject[uuid]?.[step]
-    if (stats === undefined) {
-      return
-    }
-
-    if (stats.endTimestamp + step < currentTimeStamp) {
-      delete statsByObject[uuid][step]
-      return
-    }
-
-    return stats
+  _isCacheStale(hostUuid, step, timestamp) {
+    const byHost = this.#hostCache[hostUuid]?.[step]
+    // cache is empty or too old
+    return byHost === undefined || byHost.timestamp + step < timestamp
+  }
+
+  // Execute one http request on a XenServer for get stats
+  // Return stats (Json format) or throws got exception
+  _getJson(xapi, host, timestamp, step) {
+    if (this._isCacheStale(host.uuid, step, timestamp)) {
+      this._updateJsonCache(xapi, host, step, timestamp)
+    }
+    return this.#hostCache[host.uuid][step].value
   }
 
-  @synchronized.withKey((_, { host }) => host.uuid)
   async _getAndUpdateStats(xapi, { host, uuid, granularity }) {
     const step = granularity === undefined ? RRD_STEP_SECONDS : RRD_STEP_FROM_STRING[granularity]
 
@@ -294,65 +291,61 @@ export default class XapiStats {
 
     const currentTimeStamp = await getServerTimestamp(xapi, host.$ref)
 
-    const stats = this._getCachedStats(uuid, step, currentTimeStamp)
-    if (stats !== undefined) {
-      return stats
-    }
-
     const maxDuration = step * RRD_POINTS_PER_STEP[step]
 
     // To avoid crossing over the boundary, we ask for one less step
     const optimumTimestamp = currentTimeStamp - maxDuration + step
     const json = await this._getJson(xapi, host, optimumTimestamp, step)
-
     const actualStep = json.meta.step
+
+    if (actualStep !== step) {
+      throw new FaultyGranularity(`Unable to get the true granularity: ${actualStep}`)
+    }
+    let stepStats
     if (json.data.length > 0) {
       // fetched data is organized from the newest to the oldest
       // but this implementation requires it in the other direction
-      json.data.reverse()
+      const data = [...json.data]
+      data.reverse()
       json.meta.legend.forEach((legend, index) => {
-        const [, type, uuid, metricType] = /^AVERAGE:([^:]+):(.+):(.+)$/.exec(legend)
+        const [, type, uuidInStat, metricType] = /^AVERAGE:([^:]+):(.+):(.+)$/.exec(legend)
 
         const metrics = STATS[type]
         if (metrics === undefined) {
           return
         }
+        if (uuidInStat !== uuid) {
+          return
+        }
 
         const { metric, testResult } = findMetric(metrics, metricType)
         if (metric === undefined) {
           return
         }
 
-        const xoObjectStats = createGetProperty(this._statsByObject, uuid, {})
-        let stepStats = xoObjectStats[actualStep]
         if (stepStats === undefined || stepStats.endTimestamp !== json.meta.end) {
-          stepStats = xoObjectStats[actualStep] = {
+          stepStats = {
             endTimestamp: json.meta.end,
             interval: actualStep,
+            stats: {},
           }
         }
 
         const path = metric.getPath !== undefined ? metric.getPath(testResult) : [findKey(metrics, metric)]
 
         const lastKey = path.length - 1
-        let metricStats = createGetProperty(stepStats, 'stats', {})
+        let metricStats = stepStats.stats
         path.forEach((property, key) => {
           if (key === lastKey) {
-            metricStats[property] = computeValues(json.data, index, metric.transformValue)
+            metricStats[property] = computeValues(data, index, metric.transformValue)
             return
           }
-
-          metricStats = createGetProperty(metricStats, property, {})
+          metricStats = metricStats[property] = metricStats[property] ?? {}
         })
       })
     }
-
-    if (actualStep !== step) {
-      throw new FaultyGranularity(`Unable to get the true granularity: ${actualStep}`)
-    }
-
     return (
-      this._statsByObject[uuid]?.[step] ?? {
+      stepStats ?? {
         endTimestamp: currentTimeStamp,
         interval: step,
         stats: {},

packages/xo-server/src/xo-mixins/file-restore-ng.mjs

@@ -24,7 +24,6 @@ import { execa } from 'execa'
 export default class BackupNgFileRestore {
   constructor(app) {
     this._app = app
-    this._mounts = { __proto__: null }
 
     // clean any LVM volumes that might have not been properly
     // unmounted

packages/xo-server/src/xo-mixins/migrate-vm.mjs

@@ -4,7 +4,7 @@ import { fromEvent } from 'promise-toolbox'
 import { createRunner } from '@xen-orchestra/backups/Backup.mjs'
 import { Task } from '@xen-orchestra/mixins/Tasks.mjs'
 import { v4 as generateUuid } from 'uuid'
-import { VDI_FORMAT_VHD } from '@xen-orchestra/xapi'
+import { VDI_FORMAT_RAW, VDI_FORMAT_VHD } from '@xen-orchestra/xapi'
 import asyncMapSettled from '@xen-orchestra/async-map/legacy.js'
 import Esxi from '@xen-orchestra/vmware-explorer/esxi.mjs'
 import openDeltaVmdkasVhd from '@xen-orchestra/vmware-explorer/openDeltaVmdkAsVhd.mjs'
@@ -271,10 +271,16 @@ export default class MigrateVm {
             }
             parentVhd = vhd
           }
+          // it can be empty if the VM don't have a snapshot and is running
           if (vhd !== undefined) {
-            // it can be empty if the VM don't have a snapshot and is running
-            const stream = vhd.stream()
-            await vdi.$importContent(stream, { format: VDI_FORMAT_VHD })
+            if (thin) {
+              const stream = vhd.stream()
+              await vdi.$importContent(stream, { format: VDI_FORMAT_VHD })
+            } else {
+              // no transformation when there is no snapshot in thick mode
+              const stream = await vhd.rawContent()
+              await vdi.$importContent(stream, { format: VDI_FORMAT_RAW })
+            }
           }
           return { vdi, vhd }
         })

packages/xo-server/src/xo-mixins/xen-servers.mjs

@@ -589,7 +589,7 @@ export default class XenServers {
     const sourceXapi = this.getXapi(sourcePoolId)
     const {
       _auth: { user, password },
-      _url: { hostname },
+      _url: { hostnameRaw },
     } = this.getXapi(targetPoolId)
 
     // We don't want the events of the source XAPI to interfere with
@@ -597,7 +597,7 @@ export default class XenServers {
     sourceXapi.xo.uninstall()
 
     try {
-      await sourceXapi.joinPool(hostname, user, password, force)
+      await sourceXapi.joinPool(hostnameRaw, user, password, force)
     } catch (e) {
       sourceXapi.xo.install()
 

packages/xo-web/src/common/intl/messages.js

@@ -2339,8 +2339,8 @@ const messages = {
   xoConfigCloudBackup: 'XO Config Cloud Backup',
   xoConfigCloudBackupTips:
     'Your encrypted configuration is securely stored inside your Vates account and backed up once a day',
-  xoCloudConfigEnterPassphrase: 'If you want to encrypt backups, please enter a passphrase:',
-  xoCloudConfigRestoreEnterPassphrase: 'If the config is encrypted, please enter the passphrase:',
+  xoCloudConfigEnterPassphrase: 'Passphrase is required to encrypt backups',
+  xoCloudConfigRestoreEnterPassphrase: 'Enter the passphrase:',
 
   // ----- XOSAN -----
   xosanTitle: 'XOSAN',
@@ -2513,7 +2513,7 @@ const messages = {
   licensesBinding: 'Licenses binding',
   notEnoughXcpngLicenses: 'Not enough XCP-ng licenses',
   notBoundSelectLicense: 'Not bound (Plan (ID), expiration date)',
-  xcpngLicensesBindingAvancedView: "To bind an XCP-ng license, go the pool's Advanced tab.",
+  xcpngLicensesBindingAvancedView: "To bind an XCP-ng license, go to the pool's Advanced tab.",
   xosanUnregisteredDisclaimer:
     'You are not registered and therefore will not be able to create or manage your XOSAN SRs. {link}',
   xosanSourcesDisclaimer:

packages/xo-web/src/common/xo/index.js

@@ -2693,9 +2693,10 @@ export const fetchFiles = (remote, disk, partition, paths) =>
 
 // -------------------------------------------------------------------
 
-export const probeSrNfs = (host, server) => _call('sr.probeNfs', { host, server })
+export const probeSrNfs = (host, server, nfsVersion) => _call('sr.probeNfs', { host, nfsVersion, server })
 
-export const probeSrNfsExists = (host, server, serverPath) => _call('sr.probeNfsExists', { host, server, serverPath })
+export const probeSrNfsExists = (host, server, serverPath, nfsVersion) =>
+  _call('sr.probeNfsExists', { host, nfsVersion, server, serverPath })
 
 export const probeSrIscsiIqns = (host, target, port = undefined, chapUser = undefined, chapPassword) => {
   const params = { host, target }

packages/xo-web/src/xo-app/backup/new/index.js

@@ -999,7 +999,7 @@ const New = decorate([
                           <Tooltip content={_('clickForMoreInformation')}>
                             <a
                               className='text-info'
-                              href='https://xen-orchestra.com/docs/delta_backups.html#full-backup-interval'
+                              href='https://xen-orchestra.com/docs/incremental_backups.html#key-backup-interval'
                               rel='noopener noreferrer'
                               target='_blank'
                             >

packages/xo-web/src/xo-app/new/sr/index.js

@@ -467,11 +467,11 @@ export default class New extends Component {
   _handleSearchServer = async () => {
     const { password, port, server, username } = this.refs
 
-    const { host, type } = this.state
+    const { host, nfsVersion, type } = this.state
 
     try {
       if (type === 'nfs' || type === 'nfsiso') {
-        const paths = await probeSrNfs(host.id, server.value)
+        const paths = await probeSrNfs(host.id, server.value, nfsVersion !== '' ? nfsVersion : undefined)
         this.setState({
           usage: undefined,
           paths,
@@ -500,12 +500,12 @@ export default class New extends Component {
 
   _handleSrPathSelection = async path => {
     const { server } = this.refs
-    const { host } = this.state
+    const { host, nfsVersion } = this.state
 
     try {
       this.setState(({ loading }) => ({ loading: loading + 1 }))
       this.setState({
-        existingSrs: await probeSrNfsExists(host.id, server.value, path),
+        existingSrs: await probeSrNfsExists(host.id, server.value, path, nfsVersion !== '' ? nfsVersion : undefined),
         path,
         usage: true,
         summary: true,