fix(@xen-orchestra/vmware-explorer): cleanup

@xen-orchestra/vmware-explorer/VhdEsxiSeSparse.mjs

@@ -4,6 +4,9 @@ import { FOOTER_SIZE } from 'vhd-lib/_constants.js'
 import { notEqual, strictEqual } from 'node:assert'
 import { unpackFooter, unpackHeader } from 'vhd-lib/Vhd/_utils.js'
 import { VhdAbstract } from 'vhd-lib'
+import { createLogger } from '@xen-orchestra/log'
+
+const { debug } = createLogger('xen-orchestra:vmware-explorer:vhdesxisesparse')
 
 // from https://github.com/qemu/qemu/commit/98eb9733f4cf2eeab6d12db7e758665d2fd5367b#
 
@@ -88,6 +91,9 @@ export default class VhdEsxiSeSparse extends VhdAbstract {
   async readHeaderAndFooter() {
     const buffer = await this.#read(0, 2048)
     strictEqual(buffer.readBigInt64LE(0), 0xcafebaben)
+    for (let i = 0; i < 2048 / 8; i++) {
+      debug(i, '> ', buffer.readBigInt64LE(8 * i).toString(16), buffer.readBigInt64LE(8 * i))
+    }
 
     strictEqual(readInt64(buffer, 1), 0x200000001) // version 2.1
 
@@ -98,6 +104,14 @@ export default class VhdEsxiSeSparse extends VhdAbstract {
     const grain_tables_size = readInt64(buffer, 19)
     this.#grainOffset = readInt64(buffer, 24)
 
+    debug({
+      capacity,
+      grain_size,
+      grain_tables_offset,
+      grain_tables_size,
+      grainSize: this.#grainSize,
+    })
+
     this.#grainSize = grain_size * 512 // 8 sectors / 4KB default
     this.#grainTableOffset = grain_tables_offset * 512
     this.#grainTableSize = grain_tables_size * 512
@@ -112,10 +126,12 @@ export default class VhdEsxiSeSparse extends VhdAbstract {
   }
 
   async readBlockAllocationTable() {
+    debug('READ BLOCK ALLOCATION', this.#grainTableSize)
     const CHUNK_SIZE = 64 * 512
 
     strictEqual(this.#grainTableSize % CHUNK_SIZE, 0)
 
+    debug(' will read ', this.#grainTableSize / CHUNK_SIZE, 'table')
     for (let chunkIndex = 0, grainIndex = 0; chunkIndex < this.#grainTableSize / CHUNK_SIZE; chunkIndex++) {
       process.stdin.write('.')
       const start = chunkIndex * CHUNK_SIZE + this.#grainTableOffset
@@ -130,11 +146,15 @@ export default class VhdEsxiSeSparse extends VhdAbstract {
             break
         }
         if (entry > 3n) {
+          const intIndex = +(((entry & 0x0fff000000000000n) >> 48n) | ((entry & 0x0000ffffffffffffn) << 12n))
+          const pos = intIndex * this.#grainSize + CHUNK_SIZE * chunkIndex + this.#grainOffset
+          debug({ indexInChunk, grainIndex, intIndex, pos })
           this.#grainMap.set(grainIndex)
           grainIndex++
         }
       }
     }
+    debug('found', this.#grainMap.size)
 
     // read grain directory and the grain tables
     const nbBlocks = this.header.maxTableEntries

@xen-orchestra/vmware-explorer/index.mjs

@@ -0,0 +1,5 @@
+import Esxi from './esxi.mjs'
+import openDeltaVmdkasVhd from './openDeltaVmdkAsVhd.mjs'
+import VhdEsxiRaw from './VhdEsxiRaw.mjs'
+
+export { openDeltaVmdkasVhd, Esxi, VhdEsxiRaw }

@xen-orchestra/vmware-explorer/package.json

@@ -4,8 +4,9 @@
   "version": "0.0.3",
   "name": "@xen-orchestra/vmware-explorer",
   "dependencies": {
-    "@vates/task": "^0.0.1",
     "@vates/read-chunk": "^1.0.1",
+    "@vates/task": "^0.0.1",
+    "@xen-orchestra/log": "^0.6.0",
     "lodash": "^4.17.21",
     "node-fetch": "^3.3.0",
     "node-vsphere-soap": "^0.0.2-5",

CHANGELOG.unreleased.md

@@ -12,8 +12,6 @@
 > Users must be able to say: “I had this issue, happy to know it's fixed”
 
 - [Continuous Replication] Fix `VDI_IO_ERROR` when after a VDI has been resized
-- [REST API] Fix VDI import
-- Fix failing imports (REST API and web UI) [Forum#58146](https://xcp-ng.org/forum/post/58146)
 
 ### Packages to release
 
@@ -33,6 +31,5 @@
 
 - @xen-orchestra/backups patch
 - xen-api patch
-- xo-server patch
 
 <!--packages-end-->

docker/Dockerfile

@@ -4,7 +4,7 @@ FROM ubuntu:xenial
 # https://qastack.fr/programming/25899912/how-to-install-nvm-in-docker
 
 RUN apt-get update
-RUN apt-get install -y curl qemu-utils vmdk-stream-converter git libxml2-utils libfuse2 nbdkit
+RUN apt-get install -y curl qemu-utils blktap-utils vmdk-stream-converter git libxml2-utils libfuse2 nbdkit
 ENV NVM_DIR /usr/local/nvm
 RUN mkdir -p /usr/local/nvm
 RUN cd /usr/local/nvm

packages/vhd-lib/tests/utils.integ.spec.js

@@ -1,40 +0,0 @@
-'use strict'
-
-/* eslint-env jest */
-
-const fs = require('fs-extra')
-const rimraf = require('rimraf')
-const tmp = require('tmp')
-const { getSyncedHandler } = require('@xen-orchestra/fs')
-const { pFromCallback } = require('promise-toolbox')
-
-const { checkFile, createRandomFile, convertFromRawToVhd } = require('./utils')
-
-let tempDir = null
-let disposeHandler
-
-beforeEach(async () => {
-  tempDir = await pFromCallback(cb => tmp.dir(cb))
-
-  const d = await getSyncedHandler({ url: `file://${tempDir}` })
-  disposeHandler = d.dispose
-})
-
-afterEach(async () => {
-  await rimraf(tempDir)
-  disposeHandler()
-})
-
-test('checkFile fails with unvalid VHD file', async () => {
-  const initalSizeInMB = 4
-  const rawFileName = `${tempDir}/randomfile`
-  await createRandomFile(rawFileName, initalSizeInMB)
-  const vhdFileName = `${tempDir}/vhdFile.vhd`
-  await convertFromRawToVhd(rawFileName, vhdFileName)
-
-  await checkFile(vhdFileName)
-
-  const sizeToTruncateInByte = 250000
-  await fs.truncate(vhdFileName, sizeToTruncateInByte)
-  await expect(async () => await checkFile(vhdFileName)).rejects.toThrow()
-})

packages/vhd-lib/tests/utils.js

@@ -5,7 +5,6 @@ const { pipeline } = require('readable-stream')
 const asyncIteratorToStream = require('async-iterator-to-stream')
 const execa = require('execa')
 const fs = require('fs-extra')
-const fsPromise = require('node:fs/promises')
 const { randomBytes } = require('crypto')
 
 const createRandomStream = asyncIteratorToStream(function* (size) {
@@ -22,11 +21,7 @@ async function createRandomFile(name, sizeMB) {
 exports.createRandomFile = createRandomFile
 
 async function checkFile(vhdName) {
-  // Since the qemu-img check command isn't compatible with vhd format, we use
-  // the convert command to do a check by conversion. Indeed, the conversion will
-  // fail if the source file isn't a proper vhd format.
-  await execa('qemu-img', ['convert', '-fvpc', '-Oqcow2', vhdName, 'outputFile.qcow2'])
-  await fsPromise.unlink('./outputFile.qcow2')
+  await execa('vhd-util', ['check', '-p', '-b', '-t', '-n', vhdName])
 }
 exports.checkFile = checkFile
 

packages/xen-api/examples/import-vdi

@@ -39,12 +39,7 @@ defer(async ($defer, argv) => {
   $defer(() => xapi.disconnect())
 
   const { cancel, token } = CancelToken.source()
-  process.once('SIGINT', () => {
-    cancel()
-    process.once('SIGINT', () => {
-      process.exit(1)
-    })
-  })
+  process.on('SIGINT', cancel)
 
   let input = createInputStream(opts._[1])
   $defer.onFailure(() => input.destroy())
@@ -80,5 +75,7 @@ defer(async ($defer, argv) => {
     },
   })
 
-  console.log(result !== undefined ? result : 'ok')
+  if (result !== undefined) {
+    console.log(result)
+  }
 })(process.argv.slice(2)).catch(console.error.bind(console, 'error'))

packages/xen-api/src/index.js

@@ -11,18 +11,8 @@ import { coalesceCalls } from '@vates/coalesce-calls'
 import { Collection } from 'xo-collection'
 import { EventEmitter } from 'events'
 import { Index } from 'xo-collection/index'
+import { cancelable, defer, fromCallback, fromEvents, ignoreErrors, pDelay, pRetry, pTimeout } from 'promise-toolbox'
 import { limitConcurrency } from 'limit-concurrency-decorator'
-import {
-  cancelable,
-  CancelToken,
-  defer,
-  fromCallback,
-  fromEvent,
-  ignoreErrors,
-  pDelay,
-  pRetry,
-  pTimeout,
-} from 'promise-toolbox'
 
 import autoTransport from './transports/auto'
 import debug from './_debug'
@@ -100,7 +90,6 @@ export class Xapi extends EventEmitter {
       opts.syncStackTraces ?? process.env.NODE_ENV === 'development' ? addSyncStackTrace : identity
     this._callTimeout = makeCallSetting(opts.callTimeout, 60 * 60 * 1e3) // 1 hour but will be reduced in the future
     this._httpInactivityTimeout = opts.httpInactivityTimeout ?? 5 * 60 * 1e3 // 5 mins
-    this._httpTimeout = opts.httpTimeout ?? 24 * 60 * 60 * 1e3 // 24 hours
     this._eventPollDelay = opts.eventPollDelay ?? 60 * 1e3 // 1 min
     this._pool = null
     this._readOnly = Boolean(opts.readOnly)
@@ -163,14 +152,13 @@ export class Xapi extends EventEmitter {
       this._resolveObjectsFetched = resolve
     })
     this._eventWatchers = { __proto__: null }
-    this._taskWatchers = undefined // set in _watchEvents
+    this._taskWatchers = { __proto__: null }
     this._watchedTypes = undefined
     const { watchEvents } = opts
     if (watchEvents !== false) {
       if (Array.isArray(watchEvents)) {
         this._watchedTypes = watchEvents
       }
-
       this.watchEvents()
     }
   }
@@ -380,13 +368,6 @@ export class Xapi extends EventEmitter {
 
   @cancelable
   async getResource($cancelToken, pathname, { host, query, task } = {}) {
-    const timeout = this._httpTimeout
-    if (timeout !== 0) {
-      const source = CancelToken.source([$cancelToken])
-      setTimeout(source.cancel, timeout)
-      $cancelToken = source.token
-    }
-
     const taskRef = await this._autoTask(task, `Xapi#getResource ${pathname}`)
 
     query = { ...query, session_id: this.sessionId }
@@ -449,13 +430,6 @@ export class Xapi extends EventEmitter {
       throw new Error('cannot put resource in read only mode')
     }
 
-    const timeout = this._httpTimeout
-    if (timeout !== 0) {
-      const source = CancelToken.source([$cancelToken])
-      setTimeout(source.cancel, timeout)
-      $cancelToken = source.token
-    }
-
     const taskRef = await this._autoTask(task, `Xapi#putResource ${pathname}`)
 
     query = { ...query, session_id: this.sessionId }
@@ -544,10 +518,6 @@ export class Xapi extends EventEmitter {
           )
         : doRequest(url.href)
     )
-    const responseEnd = fromEvent(response, 'end')
-    responseEnd.catch(noop)
-
-    console.log({ useHack })
 
     if (pTaskResult !== undefined) {
       if (useHack) {
@@ -568,12 +538,7 @@ export class Xapi extends EventEmitter {
 
     const { req } = response
     if (!req.finished) {
-      console.log('waiting for request to finish')
-      await new Promise((resolve, reject) => {
-        req.on('finish', resolve).on('error', reject)
-        response.on('error', reject)
-      })
-      console.log('request finished')
+      await fromEvents(req, ['close', 'finish'])
     }
 
     if (useHack) {
@@ -581,9 +546,6 @@ export class Xapi extends EventEmitter {
     } else {
       // consume the response
       response.resume()
-      await new Promise((resolve, reject) => {
-        response.on('end', resolve).on('error', reject)
-      })
     }
 
     return pTaskResult
@@ -997,7 +959,7 @@ export class Xapi extends EventEmitter {
       }
 
       const taskWatchers = this._taskWatchers
-      const taskWatcher = taskWatchers?.[ref]
+      const taskWatcher = taskWatchers[ref]
       if (taskWatcher !== undefined) {
         const result = getTaskResult(object)
         if (result !== undefined) {
@@ -1099,7 +1061,7 @@ export class Xapi extends EventEmitter {
     }
 
     const taskWatchers = this._taskWatchers
-    const taskWatcher = taskWatchers?.[ref]
+    const taskWatcher = taskWatchers[ref]
     if (taskWatcher !== undefined) {
       const error = new Error('task has been destroyed before completion')
       error.task = object
@@ -1117,13 +1079,6 @@ export class Xapi extends EventEmitter {
   _watchEvents = coalesceCalls(this._watchEvents)
   // eslint-disable-next-line no-dupe-class-members
   async _watchEvents() {
-    {
-      const watchedTypes = this._watchedTypes
-      if (this._taskWatchers === undefined && (watchedTypes === undefined || watchedTypes.includes('task'))) {
-        this._taskWatchers = { __proto__: null }
-      }
-    }
-
     // eslint-disable-next-line no-labels
     mainLoop: while (true) {
       if (this._resolveObjectsFetched === undefined) {

packages/xo-server/config.toml

@@ -132,10 +132,7 @@ port = 80
 #
 # This breaks a number of XO use cases, for instance uploading a VDI via the
 # REST API, therefore it's changed to 1 day.
-#
-# Completely disabled for now because it appears to be broken:
-# https://github.com/nodejs/node/issues/46574
-requestTimeout = 0
+requestTimeout = 86400000
 
 [http.mounts]
 '/' = '../xo-web/dist'

packages/xo-server/src/xo-mixins/migrate-vm.mjs

@@ -6,10 +6,8 @@ import { Task } from '@xen-orchestra/mixins/Tasks.mjs'
 import { v4 as generateUuid } from 'uuid'
 import { VDI_FORMAT_VHD } from '@xen-orchestra/xapi'
 import asyncMapSettled from '@xen-orchestra/async-map/legacy.js'
-import Esxi from '@xen-orchestra/vmware-explorer/esxi.mjs'
-import openDeltaVmdkasVhd from '@xen-orchestra/vmware-explorer/openDeltaVmdkAsVhd.mjs'
+import { openDeltaVmdkasVhd, Esxi, VhdEsxiRaw } from '@xen-orchestra/vmware-explorer'
 import OTHER_CONFIG_TEMPLATE from '../xapi/other-config-template.mjs'
-import VhdEsxiRaw from '@xen-orchestra/vmware-explorer/VhdEsxiRaw.mjs'
 
 export default class MigrateVm {
   constructor(app) {

packages/xo-server/src/xo-mixins/rest-api.mjs

@@ -150,32 +150,6 @@ export default class RestApi {
         )
       })
     )
-
-    // should go before routes /:collection/:object because they will match but
-    // will not work due to the extension being included in the object identifer
-    api.get(
-      '/:collection(vdis|vdi-snapshots)/:object.:format(vhd|raw)',
-      wrap(async (req, res) => {
-        const stream = await req.xapiObject.$exportContent({ format: req.params.format })
-
-        stream.headers['content-disposition'] = 'attachment'
-        res.writeHead(stream.statusCode, stream.statusMessage != null ? stream.statusMessage : '', stream.headers)
-
-        await fromCallback(pipeline, stream, res)
-      })
-    )
-    api.get(
-      '/:collection(vms|vm-snapshots|vm-templates)/:object.xva',
-      wrap(async (req, res) => {
-        const stream = await req.xapiObject.$export({ compress: req.query.compress })
-
-        stream.headers['content-disposition'] = 'attachment'
-        res.writeHead(stream.statusCode, stream.statusMessage != null ? stream.statusMessage : '', stream.headers)
-
-        await fromCallback(pipeline, stream, res)
-      })
-    )
-
     api.get('/:collection/:object', (req, res) => {
       res.json(req.xoObject)
     })
@@ -199,7 +173,7 @@ export default class RestApi {
     )
 
     api.post(
-      '/:collection(srs)/:object/vdis',
+      '/srs/:object/vdis',
       wrap(async (req, res) => {
         const sr = req.xapiObject
         req.length = +req.headers['content-length']
@@ -222,5 +196,29 @@ export default class RestApi {
         res.sendStatus(200)
       })
     )
+
+    api.get(
+      '/:collection(vdis|vdi-snapshots)/:object.:format(vhd|raw)',
+      wrap(async (req, res) => {
+        const stream = await req.xapiObject.$exportContent({ format: req.params.format })
+
+        stream.headers['content-disposition'] = 'attachment'
+        res.writeHead(stream.statusCode, stream.statusMessage != null ? stream.statusMessage : '', stream.headers)
+
+        await fromCallback(pipeline, stream, res)
+      })
+    )
+
+    api.get(
+      '/:collection(vms|vm-snapshots|vm-templates)/:object.xva',
+      wrap(async (req, res) => {
+        const stream = await req.xapiObject.$export({ compress: req.query.compress })
+
+        stream.headers['content-disposition'] = 'attachment'
+        res.writeHead(stream.statusCode, stream.statusMessage != null ? stream.statusMessage : '', stream.headers)
+
+        await fromCallback(pipeline, stream, res)
+      })
+    )
   }
 }

packages/xo-vmdk-to-vhd/src/vmdk-to-vhd.integ.spec.js

@@ -12,7 +12,6 @@ import { vmdkToVhd, readVmdkGrainTable } from '.'
 import VMDKDirectParser from './vmdk-read'
 import { generateVmdkData } from './vmdk-generate'
 import asyncIteratorToStream from 'async-iterator-to-stream'
-import fs from 'fs'
 
 const initialDir = process.cwd()
 jest.setTimeout(100000)
@@ -37,14 +36,6 @@ function bufferToArray(buffer) {
   return res
 }
 
-async function checkFile(vhdName) {
-  // Since the qemu-img check command isn't compatible with vhd format, we use
-  // the convert command to do a check by conversion. Indeed, the conversion will
-  // fail if the source file isn't a proper vhd format.
-  await execa('qemu-img', ['convert', '-fvpc', '-Oqcow2', vhdName, 'outputFile.qcow2'])
-  await fs.promises.unlink('./outputFile.qcow2')
-}
-
 function createFileAccessor(file) {
   return async (start, end) => {
     if (start < 0 || end < 0) {
@@ -80,7 +71,7 @@ test('VMDK to VHD can convert a random data file with VMDKDirectParser', async (
       )
     ).pipe(createWriteStream(vhdFileName))
     await fromEvent(pipe, 'finish')
-    await checkFile(vhdFileName)
+    await execa('vhd-util', ['check', '-p', '-b', '-t', '-n', vhdFileName])
     await execa('qemu-img', ['convert', '-fvmdk', '-Oraw', vmdkFileName, reconvertedFromVmdk])
     await execa('qemu-img', ['convert', '-fvpc', '-Oraw', vhdFileName, reconvertedFromVhd])
     await execa('qemu-img', ['compare', inputRawFileName, vhdFileName])