diff --git a/@xen-orchestra/vmware-explorer/.npmignore b/@xen-orchestra/vmware-explorer/.npmignore new file mode 120000 index 000000000..008d1b9b9 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/.npmignore @@ -0,0 +1 @@ +../../scripts/npmignore \ No newline at end of file diff --git a/@xen-orchestra/vmware-explorer/VhdEsxiCowd.mjs b/@xen-orchestra/vmware-explorer/VhdEsxiCowd.mjs new file mode 100644 index 000000000..95652d139 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/VhdEsxiCowd.mjs @@ -0,0 +1,162 @@ +import _computeGeometryForSize from 'vhd-lib/_computeGeometryForSize.js' +import { createFooter, createHeader } from 'vhd-lib/_createFooterHeader.js' +import { FOOTER_SIZE } from 'vhd-lib/_constants.js' +import { notEqual, strictEqual } from 'node:assert' +import { unpackFooter, unpackHeader } from 'vhd-lib/Vhd/_utils.js' +import { VhdAbstract } from 'vhd-lib' + +export default class VhdEsxiCowd extends VhdAbstract { + #esxi + #datastore + #parentVhd + #path + #lookMissingBlockInParent + + #header + #footer + + #grainDirectory + + static async open(esxi, datastore, path, parentVhd, opts) { + const vhd = new VhdEsxiCowd(esxi, datastore, path, parentVhd, opts) + await vhd.readHeaderAndFooter() + return vhd + } + constructor(esxi, datastore, path, parentVhd, { lookMissingBlockInParent = true } = {}) { + super() + this.#esxi = esxi + this.#path = path + this.#datastore = datastore + this.#parentVhd = parentVhd + this.#lookMissingBlockInParent = lookMissingBlockInParent + } + + get header() { + return this.#header + } + + get footer() { + return this.#footer + } + + containsBlock(blockId) { + notEqual(this.#grainDirectory, undefined, "bat must be loaded to use contain blocks'") + // only check if a grain table exist for on of the sector of the block + // the great news is that a grain size has 4096 entries of 512B = 2M + // and a vhd block is also 2M + // so we only need to check if a grain table exists (it's not created without data) + + // depending on the paramters we also look into the parent data + return ( + this.#grainDirectory.readInt32LE(blockId * 4) !== 0 || + (this.#lookMissingBlockInParent && this.#parentVhd.containsBlock(blockId)) + ) + } + + async #read(start, length) { + return (await this.#esxi.download(this.#datastore, this.#path, `${start}-${start + length - 1}`)).buffer() + } + + async readHeaderAndFooter() { + const buffer = await this.#read(0, 2048) + + strictEqual(buffer.slice(0, 4).toString('ascii'), 'COWD') + strictEqual(buffer.readInt32LE(4), 1) // version + strictEqual(buffer.readInt32LE(8), 3) // flags + const numSectors = buffer.readInt32LE(12) + const grainSize = buffer.readInt32LE(16) + strictEqual(grainSize, 1) // 1 grain should be 1 sector long + strictEqual(buffer.readInt32LE(20), 4) // grain directory position in sectors + + const nbGrainDirectoryEntries = buffer.readInt32LE(24) + strictEqual(nbGrainDirectoryEntries, Math.ceil(numSectors / 4096)) + const size = numSectors * 512 + // a grain directory entry contains the address of a grain table + // a grain table can adresses at most 4096 grain of 512 Bytes of data + this.#header = unpackHeader(createHeader(Math.ceil(size / (4096 * 512)))) + + const geometry = _computeGeometryForSize(size) + const actualSize = geometry.actualSize + this.#footer = unpackFooter( + createFooter(actualSize, Math.floor(Date.now() / 1000), geometry, FOOTER_SIZE, this.#parentVhd.footer.diskType) + ) + } + + async readBlockAllocationTable() { + const nbBlocks = this.header.maxTableEntries + this.#grainDirectory = await this.#read(2048 /* header length */, nbBlocks * 4) + } + + // we're lucky : a grain address can address exacty a full block + async readBlock(blockId) { + notEqual(this.#grainDirectory, undefined, 'grainDirectory is not loaded') + const sectorOffset = this.#grainDirectory.readInt32LE(blockId * 4) + + const buffer = (await this.#parentVhd.readBlock(blockId)).buffer + + if (sectorOffset === 0) { + strictEqual(this.#lookMissingBlockInParent, true, "shouldn't have empty block in a delta alone") + return { + id: blockId, + bitmap: buffer.slice(0, 512), + data: buffer.slice(512), + buffer, + } + } + const offset = sectorOffset * 512 + + const graintable = await this.#read(offset, 4096 * 4 /* grain table length */) + + strictEqual(graintable.length, 4096 * 4) + // we have no guaranty that data are ordered or contiguous + // let's construct ranges to limit the number of queries + let rangeStart, offsetStart, offsetEnd + + const changeRange = async (index, offset) => { + if (offsetStart !== undefined) { + // if there was already a branch + if (offset === offsetEnd) { + offsetEnd++ + return + } + const grains = await this.#read(offsetStart * 512, (offsetEnd - offsetStart) * 512) + grains.copy(buffer, (rangeStart + 1) /* block bitmap */ * 512) + } + // start a new range + if (offset) { + // we're at the beginning of a range present in the file + rangeStart = index + offsetStart = offset + offsetEnd = offset + 1 + } else { + // we're at the beginning of a range from the parent or empty + rangeStart = undefined + offsetStart = undefined + offsetEnd = undefined + } + } + + for (let i = 0; i < graintable.length / 4; i++) { + const grainOffset = graintable.readInt32LE(i * 4) + if (grainOffset === 0) { + // the content from parent : it is already in buffer + await changeRange() + } else if (grainOffset === 1) { + await changeRange() + // this is a emptied grain, no data, don't look into parent + buffer.fill(0, (i + 1) /* block bitmap */ * 512) + } else if (grainOffset > 1) { + // non empty grain, read from file + await changeRange(i, grainOffset) + } + } + // close last range + await changeRange() + return { + id: blockId, + bitmap: buffer.slice(0, 512), + data: buffer.slice(512), + buffer, + } + } +} diff --git a/@xen-orchestra/vmware-explorer/VhdEsxiRaw.mjs b/@xen-orchestra/vmware-explorer/VhdEsxiRaw.mjs new file mode 100644 index 000000000..6142f2de8 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/VhdEsxiRaw.mjs @@ -0,0 +1,126 @@ +import _computeGeometryForSize from 'vhd-lib/_computeGeometryForSize.js' +import { createFooter, createHeader } from 'vhd-lib/_createFooterHeader.js' +import { DISK_TYPES, FOOTER_SIZE } from 'vhd-lib/_constants.js' +import { readChunk } from '@vates/read-chunk' +import { Task } from '@vates/task' +import { unpackFooter, unpackHeader } from 'vhd-lib/Vhd/_utils.js' +import { VhdAbstract } from 'vhd-lib' +import assert from 'node:assert' + +/* eslint-disable no-console */ + +// create a thin VHD from a raw disk +const VHD_BLOCK_LENGTH = 2 * 1024 * 1024 +export default class VhdEsxiRaw extends VhdAbstract { + #esxi + #datastore + #path + #thin + + #bat + #header + #footer + + static async open(esxi, datastore, path, opts) { + const vhd = new VhdEsxiRaw(esxi, datastore, path, opts) + await vhd.readHeaderAndFooter() + return vhd + } + + get header() { + return this.#header + } + + get footer() { + return this.#footer + } + + constructor(esxi, datastore, path, { thin = true } = {}) { + super() + this.#esxi = esxi + this.#path = path + this.#datastore = datastore + this.#thin = thin + } + + async readHeaderAndFooter() { + const res = await this.#esxi.download(this.#datastore, this.#path) + const length = res.headers.get('content-length') + + this.#header = unpackHeader(createHeader(length / VHD_BLOCK_LENGTH)) + const geometry = _computeGeometryForSize(length) + const actualSize = geometry.actualSize + + this.#footer = unpackFooter( + createFooter(actualSize, Math.floor(Date.now() / 1000), geometry, FOOTER_SIZE, DISK_TYPES.DYNAMIC) + ) + } + + containsBlock(blockId) { + if (!this.#thin) { + return true + } + assert.notEqual(this.#bat, undefined, 'bat is not loaded') + return this.#bat.has(blockId) + } + + async readBlock(blockId) { + const start = blockId * VHD_BLOCK_LENGTH + const end = (blockId + 1) * VHD_BLOCK_LENGTH - 1 + + const data = await (await this.#esxi.download(this.#datastore, this.#path, `${start}-${end}`)).buffer() + + const bitmap = Buffer.alloc(512, 255) + return { + id: blockId, + bitmap, + data, + buffer: Buffer.concat([bitmap, data]), + } + } + + // this will read all the disk once to check which block contains data, it can take a long time to execute depending on the network speed + async readBlockAllocationTable() { + if (!this.#thin) { + // fast path : is we do not use thin mode, the BAT is full + return + } + const res = await this.#esxi.download(this.#datastore, this.#path) + const length = res.headers.get('content-length') + const stream = res.body + const empty = Buffer.alloc(VHD_BLOCK_LENGTH, 0) + let pos = 0 + this.#bat = new Set() + let nextChunkLength = Math.min(VHD_BLOCK_LENGTH, length) + Task.set('total', length / VHD_BLOCK_LENGTH) + const progress = setInterval(() => { + Task.set('progress', Math.round((pos * 100) / length)) + console.log('reading blocks', pos / VHD_BLOCK_LENGTH, '/', length / VHD_BLOCK_LENGTH) + }, 30 * 1000) + + while (nextChunkLength > 0) { + try { + const chunk = await readChunk(stream, nextChunkLength) + let isEmpty + if (nextChunkLength === VHD_BLOCK_LENGTH) { + isEmpty = empty.equals(chunk) + } else { + // last block can be smaller + isEmpty = Buffer.alloc(nextChunkLength, 0).equals(chunk) + } + if (!isEmpty) { + this.#bat.add(pos / VHD_BLOCK_LENGTH) + } + pos += VHD_BLOCK_LENGTH + nextChunkLength = Math.min(VHD_BLOCK_LENGTH, length - pos) + } catch (error) { + clearInterval(progress) + throw error + } + } + console.log('BAT reading done, remaining ', this.#bat.size, '/', Math.ceil(length / VHD_BLOCK_LENGTH)) + clearInterval(progress) + } +} + +/* eslint-enable no-console */ diff --git a/@xen-orchestra/vmware-explorer/VhdEsxiSeSparse.mjs b/@xen-orchestra/vmware-explorer/VhdEsxiSeSparse.mjs new file mode 100644 index 000000000..e16b7a0e2 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/VhdEsxiSeSparse.mjs @@ -0,0 +1,243 @@ +import _computeGeometryForSize from 'vhd-lib/_computeGeometryForSize.js' +import { createFooter, createHeader } from 'vhd-lib/_createFooterHeader.js' +import { FOOTER_SIZE } from 'vhd-lib/_constants.js' +import { notEqual, strictEqual } from 'node:assert' +import { unpackFooter, unpackHeader } from 'vhd-lib/Vhd/_utils.js' +import { VhdAbstract } from 'vhd-lib' + +// from https://github.com/qemu/qemu/commit/98eb9733f4cf2eeab6d12db7e758665d2fd5367b# + +function readInt64(buffer, index) { + const n = buffer.readBigInt64LE(index * 8 /* size of an int64 in bytes */) + if (n > Number.MAX_SAFE_INTEGER) { + throw new Error(`can't handle ${n} ${Number.MAX_SAFE_INTEGER} ${n & 0x00000000ffffffffn}`) + } + return +n +} + +export default class VhdEsxiSeSparse extends VhdAbstract { + #esxi + #datastore + #parentVhd + #path + #lookMissingBlockInParent + + #header + #footer + + #grainDirectory + // as we will read all grain with data with load everything in memory + // in theory , that can be 512MB of data for a 2TB fully allocated + // but our use case is to transfer a relatively small diff + // and random access is expensive in HTTP, and migration is a one time cors + // so let's go with naive approach, and future me will have to handle a more + // clever approach if necessary + // grain at zero won't be stored + + #grainMap = new Map() + + #grainSize + #grainTableSize + #grainTableOffset + #grainOffset + + static async open(esxi, datastore, path, parentVhd, opts) { + const vhd = new VhdEsxiSeSparse(esxi, datastore, path, parentVhd, opts) + await vhd.readHeaderAndFooter() + return vhd + } + constructor(esxi, datastore, path, parentVhd, { lookMissingBlockInParent = true } = {}) { + super() + this.#esxi = esxi + this.#path = path + this.#datastore = datastore + this.#parentVhd = parentVhd + this.#lookMissingBlockInParent = lookMissingBlockInParent + } + + get header() { + return this.#header + } + + get footer() { + return this.#footer + } + + async #readGrain(start, length = 4 * 1024) { + return (await this.#esxi.download(this.#datastore, this.#path, `${start}-${start + length - 1}`)).buffer() + } + + containsBlock(blockId) { + notEqual(this.#grainDirectory, undefined, "bat must be loaded to use contain blocks'") + + // a grain table is 4096 entries of 4KB + // a grain table cover 8 vhd blocks + // grain table always exists in sespars + + // depending on the paramters we also look into the parent data + return ( + this.#grainDirectory.readInt32LE(blockId * 4) !== 0 || + (this.#lookMissingBlockInParent && this.#parentVhd.containsBlock(blockId)) + ) + } + + async #read(start, end) { + return (await this.#esxi.download(this.#datastore, this.#path, `${start}-${end}`)).buffer() + } + + async readHeaderAndFooter() { + const buffer = await this.#read(0, 2048) + strictEqual(buffer.readBigInt64LE(0), 0xcafebaben) + for (let i = 0; i < 2048 / 8; i++) { + console.log(i, '> ', buffer.readBigInt64LE(8 * i).toString(16), buffer.readBigInt64LE(8 * i)) + } + + strictEqual(readInt64(buffer, 1), 0x200000001) // version 2.1 + + const capacity = readInt64(buffer, 2) + const grain_size = readInt64(buffer, 3) + const grain_table_size = readInt64(buffer, 4) + const flags = readInt64(buffer, 5) + + const grain_dir_offset = readInt64(buffer, 16) + const grain_dir_size = readInt64(buffer, 17) + const grain_tables_offset = readInt64(buffer, 18) + const grain_tables_size = readInt64(buffer, 19) + this.#grainOffset = readInt64(buffer, 24) + + console.log({ + capacity, + grain_size, + grain_table_size, + flags, + grain_dir_offset, + grain_dir_size, + grain_tables_offset, + grain_tables_size, + grainSize: this.#grainSize, + }) + + this.#grainSize = grain_size * 512 // 8 sectors / 4KB default + this.#grainTableOffset = grain_tables_offset * 512 + this.#grainTableSize = grain_tables_size * 512 + + const size = capacity * grain_size * 512 + this.#header = unpackHeader(createHeader(Math.ceil(size / (4096 * 512)))) + const geometry = _computeGeometryForSize(size) + const actualSize = geometry.actualSize + this.#footer = unpackFooter( + createFooter(actualSize, Math.floor(Date.now() / 1000), geometry, FOOTER_SIZE, this.#parentVhd.footer.diskType) + ) + } + + async readBlockAllocationTable() { + console.log('READ BLOCK ALLOCATION', this.#grainTableSize) + const CHUNK_SIZE = 64 * 512 + + strictEqual(this.#grainTableSize % CHUNK_SIZE, 0) + + console.log(' will read ', this.#grainTableSize / CHUNK_SIZE, 'table') + for (let chunkIndex = 0, grainIndex = 0; chunkIndex < this.#grainTableSize / CHUNK_SIZE; chunkIndex++) { + process.stdin.write('.') + const start = chunkIndex * CHUNK_SIZE + this.#grainTableOffset + const end = start + 4096 * 8 - 1 + const buffer = await this.#read(start, end) + for (let indexInChunk = 0; indexInChunk < 4096; indexInChunk++) { + const entry = buffer.readBigInt64LE(indexInChunk * 8) + switch (entry) { + case 0n: // not allocated, go to parent + break + case 1n: // unmapped + break + } + if (entry > 3n) { + const intIndex = +(((entry & 0x0fff000000000000n) >> 48n) | ((entry & 0x0000ffffffffffffn) << 12n)) + let pos = intIndex * this.#grainSize + CHUNK_SIZE * chunkIndex + this.#grainOffset + console.log({ indexInChunk, grainIndex, intIndex, pos }) + this.#grainMap.set(grainIndex) + grainIndex++ + } + } + } + console.log('found', this.#grainMap.size) + + // read grain directory and the grain tables + const nbBlocks = this.header.maxTableEntries + this.#grainDirectory = await this.#read(2048 /* header length */, 2048 + nbBlocks * 4 - 1) + } + + // we're lucky : a grain address can address exacty a full block + async readBlock(blockId) { + notEqual(this.#grainDirectory, undefined, 'grainDirectory is not loaded') + const sectorOffset = this.#grainDirectory.readInt32LE(blockId * 4) + + const buffer = (await this.#parentVhd.readBlock(blockId)).buffer + + if (sectorOffset === 0) { + strictEqual(this.#lookMissingBlockInParent, true, "shouldn't have empty block in a delta alone") + return { + id: blockId, + bitmap: buffer.slice(0, 512), + data: buffer.slice(512), + buffer, + } + } + const offset = sectorOffset * 512 + + const graintable = await this.#read(offset, offset + 4096 * 4 /* grain table length */ - 1) + + strictEqual(graintable.length, 4096 * 4) + // we have no guaranty that data are order or contiguous + // let's construct ranges to limit the number of queries + let rangeStart, offsetStart, offsetEnd, lastOffset + + const changeRange = async (index, offset) => { + if (offsetStart !== undefined) { + // if there was a + if (offset === offsetEnd) { + offsetEnd++ + return + } + const grains = await this.#read(offsetStart * 512, offsetEnd * 512 - 1) + grains.copy(buffer, (rangeStart + 1) /* block bitmap */ * 512) + } + if (offset) { + // we're at the beginning of a range present in the file + rangeStart = index + offsetStart = offset + offsetEnd = offset + 1 + } else { + // we're at the beginning of a range from the parent or empty + rangeStart = undefined + offsetStart = undefined + offsetEnd = undefined + } + } + + for (let i = 0; i < graintable.length / 4; i++) { + const grainOffset = graintable.readInt32LE(i * 4) + if (grainOffset === 0) { + await changeRange() + // from parent + continue + } + if (grainOffset === 1) { + await changeRange() + // this is a emptied grain, no data, don't look into parent + buffer.fill(0, (i + 1) /* block bitmap */ * 512) + } + + if (grainOffset > 1) { + // non empty grain + await changeRange(i, grainOffset) + } + } + await changeRange() + return { + id: blockId, + bitmap: buffer.slice(0, 512), + data: buffer.slice(512), + buffer, + } + } +} diff --git a/@xen-orchestra/vmware-explorer/esxi.mjs b/@xen-orchestra/vmware-explorer/esxi.mjs new file mode 100644 index 000000000..fb8a52f52 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/esxi.mjs @@ -0,0 +1,260 @@ +import { Client } from 'node-vsphere-soap' +import { dirname } from 'node:path' +import { EventEmitter } from 'node:events' +import { strictEqual } from 'node:assert' +import fetch from 'node-fetch' + +import parseVmdk from './parsers/vmdk.mjs' +import parseVmsd from './parsers/vmsd.mjs' +import parseVmx from './parsers/vmx.mjs' + +export default class Esxi extends EventEmitter { + #client + #cookies + #host + #user + #password + #ready = false + + constructor(host, user, password, sslVerify = true) { + super() + this.#host = host + this.#user = user + this.#password = password + // @FIXME this module inject NODE_TLS_REJECT_UNAUTHORIZED into the process env, which is problematic because it disables globally SSL certificate verification + // + // we need to find a fix for this, maybe forking the library + this.#client = new Client(host, user, password, sslVerify) + this.#client.once('ready', () => { + this.#ready = true + this.emit('ready') + }) + this.#client.on('error', err => { + this.emit('error', err) + }) + } + + #exec(cmd, args) { + strictEqual(this.#ready, true) + const client = this.#client + return new Promise(function (resolve, reject) { + client.once('error', function (error) { + client.off('result', resolve) + reject(error) + }) + client.runCommand(cmd, args).once('result', function () { + client.off('error', reject) + resolve(...arguments) + }) + }) + } + + async download(dataStore, path, range) { + strictEqual(this.#ready, true) + const url = `https://${this.#host}/folder/${path}?dsName=${dataStore}` + const headers = {} + if (this.#cookies) { + headers.cookie = this.#cookies + } else { + headers.Authorization = 'Basic ' + Buffer.from(this.#user + ':' + this.#password).toString('base64') + } + if (range) { + headers['content-type'] = 'multipart/byteranges' + headers.Range = 'bytes=' + range + } + const res = await fetch(url, { + method: 'GET', + headers, + highWaterMark: 10 * 1024 * 1024, + }) + if (res.status < 200 || res.status >= 300) { + const error = new Error(res.status + ' ' + res.statusText + ' ' + url) + error.cause = res + throw error + } + if (res.headers.raw()['set-cookie']) { + this.#cookies = res.headers + .raw() + ['set-cookie'].map(cookie => cookie.split(';')[0]) + .join('; ') + } + return res + } + + // inspired from https://github.com/reedog117/node-vsphere-soap/blob/master/test/vsphere-soap.test.js#L95 + async search(type, properties) { + // get property collector + const propertyCollector = this.#client.serviceContent.propertyCollector + // get view manager + const viewManager = this.#client.serviceContent.viewManager + // get root folder + const rootFolder = this.#client.serviceContent.rootFolder + let result = await this.#exec('CreateContainerView', { + _this: viewManager, + container: rootFolder, + type: [type], + recursive: true, + }) + + // build all the data structures needed to query all the vm names + const containerView = result.returnval + + const objectSpec = { + attributes: { 'xsi:type': 'ObjectSpec' }, // setting attributes xsi:type is important or else the server may mis-recognize types! + obj: containerView, + skip: true, + selectSet: [ + { + attributes: { 'xsi:type': 'TraversalSpec' }, + name: 'traverseEntities', + type: 'ContainerView', + path: 'view', + skip: false, + }, + ], + } + + const propertyFilterSpec = { + attributes: { 'xsi:type': 'PropertyFilterSpec' }, + propSet: properties.map(p => ({ + attributes: { 'xsi:type': 'PropertySpec' }, + type, + pathSet: [p], + })), + objectSet: [objectSpec], + } + + result = await this.#exec('RetrievePropertiesEx', { + _this: propertyCollector, + specSet: [propertyFilterSpec], + options: { attributes: { type: 'RetrieveOptions' } }, + }) + + const objects = {} + const returnObj = Array.isArray(result.returnval.objects) ? result.returnval.objects : [result.returnval.objects] + + returnObj.forEach(({ obj, propSet }) => { + objects[obj.$value] = {} + propSet = Array.isArray(propSet) ? propSet : [propSet] + propSet.forEach(({ name, val }) => { + // don't care about the type for now + delete val.attributes + // a scalar value : simplify it + if (val.$value) { + objects[obj.$value][name] = val.$value + } else { + objects[obj.$value][name] = val + } + }) + }) + return objects + } + + async #inspectVmdk(dataStores, currentDataStore, currentPath, filePath) { + let diskDataStore, diskPath + if (filePath.startsWith('/')) { + // disk is on another datastore + Object.keys(dataStores).forEach(dataStoreUrl => { + if (filePath.startsWith(dataStoreUrl)) { + diskDataStore = dataStores[dataStoreUrl].name + diskPath = filePath.substring(dataStoreUrl.length + 1) + } + }) + } else { + diskDataStore = currentDataStore + diskPath = currentPath + '/' + filePath + } + const vmdkRes = await this.download(diskDataStore, diskPath) + const text = await vmdkRes.text() + const parsed = parseVmdk(text) + + return { + ...parsed, + datastore: diskDataStore, + path: dirname(diskPath), + descriptionLabel: ' from esxi', + } + } + + async getTransferableVmMetadata(vmId) { + const search = await this.search('VirtualMachine', ['name', 'config', 'storage', 'runtime', 'snapshot']) + if (search[vmId] === undefined) { + throw new Error(`VM ${vmId} not found `) + } + const { config, runtime } = search[vmId] + + const [, dataStore, vmxPath] = config.files.vmPathName.match(/^\[(.*)\] (.+.vmx)$/) + const res = await this.download(dataStore, vmxPath) + const vmx = parseVmx(await res.text()) + // list datastores + const dataStores = {} + Object.values(await this.search('Datastore', ['summary'])).forEach(({ summary }) => { + dataStores[summary.url] = summary + }) + + const disks = [] + const networks = [] + + for (const key of Object.keys(vmx)) { + if (key.match('^scsi([0-9]+)$') !== null) { + const scsiChannel = vmx[key] + for (const diskIndex in Object.values(scsiChannel)) { + const disk = scsiChannel[diskIndex] + if (typeof disk !== 'object' || disk.deviceType !== 'scsi-hardDisk') { + continue + } + disks.push({ + ...(await this.#inspectVmdk(dataStores, dataStore, dirname(vmxPath), disk.fileName)), + node: `${key}:${diskIndex}`, + }) + } + } + if (key.match('^ethernet([0-9]+)$') !== null) { + const ethernet = vmx[key] + + networks.push({ + label: ethernet.networkName, + macAddress: ethernet.generatedAddress, + isGenerated: ethernet.addressType === 'generated', + }) + } + } + let snapshots + try { + const vmsd = await (await this.download(dataStore, vmxPath.replace(/\.vmx$/, '.vmsd'))).text() + snapshots = parseVmsd(vmsd) + + for (const snapshotIndex in snapshots?.snapshots) { + const snapshot = snapshots.snapshots[snapshotIndex] + for (const diskIndex in snapshot.disks) { + const fileName = snapshot.disks[diskIndex].fileName + snapshot.disks[diskIndex] = { + node: snapshot.disks[diskIndex]?.node, // 'scsi0:0', + ...(await this.#inspectVmdk(dataStores, dataStore, dirname(vmxPath), fileName)), + } + } + } + } catch (error) { + // no vmsd file :fall back to a full withou snapshots + } + + return { + name_label: config.name, + memory: +config.hardware.memoryMB * 1024 * 1024, + numCpu: +config.hardware.numCPU, + guestToolsInstalled: false, + firmware: config.firmware === 'efi' ? 'uefi' : config.firmware, // bios or uefi + powerState: runtime.powerState, + snapshots, + disks, + networks, + } + } + + powerOff(vmId) { + return this.#exec('PowerOffVM_Task', { _this: vmId }) + } + powerOn(vmId) { + return this.#exec('PowerOnVM_Task', { _this: vmId }) + } +} diff --git a/@xen-orchestra/vmware-explorer/index.mjs b/@xen-orchestra/vmware-explorer/index.mjs new file mode 100644 index 000000000..bbabf053e --- /dev/null +++ b/@xen-orchestra/vmware-explorer/index.mjs @@ -0,0 +1,16 @@ +import Esxi from './esxi.mjs' +const host = '10.10.0.62' +const user = 'root' +const password = 'vateslab' +const sslVerify = false + +console.log(Esxi) +const esxi = new Esxi(host, user, password, sslVerify) +console.log(esxi) +esxi.on('ready', async function () { + //const metadata = await esxi.getTransferableVmMetadata('4') + //console.log('metadata', metadata) + + const res = await esxi.powerOn(9) + console.log(res) +}) diff --git a/@xen-orchestra/vmware-explorer/openDeltaVmdkAsVhd.mjs b/@xen-orchestra/vmware-explorer/openDeltaVmdkAsVhd.mjs new file mode 100644 index 000000000..b0ba83ca3 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/openDeltaVmdkAsVhd.mjs @@ -0,0 +1,19 @@ +import VhdEsxiCowd from './VhdEsxiCowd.mjs' +// import VhdEsxiSeSparse from "./VhdEsxiSeSparse.mjs"; + +export default async function openDeltaVmdkasVhd(esxi, datastore, path, parentVhd, opts) { + let vhd + if (path.endsWith('-sesparse.vmdk')) { + throw new Error(`sesparse Vmdk reading is not functionnal yet ${path}`) + // vhd = new VhdEsxiSeSparse(esxi, datastore, path, parentVhd, opts) + } else { + if (path.endsWith('-delta.vmdk')) { + vhd = new VhdEsxiCowd(esxi, datastore, path, parentVhd, opts) + } else { + throw new Error(`Vmdk ${path} does not seems to be a delta vmdk`) + } + } + await vhd.readHeaderAndFooter() + await vhd.readBlockAllocationTable() + return vhd +} diff --git a/@xen-orchestra/vmware-explorer/package.json b/@xen-orchestra/vmware-explorer/package.json new file mode 100644 index 000000000..0d4659cf0 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/package.json @@ -0,0 +1,31 @@ +{ + "license": "ISC", + "private": false, + "version": "0.0.2", + "name": "@xen-orchestra/vmware-explorer", + "dependencies": { + "@vates/task": "^0.0.1", + "@vates/read-chunk": "^1.0.1", + "lodash": "^4.17.21", + "node-fetch": "^3.3.0", + "node-vsphere-soap": "^0.0.2-5", + "vhd-lib": "^4.2.0" + }, + "engines": { + "node": ">=18" + }, + "homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/@xen-orchestra/vmware-explorer", + "bugs": "https://github.com/vatesfr/xen-orchestra/issues", + "repository": { + "directory": "@xen-orchestra/vmware-explorer", + "type": "git", + "url": "https://github.com/vatesfr/xen-orchestra.git" + }, + "author": { + "name": "Vates SAS", + "url": "https://vates.fr" + }, + "scripts": { + "postversion": "npm publish --access public" + } +} diff --git a/@xen-orchestra/vmware-explorer/parsers/vmdk.mjs b/@xen-orchestra/vmware-explorer/parsers/vmdk.mjs new file mode 100644 index 000000000..2274cfc02 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/parsers/vmdk.mjs @@ -0,0 +1,71 @@ +import { strictEqual } from 'node:assert' + +// the vmdk files each contains a disk metadata +export function parseDescriptor(text) { + const descriptorText = text.toString('ascii').replace(/\x00+$/, '') // eslint-disable-line no-control-regex + strictEqual(descriptorText.substr(0, 21), '# Disk DescriptorFile') + const descriptorDict = {} + const extentList = [] + const lines = descriptorText.split(/\r?\n/).filter(line => { + return line.trim().length > 0 && line[0] !== '#' + }) + for (const line of lines) { + const defLine = line.split('=') + // the wonky quote test is to avoid having an equal sign in the name of an extent + if (defLine.length === 2 && defLine[0].indexOf('"') === -1) { + descriptorDict[defLine[0].toLowerCase()] = defLine[1].replace(/['"]+/g, '') + } else { + const [, access, sizeSectors, type, name, offset] = line.match(/([A-Z]+) ([0-9]+) ([A-Z]+) "(.*)" ?(.*)$/) + extentList.push({ access, sizeSectors, type, name, offset }) + } + } + strictEqual(extentList.length, 1, 'only one extent per vmdk is supported') + return { ...descriptorDict, extent: extentList[0] } +} + +// https://github.com/libyal/libvmdk/blob/main/documentation/VMWare%20Virtual%20Disk%20Format%20(VMDK).asciidoc#5-the-cowd-sparse-extent-data-file +// this parser will only handle vmdk files that contains the descriptor but not the data +// the data must be in a separated file ( fileName ) +export default function parseVmdk(raw) { + strictEqual(typeof raw, 'string') + + const descriptor = parseDescriptor(raw) + const isFull = !descriptor.parentfilenamehint + return { + capacity: descriptor.extent.sizeSectors * 512, + isFull, + uid: descriptor.cid, + fileName: descriptor.extent.name, + parentId: descriptor.parentcid, + parentFileName: descriptor.parentfilenamehint, + vmdFormat: descriptor.extent.type, + nameLabel: descriptor.extent.name, + } +} + +/** file content example + * +# Disk DescriptorFile +version=1 +encoding="UTF-8" +CID=d7980f7a +parentCID=ffffffff +isNativeSnapshot="no" +createType="vmfs" + +# Extent description +RW 67108864 VMFS "test flo_0-flat.vmdk" + +# The Disk Data Base +#DDB + +ddb.adapterType = "lsilogic" +ddb.geometry.cylinders = "4177" +ddb.geometry.heads = "255" +ddb.geometry.sectors = "63" +ddb.longContentID = "741a56c9a09190258e814e7ed7980f7a" +ddb.thinProvisioned = "1" +ddb.toolsVersion = "2147483647" +ddb.uuid = "60 00 C2 90 6f ee 5b 7e-d5 51 99 ed 52 8e b8 b0" +ddb.virtualHWVersion = "11" + */ diff --git a/@xen-orchestra/vmware-explorer/parsers/vmsd.mjs b/@xen-orchestra/vmware-explorer/parsers/vmsd.mjs new file mode 100644 index 000000000..dbeeee07c --- /dev/null +++ b/@xen-orchestra/vmware-explorer/parsers/vmsd.mjs @@ -0,0 +1,109 @@ +// the vmsd file contain the snapshot history of the VM , and their chaining + +function set(obj, keyPath, val) { + const [key, ...other] = keyPath + // key like snapshot0->snapshot9 are grouped in an array snapshots[] + const match = key.match(/^(.+)([0-9])$/) + if (match) { + // an array + let [, label, index] = match + // I like my array names in plural form + label += 's' + if (!obj[label]) { + // create array if not exists + obj[label] = [] + } + if (other.length) { + // it contains objects + if (!obj[label][index]) { + // and this object is not already initialized + obj[label][parseInt(index)] = {} + } + set(obj[label][index], other, val) + } else { + // it contains a scalar value + obj[label][index] = val + } + } else { + if (other.length) { + // an object + if (!obj[key]) { + // first time + obj[key] = {} + } + set(obj[key], other, val) + } else { + // a scalar + obj[key] = val + } + } +} + +export default function parseVmsd(text) { + const parsed = {} + text.split('\n').forEach(line => { + const [key, val] = line.split(' = ') + if (!key.startsWith('snapshot')) { + // we only look for the snapshot part of the file + return + } + // remove the " around value + set(parsed, key.split('.'), val?.substring(1, val.length - 1)) + }) + if (parsed.snapshot?.current == undefined) { + return + } + + return { + lastUID: parsed.snapshot.current, + current: parsed.snapshot.current, + numSnapshots: parsed.snapshot.numSnapshots, + snapshots: Object.values(parsed.snapshots) || [], + } +} + +/** file content example + * .encoding = "UTF-8" +snapshot.lastUID = "4" +snapshot.current = "4" +snapshot0.uid = "1" +snapshot0.filename = "test flo-Snapshot1.vmsn" +snapshot0.displayName = "SNAPSHOT POST INSTALL" +snapshot0.description = "blablabla" +snapshot0.createTimeHigh = "388745" +snapshot0.createTimeLow = "1991180826" +snapshot0.numDisks = "1" +snapshot0.disk0.fileName = "test flo_0.vmdk" +snapshot0.disk0.node = "scsi0:0" +snapshot.numSnapshots = "4" +snapshot1.uid = "2" +snapshot1.filename = "test flo-Snapshot2.vmsn" +snapshot1.parent = "1" +snapshot1.displayName = "SECOND" +snapshot1.description = "small" +snapshot1.createTimeHigh = "388801" +snapshot1.createTimeLow = "-814770620" +snapshot1.numDisks = "1" +snapshot1.disk0.fileName = "test flo_0-000001.vmdk" +snapshot1.disk0.node = "scsi0:0" +snapshot2.uid = "3" +snapshot2.filename = "test flo-Snapshot3.vmsn" +snapshot2.parent = "2" +snapshot2.displayName = "third" +snapshot2.type = "1" +snapshot2.createTimeHigh = "388802" +snapshot2.createTimeLow = "96936632" +snapshot2.numDisks = "1" +snapshot2.disk0.fileName = "test flo_0-000002.vmdk" +snapshot2.disk0.node = "scsi0:0" +snapshot3.uid = "4" +snapshot3.filename = "test flo-Snapshot4.vmsn" +snapshot3.parent = "3" +snapshot3.displayName = "from cli" +snapshot3.type = "1" +snapshot3.createTimeHigh = "388804" +snapshot3.createTimeLow = "1541463919" +snapshot3.numDisks = "1" +snapshot3.disk0.fileName = "test flo_0-000003.vmdk" +snapshot3.disk0.node = "scsi0:0" + */ diff --git a/@xen-orchestra/vmware-explorer/parsers/vmx.mjs b/@xen-orchestra/vmware-explorer/parsers/vmx.mjs new file mode 100644 index 000000000..74e885be5 --- /dev/null +++ b/@xen-orchestra/vmware-explorer/parsers/vmx.mjs @@ -0,0 +1,162 @@ +// the VMX file contains the VM metadata + +function set(obj, keyPath, val) { + let [key, ...other] = keyPath + + if (key.includes(':')) { + // it's an array + let index + ;[key, index] = key.split(':') + index = parseInt(index) + if (!obj[key]) { + // first time on this array + obj[key] = [] + } + if (!other.length) { + // without descendant + obj[key][index] = val + } else { + // with descendant + if (!obj[key][index]) { + // first time on this descendant + obj[key][index] = {} + } + set(obj[key][index], other, val) + } + } else { + // it's an object + if (!other.length) { + // without descendant + obj[key] = val + } else { + if (obj[key] === undefined) { + // first time + obj[key] = {} + } + // with descendant + if (typeof obj[key] !== 'object') { + // sometimes there is additionnal properties on a string ( like guestOS="ubuntut " and then guestOS.detailed =) + // we ignore these data for now + return + } + set(obj[key], other, val) + } + } +} + +// this file contains the vm configuration +export default function parseVmx(text) { + const vmx = {} + text.split('\n').forEach(line => { + const [key, val] = line.split(' = ') + set(vmx, key.split('.'), val?.substring(1, val.length - 1)) + }) + return vmx +} + +/** + * example data files : + .encoding = "UTF-8" +config.version = "8" +virtualHW.version = "11" +nvram = "test flo.nvram" +pciBridge0.present = "TRUE" +svga.present = "TRUE" +pciBridge4.present = "TRUE" +pciBridge4.virtualDev = "pcieRootPort" +pciBridge4.functions = "8" +pciBridge5.present = "TRUE" +pciBridge5.virtualDev = "pcieRootPort" +pciBridge5.functions = "8" +pciBridge6.present = "TRUE" +pciBridge6.virtualDev = "pcieRootPort" +pciBridge6.functions = "8" +pciBridge7.present = "TRUE" +pciBridge7.virtualDev = "pcieRootPort" +pciBridge7.functions = "8" +vmci0.present = "TRUE" +hpet0.present = "TRUE" +floppy0.present = "FALSE" +RemoteDisplay.maxConnections = "-1" +numvcpus = "2" +memSize = "1024" +bios.bootRetry.delay = "10" +sched.cpu.units = "mhz" +sched.cpu.affinity = "all" +sched.cpu.latencySensitivity = "normal" +powerType.powerOff = "soft" +powerType.suspend = "soft" +powerType.reset = "soft" +tools.upgrade.policy = "manual" +scsi0.virtualDev = "lsilogic" +scsi0.present = "TRUE" +sata0.present = "TRUE" +usb.present = "TRUE" +ehci.present = "TRUE" +scsi0:0.deviceType = "scsi-hardDisk" +scsi0:0.fileName = "test flo_0-000004.vmdk" +sched.scsi0:0.shares = "normal" +sched.scsi0:0.throughputCap = "off" +scsi0:0.present = "TRUE" +ethernet0.virtualDev = "vmxnet3" +ethernet0.networkName = "VM Network" +ethernet0.addressType = "generated" +ethernet0.wakeOnPcktRcv = "FALSE" +ethernet0.uptCompatibility = "TRUE" +ethernet0.present = "TRUE" +sata0:0.deviceType = "cdrom-image" +sata0:0.fileName = "/vmfs/volumes/636a25ed-0b37afee-60ef-3cfdfe75e770/ISO/ubuntu-22.04.1-live-server-amd64.iso" +sata0:0.present = "TRUE" +displayName = "test flo" +guestOS = "ubuntu-64" +toolScripts.afterPowerOn = "TRUE" +toolScripts.afterResume = "TRUE" +toolScripts.beforeSuspend = "TRUE" +toolScripts.beforePowerOff = "TRUE" +tools.syncTime = "FALSE" +uuid.bios = "56 4d fd 52 65 f4 71 df-e4 c8 66 8e b1 27 c4 8b" +uuid.location = "56 4d fd 52 65 f4 71 df-e4 c8 66 8e b1 27 c4 8b" +vc.uuid = "52 53 bd 47 4e a1 ba 91-dd c2 15 04 b1 06 35 92" +sched.cpu.min = "0" +sched.cpu.shares = "normal" +sched.mem.min = "0" +sched.mem.minSize = "0" +sched.mem.shares = "normal" +virtualHW.productCompatibility = "hosted" +sched.swap.derivedName = "/vmfs/volumes/636a25ed-0b37afee-60ef-3cfdfe75e770/test flo/test flo-988abbf8.vswp" +replay.supported = "FALSE" +replay.filename = "" +migrate.hostlog = "./test flo-988abbf8.hlog" +scsi0:0.redo = "" +pciBridge0.pciSlotNumber = "17" +pciBridge4.pciSlotNumber = "21" +pciBridge5.pciSlotNumber = "22" +pciBridge6.pciSlotNumber = "23" +pciBridge7.pciSlotNumber = "24" +scsi0.pciSlotNumber = "16" +usb.pciSlotNumber = "32" +ethernet0.pciSlotNumber = "160" +ehci.pciSlotNumber = "33" +vmci0.pciSlotNumber = "34" +sata0.pciSlotNumber = "35" +ethernet0.generatedAddress = "00:0c:29:27:c4:8b" +ethernet0.generatedAddressOffset = "0" +vmci0.id = "-1322793845" +monitor.phys_bits_used = "42" +vmotion.checkpointFBSize = "4194304" +vmotion.checkpointSVGAPrimarySize = "4194304" +cleanShutdown = "TRUE" +softPowerOff = "FALSE" +usb:1.speed = "2" +usb:1.present = "TRUE" +usb:1.deviceType = "hub" +usb:1.port = "1" +usb:1.parent = "-1" +svga.guestBackedPrimaryAware = "TRUE" +extendedConfigFile = "test flo.vmxf" +tools.remindInstall = "TRUE" +usb:0.present = "TRUE" +usb:0.deviceType = "hid" +usb:0.port = "0" +usb:0.parent = "-1" + */ diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 77ab5de5a..784473a9c 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -27,4 +27,7 @@ +- @xen-orchestra/vmware-explorer patch +- xo-server minor + diff --git a/packages/xo-server/package.json b/packages/xo-server/package.json index ff3e8ca0b..e3938ea84 100644 --- a/packages/xo-server/package.json +++ b/packages/xo-server/package.json @@ -51,6 +51,7 @@ "@xen-orchestra/mixins": "^0.9.0", "@xen-orchestra/self-signed": "^0.1.3", "@xen-orchestra/template": "^0.1.0", + "@xen-orchestra/vmware-explorer": "^0.0.2", "@xen-orchestra/xapi": "^1.6.1", "ajv": "^8.0.3", "app-conf": "^2.3.0", diff --git a/packages/xo-server/src/api/vm.mjs b/packages/xo-server/src/api/vm.mjs index 83279b41f..bdfb8db78 100644 --- a/packages/xo-server/src/api/vm.mjs +++ b/packages/xo-server/src/api/vm.mjs @@ -1300,6 +1300,33 @@ import_.resolve = { export { import_ as import } +export async function importFromEsxi({ + host, + network, + password, + sr, + sslVerify = true, + stopSource = false, + thin = false, + user, + vm, +}) { + const task = await this.tasks.create({ name: `importing vm ${vm}` }) + return task.run(() => this.migrationfromEsxi({ host, user, password, sslVerify, thin, vm, sr, network, stopSource })) +} + +importFromEsxi.params = { + host: { type: 'string' }, + network: { type: 'string' }, + password: { type: 'string' }, + sr: { type: 'string' }, + sslVerify: { type: 'boolean', optional: true }, + stopSource: { type: 'boolean', optional: true }, + thin: { type: 'boolean', optional: true }, + user: { type: 'string' }, + vm: { type: 'string' }, +} + // ------------------------------------------------------------------- // FIXME: if position is used, all other disks after this position diff --git a/packages/xo-server/src/xo-mixins/migrate-vm.mjs b/packages/xo-server/src/xo-mixins/migrate-vm.mjs index 0de75059e..dbf8633d3 100644 --- a/packages/xo-server/src/xo-mixins/migrate-vm.mjs +++ b/packages/xo-server/src/xo-mixins/migrate-vm.mjs @@ -1,5 +1,15 @@ import { Backup } from '@xen-orchestra/backups/Backup.js' +import { decorateWith } from '@vates/decorate-with' +import { defer as deferrable } from 'golike-defer' +import { fromEvent } from 'promise-toolbox' +import { Task } from '@xen-orchestra/mixins/Tasks.mjs' import { v4 as generateUuid } from 'uuid' +import { VDI_FORMAT_VHD } from '@xen-orchestra/xapi' +import asyncMapSettled from '@xen-orchestra/async-map/legacy.js' +import Esxi from '@xen-orchestra/vmware-explorer/esxi.mjs' +import openDeltaVmdkasVhd from '@xen-orchestra/vmware-explorer/openDeltaVmdkAsVhd.mjs' +import OTHER_CONFIG_TEMPLATE from '../xapi/other-config-template.mjs' +import VhdEsxiRaw from '@xen-orchestra/vmware-explorer/VhdEsxiRaw.mjs' export default class MigrateVm { constructor(app) { @@ -106,4 +116,196 @@ export default class MigrateVm { } } } + + #buildDiskChainByNode(disks, snapshots) { + let chain = [] + if (snapshots && snapshots.current) { + const currentSnapshotId = snapshots.current + + let currentSnapshot = snapshots.snapshots.find(({ uid }) => uid === currentSnapshotId) + + chain = [currentSnapshot.disks] + while ((currentSnapshot = snapshots.snapshots.find(({ uid }) => uid === currentSnapshot.parent))) { + chain.push(currentSnapshot.disks) + } + chain.reverse() + } + + chain.push(disks) + + for (const disk of chain) { + if (disk.capacity > 2 * 1024 * 1024 * 1024 * 1024) { + /* 2TO */ + throw new Error("Can't migrate disks larger than 2TiB") + } + } + + const chainsByNodes = {} + chain.forEach(disks => { + disks.forEach(disk => { + chainsByNodes[disk.node] = chainsByNodes[disk.node] || [] + chainsByNodes[disk.node].push(disk) + }) + }) + + return chainsByNodes + } + + #connectToEsxi(host, user, password, sslVerify = true) { + return new Task({ name: `connecting to ${host}` }).run(async () => { + const esxi = new Esxi(host, user, password, sslVerify) + await fromEvent(esxi, 'ready') + return esxi + }) + } + + @decorateWith(deferrable) + async migrationfromEsxi( + $defer, + { host, user, password, sslVerify, sr: srId, network: networkId, vm: vmId, thin, stopSource } + ) { + const app = this._app + const esxi = await this.#connectToEsxi(host, user, password, sslVerify) + + const esxiVmMetadata = await new Task({ name: `get metadata of ${vmId}` }).run(async () => { + return esxi.getTransferableVmMetadata(vmId) + }) + + const { disks, firmware, memory, name_label, networks, numCpu, powerState, snapshots } = esxiVmMetadata + const isRunning = powerState !== 'poweredOff' + + const chainsByNodes = await new Task({ name: `build disks and snapshots chains for ${vmId}` }).run(async () => { + return this.#buildDiskChainByNode(disks, snapshots) + }) + + const sr = app.getXapiObject(srId) + const xapi = sr.$xapi + + const vm = await new Task({ name: 'creating MV on XCP side ' }).run(async () => { + // got data, ready to start creating + const vm = await xapi._getOrWaitObject( + await xapi.VM_create({ + ...OTHER_CONFIG_TEMPLATE, + memory_dynamic_max: memory, + memory_dynamic_min: memory, + memory_static_max: memory, + memory_static_min: memory, + name_description: 'from esxi', + name_label, + VCPUs_at_startup: numCpu, + VCPUs_max: numCpu, + }) + ) + await Promise.all([ + vm.update_HVM_boot_params('firmware', firmware), + vm.update_platform('device-model', 'qemu-upstream-' + (firmware === 'uefi' ? 'uefi' : 'compat')), + asyncMapSettled(['start', 'start_on'], op => vm.update_blocked_operations(op, 'Esxi migration in progress...')), + vm.set_name_label(`[Importing...] ${name_label}`), + ]) + + const vifDevices = await xapi.call('VM.get_allowed_VIF_devices', vm.$ref) + + await Promise.all( + networks.map((network, i) => + xapi.VIF_create( + { + device: vifDevices[i], + network: xapi.getObject(networkId).$ref, + VM: vm.$ref, + }, + { + MAC: network.macAddress, + } + ) + ) + ) + return vm + }) + + $defer.onFailure.call(xapi, 'VM_destroy', vm.$ref) + + const vhds = await Promise.all( + Object.keys(chainsByNodes).map(async (node, userdevice) => + new Task({ name: `Cold import of disks ${node} ` }).run(async () => { + const chainByNode = chainsByNodes[node] + const vdi = await xapi._getOrWaitObject( + await xapi.VDI_create({ + name_description: 'fromESXI' + chainByNode[0].descriptionLabel, + name_label: '[ESXI]' + chainByNode[0].nameLabel, + SR: sr.$ref, + virtual_size: chainByNode[0].capacity, + }) + ) + // it can fail before the vdi is connected to the vm + + $defer.onFailure.call(xapi, 'VDI_destroy', vdi.$ref) + + await xapi.VBD_create({ + VDI: vdi.$ref, + VM: vm.$ref, + }) + let parentVhd, vhd + // if the VM is running we'll transfer everything before the last , which is an active disk + // the esxi api does not allow us to read an active disk + // later we'll stop the VM and transfer this snapshot + const nbColdDisks = isRunning ? chainByNode.length - 1 : chainByNode.length + for (let diskIndex = 0; diskIndex < nbColdDisks; diskIndex++) { + // the first one is a RAW disk ( full ) + const disk = chainByNode[diskIndex] + const { fileName, path, datastore, isFull } = disk + if (isFull) { + vhd = await VhdEsxiRaw.open(esxi, datastore, path + '/' + fileName, { thin }) + await vhd.readBlockAllocationTable() + } else { + vhd = await openDeltaVmdkasVhd(esxi, datastore, path + '/' + fileName, parentVhd) + } + parentVhd = vhd + } + + const stream = vhd.stream() + await vdi.$importContent(stream, { format: VDI_FORMAT_VHD }) + return { vdi, vhd } + }) + ) + ) + + if (isRunning && stopSource) { + // it the vm was running, we stop it and transfer the data in the active disk + await new Task({ name: 'powering down source VM' }).run(() => esxi.powerOff(vmId)) + + await Promise.all( + Object.keys(chainsByNodes).map(async (node, userdevice) => { + await new Task({ name: `Transfering deltas of ${userdevice}` }).run(async () => { + const chainByNode = chainsByNodes[node] + const disk = chainByNode[chainByNode.length - 1] + const { fileName, path, datastore, isFull } = disk + const { vdi, vhd: parentVhd } = vhds[userdevice] + let vhd + if (isFull) { + vhd = await VhdEsxiRaw.open(esxi, datastore, path + '/' + fileName, { thin }) + await vhd.readBlockAllocationTable() + } else { + // we only want to transfer blocks present in the delta vhd, not the full vhd chain + vhd = await openDeltaVmdkasVhd(esxi, datastore, path + '/' + fileName, parentVhd, { + lookMissingBlockInParent: false, + }) + } + const stream = vhd.stream() + + await vdi.$importContent(stream, { format: VDI_FORMAT_VHD }) + }) + }) + ) + } + + await new Task({ name: 'Finishing transfer' }).run(async () => { + // remove the importing in label + await vm.set_name_label(esxiVmMetadata.name_label) + + // remove lock on start + await asyncMapSettled(['start', 'start_on'], op => vm.update_blocked_operations(op, null)) + }) + + return vm.uuid + } } diff --git a/yarn.lock b/yarn.lock index c02284c64..3f9c6ce63 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7181,6 +7181,11 @@ data-uri-to-buffer@3: resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz#594b8973938c5bc2c33046535785341abc4f3636" integrity sha512-WboRycPNsVw3B3TL559F7kuBUM4d8CgMEvk6xEJlOp7OBPjt6G7z8WMWlD2rOFZLk6OYfFIUGsCOWzcQH9K2og== +data-uri-to-buffer@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz#d8feb2b2881e6a4f58c2e08acfd0e2834e26222e" + integrity sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A== + dateformat@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-2.2.0.tgz#4065e2013cf9fb916ddfd82efb506ad4c6769062" @@ -9205,6 +9210,14 @@ feature-policy@0.3.0: resolved "https://registry.yarnpkg.com/feature-policy/-/feature-policy-0.3.0.tgz#7430e8e54a40da01156ca30aaec1a381ce536069" integrity sha512-ZtijOTFN7TzCujt1fnNhfWPFPSHeZkesff9AXZj+UEjYBynWNUIYpC87Ve4wHzyexQsImicLu7WsC2LHq7/xrQ== +fetch-blob@^3.1.2, fetch-blob@^3.1.4: + version "3.2.0" + resolved "https://registry.yarnpkg.com/fetch-blob/-/fetch-blob-3.2.0.tgz#f09b8d4bbd45adc6f0c20b7e787e793e309dcce9" + integrity sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ== + dependencies: + node-domexception "^1.0.0" + web-streams-polyfill "^3.0.3" + fifolock@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fifolock/-/fifolock-1.0.0.tgz#a37e54f3ebe69d13480d95a82abc42b7a5c1792d" @@ -9392,6 +9405,11 @@ fined@^1.0.1: object.pick "^1.2.0" parse-filepath "^1.0.1" +first-chunk-stream@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/first-chunk-stream/-/first-chunk-stream-0.1.0.tgz#755d3ec14d49a86e3d2fcc08beead5c0ca2b9c0a" + integrity sha512-o7kVqimu9cl+XNeEGqDPI8Ms4IViicBnjIDZ5uU+7aegfDhJJiU1Da9y52Qt0TfBO3rpKA5hW2cqwp4EkCfl9w== + first-chunk-stream@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/first-chunk-stream/-/first-chunk-stream-2.0.0.tgz#1bdecdb8e083c0664b91945581577a43a9f31d70" @@ -9502,6 +9520,13 @@ form-data@~2.3.2: combined-stream "^1.0.6" mime-types "^2.1.12" +formdata-polyfill@^4.0.10: + version "4.0.10" + resolved "https://registry.yarnpkg.com/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz#24807c31c9d402e002ab3d8c720144ceb8848423" + integrity sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g== + dependencies: + fetch-blob "^3.1.2" + forwarded@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" @@ -13180,11 +13205,21 @@ lodash.uniq@^4.5.0: resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773" integrity sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ== +lodash@3.x.x: + version "3.10.1" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.10.1.tgz#5bf45e8e49ba4189e17d482789dfd15bd140b7b6" + integrity sha512-9mDDwqVIma6OZX79ZlDACZl8sBm0TEnkf99zV3iMA4GzkIT/9hiqP5mY0HoT1iNLCrKc/R1HByV+yJfRWVJryQ== + lodash@^4.13.1, lodash@^4.15.0, lodash@^4.16.2, lodash@^4.16.6, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.2, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.17.4, lodash@^4.17.5, lodash@^4.2.0, lodash@^4.2.1, lodash@^4.6.1: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== +lodash@~2.4.1: + version "2.4.2" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-2.4.2.tgz#fadd834b9683073da179b3eae6d9c0d15053f73e" + integrity sha512-Kak1hi6/hYHGVPmdyiZijoQyz5x2iGVzs6w9GYB/HiXEtylY7tIoYEROMjvM1d9nXJqPOrG2MNPMn01bJ+S0Rw== + log-symbols@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-1.0.2.tgz#376ff7b58ea3086a0f09facc74617eca501e1a18" @@ -14028,6 +14063,11 @@ node-addon-api@^5.0.0: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-5.1.0.tgz#49da1ca055e109a23d537e9de43c09cca21eb762" integrity sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA== +node-domexception@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" + integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== + node-fetch@^1.0.1: version "1.7.3" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.7.3.tgz#980f6f72d85211a5347c6b2bc18c5b84c3eb47ef" @@ -14043,6 +14083,15 @@ node-fetch@^2.6.7: dependencies: whatwg-url "^5.0.0" +node-fetch@^3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.3.0.tgz#37e71db4ecc257057af828d523a7243d651d91e4" + integrity sha512-BKwRP/O0UvoMKp7GNdwPlObhYGB5DQqwhEDQlNKuoqwVYSxkSZCSbHjnFFmUEtwSKRPU4kNK8PbDYYitwaE3QA== + dependencies: + data-uri-to-buffer "^4.0.0" + fetch-blob "^3.1.4" + formdata-polyfill "^4.0.10" + node-forge@^0.10.0: version "0.10.0" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.10.0.tgz#32dea2afb3e9926f02ee5ce8794902691a676bf3" @@ -14118,6 +14167,15 @@ node-version@^1.0.0: resolved "https://registry.yarnpkg.com/node-version/-/node-version-1.2.0.tgz#34fde3ffa8e1149bd323983479dda620e1b5060d" integrity sha512-ma6oU4Sk0qOoKEAymVoTvk8EdXEobdS7m/mAGhDJ8Rouugho48crHBORAmy5BoOcv8wraPM6xumapQp5hl4iIQ== +node-vsphere-soap@^0.0.2-5: + version "0.0.2-5" + resolved "https://registry.yarnpkg.com/node-vsphere-soap/-/node-vsphere-soap-0.0.2-5.tgz#e055a17d23452276b0755949b163e16b4214755c" + integrity sha512-FC0QHZMV1QWuCPKdUmYWAX2yVnHNybEGblKOwkFow6DS6xAejIAqRn+hd5imK+VLt2yBT+0XprP44zl2+eTfzw== + dependencies: + lodash "3.x.x" + soap "0.8.0" + soap-cookie "0.10.x" + node-xmpp-client@^3.0.0: version "3.2.0" resolved "https://registry.yarnpkg.com/node-xmpp-client/-/node-xmpp-client-3.2.0.tgz#af4527df0cc5abd2690cba2139cc1ecdc81ea189" @@ -17041,7 +17099,7 @@ replace-homedir@^1.0.0: is-absolute "^1.0.0" remove-trailing-separator "^1.1.0" -request@^2.65.0, request@^2.74.0, request@^2.87.0: +request@>=2.9.0, request@^2.65.0, request@^2.74.0, request@^2.87.0: version "2.88.2" resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== @@ -17366,7 +17424,7 @@ sass@^1.38.1: immutable "^4.0.0" source-map-js ">=0.6.2 <2.0.0" -sax@1.2.x, sax@>=0.6.0, sax@~1.2.4: +sax@1.2.x, sax@>=0.6, sax@>=0.6.0, sax@~1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== @@ -17759,6 +17817,21 @@ snapdragon@^0.8.1: source-map-resolve "^0.5.0" use "^3.1.0" +soap-cookie@0.10.x: + version "0.10.1" + resolved "https://registry.yarnpkg.com/soap-cookie/-/soap-cookie-0.10.1.tgz#7c7e62f3779b3e42e6b584d35ecabee92121a23f" + integrity sha512-lG3/Vozl7otPEFbEWLIDOyArDMAUZBJhMQBXW/L5cfGh88GMBtBItOw28zcMLO0o6Y7RC2vkUtZ7CWauTv0a7w== + +soap@0.8.0: + version "0.8.0" + resolved "https://registry.yarnpkg.com/soap/-/soap-0.8.0.tgz#ab2766a7515fa5069f264a094e087e3fe74e2a78" + integrity sha512-rQpzOrol1pQpvhn9CdxDJg/D0scOwlr+DcdMFMAm/Q1cWbjaYKEMCl1dcfW5JjMFdf5esKUqGplDPEEB0Z2RZA== + dependencies: + lodash "~2.4.1" + request ">=2.9.0" + sax ">=0.6" + strip-bom "~0.3.1" + sockjs-client@^1.5.0: version "1.6.1" resolved "https://registry.yarnpkg.com/sockjs-client/-/sockjs-client-1.6.1.tgz#350b8eda42d6d52ddc030c39943364c11dcad806" @@ -18342,6 +18415,14 @@ strip-bom@^4.0.0: resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-4.0.0.tgz#9c3505c1db45bcedca3d9cf7a16f5c5aa3901878" integrity sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w== +strip-bom@~0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-0.3.1.tgz#9e8a39eff456ff9abc2f059f5f2225bb0f3f7ca5" + integrity sha512-8m24eJUyKXllSCydAwFVbr4QRZrRb82T2QfwtbO9gTLWhWIOxoDEZESzCGMgperFNyLhly6SDOs+LPH6/seBfw== + dependencies: + first-chunk-stream "^0.1.0" + is-utf8 "^0.2.0" + strip-eof@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf" @@ -19984,6 +20065,11 @@ wcwidth@^1.0.1: dependencies: defaults "^1.0.3" +web-streams-polyfill@^3.0.3: + version "3.2.1" + resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz#71c2718c52b45fd49dbeee88634b3a60ceab42a6" + integrity sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q== + webidl-conversions@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"