NuKVM/xen-orchestra
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cleanup

Browse Source
This commit is contained in:
Florent Beauchamp
2022-12-16 16:34:16 +01:00
parent 0876de77f5
commit a609772415
12 changed files with 164 additions and 309 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
@xen-orchestra/vmware-explorer/VhdEsxiRaw.mjs
View File

@@ -1,16 +1,13 @@
import assert from 'node:assert'
import { VhdAbstract} from 'vhd-lib'
import { VhdAbstract } from 'vhd-lib'
import { createFooter, createHeader } from 'vhd-lib/_createFooterHeader.js'
import _computeGeometryForSize from 'vhd-lib/_computeGeometryForSize.js'
import { DISK_TYPES, FOOTER_SIZE } from 'vhd-lib/_constants.js'
import { unpackFooter, unpackHeader } from 'vhd-lib/Vhd/_utils.js'
import { readChunk } from '@vates/read-chunk'
import { fromEvent } from 'promise-toolbox'
const VHD_BLOCK_LENGTH = 2*1024*1024
export default class VhdEsxiRaw extends VhdAbstract{
const VHD_BLOCK_LENGTH = 2 * 1024 * 1024
export default class VhdEsxiRaw extends VhdAbstract {
#esxi
#datastore
#path
@@ -22,13 +19,9 @@ export default class VhdEsxiRaw extends VhdAbstract{
#stream
#bytesRead = 0
static async open(esxi, datastore, path) {
console.log('VhdEsxiRaw.open ', datastore, path)
const vhd = new VhdEsxiRaw(esxi, datastore, path)
console.log('VhdEsxiRaw.open instantiated ')
await vhd.readHeaderAndFooter()
console.log('VhdEsxiRaw.open readHeaderAndFooter ')
return vhd
}
@@ -40,7 +33,7 @@ export default class VhdEsxiRaw extends VhdAbstract{
return this.#footer
}
constructor(esxi, datastore, path){
constructor(esxi, datastore, path) {
super()
this.#esxi = esxi
this.#path = path
@@ -50,54 +43,46 @@ export default class VhdEsxiRaw extends VhdAbstract{
async readHeaderAndFooter(checkSecondFooter = true) {
const res = await this.#esxi.download(this.#datastore, this.#path)
const length = res.headers.get('content-length')
console.log(res.headers, length)
this.#header = unpackHeader(createHeader(length / VHD_BLOCK_LENGTH))
console.log('headzer ok')
const geometry = _computeGeometryForSize(length)
console.log('geometry ok')
const actualSize = geometry.actualSize
this.#footer = unpackFooter(createFooter(actualSize, Math.floor(Date.now() / 1000), geometry, FOOTER_SIZE, DISK_TYPES.DYNAMIC))
console.log('readHeaderAndFooter ok')
this.#footer = unpackFooter(
createFooter(actualSize, Math.floor(Date.now() / 1000), geometry, FOOTER_SIZE, DISK_TYPES.DYNAMIC)
)
}
containsBlock(blockId) {
assert.notEqual(this.#bat, undefined)
return this.#bat.has(blockId)
}
async readBlock(blockId){
const start = blockId * VHD_BLOCK_LENGTH
if(!this.#stream){
async readBlock(blockId) {
const start = blockId * VHD_BLOCK_LENGTH
if (!this.#stream) {
this.#stream = (await this.#esxi.download(this.#datastore, this.#path)).body
this.#bytesRead = 0
}
if(this.#bytesRead > start){
console.log('back ')
if (this.#bytesRead > start) {
this.#stream.destroy()
this.#stream = (await this.#esxi.download(this.#datastore, this.#path,`${start}-${this.footer.currentSize}`)).body
this.#stream = (
await this.#esxi.download(this.#datastore, this.#path, `${start}-${this.footer.currentSize}`)
).body
this.#bytesRead = start
}
if(start- this.#bytesRead > 0){
console.log('fast forward',`${start}-${this.footer.currentSize}`)
if (start - this.#bytesRead > 0) {
this.#stream.destroy()
this.#stream = (await this.#esxi.download(this.#datastore, this.#path,`${start}-${this.footer.currentSize}`)).body
this.#stream = (
await this.#esxi.download(this.#datastore, this.#path, `${start}-${this.footer.currentSize}`)
).body
this.#bytesRead = start
}
console.log('before', {bytesRead: this.#bytesRead, remaining:start- this.#bytesRead })
/*
while(this.#bytesRead < start ){
const buf = await readChunk(this.#stream,Math.min(start- this.#bytesRead, VHD_BLOCK_LENGTH))
this.#bytesRead += buf.length
}*/
const data = await readChunk(this.#stream, VHD_BLOCK_LENGTH)
this.#bytesRead += data.length
const bitmap= Buffer.alloc(512,255)
console.log(blockId, {start, length: data.length})
const bitmap = Buffer.alloc(512, 255)
return {
id: blockId,
bitmap,
@@ -107,40 +92,33 @@ export default class VhdEsxiRaw extends VhdAbstract{
}
async readBlockAllocationTable() {
console.log('readBlockAllocationTable ')
const res = await this.#esxi.download(this.#datastore, this.#path)
const length = res.headers.get('content-length')
const stream = res.body
console.log({headers: res.headers, length})
const empty = Buffer.alloc(VHD_BLOCK_LENGTH, 0)
let pos = 0
this.#bat = new Set()
let nextChunkLength = Math.min(VHD_BLOCK_LENGTH, length)
const progress = setInterval(()=>{
console.log(pos/VHD_BLOCK_LENGTH, pos, this.#bat.size)
}, 10 * 1000)
let nextChunkLength = Math.min(VHD_BLOCK_LENGTH, length)
const progress = setInterval(() => {
console.log(pos / VHD_BLOCK_LENGTH, pos, this.#bat.size)
}, 30 * 1000)
await fromEvent(stream, 'readable')
while(nextChunkLength > 0 ){
while (nextChunkLength > 0) {
const chunk = await readChunk(stream, nextChunkLength)
let isEmpty
if(nextChunkLength === VHD_BLOCK_LENGTH){
if (nextChunkLength === VHD_BLOCK_LENGTH) {
isEmpty = empty.equals(chunk)
} else {
// last block can be smaller
isEmpty = Buffer.alloc(nextChunkLength , 0).equals(chunk)
isEmpty = Buffer.alloc(nextChunkLength, 0).equals(chunk)
}
if(!isEmpty){
this.#bat.add(pos/VHD_BLOCK_LENGTH)
if (!isEmpty) {
this.#bat.add(pos / VHD_BLOCK_LENGTH)
}
pos += VHD_BLOCK_LENGTH
nextChunkLength = Math.min(VHD_BLOCK_LENGTH, length - pos)
}
clearInterval(progress)
}
}

85
@xen-orchestra/vmware-explorer/esxi.mjs
View File

@@ -1,40 +1,35 @@
import { EventEmitter } from 'node:events'
import { Client } from 'node-vsphere-soap'
import { strictEqual } from 'node:assert'
import fetch from 'node-fetch';
import fetch from 'node-fetch'
import parseVmx from './parsers/vmx.mjs'
import { dirname } from 'node:path'
import { dirname } from 'node:path'
import parseVmdk, { VhdCowd } from './parsers/vmdk.mjs'
import parseVmsd from './parsers/vmsd.mjs';
import VhdEsxiRaw from './VhdEsxiRaw.mjs';
import parseVmsd from './parsers/vmsd.mjs'
import VhdEsxiRaw from './VhdEsxiRaw.mjs'
const MAX_SCSI = 9
const MAX_ETHERNET = 9
export default class Esxi extends EventEmitter {
#client
#host
#user
#password
#ready = false
#sslVerify
constructor(host, user, password, sslVerify = true) {
super()
this.#host = host
this.#user = user
this.#password = password
this.#sslVerify = sslVerify
this.#client = new Client(host, user, password, sslVerify)
this.#client.once('ready', () => {
this.#ready = true
this.emit('ready')
})
this.#client.on('error', err=>{
this.#client.on('error', err => {
console.error(err)
this.emit('error', err)
})
@@ -59,19 +54,19 @@ export default class Esxi extends EventEmitter {
strictEqual(this.#ready, true)
const url = `https://${this.#host}/folder/${path}?dsName=${dataStore}`
const headers = {
'Authorization': 'Basic ' + Buffer.from(this.#user + ':' + this.#password).toString('base64')
Authorization: 'Basic ' + Buffer.from(this.#user + ':' + this.#password).toString('base64'),
}
if(range){
headers['content-type'] = 'multipart/byteranges'
headers.Range = 'bytes='+ range
if (range) {
headers['content-type'] = 'multipart/byteranges'
headers.Range = 'bytes=' + range
}
const res = await fetch(url, {
method: 'GET',
headers,
highWaterMark: 10 * 1024 * 1024
highWaterMark: 10 * 1024 * 1024,
})
if(res.status < 200 || res.status >=300){
const error = new Error(res.status+ ' '+res.statusText+ ' '+ url )
if (res.status < 200 || res.status >= 300) {
const error = new Error(res.status + ' ' + res.statusText + ' ' + url)
error.cause = res
throw error
}
@@ -146,8 +141,7 @@ export default class Esxi extends EventEmitter {
return objects
}
async #inspectVmdk(dataStores, currentDataStore, currentPath, filePath){
async #inspectVmdk(dataStores, currentDataStore, currentPath, filePath) {
let diskDataStore, diskPath
if (filePath.startsWith('/')) {
// disk is on another datastore
@@ -164,7 +158,7 @@ export default class Esxi extends EventEmitter {
const vmdkRes = await this.download(diskDataStore, diskPath)
const text = await vmdkRes.text()
const parsed = parseVmdk(text)
const {fileName, parentFileName, capacity} = parsed
const { fileName, parentFileName, capacity } = parsed
return {
...parsed,
@@ -172,35 +166,24 @@ export default class Esxi extends EventEmitter {
path: dirname(diskPath),
descriptionLabel: ' from esxi',
vhd: async () => {
console.log('vhd from snapshot')
if(fileName.endsWith('-flat.vmdk')){
console.log('snapshot flat.vmdk vhd ')
const vhd = await VhdEsxiRaw.open(this,diskDataStore, dirname(diskPath) + '/' + fileName)
console.log('snapshot flat.vmdk vhd openned')
if (fileName.endsWith('-flat.vmdk')) {
const vhd = await VhdEsxiRaw.open(this, diskDataStore, dirname(diskPath) + '/' + fileName)
await vhd.readBlockAllocationTable()
console.log('snapshot flat.vmdk bat openned')
return vhd.stream()
}
console.log('snapshot current.vmdk filenameok')
// last snasphot only works when vm is powered off
const vhd = await VhdCowd.open(this, diskDataStore, dirname(diskPath) + '/' + fileName, parentFileName)
console.log('snapshot current.vmdk vhd openned')
await vhd.readBlockAllocationTable()
console.log('snapshot current.vmdk bat openned')
return vhd.stream()
},
rawStream: async () => {
console.log('snapshot rawStream from snapshot')
if(!fileName.endsWith('-flat.vmdk')){
if (!fileName.endsWith('-flat.vmdk')) {
return
}
console.log('snapshot rawStream from snapshot filename ok')
// @todo : only if vm is powered off
const stream = (await this.download(diskDataStore, dirname(diskPath) + '/' + fileName)).body
console.log('snapshot realy vhd from snapshot got stream ')
stream.length = capacity
return stream
},
@@ -214,7 +197,7 @@ export default class Esxi extends EventEmitter {
}
const { config, runtime } = search[vmId]
const [_, dataStore, vmxPath] = config.files.vmPathName.match(/^\[(.*)\] (.+.vmx)$/)
const [, dataStore, vmxPath] = config.files.vmPathName.match(/^\[(.*)\] (.+.vmx)$/)
const res = await this.download(dataStore, vmxPath)
const vmx = parseVmx(await res.text())
@@ -238,8 +221,8 @@ export default class Esxi extends EventEmitter {
continue
}
disks.push({
...await this.#inspectVmdk(dataStores, dataStore, dirname(vmxPath), disk.fileName),
node: `scsi${scsiIndex}:${diskIndex}`
...(await this.#inspectVmdk(dataStores, dataStore, dirname(vmxPath), disk.fileName)),
node: `scsi${scsiIndex}:${diskIndex}`,
})
}
}
@@ -258,13 +241,13 @@ export default class Esxi extends EventEmitter {
const snapshots = parseVmsd(await (await this.download(dataStore, vmxPath.replace('.vmx', '.vmsd'))).text())
for(const snapshotIndex in snapshots.snapshots){
for (const snapshotIndex in snapshots.snapshots) {
const snapshot = snapshots.snapshots[snapshotIndex]
for(const diskIndex in snapshot.disks){
for (const diskIndex in snapshot.disks) {
const fileName = snapshot.disks[diskIndex].fileName
snapshot.disks[diskIndex] = {
node:snapshot.disks[diskIndex]?.node , // 'scsi0:0',
... await this.#inspectVmdk(dataStores, dataStore, dirname(vmxPath),fileName),
node: snapshot.disks[diskIndex]?.node, // 'scsi0:0',
...(await this.#inspectVmdk(dataStores, dataStore, dirname(vmxPath), fileName)),
}
}
}
@@ -277,40 +260,30 @@ export default class Esxi extends EventEmitter {
firmware: config.firmware, // bios or uefi
powerState: runtime.powerState,
snapshots,
disks: disks.map(({ fileName, rawDiskFileName, datastore, path,parentFileName, ...other }) => {
disks: disks.map(({ fileName, rawDiskFileName, datastore, path, parentFileName, ...other }) => {
return {
...other,
vhd: async () => {
console.log('current.vmdk')
if(fileName.endsWith('-flat.vmdk')){
console.log('flat.vmdk vhd ')
const vhd = await VhdEsxiRaw.open(this, datastore, path + '/' + fileName)
console.log('flat.vmdk vhd openned')
if (fileName.endsWith('-flat.vmdk')) {
const vhd = await VhdEsxiRaw.open(this, datastore, path + '/' + fileName)
await vhd.readBlockAllocationTable()
console.log('flat.vmdk bat openned')
return vhd.stream()
}
console.log('current.vmdk filenameok')
// last snasphot only works when vm is powered off
const vhd = await VhdCowd.open(this, datastore, path + '/' + fileName, parentFileName)
console.log('current.vmdk vhd openned')
await vhd.readBlockAllocationTable()
console.log('current.vmdk bat openned')
return vhd.stream()
},
rawStream: async () => {
if(fileName.endsWith('-flat.vmdk')){
if (fileName.endsWith('-flat.vmdk')) {
return
}
console.log('rawStream from snapshot filename ok')
// @todo : only if vm is powered off
const stream = (await this.download(datastore, path + '/' + fileName)).body
console.log('realy vhd from snapshot got stream ')
const stream = (await this.download(datastore, path + '/' + fileName)).body
stream.length = other.capacity
return stream
},
}
}),

9
@xen-orchestra/vmware-explorer/package.json
View File

@@ -1,12 +1,14 @@
{
"license": "ISC",
"private": true,
"private": false,
"version": "0.0.1",
"name": "@xen-orchestra/vmware-explorer",
"dependencies": {
"@vates/read-chunk": "^1.0.1",
"lodash": "^4.17.21",
"node-fetch": "^3.3.0",
"node-vsphere-soap": "^0.0.2-5"
"node-vsphere-soap": "^0.0.2-5",
"vhd-lib": "^4.2.0"
},
"engines": {
"node": ">=18"
@@ -21,5 +23,8 @@
"author": {
"name": "Vates SAS",
"url": "https://vates.fr"
},
"scripts": {
"postversion": "npm publish --access public"
}
}

106
@xen-orchestra/vmware-explorer/parsers/vmdk.mjs
View File

@@ -1,17 +1,11 @@
import { strictEqual } from 'node:assert'
import assert from 'node:assert'
import { VhdAbstract} from 'vhd-lib'
import { notEqual, strictEqual } from 'node:assert'
import { VhdAbstract } from 'vhd-lib'
import { createFooter, createHeader } from 'vhd-lib/_createFooterHeader.js'
import _computeGeometryForSize from 'vhd-lib/_computeGeometryForSize.js'
import { DISK_TYPES, FOOTER_SIZE } from 'vhd-lib/_constants.js'
import { unpackFooter, unpackHeader } from 'vhd-lib/Vhd/_utils.js'
import { readChunk } from '@vates/read-chunk'
export class VhdCowd extends VhdAbstract{
export class VhdCowd extends VhdAbstract {
#esxi
#datastore
#parentFileName
@@ -27,7 +21,7 @@ export class VhdCowd extends VhdAbstract{
await vhd.readHeaderAndFooter()
return vhd
}
constructor(esxi, datastore, path, parentFileName){
constructor(esxi, datastore, path, parentFileName) {
super()
this.#esxi = esxi
this.#path = path
@@ -44,65 +38,65 @@ export class VhdCowd extends VhdAbstract{
}
containsBlock(blockId) {
assert.notEqual(this.#grainDirectory, undefined)
notEqual(this.#grainDirectory, undefined)
// only check if a grain table exist for on of the sector of the block
// the great news is that a grain size has 4096 entries of 512B = 2M
// and a vhd block is also 2M
// so we only need to check if a grain table exists (it's not created without data)
return this.#grainDirectory.readInt32LE(blockId * 4 ) !== 0
return this.#grainDirectory.readInt32LE(blockId * 4) !== 0
}
async #read(start, end){
async #read(start, end) {
return (await this.#esxi.download(this.#datastore, this.#path, `${start}-${end}`)).buffer()
}
async #readMultiple(ranges){
let range = ranges.map(([start, end])=> `${start}-${end}`).join(',')
return (await this.#esxi.download(this.#datastore, this.#path, range)).buffer()
}
async readHeaderAndFooter(checkSecondFooter = true) {
const buffer = await this.#read(0, 2048)
assert.strictEqual(buffer.slice(0,4).toString('ascii'),'COWD')
assert.strictEqual(buffer.readInt32LE(4), 1) // version
assert.strictEqual(buffer.readInt32LE(8), 3) // flags
strictEqual(buffer.slice(0, 4).toString('ascii'), 'COWD')
strictEqual(buffer.readInt32LE(4), 1) // version
strictEqual(buffer.readInt32LE(8), 3) // flags
const sectorCapacity = buffer.readInt32LE(12)
// const sectorGrainNumber = buffer.readInt32LE(16)
assert.strictEqual(buffer.readInt32LE(20), 4) // grain directory position in sectors
strictEqual(buffer.readInt32LE(20), 4) // grain directory position in sectors
// const nbGrainDirectoryEntries = buffer.readInt32LE(24)
// const nextFreeSector = buffer.readInt32LE(28)
const size = sectorCapacity * 512
// a grain directory entry represent a grain table
// a grain table can adresse, at most 4096 grain of 512 B
this.#header = unpackHeader(createHeader(Math.ceil(size / (4096 * 512 ))))
this.#header = unpackHeader(createHeader(Math.ceil(size / (4096 * 512))))
this.#header.parentUnicodeName = this.#parentFileName
const geometry = _computeGeometryForSize(size)
const actualSize = geometry.actualSize
this.#footer = unpackFooter(createFooter(actualSize, Math.floor(Date.now() / 1000), geometry, FOOTER_SIZE, this.#parentFileName ? DISK_TYPES.DIFFERENCING : DISK_TYPES.DYNAMIC))
this.#footer = unpackFooter(
createFooter(
actualSize,
Math.floor(Date.now() / 1000),
geometry,
FOOTER_SIZE,
this.#parentFileName ? DISK_TYPES.DIFFERENCING : DISK_TYPES.DYNAMIC
)
)
}
async readBlockAllocationTable() {
const nbBlocks = this.header.maxTableEntries
this.#grainDirectory = await this.#read(2048, 2048+nbBlocks*4 -1)
this.#grainDirectory = await this.#read(2048, 2048 + nbBlocks * 4 - 1)
}
async readBlock(blockId){
const sectorOffset = this.#grainDirectory.readInt32LE( blockId * 4)
if(sectorOffset === 1){
async readBlock(blockId) {
const sectorOffset = this.#grainDirectory.readInt32LE(blockId * 4)
if (sectorOffset === 1) {
return Promise.resolve(Buffer.alloc(4096 * 512, 0))
}
const offset= sectorOffset * 512
const offset = sectorOffset * 512
const graintable = await this.#read(offset, offset + 2048 - 1)
const buf = Buffer.concat([
Buffer.alloc(512, 255), // vhd block bitmap,
Buffer.alloc(512*4096, 0 ) // empty data
Buffer.alloc(512 * 4096, 0), // empty data
])
// we have no guaranty that data are order or contiguous
@@ -110,16 +104,16 @@ export class VhdCowd extends VhdAbstract{
const fileOffsetToIndexInGrainTable = {}
let nbNonEmptyGrain = 0
for(let i=0;i < graintable.length / 4 ;i++){
const grainOffset = graintable.readInt32LE( i * 4)
if(grainOffset !==0){
for (let i = 0; i < graintable.length / 4; i++) {
const grainOffset = graintable.readInt32LE(i * 4)
if (grainOffset !== 0) {
// non empty grain
fileOffsetToIndexInGrainTable[grainOffset] = i
nbNonEmptyGrain++
}
}
// grain table exists but only contains empty grains
if(nbNonEmptyGrain ===0){
if (nbNonEmptyGrain === 0) {
return {
id: blockId,
bitmap: buf.slice(0, this.bitmapSize),
@@ -128,28 +122,27 @@ export class VhdCowd extends VhdAbstract{
}
}
const offsets = Object.keys(fileOffsetToIndexInGrainTable).map(offset=>parseInt(offset))
offsets.sort((a,b) => a-b)
const offsets = Object.keys(fileOffsetToIndexInGrainTable).map(offset => parseInt(offset))
offsets.sort((a, b) => a - b)
let startOffset = offsets[0]
const ranges = []
const OVERPROVISION = 3
for(let i=1; i < offsets.length; i ++){
if(offsets[i-1] + OVERPROVISION < offsets[i]){
ranges.push({startOffset, endOffset: offsets[i-1]})
for (let i = 1; i < offsets.length; i++) {
if (offsets[i - 1] + OVERPROVISION < offsets[i]) {
ranges.push({ startOffset, endOffset: offsets[i - 1] })
startOffset = offsets[i]
}
}
ranges.push({startOffset, endOffset: offsets[offsets.length-1]})
ranges.push({ startOffset, endOffset: offsets[offsets.length - 1] })
for(const {startOffset, endOffset} of ranges){
for (const { startOffset, endOffset } of ranges) {
const startIndex = fileOffsetToIndexInGrainTable[startOffset]
const startInBlock = startIndex * 512 + 512 /* block bitmap */
const sectors = await this.#read(startOffset*512,endOffset*512 - 1)
const sectors = await this.#read(startOffset * 512, endOffset * 512 - 1)
// @todo : if overprovision > 1 , it may copy random data from the vmdk
sectors.copy(buf,startInBlock)
sectors.copy(buf, startInBlock)
}
return {
id: blockId,
@@ -158,16 +151,12 @@ export class VhdCowd extends VhdAbstract{
buffer: buf,
}
}
}
// this file contains the disk metadata
export function parseDescriptor(text){
export function parseDescriptor(text) {
const descriptorText = text.toString('ascii').replace(/\x00+$/, '') // eslint-disable-line no-control-regex
strictEqual(descriptorText.substr(0,21), '# Disk DescriptorFile')
strictEqual(descriptorText.substr(0, 21), '# Disk DescriptorFile')
const descriptorDict = {}
const extentList = []
const lines = descriptorText.split(/\r?\n/).filter(line => {
@@ -179,18 +168,17 @@ export function parseDescriptor(text){
if (defLine.length === 2 && defLine[0].indexOf('"') === -1) {
descriptorDict[defLine[0].toLowerCase()] = defLine[1].replace(/['"]+/g, '')
} else {
const [_, access, sizeSectors, type, name, offset] = line.match(/([A-Z]+) ([0-9]+) ([A-Z]+) "(.*)" ?(.*)$/)
extentList.push({access, sizeSectors, type, name, offset})
const [, access, sizeSectors, type, name, offset] = line.match(/([A-Z]+) ([0-9]+) ([A-Z]+) "(.*)" ?(.*)$/)
extentList.push({ access, sizeSectors, type, name, offset })
}
}
strictEqual(extentList.length, 1, 'only one extent per vmdk is supported')
return { ... descriptorDict, extent: extentList[0]}
return { ...descriptorDict, extent: extentList[0] }
}
// https://github.com/libyal/libvmdk/blob/main/documentation/VMWare%20Virtual%20Disk%20Format%20(VMDK).asciidoc#5-the-cowd-sparse-extent-data-file
// vmdk file can be only a descriptor, or a
export default function parseVmdk(raw){
export default function parseVmdk(raw) {
strictEqual(typeof raw, 'string')
const descriptor = parseDescriptor(raw)
@@ -203,6 +191,6 @@ export default function parseVmdk(raw){
parentId: descriptor.parentcid,
parentFileName: descriptor.parentfilenamehint,
vmdFormat: descriptor.extent.type,
nameLabel: descriptor.extent.name
nameLabel: descriptor.extent.name,
}
}

41
@xen-orchestra/vmware-explorer/parsers/vmsd.mjs
View File

@@ -1,29 +1,29 @@
// these files contains the snapshot history of the VM
function set(obj, keyPath, val){
function set(obj, keyPath, val) {
const [key, ...other] = keyPath
const match = key.match(/^(.+)([0-9])$/)
if(match){
if (match) {
// an array
let [_,label, index] = match
label +='s'
if(!obj[label]){
let [, label, index] = match
label += 's'
if (!obj[label]) {
obj[label] = []
}
if(other.length){
if(!obj[label][index]){
if (other.length) {
if (!obj[label][index]) {
obj[label][parseInt(index)] = {}
}
set(obj[label][index], other, val)
}else {
} else {
obj[label][index] = val
}
} else{
if(other.length){
} else {
if (other.length) {
// an object
if(!obj[key]){
// first time
obj[key] = {}
if (!obj[key]) {
// first time
obj[key] = {}
}
set(obj[key], other, val)
} else {
@@ -33,28 +33,21 @@ function set(obj, keyPath, val){
}
}
export default function parseVmsd(text){
const parsed ={}
export default function parseVmsd(text) {
const parsed = {}
text.split('\n').forEach(line => {
const [key, val] = line.split(' = ')
if(!key.startsWith('snapshot')){
if (!key.startsWith('snapshot')) {
return
}
set(parsed, key.split('.'), val?.substring(1, val.length - 1))
})
console.log('vmsd',{
lastUID: parsed.snapshot.current,
current: parsed.snapshot.current,
numSnapshots: parsed.snapshot.numSnapshots,
})
return {
lastUID: parsed.snapshot.current,
current: parsed.snapshot.current,
numSnapshots: parsed.snapshot.numSnapshots,
snapshots: Object.values(parsed.snapshots) || []
snapshots: Object.values(parsed.snapshots) || [],
}
}

3
@xen-orchestra/vmware-explorer/parsers/vmx.mjs
View File

@@ -16,8 +16,7 @@ function set(obj, keyPath, val) {
} else {
// with descendant
if (!obj[key][index]) {
// first time on this descendant
obj[key][index] = {}
// first time on this descendantD2
}
set(obj[key][index], other, val)
}

1
@xen-orchestra/xapi/vdi.js
View File

@@ -101,7 +101,6 @@ class Vdi {
task: await this.task_create(`Importing content into VDI ${await this.getField('VDI', ref, 'name_label')}`),
})
} catch (error) {
console.error({error})
// augment the error with as much relevant info as possible
const [poolMaster, vdi] = await Promise.all([
this.getRecord('host', this.pool.master),

12
packages/vhd-lib/Vhd/VhdAbstract.js
View File

@@ -274,10 +274,8 @@ exports.VhdAbstract = class VhdAbstract {
bat.writeUInt32BE(offsetSector, i * 4)
offsetSector += blockSizeInSectors
fileSize += this.fullBlockSize
console.log('has ', i)
} else {
bat.writeUInt32BE(BLOCK_UNUSED, i * 4)
// console.log('has NOT ', i)
}
}
@@ -299,25 +297,17 @@ exports.VhdAbstract = class VhdAbstract {
yield buffer
}
}
let i
const progress = setInterval(()=>{
console.log('yield blocks',i,header.maxTableEntries )
}, 60 * 1000)
// yield all blocks
// since contains() can be costly for synthetic vhd, use the computed bat
for ( i = 0; i < header.maxTableEntries; i++) {
for (let i = 0; i < header.maxTableEntries; i++) {
if (bat.readUInt32BE(i * 4) !== BLOCK_UNUSED) {
const block = await self.readBlock(i)
console.log(i, block.buffer.length)
yield block.buffer
}
}
clearInterval(progress)
console.log('block done')
// yield footer again
yield rawFooter
console.log('footer done', fileSize)
}
const stream = asyncIteratorToStream(iterator())

5
packages/xo-server/src/api/vm.mjs
View File

@@ -1301,11 +1301,10 @@ export { import_ as import }
export async function importFomEsxi({host, user, password, sslVerify=true, sr, network, vm, thin=false}){
console.log({host, user, password, sslVerify, sr, network, vm, thin})
await this.migrationfromEsxi({host, user, password, sslVerify, thin, vm, sr, network})
return await this.migrationfromEsxi({host, user, password, sslVerify, thin, vm, sr, network})
return
}
importFomEsxi.params = {
host: { type: 'string' },
network: { type: 'string' },

81
packages/xo-server/src/xo-mixins/migrate-vm.mjs
View File

@@ -112,20 +112,15 @@ export default class MigrateVm {
}
}
async migrationfromEsxi({ host, user, password, sslVerify, sr: srId, network: networkId, vm:vmId ,thin}) {
async migrationfromEsxi({ host, user, password, sslVerify, sr: srId, network: networkId, vm: vmId, thin }) {
const esxi = new Esxi(host, user, password, sslVerify)
const app = this._app
const sr = app.getXapiObject(srId)
const xapi = sr.$xapi
await fromEvent(esxi, 'ready')
console.log('connected')
const esxiVmMetadata = await esxi.getTransferableVmMetadata(
vmId
)
const { memory,name_label, networks, numCpu } = esxiVmMetadata
const esxiVmMetadata = await esxi.getTransferableVmMetadata(vmId)
const { memory, name_label, networks, numCpu } = esxiVmMetadata
const vm = await xapi._getOrWaitObject(
await xapi.VM_create({
...OTHER_CONFIG_TEMPLATE,
@@ -139,8 +134,6 @@ export default class MigrateVm {
VCPUs_max: numCpu,
})
)
console.log('VM created', vm.uuid, vm.$ref)
await Promise.all([
asyncMapSettled(['start', 'start_on'], op => vm.update_blocked_operations(op, 'OVA import in progress...')),
vm.set_name_label(`[Importing...] ${name_label}`),
@@ -148,44 +141,45 @@ export default class MigrateVm {
const vifDevices = await xapi.call('VM.get_allowed_VIF_devices', vm.$ref)
await Promise.all(networks.map((network, i) =>
xapi.VIF_create({
device: vifDevices[i],
network: xapi.getObject(networkId).$ref,
VM: vm.$ref,
})
))
console.log('network created')
await Promise.all(
networks.map((network, i) =>
xapi.VIF_create({
device: vifDevices[i],
network: xapi.getObject(networkId).$ref,
VM: vm.$ref,
})
)
)
console.log('network created')
// get the snapshot to migrate
const snapshots = esxiVmMetadata.snapshots
const currentSnapshotId = snapshots.current
let currentSnapshot = snapshots.snapshots.find(({uid})=> uid === currentSnapshotId)
let currentSnapshot = snapshots.snapshots.find(({ uid }) => uid === currentSnapshotId)
const chain = [currentSnapshot.disks]
while(currentSnapshot = snapshots.snapshots.find(({uid})=> uid === currentSnapshot.parent)){
// chain.push(currentSnapshot.disks)
while ((currentSnapshot = snapshots.snapshots.find(({ uid }) => uid === currentSnapshot.parent))) {
chain.push(currentSnapshot.disks)
}
chain.reverse()
chain.push( esxiVmMetadata.disks)
chain.push(esxiVmMetadata.disks)
const chainsByNodes = {}
chain.forEach(disks=>{
disks.forEach(disk=>{
chain.forEach(disks => {
disks.forEach(disk => {
chainsByNodes[disk.node] = chainsByNodes[disk.node] || []
chainsByNodes[disk.node].push(disk)
})
})
for(const node in chainsByNodes){
let chainByNode = chainsByNodes[node]
for (const node in chainsByNodes) {
const chainByNode = chainsByNodes[node]
const vdi = await xapi._getOrWaitObject(
await xapi.VDI_create({
name_description: 'fromESXI'+chainByNode[0].descriptionLabel,
name_label: '[ESXI]'+chainByNode[0].nameLabel,
name_description: 'fromESXI' + chainByNode[0].descriptionLabel,
name_label: '[ESXI]' + chainByNode[0].nameLabel,
SR: sr.$ref,
virtual_size: chainByNode[0].capacity,
})
@@ -196,44 +190,31 @@ export default class MigrateVm {
VDI: vdi.$ref,
VM: vm.$ref,
})
for(const disk of chainByNode){
for (const disk of chainByNode) {
// the first one is a RAW disk ( full )
// the live disk can only be migrated on a powerdoff vm
console.log('will import ',{ disk})
console.log('will import ', { disk })
let format = VDI_FORMAT_VHD
let stream
if(!thin){
if (!thin) {
stream = await disk.rawStream()
format = VDI_FORMAT_RAW
}
if(!stream){
if (!stream) {
stream = await disk.vhd()
}
console.log('will import in format ',{format})
console.log('will import in format ', { disk, format })
await vdi.$importContent(stream, { format })
console.log('disk imported')
}
// and then we can import the running disk ( after shutting down the VM)
}
// esxiVmMetadata.disks}
// remove the importing in label
await vm.set_name_label(esxiVmMetadata.name_label)
// take the current snaptshot
// remove lock on start
await asyncMapSettled(['start', 'start_on'], op => vm.update_blocked_operations(op, null))
return vm.uuid
}
}
// Vdis is 1874b686-0f27-45a4-a3d6-3cd2e248ac91

5
packages/xo-web/src/xo-app/import/index.js
View File

@@ -8,7 +8,6 @@ import { routes } from 'utils'
import DiskImport from '../disk-import'
import VmImport from '../vm-import'
import ImportVmFromEsxi from '../vm-import/esxi'
const HEADER = (
<Container>
@@ -26,9 +25,6 @@ const HEADER = (
<NavLink to='/import/disk'>
<Icon icon='disk' /> {_('labelDisk')}
</NavLink>
<NavLink to='/import/esxi'>
from esxi
</NavLink>
</NavTabs>
</Col>
</Row>
@@ -38,7 +34,6 @@ const HEADER = (
const Import = routes('vm', {
disk: DiskImport,
vm: VmImport,
esxi: ImportVmFromEsxi,
})(({ children }) => (
<Page header={HEADER} title='newImport' formatTitle>
{children}

45
packages/xo-web/src/xo-app/vm-import/esxi/index.js
View File

@@ -1,45 +0,0 @@
import React, { Component } from "react";
import { Container } from "../../../common/grid";
class VmSetting extends Component{
render(){
return 'network map , description, label, start after import , stop source after import'
}
}
class VMPicker extends Component{
render(){
return 'vmlist'
}
}
// connexion form to esxi
class Esxi extends Component{
_connect(host, login, password, sslVerify){
}
render(){
const {esxi} = this.props
return 'host, login password'
}
}
export default class ImportVmFromEsxi extends Component{
render(){
const { esxi, vms, vmId } = this.state || {}
if(vmId){
return <VmSetting vms={vms} vmId={vmId} esxi={esxi}/>
}
if(vms){
return <VMPicker vms={vms} esxi={esxi}/>
}
return <Esxi onConnect={esxi=>this.setState({esxi})} esxi={esxi} onVmList={vms=>this.setState({vms})}/>
}
}