feat(vhd-lib): handle file alias (#5962)

@xen-orchestra/fs/src/s3.js

@@ -211,6 +211,23 @@ export default class S3Handler extends RemoteHandlerAbstract {
     // nothing to do, directories do not exist, they are part of the files' path
   }
 
+  // reimplement _rmTree to handle efficiantly path with more than 1000 entries in trees
+  // @todo : use parallel processing for unlink
+  async _rmTree(path) {
+    let NextContinuationToken
+    do {
+      const result = await this._s3.listObjectsV2({
+        Bucket: this._bucket,
+        Prefix: this._dir + path + '/',
+        ContinuationToken: NextContinuationToken,
+      })
+      NextContinuationToken = result.isTruncated ? null : result.NextContinuationToken
+      for (const path of result.Contents) {
+        await this._unlink(path)
+      }
+    } while (NextContinuationToken !== null)
+  }
+
   async _write(file, buffer, position) {
     if (typeof file !== 'string') {
       file = file.fd

CHANGELOG.unreleased.md

@@ -31,5 +31,6 @@
 > In case of conflict, the highest (lowest in previous list) `$version` wins.
 
 - @xen-orchestra/fs minor
+- vhd-lib minor
 - xo-server patch
 - vhd-cli minor

packages/vhd-lib/src/Vhd/VhdAbstract.integ.spec.js

@@ -0,0 +1,141 @@
+/* eslint-env jest */
+
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import fs from 'fs-extra'
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { Disposable, pFromCallback } from 'promise-toolbox'
+
+import { openVhd } from '../index'
+import { createRandomFile, convertFromRawToVhd, createRandomVhdDirectory } from '../tests/utils'
+import { VhdAbstract } from './VhdAbstract'
+
+let tempDir
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+test('It creates an alias', async () => {
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file://' + tempDir })
+    const aliasPath = `alias/alias.alias.vhd`
+    const aliasFsPath = `${tempDir}/${aliasPath}`
+    await fs.mkdirp(`${tempDir}/alias`)
+
+    const testOneCombination = async ({ targetPath, targetContent }) => {
+      await VhdAbstract.createAlias(handler, aliasPath, targetPath)
+      // alias file is created
+      expect(await fs.exists(aliasFsPath)).toEqual(true)
+      // content is the target path relative to the alias location
+      const content = await fs.readFile(aliasFsPath, 'utf-8')
+      expect(content).toEqual(targetContent)
+      // create alias fails if alias already exists, remove it before next loop step
+      await fs.unlink(aliasFsPath)
+    }
+
+    const combinations = [
+      { targetPath: `targets.vhd`, targetContent: `../targets.vhd` },
+      { targetPath: `alias/targets.vhd`, targetContent: `targets.vhd` },
+      { targetPath: `alias/sub/targets.vhd`, targetContent: `sub/targets.vhd` },
+      { targetPath: `sibling/targets.vhd`, targetContent: `../sibling/targets.vhd` },
+    ]
+
+    for (const { targetPath, targetContent } of combinations) {
+      await testOneCombination({ targetPath, targetContent })
+    }
+  })
+})
+
+test('alias must have *.alias.vhd extension', async () => {
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    const aliasPath = `${tempDir}/invalidalias.vhd`
+    const targetPath = `${tempDir}/targets.vhd`
+    expect(async () => await VhdAbstract.createAlias(handler, aliasPath, targetPath)).rejects.toThrow()
+
+    expect(await fs.exists(aliasPath)).toEqual(false)
+  })
+})
+
+test('alias must not be chained', async () => {
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    const aliasPath = `${tempDir}/valid.alias.vhd`
+    const targetPath = `${tempDir}/an.other.valid.alias.vhd`
+    expect(async () => await VhdAbstract.createAlias(handler, aliasPath, targetPath)).rejects.toThrow()
+    expect(await fs.exists(aliasPath)).toEqual(false)
+  })
+})
+
+test('It rename and unlink a VHDFile', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    const { size } = await fs.stat(vhdFileName)
+    const targetFileName = `${tempDir}/renamed.vhd`
+
+    await VhdAbstract.rename(handler, vhdFileName, targetFileName)
+    expect(await fs.exists(vhdFileName)).toEqual(false)
+    const { size: renamedSize } = await fs.stat(targetFileName)
+    expect(size).toEqual(renamedSize)
+    await VhdAbstract.unlink(handler, targetFileName)
+    expect(await fs.exists(targetFileName)).toEqual(false)
+  })
+})
+
+test('It rename and unlink a VhdDirectory', async () => {
+  const initalSize = 4
+  const vhdDirectory = `${tempDir}/randomfile.dir`
+  await createRandomVhdDirectory(vhdDirectory, initalSize)
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    const vhd = yield openVhd(handler, vhdDirectory)
+    expect(vhd.header.cookie).toEqual('cxsparse')
+    expect(vhd.footer.cookie).toEqual('conectix')
+
+    const targetFileName = `${tempDir}/renamed.vhd`
+    await VhdAbstract.rename(handler, vhdDirectory, targetFileName)
+    expect(await fs.exists(vhdDirectory)).toEqual(false)
+    await VhdAbstract.unlink(handler, targetFileName)
+    expect(await fs.exists(targetFileName)).toEqual(false)
+  })
+})
+
+test('It create , rename and unlink alias', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  const aliasFileName = `${tempDir}/aliasFileName.alias.vhd`
+  const aliasFileNameRenamed = `${tempDir}/aliasFileNameRenamed.alias.vhd`
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    await VhdAbstract.createAlias(handler, aliasFileName, vhdFileName)
+    expect(await fs.exists(aliasFileName)).toEqual(true)
+    expect(await fs.exists(vhdFileName)).toEqual(true)
+
+    await VhdAbstract.rename(handler, aliasFileName, aliasFileNameRenamed)
+    expect(await fs.exists(aliasFileName)).toEqual(false)
+    expect(await fs.exists(vhdFileName)).toEqual(true)
+    expect(await fs.exists(aliasFileNameRenamed)).toEqual(true)
+
+    await VhdAbstract.unlink(handler, aliasFileNameRenamed)
+    expect(await fs.exists(aliasFileName)).toEqual(false)
+    expect(await fs.exists(vhdFileName)).toEqual(false)
+    expect(await fs.exists(aliasFileNameRenamed)).toEqual(false)
+  })
+})

packages/vhd-lib/src/Vhd/VhdAbstract.js

@@ -1,6 +1,9 @@
 import { computeBatSize, sectorsRoundUpNoZero, sectorsToBytes } from './_utils'
 import { PLATFORM_NONE, SECTOR_SIZE, PLATFORM_W2KU, PARENT_LOCATOR_ENTRIES } from '../_constants'
+import { resolveAlias, isVhdAlias } from '../_resolveAlias'
+
 import assert from 'assert'
+import path from 'path'
 
 export class VhdAbstract {
   #header
@@ -164,4 +167,41 @@ export class VhdAbstract {
       }
     }
   }
+
+  static async rename(handler, sourcePath, targetPath) {
+    await handler.rename(sourcePath, targetPath)
+  }
+
+  static async unlink(handler, path) {
+    const resolved = await resolveAlias(handler, path)
+    try {
+      await handler.unlink(resolved)
+    } catch (err) {
+      if (err.code === 'EISDIR') {
+        await handler.rmtree(resolved)
+      } else {
+        throw err
+      }
+    }
+
+    // also delete the alias file
+    if (path !== resolved) {
+      await handler.unlink(path)
+    }
+  }
+
+  static async createAlias(handler, aliasPath, targetPath) {
+    if (!isVhdAlias(aliasPath)) {
+      throw new Error(`Alias must be named *.alias.vhd,  ${aliasPath} given`)
+    }
+    if (isVhdAlias(targetPath)) {
+      throw new Error(`Chaining alias is forbidden ${aliasPath} to ${targetPath}`)
+    }
+    // aliasPath and targetPath are absolute path from the root of the handler
+    // normalize them so they can't  escape this dir
+    const aliasDir = path.dirname(path.resolve('/', aliasPath))
+    // only store the relative path from alias to target
+    const relativePathToTarget = path.relative(aliasDir, path.resolve('/', targetPath))
+    await handler.writeFile(aliasPath, relativePathToTarget)
+  }
 }

packages/vhd-lib/src/_resolveAlias.integ.spec.js

@@ -0,0 +1,56 @@
+/* eslint-env jest */
+
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { Disposable, pFromCallback } from 'promise-toolbox'
+
+import { isVhdAlias, resolveAlias } from './_resolveAlias'
+
+let tempDir
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+test('is vhd alias recognize only *.alias.vhd files', () => {
+  expect(isVhdAlias('filename.alias.vhd')).toEqual(true)
+  expect(isVhdAlias('alias.vhd')).toEqual(false)
+  expect(isVhdAlias('filename.vhd')).toEqual(false)
+  expect(isVhdAlias('filename.alias.vhd.other')).toEqual(false)
+})
+
+test('resolve return the path in argument for a non alias file ', async () => {
+  expect(await resolveAlias(null, 'filename.vhd')).toEqual('filename.vhd')
+})
+test('resolve get the path of the target file for an alias', async () => {
+  await Disposable.use(async function* () {
+    // same directory
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    const tempDirFomRemoteUrl = tempDir.slice(1) // remove the / which is included in the remote url
+    const alias = `${tempDirFomRemoteUrl}/alias.alias.vhd`
+    await handler.writeFile(alias, 'target.vhd')
+    expect(await resolveAlias(handler, alias)).toEqual(`${tempDirFomRemoteUrl}/target.vhd`)
+
+    // different directory
+    await handler.mkdir(`${tempDirFomRemoteUrl}/sub/`)
+    await handler.writeFile(alias, 'sub/target.vhd', { flags: 'w' })
+    expect(await resolveAlias(handler, alias)).toEqual(`${tempDirFomRemoteUrl}/sub/target.vhd`)
+  })
+})
+
+test('resolve throws an error an alias to an alias', async () => {
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file:///' })
+    const alias = `${tempDir}/alias.alias.vhd`
+    const target = `${tempDir}/target.alias.vhd`
+    await handler.writeFile(alias, target)
+    expect(async () => await resolveAlias(handler, alias)).rejects.toThrow(Error)
+  })
+})

packages/vhd-lib/src/_resolveAlias.js

@@ -0,0 +1,18 @@
+import resolveRelativeFromFile from './_resolveRelativeFromFile'
+
+export function isVhdAlias(filename) {
+  return filename.endsWith('.alias.vhd')
+}
+
+export async function resolveAlias(handler, filename) {
+  if (!isVhdAlias(filename)) {
+    return filename
+  }
+  const aliasContent = (await handler.readFile(filename)).toString().trim()
+  // also handle circular references and unreasonnably long chains
+  if (isVhdAlias(aliasContent)) {
+    throw new Error(`Chaining alias is forbidden ${filename} to ${aliasContent}`)
+  }
+  // the target is relative to the alias location
+  return resolveRelativeFromFile(filename, aliasContent)
+}

packages/vhd-lib/src/openVhd.integ.spec.js

@@ -0,0 +1,62 @@
+/* eslint-env jest */
+
+import rimraf from 'rimraf'
+import tmp from 'tmp'
+import { getSyncedHandler } from '@xen-orchestra/fs'
+import { Disposable, pFromCallback } from 'promise-toolbox'
+
+import { openVhd } from './index'
+import { createRandomFile, convertFromRawToVhd, createRandomVhdDirectory } from './tests/utils'
+
+import { VhdAbstract } from './Vhd/VhdAbstract'
+
+let tempDir
+
+jest.setTimeout(60000)
+
+beforeEach(async () => {
+  tempDir = await pFromCallback(cb => tmp.dir(cb))
+})
+
+afterEach(async () => {
+  await pFromCallback(cb => rimraf(tempDir, cb))
+})
+
+test('It opens a vhd file ( alias or not)', async () => {
+  const initalSize = 4
+  const rawFileName = `${tempDir}/randomfile`
+  await createRandomFile(rawFileName, initalSize)
+  const vhdFileName = `${tempDir}/randomfile.vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file://' })
+    const vhd = yield openVhd(handler, vhdFileName)
+    expect(vhd.header.cookie).toEqual('cxsparse')
+    expect(vhd.footer.cookie).toEqual('conectix')
+
+    const aliasFileName = `${tempDir}/out.alias.vhd`
+    await VhdAbstract.createAlias(handler, aliasFileName, vhdFileName)
+    const alias = yield openVhd(handler, aliasFileName)
+    expect(alias.header.cookie).toEqual('cxsparse')
+    expect(alias.footer.cookie).toEqual('conectix')
+  })
+})
+
+test('It opens a vhd directory', async () => {
+  const initalSize = 4
+  const vhdDirectory = `${tempDir}/randomfile.dir`
+  await createRandomVhdDirectory(vhdDirectory, initalSize)
+
+  await Disposable.use(async function* () {
+    const handler = yield getSyncedHandler({ url: 'file://' })
+    const vhd = yield openVhd(handler, vhdDirectory)
+    expect(vhd.header.cookie).toEqual('cxsparse')
+    expect(vhd.footer.cookie).toEqual('conectix')
+
+    const aliasFileName = `${tempDir}/out.alias.vhd`
+    await VhdAbstract.createAlias(handler, aliasFileName, vhdDirectory)
+    const alias = yield openVhd(handler, aliasFileName)
+    expect(alias.header.cookie).toEqual('cxsparse')
+    expect(alias.footer.cookie).toEqual('conectix')
+  })
+})

packages/vhd-lib/src/openVhd.js

@@ -1,12 +1,14 @@
+import { resolveAlias } from './_resolveAlias'
 import { VhdFile, VhdDirectory } from './'
 
 export async function openVhd(handler, path) {
+  const resolved = await resolveAlias(handler, path)
   try {
-    return await VhdFile.open(handler, path)
+    return await VhdFile.open(handler, resolved)
   } catch (e) {
     if (e.code !== 'EISDIR') {
       throw e
     }
-    return await VhdDirectory.open(handler, path)
+    return await VhdDirectory.open(handler, resolved)
   }
 }

packages/vhd-lib/src/tests/utils.js

@@ -48,3 +48,40 @@ export async function recoverRawContent(vhdName, rawName, originalSize) {
     await execa('truncate', ['-s', originalSize, rawName])
   }
 }
+
+export async function createRandomVhdDirectory(path, sizeMB) {
+  fs.mkdir(path)
+  const rawFileName = `${path}/temp.raw`
+  await createRandomFile(rawFileName, sizeMB)
+  const vhdFileName = `${path}/vhd`
+  await convertFromRawToVhd(rawFileName, vhdFileName)
+
+  const srcVhd = await fs.open(vhdFileName, 'r')
+
+  const footer = Buffer.alloc(512)
+  await fs.read(srcVhd, footer, 0, footer.length, 0)
+  await fs.writeFile(path + '/footer', footer)
+
+  const header = Buffer.alloc(1024)
+  await fs.read(srcVhd, header, 0, header.length, 512)
+  await fs.writeFile(path + '/header', header)
+
+  await fs.close(srcVhd)
+
+  // a BAT , with at most 512 blocks of 2MB
+  const bat = Buffer.alloc(512, 1)
+  await fs.writeFile(path + '/bat', bat)
+
+  // copy blocks
+  const srcRaw = await fs.open(rawFileName, 'r')
+  const blockDataSize = 512 * 4096
+  const bitmap = Buffer.alloc(4096)
+  await fs.mkdir(path + '/blocks/')
+  await fs.mkdir(path + '/blocks/1/')
+  for (let i = 0, offset = 0; i < sizeMB; i++, offset += blockDataSize) {
+    const blockData = Buffer.alloc(blockDataSize)
+    await fs.read(srcRaw, blockData, offset)
+    await fs.writeFile(path + '/blocks/1/' + i, Buffer.concat([bitmap, blockData]))
+  }
+  await fs.close(srcRaw)
+}