feat(xo-server-load-balancer): limit concurrent migrations (#7297)

Fixes #7084

CHANGELOG.unreleased.md

@@ -16,6 +16,7 @@
 - [VM/Advanced] Admin can change VM creator [Forum#7313](https://xcp-ng.org/forum/topic/7313/change-created-by-and-date-information) (PR [#7276](https://github.com/vatesfr/xen-orchestra/pull/7276))
 - [Host/Reboot] Confirmation modal to reboot an updated slave host if the master is not [#7059](https://github.com/vatesfr/xen-orchestra/issues/7059) (PR [#7293](https://github.com/vatesfr/xen-orchestra/pull/7293))
 - [Backup/Restore] Show whether the memory was backed up (PR [#7315](https://github.com/vatesfr/xen-orchestra/pull/7315))
+- [Plugin/load-balancer] Limit concurrent VM migrations to 2 (configurable) to avoid long paused VMs [#7084](https://github.com/vatesfr/xen-orchestra/issues/7084) (PR [#7297](https://github.com/vatesfr/xen-orchestra/pull/7297))
 
 ### Bug fixes
 
@@ -57,7 +58,7 @@
 - @xen-orchestra/xapi minor
 - xen-api patch
 - xo-server minor
-- xo-server-load-balancer patch
+- xo-server-load-balancer minor
 - xo-web minor
 
 <!--packages-end-->

packages/xo-server-load-balancer/package.json

@@ -29,6 +29,7 @@
   "dependencies": {
     "@xen-orchestra/cron": "^1.0.6",
     "@xen-orchestra/log": "^0.6.0",
+    "limit-concurrency-decorator": "^0.6.0",
     "lodash": "^4.16.2"
   },
   "devDependencies": {

packages/xo-server-load-balancer/src/density-plan.js

@@ -208,7 +208,13 @@ export default class DensityPlan extends Plan {
         debug(
           `Migrate VM (${vm.id} "${vm.name_label}") to Host (${destination.id} "${destination.name_label}") from Host (${fmtSrcHost}).`
         )
-        return xapiDest.migrateVm(vm._xapiId, this.xo.getXapi(destination), destination._xapiId)
+        return this._concurrentMigrationLimiter.call(
+          xapiDest,
+          'migrateVm',
+          vm._xapiId,
+          this.xo.getXapi(destination),
+          destination._xapiId
+        )
       })
     )
 

packages/xo-server-load-balancer/src/index.js

@@ -1,5 +1,6 @@
 import { createSchedule } from '@xen-orchestra/cron'
 import { intersection, uniq } from 'lodash'
+import { limitConcurrency } from 'limit-concurrency-decorator'
 
 import DensityPlan from './density-plan'
 import PerformancePlan from './performance-plan'
@@ -104,6 +105,20 @@ export const configurationSchema = {
         $type: 'Tag',
       },
     },
+    advanced: {
+      title: 'Advanced',
+      type: 'object',
+      default: {},
+      properties: {
+        maxConcurrentMigrations: {
+          default: 2,
+          description: 'Limit maximum number of simultaneous migrations for faster migrations',
+          minimum: 1,
+          title: 'Maximum concurrent migrations',
+          type: 'integer',
+        },
+      },
+    },
   },
 
   additionalProperties: false,
@@ -124,10 +139,11 @@ class LoadBalancerPlugin {
     })
   }
 
-  async configure({ plans, ignoredVmTags = [] }) {
+  async configure({ plans, advanced, ignoredVmTags = [] }) {
     this._plans = []
     this._poolIds = [] // Used pools.
     this._globalOptions = { ignoredVmTags: new Set(ignoredVmTags) }
+    this._concurrentMigrationLimiter = limitConcurrency(advanced.maxConcurrentMigrations)()
 
     if (plans) {
       for (const plan of plans) {
@@ -155,11 +171,11 @@ class LoadBalancerPlugin {
     this._poolIds = this._poolIds.concat(pools)
     let plan
     if (mode === PERFORMANCE_MODE) {
-      plan = new PerformancePlan(this.xo, name, pools, options, this._globalOptions)
+      plan = new PerformancePlan(this.xo, name, pools, options, this._globalOptions, this._concurrentMigrationLimiter)
     } else if (mode === DENSITY_MODE) {
-      plan = new DensityPlan(this.xo, name, pools, options, this._globalOptions)
+      plan = new DensityPlan(this.xo, name, pools, options, this._globalOptions, this._concurrentMigrationLimiter)
     } else {
-      plan = new SimplePlan(this.xo, name, pools, options, this._globalOptions)
+      plan = new SimplePlan(this.xo, name, pools, options, this._globalOptions, this._concurrentMigrationLimiter)
     }
     this._plans.push(plan)
   }

packages/xo-server-load-balancer/src/performance-plan.js

@@ -205,7 +205,15 @@ export default class PerformancePlan extends Plan {
       )
       optimizationCount++
 
-      promises.push(xapiSrc.migrateVm(vm._xapiId, this.xo.getXapi(destination), destination._xapiId))
+      promises.push(
+        this._concurrentMigrationLimiter.call(
+          xapiSrc,
+          'migrateVm',
+          vm._xapiId,
+          this.xo.getXapi(destination),
+          destination._xapiId
+        )
+      )
     }
 
     await Promise.all(promises)

packages/xo-server-load-balancer/src/plan.js

@@ -97,7 +97,14 @@ function setRealCpuAverageOfVms(vms, vmsAverages, nCpus) {
 // ===================================================================
 
 export default class Plan {
-  constructor(xo, name, poolIds, { excludedHosts, thresholds, antiAffinityTags = [] }, globalOptions) {
+  constructor(
+    xo,
+    name,
+    poolIds,
+    { excludedHosts, thresholds, antiAffinityTags = [] },
+    globalOptions,
+    concurrentMigrationLimiter
+  ) {
     this.xo = xo
     this._name = name
     this._poolIds = poolIds
@@ -113,6 +120,7 @@ export default class Plan {
     }
     this._antiAffinityTags = antiAffinityTags
     this._globalOptions = globalOptions
+    this._concurrentMigrationLimiter = concurrentMigrationLimiter
 
     for (const key in this._thresholds) {
       const attr = this._thresholds[key]
@@ -295,7 +303,6 @@ export default class Plan {
         tagsDiff[watchedTag] = diff - 1
       }
     }
-
     if (isEmpty(tagsDiff)) {
       return
     }
@@ -412,7 +419,15 @@ export default class Plan {
         delete sourceHost.vms[vm.id]
 
         // 4. Migrate.
-        promises.push(this.xo.getXapi(source).migrateVm(vm._xapiId, this.xo.getXapi(destination), destination._xapiId))
+        promises.push(
+          this._concurrentMigrationLimiter.call(
+            this.xo.getXapi(source),
+            'migrateVm',
+            vm._xapiId,
+            this.xo.getXapi(destination),
+            destination._xapiId
+          )
+        )
 
         break // Continue with the same tag, the source can be different.
       }

yarn.lock

@@ -14116,6 +14116,11 @@ limit-concurrency-decorator@^0.5.0:
   resolved "https://registry.yarnpkg.com/limit-concurrency-decorator/-/limit-concurrency-decorator-0.5.0.tgz#7455fc7c8d12e93ce725cb98dc18b861397dd726"
   integrity sha512-s5HqdnTpRJhvK/vleMY3qJ3yEfIQ1BUCUqbBJwtXCKngMSc+qpS1Rl6/rxdhr1Z/oQz3keYho6G4XCFSHb7nbA==
 
+limit-concurrency-decorator@^0.6.0:
+  version "0.6.0"
+  resolved "https://registry.yarnpkg.com/limit-concurrency-decorator/-/limit-concurrency-decorator-0.6.0.tgz#f356e543b5c4cec6b03948a68e6a57821b78866d"
+  integrity sha512-KYye2PB+skq7b6iv+k6Dwqs3AwFoMi75Zqoe3PJkGDwvt0ads215S+nJ68SPOlZNSMV6hOvz7M8Ddui67TTn+Q==
+
 limiter@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/limiter/-/limiter-2.1.0.tgz#d38d7c5b63729bb84fb0c4d8594b7e955a5182a2"