feat(job-executor): supports dynamic param vectors (#369)

See vatesfr/xo-web#837

src/api/job.js

@@ -38,14 +38,7 @@ create.params = {
           items: {
             type: 'array',
             items: {
-              type: 'object',
-              properties: {
-                type: {type: 'string'},
-                values: {
-                  type: 'array',
-                  items: {type: 'object'}
-                }
-              }
+              type: 'object'
             }
           }
         },
@@ -77,14 +70,7 @@ set.params = {
           items: {
             type: 'array',
             items: {
-              type: 'object',
-              properties: {
-                type: {type: 'string'},
-                values: {
-                  type: 'array',
-                  items: {type: 'object'}
-                }
-              }
+              type: 'object'
             }
           }
         },

src/job-executor.js

@@ -1,9 +1,18 @@
 import assign from 'lodash/assign'
-import {BaseError} from 'make-error'
+import every from 'lodash/every'
+import filter from 'lodash/filter'
+import isArray from 'lodash/isArray'
+import isPlainObject from 'lodash/isPlainObject'
+import map from 'lodash/map'
+import mapValues from 'lodash/mapValues'
+import size from 'lodash/size'
+import some from 'lodash/some'
+import { BaseError } from 'make-error'
 
+import { crossProduct } from './math'
 import {
-  createRawObject,
-  forEach
+  forEach,
+  thunkToArray
 } from './utils'
 
 export class JobExecutorError extends BaseError {}
@@ -18,30 +27,67 @@ export class UnsupportedVectorType extends JobExecutorError {
   }
 }
 
-export const productParams = (...args) => {
-  let product = createRawObject()
-  assign(product, ...args)
-  return product
+// ===================================================================
+
+const match = (pattern, value) => {
+  if (isPlainObject(pattern)) {
+    if (pattern.__or && size(pattern) === 1) {
+      return some(pattern.__or, subpattern => match(subpattern, value))
+    }
+
+    return isPlainObject(value) && every(pattern, (subpattern, key) => (
+      value[key] !== undefined && match(subpattern, value[key])
+    ))
+  }
+
+  if (isArray(pattern)) {
+    return isArray(value) && every(pattern, subpattern =>
+      some(value, subvalue => match(subpattern, subvalue))
+    )
+  }
+
+  return pattern === value
 }
 
-export function _computeCrossProduct (items, productCb, extractValueMap = {}) {
-  const upstreamValues = []
-  const itemsCopy = items.slice()
-  const item = itemsCopy.pop()
-  const values = extractValueMap[item.type] && extractValueMap[item.type](item) || item
-  forEach(values, value => {
-    if (itemsCopy.length) {
-      let downstreamValues = _computeCrossProduct(itemsCopy, productCb, extractValueMap)
-      forEach(downstreamValues, downstreamValue => {
-        upstreamValues.push(productCb(value, downstreamValue))
+const paramsVectorActionsMap = {
+  extractProperties ({ mapping, value }) {
+    return mapValues(mapping, key => value[key])
+  },
+  crossProduct ({ items }) {
+    return thunkToArray(crossProduct(
+      map(items, value => resolveParamsVector.call(this, value))
+    ))
+  },
+  fetchObjects ({ pattern }) {
+    return filter(this.xo.getObjects(), object => match(pattern, object))
+  },
+  map ({ collection, iteratee }) {
+    return map(resolveParamsVector.call(this, collection), value => {
+      const {
+        paramName = 'value',
+        ...iterateeConf
+      } = iteratee
+
+      return resolveParamsVector.call(this, {
+        ...iterateeConf,
+        [paramName]: value
       })
-    } else {
-      upstreamValues.push(value)
-    }
-  })
-  return upstreamValues
+    })
+  },
+  set: ({ values }) => values
 }
 
+export function resolveParamsVector (paramsVector) {
+  const visitor = paramsVectorActionsMap[paramsVector.type]
+  if (!visitor) {
+    throw new Error(`Unsupported function '${paramsVector.type}'.`)
+  }
+
+  return visitor.call(this, paramsVector)
+}
+
+// ===================================================================
+
 export default class JobExecutor {
   constructor (xo) {
     this.xo = xo
@@ -86,17 +132,10 @@ export default class JobExecutor {
   }
 
   async _execCall (job, runJobId) {
-    let paramsFlatVector
-
-    if (job.paramsVector) {
-      if (job.paramsVector.type === 'crossProduct') {
-        paramsFlatVector = _computeCrossProduct(job.paramsVector.items, productParams, this._extractValueCb)
-      } else {
-        throw new UnsupportedVectorType(job.paramsVector)
-      }
-    } else {
-      paramsFlatVector = [{}] // One call with no parameters
-    }
+    const { paramsVector } = job
+    const paramsFlatVector = paramsVector
+      ? resolveParamsVector.call(this, paramsVector)
+      : [{}] // One call with no parameters
 
     const connection = this.xo.createUserConnection()
     const promises = []

src/job-executor.spec.js

@@ -1,71 +1,100 @@
 /* eslint-env mocha */
 
-import {expect} from 'chai'
 import leche from 'leche'
+import { expect } from 'chai'
 
-import {
-  _computeCrossProduct,
-  productParams
-} from './job-executor'
+import { resolveParamsVector } from './job-executor'
 
-describe('productParams', function () {
+describe('resolveParamsVector', function () {
   leche.withData({
-    'Two sets of one': [
-      {a: 1, b: 2}, {a: 1}, {b: 2}
+    'cross product with three sets': [
+      // Expected result.
+      [ { id: 3, value: 'foo', remote: 'local' },
+        { id: 7, value: 'foo', remote: 'local' },
+        { id: 10, value: 'foo', remote: 'local' },
+        { id: 3, value: 'bar', remote: 'local' },
+        { id: 7, value: 'bar', remote: 'local' },
+        { id: 10, value: 'bar', remote: 'local' } ],
+      // Entry.
+      {
+        type: 'crossProduct',
+        items: [{
+          type: 'set',
+          values: [ { id: 3 }, { id: 7 }, { id: 10 } ]
+        }, {
+          type: 'set',
+          values: [ { value: 'foo' }, { value: 'bar' } ]
+        }, {
+          type: 'set',
+          values: [ { remote: 'local' } ]
+        }]
+      }
     ],
-    'Two sets of two': [
-      {a: 1, b: 2, c: 3, d: 4}, {a: 1, b: 2}, {c: 3, d: 4}
-    ],
-    'Three sets': [
-      {a: 1, b: 2, c: 3, d: 4, e: 5, f: 6}, {a: 1}, {b: 2, c: 3}, {d: 4, e: 5, f: 6}
-    ],
-    'One set': [
-      {a: 1, b: 2}, {a: 1, b: 2}
-    ],
-    'Empty set': [
-      {a: 1}, {a: 1}, {}
-    ],
-    'All empty': [
-      {}, {}, {}
-    ],
-    'No set': [
-      {}
+    'cross product with `set` and `map`': [
+      // Expected result.
+      [
+        { remote: 'local', id: 'vm:2' },
+        { remote: 'smb', id: 'vm:2' }
+      ],
+
+      // Entry.
+      {
+        type: 'crossProduct',
+        items: [{
+          type: 'set',
+          values: [ { remote: 'local' }, { remote: 'smb' } ]
+        }, {
+          type: 'map',
+          collection: {
+            type: 'fetchObjects',
+            pattern: {
+              $pool: { __or: [ 'pool:1', 'pool:8', 'pool:12' ] },
+              power_state: 'Running',
+              tags: [ 'foo' ],
+              type: 'VM'
+            }
+          },
+          iteratee: {
+            type: 'extractProperties',
+            mapping: { id: 'id' }
+          }
+        }]
+      },
+
+      // Context.
+      {
+        xo: {
+          getObjects: function () {
+            return [{
+              id: 'vm:1',
+              $pool: 'pool:1',
+              tags: [],
+              type: 'VM',
+              power_state: 'Halted'
+            }, {
+              id: 'vm:2',
+              $pool: 'pool:1',
+              tags: [ 'foo' ],
+              type: 'VM',
+              power_state: 'Running'
+            }, {
+              id: 'host:1',
+              type: 'host',
+              power_state: 'Running'
+            }, {
+              id: 'vm:3',
+              $pool: 'pool:8',
+              tags: [ 'foo' ],
+              type: 'VM',
+              power_state: 'Halted'
+            }]
+          }
+        }
+      }
     ]
-  }, function (resultSet, ...sets) {
-    it('Assembles all given param sets in on set', function () {
-      expect(productParams(...sets)).to.eql(resultSet)
-    })
-  })
-})
-
-describe('_computeCrossProduct', function () {
-  // Gives the sum of all args
-  const addTest = (...args) => args.reduce((prev, curr) => prev + curr, 0)
-  // Gives the product of all args
-  const multiplyTest = (...args) => args.reduce((prev, curr) => prev * curr, 1)
-
-  leche.withData({
-    '2 sets of 2 items to multiply': [
-      [10, 14, 15, 21], [[2, 3], [5, 7]], multiplyTest
-    ],
-    '3 sets of 2 items to multiply': [
-      [110, 130, 154, 182, 165, 195, 231, 273], [[2, 3], [5, 7], [11, 13]], multiplyTest
-    ],
-    '2 sets of 3 items to multiply': [
-      [14, 22, 26, 21, 33, 39, 35, 55, 65], [[2, 3, 5], [7, 11, 13]], multiplyTest
-    ],
-    '2 sets of 2 items to add': [
-      [7, 9, 8, 10], [[2, 3], [5, 7]], addTest
-    ],
-    '3 sets of 2 items to add': [
-      [18, 20, 20, 22, 19, 21, 21, 23], [[2, 3], [5, 7], [11, 13]], addTest
-    ],
-    '2 sets of 3 items to add': [
-      [9, 13, 15, 10, 14, 16, 12, 16, 18], [[2, 3, 5], [7, 11, 13]], addTest
-    ]
-  }, function (product, items, cb) {
-    it('Crosses sets of values with a crossProduct callback', function () {
-      expect(_computeCrossProduct(items, cb)).to.have.members(product)
+  }, function (expectedResult, entry, context) {
+    it('Resolves params vector', function () {
+      expect(resolveParamsVector.call(context, entry)).to.deep.have.members(expectedResult)
     })
   })
 })

src/math.js

@@ -0,0 +1,48 @@
+import assign from 'lodash/assign'
+
+const _combine = (vectors, n, cb) => {
+  if (!n) {
+    return
+  }
+
+  const nLast = n - 1
+
+  const vector = vectors[nLast]
+  const m = vector.length
+  if (n === 1) {
+    for (let i = 0; i < m; ++i) {
+      cb([ vector[i] ])
+    }
+    return
+  }
+
+  for (let i = 0; i < m; ++i) {
+    const value = vector[i]
+
+    _combine(vectors, nLast, (vector) => {
+      vector.push(value)
+      cb(vector)
+    })
+  }
+}
+
+// Compute all combinations from vectors.
+//
+// Ex: combine([[2, 3], [5, 7]])
+// => [ [ 2, 5 ], [ 3, 5 ], [ 2, 7 ], [ 3, 7 ] ]
+export const combine = vectors => cb => _combine(vectors, vectors.length, cb)
+
+// Merge the properties of an objects set in one object.
+//
+// Ex: mergeObjects([ { a: 1 }, { b: 2 } ]) => { a: 1, b: 2 }
+export const mergeObjects = objects => assign({}, ...objects)
+
+// Compute a cross product between vectors.
+//
+// Ex: crossProduct([ [ { a: 2 }, { b: 3 } ], [ { c: 5 }, { d: 7 } ] ] )
+// => [ { a: 2, c: 5 }, { b: 3, c: 5 }, { a: 2, d: 7 }, { b: 3, d: 7 } ]
+export const crossProduct = (vectors, mergeFn = mergeObjects) => cb => (
+  combine(vectors)(vector => {
+    cb(mergeFn(vector))
+  })
+)

src/math.spec.js

@@ -0,0 +1,72 @@
+/* eslint-env mocha */
+
+import { expect } from 'chai'
+import leche from 'leche'
+
+import { thunkToArray } from './utils'
+import {
+  crossProduct,
+  mergeObjects
+} from './math'
+
+describe('mergeObjects', function () {
+  leche.withData({
+    'Two sets of one': [
+      {a: 1, b: 2}, {a: 1}, {b: 2}
+    ],
+    'Two sets of two': [
+      {a: 1, b: 2, c: 3, d: 4}, {a: 1, b: 2}, {c: 3, d: 4}
+    ],
+    'Three sets': [
+      {a: 1, b: 2, c: 3, d: 4, e: 5, f: 6}, {a: 1}, {b: 2, c: 3}, {d: 4, e: 5, f: 6}
+    ],
+    'One set': [
+      {a: 1, b: 2}, {a: 1, b: 2}
+    ],
+    'Empty set': [
+      {a: 1}, {a: 1}, {}
+    ],
+    'All empty': [
+      {}, {}, {}
+    ],
+    'No set': [
+      {}
+    ]
+  }, function (resultSet, ...sets) {
+    it('Assembles all given param sets in on set', function () {
+      expect(mergeObjects(sets)).to.eql(resultSet)
+    })
+  })
+})
+
+describe('crossProduct', function () {
+  // Gives the sum of all args
+  const addTest = args => args.reduce((prev, curr) => prev + curr, 0)
+  // Gives the product of all args
+  const multiplyTest = args => args.reduce((prev, curr) => prev * curr, 1)
+
+  leche.withData({
+    '2 sets of 2 items to multiply': [
+      [10, 14, 15, 21], [[2, 3], [5, 7]], multiplyTest
+    ],
+    '3 sets of 2 items to multiply': [
+      [110, 130, 154, 182, 165, 195, 231, 273], [[2, 3], [5, 7], [11, 13]], multiplyTest
+    ],
+    '2 sets of 3 items to multiply': [
+      [14, 22, 26, 21, 33, 39, 35, 55, 65], [[2, 3, 5], [7, 11, 13]], multiplyTest
+    ],
+    '2 sets of 2 items to add': [
+      [7, 9, 8, 10], [[2, 3], [5, 7]], addTest
+    ],
+    '3 sets of 2 items to add': [
+      [18, 20, 20, 22, 19, 21, 21, 23], [[2, 3], [5, 7], [11, 13]], addTest
+    ],
+    '2 sets of 3 items to add': [
+      [9, 13, 15, 10, 14, 16, 12, 16, 18], [[2, 3, 5], [7, 11, 13]], addTest
+    ]
+  }, function (product, items, cb) {
+    it('Crosses sets of values with a crossProduct callback', function () {
+      expect(thunkToArray(crossProduct(items, cb))).to.have.members(product)
+    })
+  })
+})

src/schemas/job.js

@@ -1,5 +1,3 @@
-import paramsVector from 'job/params-vector'
-
 export default {
   $schema: 'http://json-schema.org/draft-04/schema#',
   type: 'object',
@@ -27,7 +25,9 @@ export default {
       type: 'string',
       description: 'called method'
     },
-    paramsVector
+    paramsVector: {
+      type: 'object'
+    }
   },
   required: [
     'type',

src/schemas/job/params-vector.js

@@ -1,59 +0,0 @@
-export default {
-  $schema: 'http://json-schema.org/draft-04/schema#',
-  type: 'object',
-  properties: {
-    type: {
-      enum: ['crossProduct']
-    },
-    items: {
-      type: 'array',
-      description: 'vector of values to multiply with others vectors',
-      items: {
-        type: 'object',
-        properties: {
-          type: {
-            enum: ['set']
-          },
-          values: {
-            type: 'array',
-            items: {
-              type: 'object'
-            },
-            minItems: 1
-          }
-        },
-        required: [
-          'type',
-          'values'
-        ]
-      },
-      minItems: 1
-    }
-  },
-  required: [
-    'type',
-    'items'
-  ]
-}
-
-/* Example:
-{
-  "type": "cross product",
-  "items": [
-    {
-      "type": "set",
-      "values":  [
-        {"id": 0, "name": "snapshost de 0"},
-        {"id": 1, "name": "snapshost de 1"}
-      ],
-    },
-    {
-      "type": "set",
-      "values": [
-        {"force": true}
-      ]
-    }
-  ]
-}
-
-*/

src/utils.js

@@ -484,5 +484,15 @@ export const scheduleFn = (cronTime, fn, timeZone) => {
 
 // -------------------------------------------------------------------
 
+// Create an array which contains the results of one thunk function.
+// Only works with synchronous thunks.
+export const thunkToArray = thunk => {
+  const values = []
+  thunk(::values.push)
+  return values
+}
+
+// -------------------------------------------------------------------
+
 // Wrap a value in a function.
 export const wrap = value => () => value