feat: add scheduler

2026-08-26 13:17:10 -05:00 · 2022-09-19 06:54:03 +00:00
parent 55b0b00cee
commit 6ce29bdb77
7 changed files with 103 additions and 153 deletions
@@ -148,6 +148,7 @@
    "passport-twitch-strategy": "2.2.0",
    "pem-jwk": "2.0.0",
    "pg": "8.8.0",
+    "pg-boss": "8.0.0",
    "pg-hstore": "2.3.4",
    "pg-pubsub": "0.8.0",
    "pg-query-stream": "4.2.4",
@@ -75,7 +75,7 @@ module.exports = {
    await WIKI.models.commentProviders.initProvider()
    await WIKI.models.sites.reloadCache()
    await WIKI.models.storage.initTargets()
-    // WIKI.scheduler.start()
+    WIKI.scheduler.start()

    await WIKI.models.subscribeToNotifications()
  },
@@ -4,8 +4,7 @@ const _ = require('lodash')
 const moment = require('moment')
 const CSR = require('@root/csr')
 const PEM = require('@root/pem')
-// eslint-disable-next-line node/no-deprecated-api
-const punycode = require('punycode')
+const punycode = require('punycode/')

 /* global WIKI */

@@ -1,134 +1,28 @@
-const moment = require('moment')
-const childProcess = require('child_process')
-const _ = require('lodash')
-const configHelper = require('../helpers/config')
+const PgBoss = require('pg-boss')

 /* global WIKI */

-class Job {
-  constructor({
-    name,
-    immediate = false,
-    schedule = 'P1D',
-    repeat = false,
-    worker = false
-  }, queue) {
-    this.queue = queue
-    this.finished = Promise.resolve()
-    this.name = name
-    this.immediate = immediate
-    this.schedule = moment.duration(schedule)
-    this.repeat = repeat
-    this.worker = worker
-  }
-
-  /**
-   * Start Job
-   *
-   * @param {Object} data Job Data
-   */
-  start(data) {
-    this.queue.jobs.push(this)
-    if (this.immediate) {
-      this.invoke(data)
-    } else {
-      this.enqueue(data)
-    }
-  }
-
-  /**
-   * Queue the next job run according to the wait duration
-   *
-   * @param {Object} data Job Data
-   */
-  enqueue(data) {
-    this.timeout = setTimeout(this.invoke.bind(this), this.schedule.asMilliseconds(), data)
-  }
-
-  /**
-   * Run the actual job
-   *
-   * @param {Object} data Job Data
-   */
-  async invoke(data) {
-    try {
-      if (this.worker) {
-        const proc = childProcess.fork(`server/core/worker.js`, [
-          `--job=${this.name}`,
-          `--data=${data}`
-        ], {
-          cwd: WIKI.ROOTPATH,
-          stdio: ['inherit', 'inherit', 'pipe', 'ipc']
-        })
-        const stderr = [];
-        proc.stderr.on('data', chunk => stderr.push(chunk))
-        this.finished = new Promise((resolve, reject) => {
-          proc.on('exit', (code, signal) => {
-            const data = Buffer.concat(stderr).toString()
-            if (code === 0) {
-              resolve(data)
-            } else {
-              const err = new Error(`Error when running job ${this.name}: ${data}`)
-              err.exitSignal = signal
-              err.exitCode = code
-              err.stderr = data
-              reject(err)
-            }
-            proc.kill()
-          })
-        })
-      } else {
-        this.finished = require(`../jobs/${this.name}`)(data)
-      }
-      await this.finished
-    } catch (err) {
-      WIKI.logger.warn(err)
-    }
-    if (this.repeat && this.queue.jobs.includes(this)) {
-      this.enqueue(data)
-    } else {
-      this.stop().catch(() => {})
-    }
-  }
-
-  /**
-   * Stop any future job invocation from occuring
-   */
-  async stop() {
-    clearTimeout(this.timeout)
-    this.queue.jobs = this.queue.jobs.filter(x => x !== this)
-    return this.finished
-  }
-}
-
 module.exports = {
+  scheduler: null,
  jobs: [],
-  init() {
+  init () {
+    WIKI.logger.info('Initializing Scheduler...')
+    this.scheduler = new PgBoss({
+      ...WIKI.models.knex.client.connectionSettings,
+      application_name: 'Wiki.js Scheduler',
+      schema: WIKI.config.db.schemas.scheduler,
+      uuid: 'v4'
+    })
    return this
  },
-  start() {
-    _.forOwn(WIKI.data.jobs, (queueParams, queueName) => {
-      if (WIKI.config.offline && queueParams.offlineSkip) {
-        WIKI.logger.warn(`Skipping job ${queueName} because offline mode is enabled. [SKIPPED]`)
-        return
-      }
-
-      const schedule = (configHelper.isValidDurationString(queueParams.schedule)) ? queueParams.schedule : 'P1D'
-      this.registerJob({
-        name: _.kebabCase(queueName),
-        immediate: _.get(queueParams, 'onInit', false),
-        schedule: schedule,
-        repeat: _.get(queueParams, 'repeat', false),
-        worker: _.get(queueParams, 'worker', false)
-      })
-    })
+  async start () {
+    WIKI.logger.info('Starting Scheduler...')
+    await this.scheduler.start()
+    WIKI.logger.info('Scheduler: [ STARTED ]')
  },
-  registerJob(opts, data) {
-    const job = new Job(opts, this)
-    job.start(data)
-    return job
-  },
-  async stop() {
-    return Promise.all(this.jobs.map(job => job.stop()))
+  async stop () {
+    WIKI.logger.info('Stopping Scheduler...')
+    await this.scheduler.stop()
+    WIKI.logger.info('Scheduler: [ STOPPED ]')
  }
 }
@@ -1,24 +0,0 @@
-const path = require('path')
-
-let WIKI = {
-  IS_DEBUG: process.env.NODE_ENV === 'development',
-  ROOTPATH: process.cwd(),
-  SERVERPATH: path.join(process.cwd(), 'server'),
-  Error: require('../helpers/error'),
-  configSvc: require('./config')
-}
-global.WIKI = WIKI
-
-WIKI.configSvc.init()
-WIKI.logger = require('./logger').init('JOB')
-const args = require('yargs').argv
-
-;(async () => {
-  try {
-    await require(`../jobs/${args.job}`)(args.data)
-    process.exit(0)
-  } catch (e) {
-    await new Promise(resolve => process.stderr.write(e.message, resolve))
-    process.exit(1)
-  }
-})()
@@ -77,6 +77,7 @@ useMeta({

      > img {
        height: 200px;
+        user-select: none;
      }
    }

@@ -3812,6 +3812,14 @@ aggregate-error@^3.0.0:
    clean-stack "^2.0.0"
    indent-string "^4.0.0"

+aggregate-error@^4.0.0:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-4.0.1.tgz#25091fe1573b9e0be892aeda15c7c66a545f758e"
+  integrity sha512-0poP0T7el6Vq3rstR8Mn4V/IQrpBLO6POkUSrN7RhyY+GF/InCFShQzsQ39T25gkHhLgSLByyAz+Kjb+c2L98w==
+  dependencies:
+    clean-stack "^4.0.0"
+    indent-string "^5.0.0"
+
 ajv-errors@^1.0.0:
  version "1.0.1"
  resolved "https://registry.yarnpkg.com/ajv-errors/-/ajv-errors-1.0.1.tgz#f35986aceb91afadec4102fbd85014950cefa64d"
@@ -5720,6 +5728,13 @@ clean-stack@^2.0.0:
  resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b"
  integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==

+clean-stack@^4.0.0:
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-4.2.0.tgz#c464e4cde4ac789f4e0735c5d75beb49d7b30b31"
+  integrity sha512-LYv6XPxoyODi36Dp976riBtSY27VmFo+MKqEU9QCCWyTrdEPDog+RWA7xQWHi6Vbp61j5c4cdzzX1NidnwtUWg==
+  dependencies:
+    escape-string-regexp "5.0.0"
+
 clean-webpack-plugin@3.0.0:
  version "3.0.0"
  resolved "https://registry.yarnpkg.com/clean-webpack-plugin/-/clean-webpack-plugin-3.0.0.tgz#a99d8ec34c1c628a4541567aa7b457446460c62b"
@@ -6386,6 +6401,13 @@ create-hmac@^1.1.0, create-hmac@^1.1.2, create-hmac@^1.1.4:
    safe-buffer "^5.0.1"
    sha.js "^2.4.8"

+cron-parser@^4.0.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.6.0.tgz#404c3fdbff10ae80eef6b709555d577ef2fd2e0d"
+  integrity sha512-guZNLMGUgg6z4+eGhmHGw7ft+v6OQeuHzd1gcLxCo9Yg/qoxmG3nindp2/uwGCLizEisf2H0ptqeVXeoCpP6FA==
+  dependencies:
+    luxon "^3.0.1"
+
 cross-fetch@3.1.5:
  version "3.1.5"
  resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.5.tgz#e1389f44d9e7ba767907f7af8454787952ab534f"
@@ -7486,6 +7508,11 @@ delaunator@4:
  resolved "https://registry.yarnpkg.com/delaunator/-/delaunator-4.0.1.tgz#3d779687f57919a7a418f8ab947d3bddb6846957"
  integrity sha512-WNPWi1IRKZfCt/qIDMfERkDp93+iZEmOxN2yy4Jg+Xhv8SLk2UTqqbe1sfiipn0and9QrE914/ihdx82Y/Giag==

+delay@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/delay/-/delay-5.0.0.tgz#137045ef1b96e5071060dd5be60bf9334436bd1d"
+  integrity sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw==
+
 delayed-stream@~1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
@@ -8166,6 +8193,11 @@ escape-html@^1.0.3, escape-html@~1.0.3:
  resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988"
  integrity sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=

+escape-string-regexp@5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz#4683126b500b61762f2dbebace1806e8be31b1c8"
+  integrity sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==
+
 escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5:
  version "1.0.5"
  resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4"
@@ -10225,6 +10257,11 @@ indent-string@^4.0.0:
  resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-4.0.0.tgz#624f8f4497d619b2d9768531d58f4122854d7251"
  integrity sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==

+indent-string@^5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-5.0.0.tgz#4fd2980fccaf8622d14c64d694f4cf33c81951a5"
+  integrity sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==
+
 indexes-of@^1.0.1:
  version "1.0.1"
  resolved "https://registry.yarnpkg.com/indexes-of/-/indexes-of-1.0.1.tgz#f30f716c8e2bd346c7b67d3df3915566a7c05607"
@@ -11884,6 +11921,11 @@ lodash.clonedeep@^4.5.0:
  resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef"
  integrity sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==

+lodash.debounce@^4.0.8:
+  version "4.0.8"
+  resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af"
+  integrity sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==
+
 lodash.includes@^4.3.0:
  version "4.3.0"
  resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f"
@@ -12073,6 +12115,11 @@ luxon@2.3.1:
  resolved "https://registry.yarnpkg.com/luxon/-/luxon-2.3.1.tgz#f276b1b53fd9a740a60e666a541a7f6dbed4155a"
  integrity sha512-I8vnjOmhXsMSlNMZlMkSOvgrxKJl0uOsEzdGgGNZuZPaS9KlefpE9KV95QFftlJSC+1UyCC9/I69R02cz/zcCA==

+luxon@^3.0.1:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.0.3.tgz#573e65531efd3d92265feb640f02ba7a192e2388"
+  integrity sha512-+EfHWnF+UT7GgTnq5zXg3ldnTKL2zdv7QJgsU5bjjpbH17E3qi/puMhQyJVYuCq+FRkogvB5WB6iVvUr+E4a7w==
+
 make-dir@^2.0.0, make-dir@^2.1.0:
  version "2.1.0"
  resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-2.1.0.tgz#5f0310e18b8be898cc07009295a30ae41e91e6f5"
@@ -13467,6 +13514,13 @@ p-map@^4.0.0:
  dependencies:
    aggregate-error "^3.0.0"

+p-map@^5.3.0:
+  version "5.5.0"
+  resolved "https://registry.yarnpkg.com/p-map/-/p-map-5.5.0.tgz#054ca8ca778dfa4cf3f8db6638ccb5b937266715"
+  integrity sha512-VFqfGDHlx87K66yZrNdI4YGtD70IRyd+zSvgks6mzHPRNkoKy+9EKP4SFC77/vTTQYmRmti7dvqC+m5jBrBAcg==
+  dependencies:
+    aggregate-error "^4.0.0"
+
 p-try@^1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3"
@@ -13957,6 +14011,19 @@ persistgraphql@^0.3.11:
    "@types/graphql" "^0.9.0"
    "@types/isomorphic-fetch" "0.0.34"

+pg-boss@8.0.0:
+  version "8.0.0"
+  resolved "https://registry.yarnpkg.com/pg-boss/-/pg-boss-8.0.0.tgz#a32cd2c6f09c894b9e3ace027ecdecc06929edf6"
+  integrity sha512-WTchkRcTS9/AFuXhNzCQ9KlHgvi9VI3YpPk2EqGDhf2Od+/5Ug1e8b+NB+A81swe8LusAoQ6ka6n4pBkpkgrkw==
+  dependencies:
+    cron-parser "^4.0.0"
+    delay "^5.0.0"
+    lodash.debounce "^4.0.8"
+    p-map "^5.3.0"
+    pg "^8.5.1"
+    serialize-error "^11.0.0"
+    uuid "^8.3.2"
+
 pg-connection-string@2.5.0, pg-connection-string@^2.5.0:
  version "2.5.0"
  resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34"
@@ -14027,7 +14094,7 @@ pg-types@^2.1.0:
    postgres-date "~1.0.4"
    postgres-interval "^1.1.0"

-pg@8.8.0, pg@^8.7.3:
+pg@8.8.0, pg@^8.5.1, pg@^8.7.3:
  version "8.8.0"
  resolved "https://registry.yarnpkg.com/pg/-/pg-8.8.0.tgz#a77f41f9d9ede7009abfca54667c775a240da686"
  integrity sha512-UXYN0ziKj+AeNNP7VDMwrehpACThH7LUl/p8TDFpEUuSejCUIwGSfxpHsPvtM6/WXFy6SU4E5RG4IJV/TZAGjw==
@@ -16732,6 +16799,13 @@ send@0.18.0:
    range-parser "~1.2.1"
    statuses "2.0.1"

+serialize-error@^11.0.0:
+  version "11.0.0"
+  resolved "https://registry.yarnpkg.com/serialize-error/-/serialize-error-11.0.0.tgz#0129f2b07b19b09bc7a5f2d850ffe9cd2d561582"
+  integrity sha512-YKrURWDqcT3VGX/s/pCwaWtpfJEEaEw5Y4gAnQDku92b/HjVj4r4UhA5QrMVMFotymK2wIWs5xthny5SMFu7Vw==
+  dependencies:
+    type-fest "^2.12.2"
+
 serialize-javascript@^2.1.2:
  version "2.1.2"
  resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-2.1.2.tgz#ecec53b0e0317bdc95ef76ab7074b7384785fa61"
@@ -18098,6 +18172,11 @@ type-fest@^0.8.1:
  resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d"
  integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==

+type-fest@^2.12.2:
+  version "2.19.0"
+  resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-2.19.0.tgz#88068015bb33036a598b952e55e9311a60fd3a9b"
+  integrity sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==
+
 type-is@^1.6.4, type-is@~1.6.17, type-is@~1.6.18:
  version "1.6.18"
  resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131"
@@ -18446,7 +18525,7 @@ uuid@8.0.0:
  resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.0.0.tgz#bc6ccf91b5ff0ac07bbcdbf1c7c4e150db4dbb6c"
  integrity sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==

-uuid@8.3.2:
+uuid@8.3.2, uuid@^8.3.2:
  version "8.3.2"
  resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
  integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==