fix(xo-server/jobs): report backups as failed if already running (#4534)
Fixes #4497
This commit is contained in:
parent
aaae2583c7
commit
30a6877f8a
@ -24,6 +24,7 @@
|
||||
- [Backup NG] Fix race conditions that could lead to disabled jobs still running (PR [4510](https://github.com/vatesfr/xen-orchestra/pull/4510))
|
||||
- [XOA] Remove "Updates" and "Licenses" tabs for non admin users (PR [#4526](https://github.com/vatesfr/xen-orchestra/pull/4526))
|
||||
- [New VM] Ability to escape [cloud config template](https://xen-orchestra.com/blog/xen-orchestra-5-21/#cloudconfigtemplates) variables [#4486](https://github.com/vatesfr/xen-orchestra/issues/4486) (PR [#4501](https://github.com/vatesfr/xen-orchestra/pull/4501))
|
||||
- [Backup NG] Properly log and report if job is already running [#4497](https://github.com/vatesfr/xen-orchestra/issues/4497) (PR [4534](https://github.com/vatesfr/xen-orchestra/pull/4534))
|
||||
|
||||
### Released packages
|
||||
|
||||
|
@ -243,38 +243,17 @@ export default class Jobs {
|
||||
}
|
||||
|
||||
async _runJob(job: Job, schedule?: Schedule, data_?: any) {
|
||||
const { id } = job
|
||||
|
||||
const runningJobs = this._runningJobs
|
||||
if (id in runningJobs) {
|
||||
throw new Error(`job ${id} is already running`)
|
||||
}
|
||||
|
||||
const { type } = job
|
||||
const executor = this._executors[type]
|
||||
if (executor === undefined) {
|
||||
throw new Error(`cannot run job ${id}: no executor for type ${type}`)
|
||||
}
|
||||
|
||||
let data
|
||||
if (type === 'backup') {
|
||||
// $FlowFixMe only defined for BackupJob
|
||||
const settings = job.settings['']
|
||||
data = {
|
||||
// $FlowFixMe only defined for BackupJob
|
||||
mode: job.mode,
|
||||
reportWhen: (settings && settings.reportWhen) || 'failure',
|
||||
}
|
||||
}
|
||||
if (type === 'metadataBackup') {
|
||||
data = {
|
||||
reportWhen: job.settings['']?.reportWhen ?? 'failure',
|
||||
}
|
||||
}
|
||||
|
||||
const logger = this._logger
|
||||
const { id, type } = job
|
||||
const runJobId = logger.notice(`Starting execution of ${id}.`, {
|
||||
data,
|
||||
data:
|
||||
type === 'backup' || type === 'metadataBackup'
|
||||
? {
|
||||
// $FlowFixMe only defined for BackupJob
|
||||
mode: job.mode,
|
||||
reportWhen: job.settings['']?.reportWhen ?? 'failure',
|
||||
}
|
||||
: undefined,
|
||||
event: 'job.start',
|
||||
userId: job.userId,
|
||||
jobId: id,
|
||||
@ -285,44 +264,64 @@ export default class Jobs {
|
||||
type,
|
||||
})
|
||||
|
||||
// runId is a temporary property used to check if the report is sent after the server interruption
|
||||
this.updateJob({ id, runId: runJobId })::ignoreErrors()
|
||||
runningJobs[id] = runJobId
|
||||
|
||||
const runs = this._runs
|
||||
|
||||
const { cancel, token } = CancelToken.source()
|
||||
runs[runJobId] = { cancel }
|
||||
|
||||
let session
|
||||
const app = this._app
|
||||
try {
|
||||
session = app.createUserConnection()
|
||||
session.set('user_id', job.userId)
|
||||
const runningJobs = this._runningJobs
|
||||
|
||||
const status = await executor({
|
||||
app,
|
||||
cancelToken: token,
|
||||
data: data_,
|
||||
job,
|
||||
logger,
|
||||
runJobId,
|
||||
schedule,
|
||||
session,
|
||||
})
|
||||
await logger.notice(
|
||||
`Execution terminated for ${job.id}.`,
|
||||
{
|
||||
event: 'job.end',
|
||||
if (id in runningJobs) {
|
||||
throw new Error(`the job (${id}) is already running`)
|
||||
}
|
||||
|
||||
const executor = this._executors[type]
|
||||
if (executor === undefined) {
|
||||
throw new Error(`cannot run job (${id}): no executor for type ${type}`)
|
||||
}
|
||||
|
||||
// runId is a temporary property used to check if the report is sent after the server interruption
|
||||
this.updateJob({ id, runId: runJobId })::ignoreErrors()
|
||||
runningJobs[id] = runJobId
|
||||
|
||||
const runs = this._runs
|
||||
let session
|
||||
try {
|
||||
const { cancel, token } = CancelToken.source()
|
||||
runs[runJobId] = { cancel }
|
||||
|
||||
session = app.createUserConnection()
|
||||
session.set('user_id', job.userId)
|
||||
|
||||
const status = await executor({
|
||||
app,
|
||||
cancelToken: token,
|
||||
data: data_,
|
||||
job,
|
||||
logger,
|
||||
runJobId,
|
||||
},
|
||||
true
|
||||
)
|
||||
schedule,
|
||||
session,
|
||||
})
|
||||
|
||||
app.emit('job:terminated', runJobId, {
|
||||
type: job.type,
|
||||
status,
|
||||
})
|
||||
await logger.notice(
|
||||
`Execution terminated for ${job.id}.`,
|
||||
{
|
||||
event: 'job.end',
|
||||
runJobId,
|
||||
},
|
||||
true
|
||||
)
|
||||
|
||||
app.emit('job:terminated', runJobId, {
|
||||
type: job.type,
|
||||
status,
|
||||
})
|
||||
} finally {
|
||||
this.updateJob({ id, runId: null })::ignoreErrors()
|
||||
delete runningJobs[id]
|
||||
delete runs[runJobId]
|
||||
if (session !== undefined) {
|
||||
session.close()
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
await logger.error(
|
||||
`The execution of ${id} has failed.`,
|
||||
@ -337,13 +336,6 @@ export default class Jobs {
|
||||
type: job.type,
|
||||
})
|
||||
throw error
|
||||
} finally {
|
||||
this.updateJob({ id, runId: null })::ignoreErrors()
|
||||
delete runningJobs[id]
|
||||
delete runs[runJobId]
|
||||
if (session !== undefined) {
|
||||
session.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -2,6 +2,7 @@
|
||||
|
||||
import asyncMap from '@xen-orchestra/async-map'
|
||||
import { createSchedule } from '@xen-orchestra/cron'
|
||||
import { ignoreErrors } from 'promise-toolbox'
|
||||
import { keyBy } from 'lodash'
|
||||
import { noSuchObject } from 'xo-common/api-errors'
|
||||
|
||||
@ -155,7 +156,9 @@ export default class Scheduling {
|
||||
this._runs[id] = createSchedule(
|
||||
schedule.cron,
|
||||
schedule.timezone
|
||||
).startJob(() => this._app.runJobSequence([schedule.jobId], schedule))
|
||||
).startJob(() => {
|
||||
ignoreErrors.call(this._app.runJobSequence([schedule.jobId], schedule))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user