fix(xo-server/jobs): report backups as failed if already running (#4534)

Fixes #4497
This commit is contained in:
badrAZ 2019-09-19 17:27:16 +02:00 committed by Julien Fontanet
parent aaae2583c7
commit 30a6877f8a
3 changed files with 67 additions and 71 deletions

View File

@ -24,6 +24,7 @@
- [Backup NG] Fix race conditions that could lead to disabled jobs still running (PR [4510](https://github.com/vatesfr/xen-orchestra/pull/4510))
- [XOA] Remove "Updates" and "Licenses" tabs for non admin users (PR [#4526](https://github.com/vatesfr/xen-orchestra/pull/4526))
- [New VM] Ability to escape [cloud config template](https://xen-orchestra.com/blog/xen-orchestra-5-21/#cloudconfigtemplates) variables [#4486](https://github.com/vatesfr/xen-orchestra/issues/4486) (PR [#4501](https://github.com/vatesfr/xen-orchestra/pull/4501))
- [Backup NG] Properly log and report if job is already running [#4497](https://github.com/vatesfr/xen-orchestra/issues/4497) (PR [4534](https://github.com/vatesfr/xen-orchestra/pull/4534))
### Released packages

View File

@ -243,38 +243,17 @@ export default class Jobs {
}
async _runJob(job: Job, schedule?: Schedule, data_?: any) {
const { id } = job
const runningJobs = this._runningJobs
if (id in runningJobs) {
throw new Error(`job ${id} is already running`)
}
const { type } = job
const executor = this._executors[type]
if (executor === undefined) {
throw new Error(`cannot run job ${id}: no executor for type ${type}`)
}
let data
if (type === 'backup') {
// $FlowFixMe only defined for BackupJob
const settings = job.settings['']
data = {
// $FlowFixMe only defined for BackupJob
mode: job.mode,
reportWhen: (settings && settings.reportWhen) || 'failure',
}
}
if (type === 'metadataBackup') {
data = {
reportWhen: job.settings['']?.reportWhen ?? 'failure',
}
}
const logger = this._logger
const { id, type } = job
const runJobId = logger.notice(`Starting execution of ${id}.`, {
data,
data:
type === 'backup' || type === 'metadataBackup'
? {
// $FlowFixMe only defined for BackupJob
mode: job.mode,
reportWhen: job.settings['']?.reportWhen ?? 'failure',
}
: undefined,
event: 'job.start',
userId: job.userId,
jobId: id,
@ -285,44 +264,64 @@ export default class Jobs {
type,
})
// runId is a temporary property used to check if the report is sent after the server interruption
this.updateJob({ id, runId: runJobId })::ignoreErrors()
runningJobs[id] = runJobId
const runs = this._runs
const { cancel, token } = CancelToken.source()
runs[runJobId] = { cancel }
let session
const app = this._app
try {
session = app.createUserConnection()
session.set('user_id', job.userId)
const runningJobs = this._runningJobs
const status = await executor({
app,
cancelToken: token,
data: data_,
job,
logger,
runJobId,
schedule,
session,
})
await logger.notice(
`Execution terminated for ${job.id}.`,
{
event: 'job.end',
if (id in runningJobs) {
throw new Error(`the job (${id}) is already running`)
}
const executor = this._executors[type]
if (executor === undefined) {
throw new Error(`cannot run job (${id}): no executor for type ${type}`)
}
// runId is a temporary property used to check if the report is sent after the server interruption
this.updateJob({ id, runId: runJobId })::ignoreErrors()
runningJobs[id] = runJobId
const runs = this._runs
let session
try {
const { cancel, token } = CancelToken.source()
runs[runJobId] = { cancel }
session = app.createUserConnection()
session.set('user_id', job.userId)
const status = await executor({
app,
cancelToken: token,
data: data_,
job,
logger,
runJobId,
},
true
)
schedule,
session,
})
app.emit('job:terminated', runJobId, {
type: job.type,
status,
})
await logger.notice(
`Execution terminated for ${job.id}.`,
{
event: 'job.end',
runJobId,
},
true
)
app.emit('job:terminated', runJobId, {
type: job.type,
status,
})
} finally {
this.updateJob({ id, runId: null })::ignoreErrors()
delete runningJobs[id]
delete runs[runJobId]
if (session !== undefined) {
session.close()
}
}
} catch (error) {
await logger.error(
`The execution of ${id} has failed.`,
@ -337,13 +336,6 @@ export default class Jobs {
type: job.type,
})
throw error
} finally {
this.updateJob({ id, runId: null })::ignoreErrors()
delete runningJobs[id]
delete runs[runJobId]
if (session !== undefined) {
session.close()
}
}
}

View File

@ -2,6 +2,7 @@
import asyncMap from '@xen-orchestra/async-map'
import { createSchedule } from '@xen-orchestra/cron'
import { ignoreErrors } from 'promise-toolbox'
import { keyBy } from 'lodash'
import { noSuchObject } from 'xo-common/api-errors'
@ -155,7 +156,9 @@ export default class Scheduling {
this._runs[id] = createSchedule(
schedule.cron,
schedule.timezone
).startJob(() => this._app.runJobSequence([schedule.jobId], schedule))
).startJob(() => {
ignoreErrors.call(this._app.runJobSequence([schedule.jobId], schedule))
})
}
}