feat(xo-server-perf-plugin): initial release (#33)
This commit is contained in:
parent
8d47c83117
commit
354fd88028
38
packages/xo-server-perf-alert/.babelrc.js
Normal file
38
packages/xo-server-perf-alert/.babelrc.js
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
'use strict'
|
||||||
|
|
||||||
|
const NODE_ENV = process.env.NODE_ENV || 'development'
|
||||||
|
const __PROD__ = NODE_ENV === 'production'
|
||||||
|
const __TEST__ = NODE_ENV === 'test'
|
||||||
|
|
||||||
|
const pkg = require('./package')
|
||||||
|
|
||||||
|
let nodeCompat = (pkg.engines || {}).node
|
||||||
|
if (nodeCompat === undefined) {
|
||||||
|
nodeCompat = '6'
|
||||||
|
} else {
|
||||||
|
const trimChars = '^=>~'
|
||||||
|
while (trimChars.includes(nodeCompat[0])) {
|
||||||
|
nodeCompat = nodeCompat.slice(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
comments: !__PROD__,
|
||||||
|
ignore: __TEST__ ? undefined : [/\.spec\.js$/],
|
||||||
|
plugins: ['lodash'],
|
||||||
|
presets: [
|
||||||
|
[
|
||||||
|
'@babel/env',
|
||||||
|
{
|
||||||
|
debug: !__TEST__,
|
||||||
|
loose: true,
|
||||||
|
shippedProposals: true,
|
||||||
|
targets: __PROD__
|
||||||
|
? { node: nodeCompat }
|
||||||
|
: { node: 'current' },
|
||||||
|
useBuiltIns: '@babel/polyfill' in (pkg.dependencies || {}) && 'usage',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
'@babel/flow',
|
||||||
|
],
|
||||||
|
}
|
24
packages/xo-server-perf-alert/.npmignore
Normal file
24
packages/xo-server-perf-alert/.npmignore
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
/benchmark/
|
||||||
|
/benchmarks/
|
||||||
|
*.bench.js
|
||||||
|
*.bench.js.map
|
||||||
|
|
||||||
|
/examples/
|
||||||
|
example.js
|
||||||
|
example.js.map
|
||||||
|
*.example.js
|
||||||
|
*.example.js.map
|
||||||
|
|
||||||
|
/fixture/
|
||||||
|
/fixtures/
|
||||||
|
*.fixture.js
|
||||||
|
*.fixture.js.map
|
||||||
|
*.fixtures.js
|
||||||
|
*.fixtures.js.map
|
||||||
|
|
||||||
|
/test/
|
||||||
|
/tests/
|
||||||
|
*.spec.js
|
||||||
|
*.spec.js.map
|
||||||
|
|
||||||
|
__snapshots__/
|
31
packages/xo-server-perf-alert/README.md
Normal file
31
packages/xo-server-perf-alert/README.md
Normal file
@ -0,0 +1,31 @@
|
|||||||
|
# xo-server-perf-alert [](https://travis-ci.org/vatesfr/xen-orchestra)
|
||||||
|
|
||||||
|
> Alert plugin for Xen Orchestra
|
||||||
|
|
||||||
|
## Install
|
||||||
|
|
||||||
|
Installation of the [npm package](https://npmjs.org/package/xo-server-perf-alert):
|
||||||
|
|
||||||
|
```
|
||||||
|
> npm install --global xo-server-perf-alert
|
||||||
|
```
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
Like all xo-server plugins, it can be configured directly via
|
||||||
|
the web interface, see [the plugin documentation](https://xen-orchestra.com/docs/plugins.html).
|
||||||
|
|
||||||
|
## Contributions
|
||||||
|
|
||||||
|
Contributions are *very* welcomed, either on the documentation or on
|
||||||
|
the code.
|
||||||
|
|
||||||
|
You may:
|
||||||
|
|
||||||
|
- report any [issue](https://github.com/vatesfr/xo-web/issues)
|
||||||
|
you've encountered;
|
||||||
|
- fork and create a pull request.
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
AGPL3 © [Vates SAS](http://vates.fr)
|
45
packages/xo-server-perf-alert/package.json
Normal file
45
packages/xo-server-perf-alert/package.json
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
{
|
||||||
|
"name": "xo-server-perf-alert",
|
||||||
|
"version": "0.0.0",
|
||||||
|
"license": "AGPL-3.0",
|
||||||
|
"description": "",
|
||||||
|
"keywords": [],
|
||||||
|
"homepage": "https://github.com/vatesfr/xen-orchestra/tree/master/packages/xo-server-perf-alert",
|
||||||
|
"bugs": "https://github.com/vatesfr/xo-web/issues",
|
||||||
|
"repository": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/vatesfr/xen-orchestra.git"
|
||||||
|
},
|
||||||
|
"preferGlobal": true,
|
||||||
|
"main": "dist/",
|
||||||
|
"bin": {},
|
||||||
|
"files": [
|
||||||
|
"dist/"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">=6"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"cron": "^1.3.0",
|
||||||
|
"d3-time-format": "^2.1.1",
|
||||||
|
"json5": "^0.5.1",
|
||||||
|
"lodash": "^4.17.4"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@babel/cli": "7.0.0-beta.38",
|
||||||
|
"@babel/core": "7.0.0-beta.38",
|
||||||
|
"@babel/preset-env": "7.0.0-beta.38",
|
||||||
|
"@babel/preset-flow": "^7.0.0-beta.38",
|
||||||
|
"babel-plugin-lodash": "^3.3.2",
|
||||||
|
"cross-env": "^5.1.3",
|
||||||
|
"rimraf": "^2.6.2"
|
||||||
|
},
|
||||||
|
"scripts": {
|
||||||
|
"build": "cross-env NODE_ENV=production babel --source-maps --out-dir=dist/ src/",
|
||||||
|
"clean": "rimraf dist/",
|
||||||
|
"dev": "cross-env NODE_ENV=development babel --watch --source-maps --out-dir=dist/ src/",
|
||||||
|
"prebuild": "yarn run clean",
|
||||||
|
"predev": "yarn run clean",
|
||||||
|
"prepublishOnly": "yarn run build"
|
||||||
|
}
|
||||||
|
}
|
839
packages/xo-server-perf-alert/src/index.js
Normal file
839
packages/xo-server-perf-alert/src/index.js
Normal file
@ -0,0 +1,839 @@
|
|||||||
|
import JSON5 from 'json5'
|
||||||
|
import { CronJob } from 'cron'
|
||||||
|
import { forOwn, map, mean } from 'lodash'
|
||||||
|
import { utcParse } from 'd3-time-format'
|
||||||
|
|
||||||
|
const VM_FUNCTIONS = {
|
||||||
|
cpuUsage: {
|
||||||
|
description:
|
||||||
|
'Raises an alarm when the average usage of any CPU is higher than the threshold',
|
||||||
|
unit: '%',
|
||||||
|
comparator: '>',
|
||||||
|
createParser: (legend, threshold) => {
|
||||||
|
const regex = /cpu[0-9]+/
|
||||||
|
const filteredLegends = legend.filter(l => l.name.match(regex))
|
||||||
|
const accumulator = Object.assign(
|
||||||
|
...filteredLegends.map(l => ({ [l.name]: [] }))
|
||||||
|
)
|
||||||
|
const getDisplayableValue = () => {
|
||||||
|
const means = Object.keys(accumulator).map(l => mean(accumulator[l]))
|
||||||
|
return Math.max(...means) * 100
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
parseRow: data => {
|
||||||
|
filteredLegends.forEach(l => {
|
||||||
|
accumulator[l.name].push(data.values[l.index])
|
||||||
|
})
|
||||||
|
},
|
||||||
|
getDisplayableValue,
|
||||||
|
shouldAlarm: () => getDisplayableValue() > threshold,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
memoryUsage: {
|
||||||
|
description:
|
||||||
|
'Raises an alarm when the used memory % is higher than the threshold',
|
||||||
|
unit: '% used',
|
||||||
|
comparator: '>',
|
||||||
|
createParser: (legend, threshold) => {
|
||||||
|
const memoryBytesLegend = legend.find(l => l.name === 'memory')
|
||||||
|
const memoryKBytesFreeLegend = legend.find(
|
||||||
|
l => l.name === 'memory_internal_free'
|
||||||
|
)
|
||||||
|
const usedMemoryRatio = []
|
||||||
|
const getDisplayableValue = () => mean(usedMemoryRatio) * 100
|
||||||
|
return {
|
||||||
|
parseRow: data => {
|
||||||
|
const memory = data.values[memoryBytesLegend.index]
|
||||||
|
usedMemoryRatio.push(
|
||||||
|
(memory - 1024 * data.values[memoryKBytesFreeLegend.index]) / memory
|
||||||
|
)
|
||||||
|
},
|
||||||
|
getDisplayableValue,
|
||||||
|
shouldAlarm: () => {
|
||||||
|
return getDisplayableValue() > threshold
|
||||||
|
},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
const HOST_FUNCTIONS = {
|
||||||
|
cpuUsage: {
|
||||||
|
description:
|
||||||
|
'Raises an alarm when the average usage of any CPU is higher than the threshold',
|
||||||
|
unit: '%',
|
||||||
|
comparator: '>',
|
||||||
|
createParser: (legend, threshold) => {
|
||||||
|
const regex = /^cpu[0-9]+$/
|
||||||
|
const filteredLegends = legend.filter(l => l.name.match(regex))
|
||||||
|
const accumulator = Object.assign(
|
||||||
|
...filteredLegends.map(l => ({ [l.name]: [] }))
|
||||||
|
)
|
||||||
|
const getDisplayableValue = () => {
|
||||||
|
const means = Object.keys(accumulator).map(l => mean(accumulator[l]))
|
||||||
|
return Math.max(...means) * 100
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
parseRow: data => {
|
||||||
|
filteredLegends.forEach(l => {
|
||||||
|
accumulator[l.name].push(data.values[l.index])
|
||||||
|
})
|
||||||
|
},
|
||||||
|
getDisplayableValue,
|
||||||
|
shouldAlarm: () => getDisplayableValue() > threshold,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
memoryUsage: {
|
||||||
|
description:
|
||||||
|
'Raises an alarm when the used memory % is higher than the threshold',
|
||||||
|
unit: '% used',
|
||||||
|
comparator: '>',
|
||||||
|
createParser: (legend, threshold) => {
|
||||||
|
const memoryKBytesLegend = legend.find(
|
||||||
|
l => l.name === 'memory_total_kib'
|
||||||
|
)
|
||||||
|
const memoryKBytesFreeLegend = legend.find(
|
||||||
|
l => l.name === 'memory_free_kib'
|
||||||
|
)
|
||||||
|
const usedMemoryRatio = []
|
||||||
|
const getDisplayableValue = () => mean(usedMemoryRatio) * 100
|
||||||
|
return {
|
||||||
|
parseRow: data => {
|
||||||
|
const memory = data.values[memoryKBytesLegend.index]
|
||||||
|
usedMemoryRatio.push(
|
||||||
|
(memory - data.values[memoryKBytesFreeLegend.index]) / memory
|
||||||
|
)
|
||||||
|
},
|
||||||
|
getDisplayableValue,
|
||||||
|
shouldAlarm: () => {
|
||||||
|
return getDisplayableValue() > threshold
|
||||||
|
},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
const TYPE_FUNCTION_MAP = {
|
||||||
|
vm: VM_FUNCTIONS,
|
||||||
|
host: HOST_FUNCTIONS,
|
||||||
|
}
|
||||||
|
|
||||||
|
// list of currently ringing alarms, to avoid double notification
|
||||||
|
const currentAlarms = {}
|
||||||
|
|
||||||
|
export const configurationSchema = {
|
||||||
|
type: 'object',
|
||||||
|
properties: {
|
||||||
|
baseUrl: {
|
||||||
|
type: 'string',
|
||||||
|
title: 'Xen Orchestra URL',
|
||||||
|
description:
|
||||||
|
'URL used in alert messages to quickly get to the VMs (ex: https://xoa.company.tld/ )',
|
||||||
|
},
|
||||||
|
hostMonitors: {
|
||||||
|
type: 'array',
|
||||||
|
title: 'Host Monitors',
|
||||||
|
description:
|
||||||
|
'Alarms checking hosts on all pools. The selected performance counter is sampled regularly and averaged. ' +
|
||||||
|
'The Average is compared to the threshold and an alarm is raised upon crossing',
|
||||||
|
items: {
|
||||||
|
type: 'object',
|
||||||
|
properties: {
|
||||||
|
uuids: {
|
||||||
|
title: 'Hosts',
|
||||||
|
type: 'array',
|
||||||
|
items: {
|
||||||
|
type: 'string',
|
||||||
|
$type: 'Host',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
variableName: {
|
||||||
|
title: 'Alarm Type',
|
||||||
|
description: Object.keys(HOST_FUNCTIONS)
|
||||||
|
.map(
|
||||||
|
k =>
|
||||||
|
` * ${k} (${HOST_FUNCTIONS[k].unit}): ${
|
||||||
|
HOST_FUNCTIONS[k].description
|
||||||
|
}`
|
||||||
|
)
|
||||||
|
.join('\n'),
|
||||||
|
type: 'string',
|
||||||
|
default: Object.keys(HOST_FUNCTIONS)[0],
|
||||||
|
enum: Object.keys(HOST_FUNCTIONS),
|
||||||
|
},
|
||||||
|
alarmTriggerLevel: {
|
||||||
|
title: 'Threshold',
|
||||||
|
description:
|
||||||
|
'The direction of the crossing is given by the Alarm type',
|
||||||
|
type: 'number',
|
||||||
|
default: 40,
|
||||||
|
},
|
||||||
|
alarmTriggerPeriod: {
|
||||||
|
title: 'Average Length (s)',
|
||||||
|
description:
|
||||||
|
'The points are averaged this number of seconds then the average is compared with the threshold',
|
||||||
|
type: 'number',
|
||||||
|
default: 60,
|
||||||
|
enum: [60, 600],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
vmMonitors: {
|
||||||
|
type: 'array',
|
||||||
|
title: 'VM Monitors',
|
||||||
|
description:
|
||||||
|
'Alarms checking all VMs on all pools. The selected performance counter is sampled regularly and averaged. ' +
|
||||||
|
'The Average is compared to the threshold and an alarm is raised upon crossing',
|
||||||
|
items: {
|
||||||
|
type: 'object',
|
||||||
|
properties: {
|
||||||
|
uuids: {
|
||||||
|
title: 'Virtual Machines',
|
||||||
|
type: 'array',
|
||||||
|
items: {
|
||||||
|
type: 'string',
|
||||||
|
$type: 'VM',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
variableName: {
|
||||||
|
title: 'Alarm Type',
|
||||||
|
description: Object.keys(VM_FUNCTIONS)
|
||||||
|
.map(
|
||||||
|
k =>
|
||||||
|
` * ${k} (${VM_FUNCTIONS[k].unit}):${
|
||||||
|
VM_FUNCTIONS[k].description
|
||||||
|
}`
|
||||||
|
)
|
||||||
|
.join('\n'),
|
||||||
|
type: 'string',
|
||||||
|
default: Object.keys(VM_FUNCTIONS)[0],
|
||||||
|
enum: Object.keys(VM_FUNCTIONS),
|
||||||
|
},
|
||||||
|
alarmTriggerLevel: {
|
||||||
|
title: 'Threshold',
|
||||||
|
description:
|
||||||
|
'The direction of the crossing is given by the Alarm type',
|
||||||
|
type: 'number',
|
||||||
|
default: 40,
|
||||||
|
},
|
||||||
|
alarmTriggerPeriod: {
|
||||||
|
title: 'Average Length (s)',
|
||||||
|
description:
|
||||||
|
'The points are averaged this number of seconds then the average is compared with the threshold',
|
||||||
|
type: 'number',
|
||||||
|
default: 60,
|
||||||
|
enum: [60, 600],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
required: ['uuids'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
toEmails: {
|
||||||
|
type: 'array',
|
||||||
|
title: 'Email addresses',
|
||||||
|
description: 'Email addresses of the alert recipients',
|
||||||
|
|
||||||
|
items: {
|
||||||
|
type: 'string',
|
||||||
|
},
|
||||||
|
minItems: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
const clearCurrentAlarms = () =>
|
||||||
|
forOwn(currentAlarms, (v, k) => {
|
||||||
|
delete currentAlarms[k]
|
||||||
|
})
|
||||||
|
|
||||||
|
const raiseOrLowerAlarm = (
|
||||||
|
alarmId,
|
||||||
|
shouldRaise,
|
||||||
|
raiseCallback,
|
||||||
|
lowerCallback
|
||||||
|
) => {
|
||||||
|
const current = currentAlarms[alarmId]
|
||||||
|
if (shouldRaise) {
|
||||||
|
if (!current) {
|
||||||
|
currentAlarms[alarmId] = true
|
||||||
|
raiseCallback(alarmId)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (current) {
|
||||||
|
try {
|
||||||
|
lowerCallback(alarmId)
|
||||||
|
} finally {
|
||||||
|
delete currentAlarms[alarmId]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getServerTimestamp (xapi, host) {
|
||||||
|
const serverLocalTime = await xapi.call('host.get_servertime', host.$ref)
|
||||||
|
return Math.floor(
|
||||||
|
utcParse('%Y%m%dT%H:%M:%SZ')(serverLocalTime).getTime() / 1000
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
class PerfAlertXoPlugin {
|
||||||
|
constructor (xo) {
|
||||||
|
this._xo = xo
|
||||||
|
this._job = new CronJob({
|
||||||
|
cronTime: '* * * * *',
|
||||||
|
start: false,
|
||||||
|
onTick: this._checkMonitors.bind(this),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async configure (configuration) {
|
||||||
|
this._configuration = configuration
|
||||||
|
clearCurrentAlarms()
|
||||||
|
}
|
||||||
|
|
||||||
|
_generateUrl (type, object) {
|
||||||
|
const map = {
|
||||||
|
vm: () => `${this._configuration.baseUrl}#/vms/${object.uuid}/stats`,
|
||||||
|
host: () => `${this._configuration.baseUrl}#/hosts/${object.uuid}/stats`,
|
||||||
|
}
|
||||||
|
return map[type]()
|
||||||
|
}
|
||||||
|
|
||||||
|
async test () {
|
||||||
|
const hostMonitorPart2 = await Promise.all(
|
||||||
|
map(this._getMonitors(), async m => {
|
||||||
|
const tableBody = (await m.snapshot()).map(entry => entry.tableItem)
|
||||||
|
return `
|
||||||
|
## Monitor for ${m.title}
|
||||||
|
|
||||||
|
${m.tableHeader}
|
||||||
|
${tableBody.join('')}`
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
this._sendAlertEmail(
|
||||||
|
'TEST',
|
||||||
|
`
|
||||||
|
# Performance Alert Test
|
||||||
|
Your alarms and their current status:
|
||||||
|
${hostMonitorPart2.join('\n')}`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
load () {
|
||||||
|
this._job.start()
|
||||||
|
}
|
||||||
|
|
||||||
|
unload () {
|
||||||
|
this._job.stop()
|
||||||
|
}
|
||||||
|
|
||||||
|
_parseDefinition (definition) {
|
||||||
|
const alarmId = `${definition.objectType}|${definition.variableName}|${
|
||||||
|
definition.alarmTriggerLevel
|
||||||
|
}`
|
||||||
|
const typeFunction =
|
||||||
|
TYPE_FUNCTION_MAP[definition.objectType][definition.variableName]
|
||||||
|
const parseData = (result, uuid) => {
|
||||||
|
const parsedLegend = result.meta.legend.map((l, index) => {
|
||||||
|
const [operation, type, uuid, name] = l.split(':')
|
||||||
|
const parsedName = name.split('_')
|
||||||
|
const lastComponent = parsedName[parsedName.length - 1]
|
||||||
|
const relatedEntity =
|
||||||
|
parsedName.length > 1 && lastComponent.match(/^[0-9a-f]{8}$/)
|
||||||
|
? lastComponent
|
||||||
|
: null
|
||||||
|
return {
|
||||||
|
operation,
|
||||||
|
type,
|
||||||
|
uuid,
|
||||||
|
name,
|
||||||
|
relatedEntity,
|
||||||
|
parsedName,
|
||||||
|
index,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
const legendTree = {}
|
||||||
|
const getNode = (element, name, defaultValue = {}) => {
|
||||||
|
const child = element[name]
|
||||||
|
if (child === undefined) {
|
||||||
|
element[name] = defaultValue
|
||||||
|
return defaultValue
|
||||||
|
}
|
||||||
|
return child
|
||||||
|
}
|
||||||
|
parsedLegend.forEach(l => {
|
||||||
|
const root = getNode(legendTree, l.uuid)
|
||||||
|
const relatedNode = getNode(root, l.relatedEntity)
|
||||||
|
relatedNode[l.name] = l
|
||||||
|
})
|
||||||
|
const parser = typeFunction.createParser(
|
||||||
|
parsedLegend.filter(l => l.uuid === uuid),
|
||||||
|
definition.alarmTriggerLevel
|
||||||
|
)
|
||||||
|
result.data.forEach(d => parser.parseRow(d))
|
||||||
|
return parser
|
||||||
|
}
|
||||||
|
const observationPeriod =
|
||||||
|
definition.alarmTriggerPeriod !== undefined
|
||||||
|
? definition.alarmTriggerPeriod
|
||||||
|
: 60
|
||||||
|
const typeText = definition.objectType === 'host' ? 'Host' : 'VM'
|
||||||
|
return {
|
||||||
|
...definition,
|
||||||
|
alarmId,
|
||||||
|
vmFunction: typeFunction,
|
||||||
|
title: `${typeText} ${definition.variableName} ${
|
||||||
|
typeFunction.comparator
|
||||||
|
} ${definition.alarmTriggerLevel}${typeFunction.unit}`,
|
||||||
|
tableHeader: `${typeText} | Value | Alert\n--- | -----:| ---:`,
|
||||||
|
snapshot: async () => {
|
||||||
|
return Promise.all(
|
||||||
|
map(definition.uuids, async uuid => {
|
||||||
|
try {
|
||||||
|
const monitoredObject = this._xo.getXapi(uuid).getObject(uuid)
|
||||||
|
const objectLink = `[${
|
||||||
|
monitoredObject.name_label
|
||||||
|
}](${this._generateUrl(definition.objectType, monitoredObject)})`
|
||||||
|
const rrd = await this.getRrd(monitoredObject, observationPeriod)
|
||||||
|
const couldFindRRD = rrd !== null
|
||||||
|
const result = {
|
||||||
|
object: monitoredObject,
|
||||||
|
couldFindRRD,
|
||||||
|
objectLink: objectLink,
|
||||||
|
listItem: ` * ${typeText} ${objectLink} ${
|
||||||
|
definition.variableName
|
||||||
|
}: **Can't read performance counters**\n`,
|
||||||
|
tableItem: `${objectLink} | - | **Can't read performance counters**\n`,
|
||||||
|
}
|
||||||
|
if (!couldFindRRD) {
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
const data = parseData(rrd, monitoredObject.uuid)
|
||||||
|
const textValue =
|
||||||
|
data.getDisplayableValue().toFixed(1) + typeFunction.unit
|
||||||
|
const shouldAlarm = data.shouldAlarm()
|
||||||
|
return {
|
||||||
|
...result,
|
||||||
|
value: data.getDisplayableValue(),
|
||||||
|
shouldAlarm: shouldAlarm,
|
||||||
|
textValue: textValue,
|
||||||
|
listItem: ` * ${typeText} ${objectLink} ${
|
||||||
|
definition.variableName
|
||||||
|
}: ${textValue}\n`,
|
||||||
|
tableItem: `${objectLink} | ${textValue} | ${
|
||||||
|
shouldAlarm ? '**Alert Ongoing**' : 'no alert'
|
||||||
|
}\n`,
|
||||||
|
}
|
||||||
|
} catch (_) {
|
||||||
|
return {
|
||||||
|
uuid,
|
||||||
|
object: null,
|
||||||
|
couldFindRRD: false,
|
||||||
|
objectLink: `cannot find object ${uuid}`,
|
||||||
|
listItem: ` * ${typeText} ${uuid} ${
|
||||||
|
definition.variableName
|
||||||
|
}: **Can't read performance counters**\n`,
|
||||||
|
tableItem: `object ${uuid} | - | **Can't read performance counters**\n`,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_getMonitors () {
|
||||||
|
return map(this._configuration.hostMonitors, def =>
|
||||||
|
this._parseDefinition({ ...def, objectType: 'host' })
|
||||||
|
).concat(
|
||||||
|
map(this._configuration.vmMonitors, def =>
|
||||||
|
this._parseDefinition({ ...def, objectType: 'vm' })
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
async _checkMonitors () {
|
||||||
|
const monitors = this._getMonitors()
|
||||||
|
for (const monitor of monitors) {
|
||||||
|
const snapshot = await monitor.snapshot()
|
||||||
|
for (const entry of snapshot) {
|
||||||
|
raiseOrLowerAlarm(
|
||||||
|
`${monitor.alarmId}|${entry.uuid}|RRD`,
|
||||||
|
!entry.couldFindRRD,
|
||||||
|
() => {
|
||||||
|
this._sendAlertEmail(
|
||||||
|
'Secondary Issue',
|
||||||
|
`
|
||||||
|
## There was an issue when trying to check ${monitor.title}
|
||||||
|
${entry.listItem}`
|
||||||
|
)
|
||||||
|
},
|
||||||
|
() => {}
|
||||||
|
)
|
||||||
|
if (!entry.couldFindRRD) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
const raiseAlarm = alarmId => {
|
||||||
|
// sample XenCenter message:
|
||||||
|
// value: 1.242087 config: <variable> <name value="mem_usage"/> </variable>
|
||||||
|
this._xo
|
||||||
|
.getXapi(entry.object.uuid)
|
||||||
|
.call(
|
||||||
|
'message.create',
|
||||||
|
'ALARM',
|
||||||
|
3,
|
||||||
|
entry.object.$type,
|
||||||
|
entry.object.uuid,
|
||||||
|
`value: ${entry.value.toFixed(
|
||||||
|
1
|
||||||
|
)} config: <variable> <name value="${
|
||||||
|
monitor.variableName
|
||||||
|
}"/> </variable>`
|
||||||
|
)
|
||||||
|
this._sendAlertEmail(
|
||||||
|
'',
|
||||||
|
`
|
||||||
|
## ALERT ${monitor.title}
|
||||||
|
${entry.listItem}
|
||||||
|
### Description
|
||||||
|
${monitor.vmFunction.description}`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
const lowerAlarm = alarmId => {
|
||||||
|
console.log('lowering Alarm', alarmId)
|
||||||
|
this._sendAlertEmail(
|
||||||
|
'END OF ALERT',
|
||||||
|
`
|
||||||
|
## END OF ALERT ${monitor.title}
|
||||||
|
${entry.listItem}
|
||||||
|
### Description
|
||||||
|
${monitor.vmFunction.description}`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
raiseOrLowerAlarm(
|
||||||
|
`${monitor.alarmId}|${entry.uuid}`,
|
||||||
|
entry.shouldAlarm,
|
||||||
|
raiseAlarm,
|
||||||
|
lowerAlarm
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_sendAlertEmail (subjectSuffix, markdownBody) {
|
||||||
|
if (
|
||||||
|
this._configuration.toEmails !== undefined &&
|
||||||
|
this._xo.sendEmail !== undefined
|
||||||
|
) {
|
||||||
|
this._xo.sendEmail({
|
||||||
|
to: this._configuration.toEmails,
|
||||||
|
subject: `[Xen Orchestra] − Performance Alert ${subjectSuffix}`,
|
||||||
|
markdown:
|
||||||
|
markdownBody +
|
||||||
|
`\n\n\nSent from Xen Orchestra [perf-alert plugin](${
|
||||||
|
this._configuration.baseUrl
|
||||||
|
}#/settings/plugins)\n`,
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
throw new Error('The email alert system has a configuration issue.')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async getRrd (xoObject, secondsAgo) {
|
||||||
|
const host = xoObject.$type === 'host' ? xoObject : xoObject.$resident_on
|
||||||
|
if (host == null) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
// we get the xapi per host, because the alarms can check VMs in various pools
|
||||||
|
const xapi = this._xo.getXapi(host.uuid)
|
||||||
|
const serverTimestamp = await getServerTimestamp(xapi, host)
|
||||||
|
const payload = {
|
||||||
|
host,
|
||||||
|
query: {
|
||||||
|
cf: 'AVERAGE',
|
||||||
|
host: (xoObject.$type === 'host').toString(),
|
||||||
|
json: 'true',
|
||||||
|
start: serverTimestamp - secondsAgo,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if (xoObject.$type === 'vm') {
|
||||||
|
payload['vm_uuid'] = xoObject.uuid
|
||||||
|
}
|
||||||
|
// JSON is not well formed, can't use the default node parser
|
||||||
|
return JSON5.parse(
|
||||||
|
await (await xapi.getResource('/rrd_updates', payload)).readAll()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
exports.default = function ({ xo }) {
|
||||||
|
return new PerfAlertXoPlugin(xo)
|
||||||
|
}
|
||||||
|
|
||||||
|
/* example legend fields:
|
||||||
|
host : memory_total_kib
|
||||||
|
host : memory_free_kib
|
||||||
|
host : cpu_avg
|
||||||
|
host : cpu3
|
||||||
|
host : cpu2
|
||||||
|
host : cpu1
|
||||||
|
host : cpu0
|
||||||
|
host : loadavg
|
||||||
|
host : CPU3-avg-freq
|
||||||
|
host : CPU2-avg-freq
|
||||||
|
host : CPU1-avg-freq
|
||||||
|
host : CPU0-avg-freq
|
||||||
|
host : cpu3-P15
|
||||||
|
host : cpu3-P14
|
||||||
|
host : cpu3-P13
|
||||||
|
host : cpu3-P12
|
||||||
|
host : cpu3-P11
|
||||||
|
host : cpu3-P10
|
||||||
|
host : cpu3-P9
|
||||||
|
host : cpu3-P8
|
||||||
|
host : cpu3-P7
|
||||||
|
host : cpu3-P6
|
||||||
|
host : cpu3-P5
|
||||||
|
host : cpu3-P4
|
||||||
|
host : cpu3-P3
|
||||||
|
host : cpu3-P2
|
||||||
|
host : cpu3-P1
|
||||||
|
host : cpu3-P0
|
||||||
|
host : cpu2-P15
|
||||||
|
host : cpu2-P14
|
||||||
|
host : cpu2-P13
|
||||||
|
host : cpu2-P12
|
||||||
|
host : cpu2-P11
|
||||||
|
host : cpu2-P10
|
||||||
|
host : cpu2-P9
|
||||||
|
host : cpu2-P8
|
||||||
|
host : cpu2-P7
|
||||||
|
host : cpu2-P6
|
||||||
|
host : cpu2-P5
|
||||||
|
host : cpu2-P4
|
||||||
|
host : cpu2-P3
|
||||||
|
host : cpu2-P2
|
||||||
|
host : cpu2-P1
|
||||||
|
host : cpu2-P0
|
||||||
|
host : cpu1-P15
|
||||||
|
host : cpu1-P14
|
||||||
|
host : cpu1-P13
|
||||||
|
host : cpu1-P12
|
||||||
|
host : cpu1-P11
|
||||||
|
host : cpu1-P10
|
||||||
|
host : cpu1-P9
|
||||||
|
host : cpu1-P8
|
||||||
|
host : cpu1-P7
|
||||||
|
host : cpu1-P6
|
||||||
|
host : cpu1-P5
|
||||||
|
host : cpu1-P4
|
||||||
|
host : cpu1-P3
|
||||||
|
host : cpu1-P2
|
||||||
|
host : cpu1-P1
|
||||||
|
host : cpu1-P0
|
||||||
|
host : cpu0-P15
|
||||||
|
host : cpu0-P14
|
||||||
|
host : cpu0-P13
|
||||||
|
host : cpu0-P12
|
||||||
|
host : cpu0-P11
|
||||||
|
host : cpu0-P10
|
||||||
|
host : cpu0-P9
|
||||||
|
host : cpu0-P8
|
||||||
|
host : cpu0-P7
|
||||||
|
host : cpu0-P6
|
||||||
|
host : cpu0-P5
|
||||||
|
host : cpu0-P4
|
||||||
|
host : cpu0-P3
|
||||||
|
host : cpu0-P2
|
||||||
|
host : cpu0-P1
|
||||||
|
host : cpu0-P0
|
||||||
|
host : cpu3-C6
|
||||||
|
host : cpu3-C5
|
||||||
|
host : cpu3-C4
|
||||||
|
host : cpu3-C3
|
||||||
|
host : cpu3-C2
|
||||||
|
host : cpu3-C1
|
||||||
|
host : cpu3-C0
|
||||||
|
host : cpu2-C6
|
||||||
|
host : cpu2-C5
|
||||||
|
host : cpu2-C4
|
||||||
|
host : cpu2-C3
|
||||||
|
host : cpu2-C2
|
||||||
|
host : cpu2-C1
|
||||||
|
host : cpu2-C0
|
||||||
|
host : cpu1-C6
|
||||||
|
host : cpu1-C5
|
||||||
|
host : cpu1-C4
|
||||||
|
host : cpu1-C3
|
||||||
|
host : cpu1-C2
|
||||||
|
host : cpu1-C1
|
||||||
|
host : cpu1-C0
|
||||||
|
host : cpu0-C6
|
||||||
|
host : cpu0-C5
|
||||||
|
host : cpu0-C4
|
||||||
|
host : cpu0-C3
|
||||||
|
host : cpu0-C2
|
||||||
|
host : cpu0-C1
|
||||||
|
host : cpu0-C0
|
||||||
|
host : Tapdisks_in_low_memory_mode
|
||||||
|
host : memory_reclaimed_max
|
||||||
|
host : memory_reclaimed
|
||||||
|
host : pif_aggr_rx
|
||||||
|
host : pif_aggr_tx
|
||||||
|
host : pif_eth2_rx
|
||||||
|
host : pif_eth2_tx
|
||||||
|
host : pif_eth0_rx
|
||||||
|
host : pif_eth0_tx
|
||||||
|
host : pif_eth1_rx
|
||||||
|
host : pif_eth1_tx
|
||||||
|
host : xapi_open_fds
|
||||||
|
host : pool_task_count
|
||||||
|
host : pool_session_count
|
||||||
|
host : xapi_memory_usage_kib
|
||||||
|
host : xapi_free_memory_kib
|
||||||
|
host : xapi_live_memory_kib
|
||||||
|
host : xapi_allocation_kib
|
||||||
|
host : inflight_2b7b1501
|
||||||
|
host : iowait_2b7b1501
|
||||||
|
host : iops_total_2b7b1501
|
||||||
|
host : iops_write_2b7b1501
|
||||||
|
host : iops_read_2b7b1501
|
||||||
|
host : io_throughput_total_2b7b1501
|
||||||
|
host : io_throughput_write_2b7b1501
|
||||||
|
host : io_throughput_read_2b7b1501
|
||||||
|
host : write_latency_2b7b1501
|
||||||
|
host : read_latency_2b7b1501
|
||||||
|
host : write_2b7b1501
|
||||||
|
host : read_2b7b1501
|
||||||
|
host : inflight_72cc0148
|
||||||
|
host : iowait_72cc0148
|
||||||
|
host : iops_total_72cc0148
|
||||||
|
host : iops_write_72cc0148
|
||||||
|
host : iops_read_72cc0148
|
||||||
|
host : io_throughput_total_72cc0148
|
||||||
|
host : io_throughput_write_72cc0148
|
||||||
|
host : io_throughput_read_72cc0148
|
||||||
|
host : write_latency_72cc0148
|
||||||
|
host : read_latency_72cc0148
|
||||||
|
host : write_72cc0148
|
||||||
|
host : read_72cc0148
|
||||||
|
host : inflight_9218facb
|
||||||
|
host : iowait_9218facb
|
||||||
|
host : iops_total_9218facb
|
||||||
|
host : iops_write_9218facb
|
||||||
|
host : iops_read_9218facb
|
||||||
|
host : io_throughput_total_9218facb
|
||||||
|
host : io_throughput_write_9218facb
|
||||||
|
host : io_throughput_read_9218facb
|
||||||
|
host : write_latency_9218facb
|
||||||
|
host : read_latency_9218facb
|
||||||
|
host : write_9218facb
|
||||||
|
host : read_9218facb
|
||||||
|
host : inflight_44f9108d
|
||||||
|
host : iowait_44f9108d
|
||||||
|
host : iops_total_44f9108d
|
||||||
|
host : iops_write_44f9108d
|
||||||
|
host : iops_read_44f9108d
|
||||||
|
host : io_throughput_total_44f9108d
|
||||||
|
host : io_throughput_write_44f9108d
|
||||||
|
host : io_throughput_read_44f9108d
|
||||||
|
host : write_latency_44f9108d
|
||||||
|
host : read_latency_44f9108d
|
||||||
|
host : write_44f9108d
|
||||||
|
host : read_44f9108d
|
||||||
|
host : inflight_438aa8dd
|
||||||
|
host : iowait_438aa8dd
|
||||||
|
host : iops_total_438aa8dd
|
||||||
|
host : iops_write_438aa8dd
|
||||||
|
host : iops_read_438aa8dd
|
||||||
|
host : io_throughput_total_438aa8dd
|
||||||
|
host : io_throughput_write_438aa8dd
|
||||||
|
host : io_throughput_read_438aa8dd
|
||||||
|
host : write_latency_438aa8dd
|
||||||
|
host : read_latency_438aa8dd
|
||||||
|
host : write_438aa8dd
|
||||||
|
host : read_438aa8dd
|
||||||
|
host : inflight_69a97fd4
|
||||||
|
host : iowait_69a97fd4
|
||||||
|
host : iops_total_69a97fd4
|
||||||
|
host : iops_write_69a97fd4
|
||||||
|
host : iops_read_69a97fd4
|
||||||
|
host : io_throughput_total_69a97fd4
|
||||||
|
host : io_throughput_write_69a97fd4
|
||||||
|
host : io_throughput_read_69a97fd4
|
||||||
|
host : write_latency_69a97fd4
|
||||||
|
host : read_latency_69a97fd4
|
||||||
|
host : write_69a97fd4
|
||||||
|
host : read_69a97fd4
|
||||||
|
host : inflight_85536572
|
||||||
|
host : iowait_85536572
|
||||||
|
host : iops_total_85536572
|
||||||
|
host : iops_write_85536572
|
||||||
|
host : iops_read_85536572
|
||||||
|
host : io_throughput_total_85536572
|
||||||
|
host : io_throughput_write_85536572
|
||||||
|
host : io_throughput_read_85536572
|
||||||
|
host : write_latency_85536572
|
||||||
|
host : read_latency_85536572
|
||||||
|
host : write_85536572
|
||||||
|
host : read_85536572
|
||||||
|
host : inflight_d4a3c32d
|
||||||
|
host : iowait_d4a3c32d
|
||||||
|
host : iops_total_d4a3c32d
|
||||||
|
host : iops_write_d4a3c32d
|
||||||
|
host : iops_read_d4a3c32d
|
||||||
|
host : io_throughput_total_d4a3c32d
|
||||||
|
host : io_throughput_write_d4a3c32d
|
||||||
|
host : io_throughput_read_d4a3c32d
|
||||||
|
host : write_latency_d4a3c32d
|
||||||
|
host : read_latency_d4a3c32d
|
||||||
|
host : write_d4a3c32d
|
||||||
|
host : read_d4a3c32d
|
||||||
|
host : inflight_c5bb6dc6
|
||||||
|
host : iowait_c5bb6dc6
|
||||||
|
host : iops_total_c5bb6dc6
|
||||||
|
host : iops_write_c5bb6dc6
|
||||||
|
host : iops_read_c5bb6dc6
|
||||||
|
host : io_throughput_total_c5bb6dc6
|
||||||
|
host : io_throughput_write_c5bb6dc6
|
||||||
|
host : io_throughput_read_c5bb6dc6
|
||||||
|
host : write_latency_c5bb6dc6
|
||||||
|
host : read_latency_c5bb6dc6
|
||||||
|
host : write_c5bb6dc6
|
||||||
|
host : read_c5bb6dc6
|
||||||
|
vm : cpu1
|
||||||
|
vm : cpu0
|
||||||
|
vm : memory
|
||||||
|
vm : vbd_xvda_inflight
|
||||||
|
vm : vbd_xvda_iowait
|
||||||
|
vm : vbd_xvda_iops_total
|
||||||
|
vm : vbd_xvda_iops_write
|
||||||
|
vm : vbd_xvda_iops_read
|
||||||
|
vm : vbd_xvda_io_throughput_total
|
||||||
|
vm : vbd_xvda_io_throughput_write
|
||||||
|
vm : vbd_xvda_io_throughput_read
|
||||||
|
vm : vbd_xvda_write_latency
|
||||||
|
vm : vbd_xvda_read_latency
|
||||||
|
vm : vbd_xvda_write
|
||||||
|
vm : vbd_xvda_read
|
||||||
|
vm : vbd_xvdb_inflight
|
||||||
|
vm : vbd_xvdb_iowait
|
||||||
|
vm : vbd_xvdb_iops_total
|
||||||
|
vm : vbd_xvdb_iops_write
|
||||||
|
vm : vbd_xvdb_iops_read
|
||||||
|
vm : vbd_xvdb_io_throughput_total
|
||||||
|
vm : vbd_xvdb_io_throughput_write
|
||||||
|
vm : vbd_xvdb_io_throughput_read
|
||||||
|
vm : vbd_xvdb_write_latency
|
||||||
|
vm : vbd_xvdb_read_latency
|
||||||
|
vm : vbd_xvdb_write
|
||||||
|
vm : vbd_xvdb_read
|
||||||
|
vm : vif_0_tx
|
||||||
|
vm : vif_0_rx
|
||||||
|
vm : memory_target
|
||||||
|
vm : memory_internal_free
|
||||||
|
*/
|
14
yarn.lock
14
yarn.lock
@ -464,7 +464,7 @@
|
|||||||
invariant "^2.2.2"
|
invariant "^2.2.2"
|
||||||
semver "^5.3.0"
|
semver "^5.3.0"
|
||||||
|
|
||||||
"@babel/preset-flow@7.0.0-beta.38":
|
"@babel/preset-flow@7.0.0-beta.38", "@babel/preset-flow@^7.0.0-beta.38":
|
||||||
version "7.0.0-beta.38"
|
version "7.0.0-beta.38"
|
||||||
resolved "https://registry.yarnpkg.com/@babel/preset-flow/-/preset-flow-7.0.0-beta.38.tgz#dda9cb14833cf353cc5e9db7a7a13fb7e5f84fba"
|
resolved "https://registry.yarnpkg.com/@babel/preset-flow/-/preset-flow-7.0.0-beta.38.tgz#dda9cb14833cf353cc5e9db7a7a13fb7e5f84fba"
|
||||||
dependencies:
|
dependencies:
|
||||||
@ -1908,7 +1908,7 @@ cosmiconfig@^4.0.0:
|
|||||||
parse-json "^4.0.0"
|
parse-json "^4.0.0"
|
||||||
require-from-string "^2.0.1"
|
require-from-string "^2.0.1"
|
||||||
|
|
||||||
cron@^1.1.0, cron@^1.2.1:
|
cron@^1.1.0, cron@^1.2.1, cron@^1.3.0:
|
||||||
version "1.3.0"
|
version "1.3.0"
|
||||||
resolved "https://registry.yarnpkg.com/cron/-/cron-1.3.0.tgz#7e459968eaf94e1a445be796ce402166c234659d"
|
resolved "https://registry.yarnpkg.com/cron/-/cron-1.3.0.tgz#7e459968eaf94e1a445be796ce402166c234659d"
|
||||||
dependencies:
|
dependencies:
|
||||||
@ -1965,6 +1965,16 @@ cssom@0.3.x, "cssom@>= 0.3.2 < 0.4.0":
|
|||||||
dependencies:
|
dependencies:
|
||||||
cssom "0.3.x"
|
cssom "0.3.x"
|
||||||
|
|
||||||
|
d3-time-format@^2.1.1:
|
||||||
|
version "2.1.1"
|
||||||
|
resolved "https://registry.yarnpkg.com/d3-time-format/-/d3-time-format-2.1.1.tgz#85b7cdfbc9ffca187f14d3c456ffda268081bb31"
|
||||||
|
dependencies:
|
||||||
|
d3-time "1"
|
||||||
|
|
||||||
|
d3-time@1:
|
||||||
|
version "1.0.8"
|
||||||
|
resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-1.0.8.tgz#dbd2d6007bf416fe67a76d17947b784bffea1e84"
|
||||||
|
|
||||||
dashdash@^1.12.0, dashdash@^1.14.0:
|
dashdash@^1.12.0, dashdash@^1.14.0:
|
||||||
version "1.14.1"
|
version "1.14.1"
|
||||||
resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0"
|
resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0"
|
||||||
|
Loading…
Reference in New Issue
Block a user