IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13947 from bergquist/alerting_for

Browse Source 
Introduce alert debouncing
This commit is contained in:
Carl Bergquist
2018-11-19 10:38:02 +01:00
committed by GitHub
parent c856c21b77 76cbd7f0de
commit 6049855dc7
36 changed files with 1083 additions and 485 deletions
Download Patch File Download Diff File Expand all files Collapse all files

796
devenv/dev-dashboards/testdata_alerts.json
View File

@@ -1,250 +1,546 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"links": [],
"panels": [
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
60
],
"type": "gt"
},
"query": {
"params": [
"A",
"5m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"enabled": true,
"frequency": "60s",
"handler": 1,
"name": "TestData - Always OK",
"noDataState": "no_data",
"notifications": []
},
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-testdata",
"editable": true,
"error": false,
"fill": 1,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 0
},
"id": 3,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"scenario": "random_walk",
"scenarioId": "csv_metric_values",
"stringInput": "1,20,90,30,5,0",
"target": ""
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt",
"value": 60
}
],
"timeFrom": null,
"timeShift": null,
"title": "Always OK",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "",
"logBase": 1,
"max": "125",
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
177
],
"type": "gt"
},
"query": {
"params": [
"A",
"5m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"enabled": true,
"executionErrorState": "alerting",
"for": "0m",
"frequency": "60s",
"handler": 1,
"name": "TestData - Always Alerting",
"noDataState": "no_data",
"notifications": []
},
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-testdata",
"editable": true,
"error": false,
"fill": 1,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 0
},
"id": 4,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"scenario": "random_walk",
"scenarioId": "csv_metric_values",
"stringInput": "200,445,100,150,200,220,190",
"target": ""
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt",
"value": 177
}
],
"timeFrom": null,
"timeShift": null,
"title": "Always Alerting",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
1
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"A",
"15m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"executionErrorState": "alerting",
"for": "5m",
"frequency": "1m",
"handler": 1,
"name": "TestData - No data",
"noDataState": "no_data",
"notifications": []
},
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-testdata",
"editable": true,
"error": false,
"fill": 1,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 7
},
"id": 5,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"scenario": "random_walk",
"scenarioId": "no_data_points",
"stringInput": "",
"target": ""
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt",
"value": 1
}
],
"timeFrom": null,
"timeShift": null,
"title": "No data",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
177
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"A",
"15m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"executionErrorState": "alerting",
"for": "1m",
"frequency": "1m",
"handler": 1,
"name": "TestData - Always Alerting with For",
"noDataState": "no_data",
"notifications": []
},
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-testdata",
"editable": true,
"error": false,
"fill": 1,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 7
},
"id": 6,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"scenario": "random_walk",
"scenarioId": "csv_metric_values",
"stringInput": "200,445,100,150,200,220,190",
"target": ""
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt",
"value": 177
}
],
"timeFrom": null,
"timeShift": null,
"title": "Always Alerting with For",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"revision": 2,
"title": "Alerting with TestData",
"schemaVersion": 16,
"style": "dark",
"tags": [
"grafana-test"
],
"style": "dark",
"timezone": "browser",
"editable": true,
"hideControls": false,
"sharedCrosshair": false,
"rows": [
{
"collapse": false,
"editable": true,
"height": 255.625,
"panels": [
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
60
],
"type": "gt"
},
"query": {
"params": [
"A",
"5m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"enabled": true,
"frequency": "60s",
"handler": 1,
"name": "TestData - Always OK",
"noDataState": "no_data",
"notifications": []
},
"aliasColors": {},
"bars": false,
"datasource": "gdev-testdata",
"editable": true,
"error": false,
"fill": 1,
"id": 3,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"span": 6,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"scenario": "random_walk",
"scenarioId": "csv_metric_values",
"stringInput": "1,20,90,30,5,0",
"target": ""
}
],
"thresholds": [
{
"value": 60,
"op": "gt",
"fill": true,
"line": true,
"colorMode": "critical"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Always OK",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "",
"logBase": 1,
"max": "125",
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
177
],
"type": "gt"
},
"query": {
"params": [
"A",
"5m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"enabled": true,
"frequency": "60s",
"handler": 1,
"name": "TestData - Always Alerting",
"noDataState": "no_data",
"notifications": []
},
"aliasColors": {},
"bars": false,
"datasource": "gdev-testdata",
"editable": true,
"error": false,
"fill": 1,
"id": 4,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"span": 6,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"scenario": "random_walk",
"scenarioId": "csv_metric_values",
"stringInput": "200,445,100,150,200,220,190",
"target": ""
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt",
"value": 177
}
],
"timeFrom": null,
"timeShift": null,
"title": "Always Alerting",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"title": "New row"
}
],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
@@ -274,14 +570,8 @@
"30d"
]
},
"templating": {
"list": []
},
"annotations": {
"list": []
},
"schemaVersion": 13,
"version": 4,
"links": [],
"gnetId": null
}
"timezone": "browser",
"title": "Alerting with TestData",
"uid": "7MeksYbmk",
"version": 1
}

2
devenv/docker/ha_test/docker-compose.yaml
View File

@@ -9,7 +9,7 @@ services:
- /var/run/docker.sock:/tmp/docker.sock:ro
db:
image: mysql
image: mysql:5.6
environment:
MYSQL_ROOT_PASSWORD: rootpass
MYSQL_DATABASE: grafana

1
devenv/docker/ha_test/grafana/provisioning/alerts.jsonnet
View File

@@ -39,6 +39,7 @@ local alertDashboardTemplate = {
"executionErrorState": "alerting",
"frequency": "10s",
"handler": 1,
"for": "1m",
"name": "bulk alerting",
"noDataState": "no_data",
"notifications": [

16
docs/sources/alerting/rules.md
View File

@@ -39,7 +39,7 @@ Currently alerting supports a limited form of high availability. Since v4.2.0 of
## Rule Config
{{< imgbox max-width="40%" img="/img/docs/v4/alerting_conditions.png" caption="Alerting Conditions" >}}
Currently only the graph panel supports alert rules but this will be added to the **Singlestat** and **Table**
panels as well in a future release.
@@ -48,6 +48,16 @@ panels as well in a future release.
Here you can specify the name of the alert rule and how often the scheduler should evaluate the alert rule.
### For
> This setting is available in Grafana 5.4 and above.
If an alert rule has a configured `For` and the query violates the configured threshold it will first go from `OK` to `Pending`. Going from `OK` to `Pending` Grafana will not send any notifications. Once the alert rule has been firing for more than `For` duration, it will change to `Alerting` and send alert notifications.
Typically, it's always a good idea to use this setting since its often worse to get false positive than wait a few minutes before the alert notification triggers.
{{< imgbox max-width="40%" img="/img/docs/v4/alerting_conditions.png" caption="Alerting Conditions" >}}
### Conditions
Currently the only condition type that exists is a `Query` condition that allows you to
@@ -57,11 +67,11 @@ specify a query letter, time range and an aggregation function.
### Query condition example
```sql
avg() OF query(A, 5m, now) IS BELOW 14
avg() OF query(A, 15m, now) IS BELOW 14
```
- `avg()` Controls how the values for **each** series should be reduced to a value that can be compared against the threshold. Click on the function to change it to another aggregation function.
- `query(A, 5m, now)` The letter defines what query to execute from the **Metrics** tab. The second two parameters define the time range, `5m, now` means 5 minutes ago to now. You can also do `10m, now-2m` to define a time range that will be 10 minutes ago to 2 minutes ago. This is useful if you want to ignore the last 2 minutes of data.
- `query(A, 15m, now)` The letter defines what query to execute from the **Metrics** tab. The second two parameters define the time range, `15m, now` means 5 minutes ago to now. You can also do `10m, now-2m` to define a time range that will be 10 minutes ago to 2 minutes ago. This is useful if you want to ignore the last 2 minutes of data.
- `IS BELOW 14` Defines the type of threshold and the threshold value. You can click on `IS BELOW` to change the type of threshold.
The query used in an alert rule cannot contain any template variables. Currently we only support `AND` and `OR` operators between conditions and they are executed serially.

2
pkg/api/alerting.go
View File

@@ -295,7 +295,7 @@ func PauseAlert(c *m.ReqContext, dto dtos.PauseAlertCommand) Response {
return Error(500, "", err)
}
var response m.AlertStateType = m.AlertStatePending
var response m.AlertStateType = m.AlertStateUnknown
pausedState := "un-paused"
if cmd.Paused {
response = m.AlertStatePaused

13
pkg/models/alert.go
View File

@@ -19,6 +19,7 @@ const (
AlertStateAlerting AlertStateType = "alerting"
AlertStateOK AlertStateType = "ok"
AlertStatePending AlertStateType = "pending"
AlertStateUnknown AlertStateType = "unknown"
)
const (
@@ -39,7 +40,12 @@ var (
)
func (s AlertStateType) IsValid() bool {
return s == AlertStateOK || s == AlertStateNoData || s == AlertStatePaused || s == AlertStatePending
return s == AlertStateOK ||
s == AlertStateNoData ||
s == AlertStatePaused ||
s == AlertStatePending ||
s == AlertStateAlerting ||
s == AlertStateUnknown
}
func (s NoDataOption) IsValid() bool {
@@ -66,12 +72,13 @@ type Alert struct {
PanelId int64
Name string
Message string
Severity string
Severity string //Unused
State AlertStateType
Handler int64
Handler int64 //Unused
Silenced bool
ExecutionError string
Frequency int64
For time.Duration
EvalData *simplejson.Json
NewStateDate time.Time

32
pkg/services/alerting/eval_context.go
View File

@@ -68,8 +68,13 @@ func (c *EvalContext) GetStateModel() *StateDescription {
Color: "#D63232",
Text: "Alerting",
}
case m.AlertStateUnknown:
return &StateDescription{
Color: "#888888",
Text: "Unknown",
}
default:
panic("Unknown rule state " + c.Rule.State)
panic("Unknown rule state for alert " + c.Rule.State)
}
}
@@ -113,7 +118,26 @@ func (c *EvalContext) GetRuleUrl() (string, error) {
return fmt.Sprintf(urlFormat, m.GetFullDashboardUrl(ref.Uid, ref.Slug), c.Rule.PanelId, c.Rule.OrgId), nil
}
// GetNewState returns the new state from the alert rule evaluation
func (c *EvalContext) GetNewState() m.AlertStateType {
ns := getNewStateInternal(c)
if ns != m.AlertStateAlerting || c.Rule.For == 0 {
return ns
}
since := time.Now().Sub(c.Rule.LastStateChange)
if c.PrevAlertState == m.AlertStatePending && since > c.Rule.For {
return m.AlertStateAlerting
}
if c.PrevAlertState == m.AlertStateAlerting {
return m.AlertStateAlerting
}
return m.AlertStatePending
}
func getNewStateInternal(c *EvalContext) m.AlertStateType {
if c.Error != nil {
c.log.Error("Alert Rule Result Error",
"ruleId", c.Rule.Id,
@@ -125,11 +149,13 @@ func (c *EvalContext) GetNewState() m.AlertStateType {
return c.PrevAlertState
}
return c.Rule.ExecutionErrorState.ToAlertState()
}
} else if c.Firing {
if c.Firing {
return m.AlertStateAlerting
}
} else if c.NoDataFound {
if c.NoDataFound {
c.log.Info("Alert Rule returned no data",
"ruleId", c.Rule.Id,
"name", c.Rule.Name,

240
pkg/services/alerting/eval_context_test.go
View File

@@ -2,11 +2,11 @@ package alerting
import (
"context"
"fmt"
"errors"
"testing"
"time"
"github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey"
)
func TestStateIsUpdatedWhenNeeded(t *testing.T) {
@@ -31,71 +31,177 @@ func TestStateIsUpdatedWhenNeeded(t *testing.T) {
})
}
func TestAlertingEvalContext(t *testing.T) {
Convey("Should compute and replace properly new rule state", t, func() {
func TestGetStateFromEvalContext(t *testing.T) {
tcs := []struct {
name string
expected models.AlertStateType
applyFn func(ec *EvalContext)
focus bool
}{
{
name: "ok -> alerting",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.Firing = true
ec.PrevAlertState = models.AlertStateOK
},
},
{
name: "ok -> error(alerting)",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Error = errors.New("test error")
ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
},
},
{
name: "ok -> pending. since its been firing for less than FOR",
expected: models.AlertStatePending,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Firing = true
ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2)
ec.Rule.For = time.Minute * 5
},
},
{
name: "ok -> pending. since it has to be pending longer than FOR and prev state is ok",
expected: models.AlertStatePending,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Firing = true
ec.Rule.LastStateChange = time.Now().Add(-(time.Hour * 5))
ec.Rule.For = time.Minute * 2
},
},
{
name: "pending -> alerting. since its been firing for more than FOR and prev state is pending",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStatePending
ec.Firing = true
ec.Rule.LastStateChange = time.Now().Add(-(time.Hour * 5))
ec.Rule.For = time.Minute * 2
},
},
{
name: "alerting -> alerting. should not update regardless of FOR",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateAlerting
ec.Firing = true
ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
ec.Rule.For = time.Minute * 2
},
},
{
name: "ok -> ok. should not update regardless of FOR",
expected: models.AlertStateOK,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
ec.Rule.For = time.Minute * 2
},
},
{
name: "ok -> error(keep_last)",
expected: models.AlertStateOK,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Error = errors.New("test error")
ec.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
},
},
{
name: "pending -> error(keep_last)",
expected: models.AlertStatePending,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStatePending
ec.Error = errors.New("test error")
ec.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
},
},
{
name: "ok -> no_data(alerting)",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Rule.NoDataState = models.NoDataSetAlerting
ec.NoDataFound = true
},
},
{
name: "ok -> no_data(keep_last)",
expected: models.AlertStateOK,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Rule.NoDataState = models.NoDataKeepState
ec.NoDataFound = true
},
},
{
name: "pending -> no_data(keep_last)",
expected: models.AlertStatePending,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStatePending
ec.Rule.NoDataState = models.NoDataKeepState
ec.NoDataFound = true
},
},
{
name: "pending -> no_data(alerting) with for duration have not passed",
expected: models.AlertStatePending,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStatePending
ec.Rule.NoDataState = models.NoDataSetAlerting
ec.NoDataFound = true
ec.Rule.For = time.Minute * 5
ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2)
},
},
{
name: "pending -> no_data(alerting) should set alerting since time passed FOR",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStatePending
ec.Rule.NoDataState = models.NoDataSetAlerting
ec.NoDataFound = true
ec.Rule.For = time.Minute * 2
ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
},
},
{
name: "pending -> error(alerting) with for duration have not passed ",
expected: models.AlertStatePending,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStatePending
ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
ec.Error = errors.New("test error")
ec.Rule.For = time.Minute * 5
ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2)
},
},
{
name: "pending -> error(alerting) should set alerting since time passed FOR",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStatePending
ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
ec.Error = errors.New("test error")
ec.Rule.For = time.Minute * 2
ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
},
},
}
for _, tc := range tcs {
ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}})
dummieError := fmt.Errorf("dummie error")
Convey("ok -> alerting", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Firing = true
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
})
Convey("ok -> error(alerting)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
})
Convey("ok -> error(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
})
Convey("pending -> error(keep_last)", func() {
ctx.PrevAlertState = models.AlertStatePending
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
})
Convey("ok -> no_data(alerting)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataSetAlerting
ctx.NoDataFound = true
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
})
Convey("ok -> no_data(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataKeepState
ctx.NoDataFound = true
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
})
Convey("pending -> no_data(keep_last)", func() {
ctx.PrevAlertState = models.AlertStatePending
ctx.Rule.NoDataState = models.NoDataKeepState
ctx.NoDataFound = true
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
})
})
tc.applyFn(ctx)
have := ctx.GetNewState()
if have != tc.expected {
t.Errorf("failed: %s \n expected '%s' have '%s'\n", tc.name, tc.expected, string(have))
}
}
}

12
pkg/services/alerting/extractor.go
View File

@@ -2,8 +2,8 @@ package alerting
import (
"errors"
"fmt"
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
@@ -115,6 +115,15 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json,
return nil, ValidationError{Reason: "Could not parse frequency"}
}
rawFor := jsonAlert.Get("for").MustString()
var forValue time.Duration
if rawFor != "" {
forValue, err = time.ParseDuration(rawFor)
if err != nil {
return nil, ValidationError{Reason: "Could not parse for"}
}
}
alert := &m.Alert{
DashboardId: e.Dash.Id,
OrgId: e.OrgID,
@@ -124,6 +133,7 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json,
Handler: jsonAlert.Get("handler").MustInt64(),
Message: jsonAlert.Get("message").MustString(),
Frequency: frequency,
For: forValue,
}
for _, condition := range jsonAlert.Get("conditions").MustArray() {

20
pkg/services/alerting/extractor_test.go
View File

@@ -3,6 +3,7 @@ package alerting
import (
"io/ioutil"
"testing"
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
@@ -46,7 +47,7 @@ func TestAlertRuleExtraction(t *testing.T) {
return nil
})
json, err := ioutil.ReadFile("./test-data/graphite-alert.json")
json, err := ioutil.ReadFile("./testdata/graphite-alert.json")
So(err, ShouldBeNil)
Convey("Extractor should not modify the original json", func() {
@@ -118,6 +119,11 @@ func TestAlertRuleExtraction(t *testing.T) {
So(alerts[1].PanelId, ShouldEqual, 4)
})
Convey("should extract for param", func() {
So(alerts[0].For, ShouldEqual, time.Minute*2)
So(alerts[1].For, ShouldEqual, time.Duration(0))
})
Convey("should extract name and desc", func() {
So(alerts[0].Name, ShouldEqual, "name1")
So(alerts[0].Message, ShouldEqual, "desc1")
@@ -140,7 +146,7 @@ func TestAlertRuleExtraction(t *testing.T) {
})
Convey("Panels missing id should return error", func() {
panelWithoutId, err := ioutil.ReadFile("./test-data/panels-missing-id.json")
panelWithoutId, err := ioutil.ReadFile("./testdata/panels-missing-id.json")
So(err, ShouldBeNil)
dashJson, err := simplejson.NewJson(panelWithoutId)
@@ -156,7 +162,7 @@ func TestAlertRuleExtraction(t *testing.T) {
})
Convey("Panel with id set to zero should return error", func() {
panelWithIdZero, err := ioutil.ReadFile("./test-data/panel-with-id-0.json")
panelWithIdZero, err := ioutil.ReadFile("./testdata/panel-with-id-0.json")
So(err, ShouldBeNil)
dashJson, err := simplejson.NewJson(panelWithIdZero)
@@ -172,7 +178,7 @@ func TestAlertRuleExtraction(t *testing.T) {
})
Convey("Parse alerts from dashboard without rows", func() {
json, err := ioutil.ReadFile("./test-data/v5-dashboard.json")
json, err := ioutil.ReadFile("./testdata/v5-dashboard.json")
So(err, ShouldBeNil)
dashJson, err := simplejson.NewJson(json)
@@ -192,7 +198,7 @@ func TestAlertRuleExtraction(t *testing.T) {
})
Convey("Parse and validate dashboard containing influxdb alert", func() {
json, err := ioutil.ReadFile("./test-data/influxdb-alert.json")
json, err := ioutil.ReadFile("./testdata/influxdb-alert.json")
So(err, ShouldBeNil)
dashJson, err := simplejson.NewJson(json)
@@ -221,7 +227,7 @@ func TestAlertRuleExtraction(t *testing.T) {
})
Convey("Should be able to extract collapsed panels", func() {
json, err := ioutil.ReadFile("./test-data/collapsed-panels.json")
json, err := ioutil.ReadFile("./testdata/collapsed-panels.json")
So(err, ShouldBeNil)
dashJson, err := simplejson.NewJson(json)
@@ -242,7 +248,7 @@ func TestAlertRuleExtraction(t *testing.T) {
})
Convey("Parse and validate dashboard without id and containing an alert", func() {
json, err := ioutil.ReadFile("./test-data/dash-without-id.json")
json, err := ioutil.ReadFile("./testdata/dash-without-id.json")
So(err, ShouldBeNil)
dashJSON, err := simplejson.NewJson(json)

47
pkg/services/alerting/notifiers/alertmanager_test.go
View File

@@ -1,13 +1,60 @@
package notifiers
import (
"context"
"testing"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/log"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting"
. "github.com/smartystreets/goconvey/convey"
)
func TestWhenAlertManagerShouldNotify(t *testing.T) {
tcs := []struct {
prevState m.AlertStateType
newState m.AlertStateType
expect bool
}{
{
prevState: m.AlertStatePending,
newState: m.AlertStateOK,
expect: false,
},
{
prevState: m.AlertStateAlerting,
newState: m.AlertStateOK,
expect: true,
},
{
prevState: m.AlertStateOK,
newState: m.AlertStatePending,
expect: false,
},
{
prevState: m.AlertStateUnknown,
newState: m.AlertStatePending,
expect: false,
},
}
for _, tc := range tcs {
am := &AlertmanagerNotifier{log: log.New("test.logger")}
evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{
State: tc.prevState,
})
evalContext.Rule.State = tc.newState
res := am.ShouldNotify(context.TODO(), evalContext, &m.AlertNotificationState{})
if res != tc.expect {
t.Errorf("got %v expected %v", res, tc.expect)
}
}
}
func TestAlertmanagerNotifier(t *testing.T) {
Convey("Alertmanager notifier tests", t, func() {

10
pkg/services/alerting/notifiers/base.go
View File

@@ -67,6 +67,16 @@ func (n *NotifierBase) ShouldNotify(ctx context.Context, context *alerting.EvalC
}
// Do not notify when we become OK for the first time.
if context.PrevAlertState == models.AlertStateUnknown && context.Rule.State == models.AlertStateOK {
return false
}
// Do not notify when we become OK for the first time.
if context.PrevAlertState == models.AlertStateUnknown && context.Rule.State == models.AlertStatePending {
return false
}
// Do not notify when we become OK from pending
if context.PrevAlertState == models.AlertStatePending && context.Rule.State == models.AlertStateOK {
return false
}

32
pkg/services/alerting/notifiers/base_test.go
View File

@@ -29,7 +29,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
newState: m.AlertStateOK,
prevState: m.AlertStatePending,
sendReminder: false,
state: &m.AlertNotificationState{},
expect: false,
},
@@ -38,7 +37,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
newState: m.AlertStateAlerting,
prevState: m.AlertStateOK,
sendReminder: false,
state: &m.AlertNotificationState{},
expect: true,
},
@@ -47,7 +45,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
newState: m.AlertStatePending,
prevState: m.AlertStateOK,
sendReminder: false,
state: &m.AlertNotificationState{},
expect: false,
},
@@ -56,7 +53,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
newState: m.AlertStateOK,
prevState: m.AlertStateOK,
sendReminder: false,
state: &m.AlertNotificationState{},
expect: false,
},
@@ -65,7 +61,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
newState: m.AlertStateOK,
prevState: m.AlertStateOK,
sendReminder: true,
state: &m.AlertNotificationState{},
expect: false,
},
@@ -74,7 +69,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
newState: m.AlertStateOK,
prevState: m.AlertStateAlerting,
sendReminder: false,
state: &m.AlertNotificationState{},
expect: true,
},
@@ -94,7 +88,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
prevState: m.AlertStateAlerting,
frequency: time.Minute * 10,
sendReminder: true,
state: &m.AlertNotificationState{},
expect: true,
},
@@ -132,6 +125,27 @@ func TestShouldSendAlertNotification(t *testing.T) {
prevState: m.AlertStateOK,
state: &m.AlertNotificationState{State: m.AlertNotificationStatePending, UpdatedAt: tnow.Add(-2 * time.Minute).Unix()},
expect: true,
},
{
name: "unknown -> ok",
prevState: m.AlertStateUnknown,
newState: m.AlertStateOK,
expect: false,
},
{
name: "unknown -> pending",
prevState: m.AlertStateUnknown,
newState: m.AlertStatePending,
expect: false,
},
{
name: "unknown -> alerting",
prevState: m.AlertStateUnknown,
newState: m.AlertStateAlerting,
expect: true,
},
}
@@ -141,6 +155,10 @@ func TestShouldSendAlertNotification(t *testing.T) {
State: tc.prevState,
})
if tc.state == nil {
tc.state = &m.AlertNotificationState{}
}
evalContext.Rule.State = tc.newState
nb := &NotifierBase{SendReminder: tc.sendReminder, Frequency: tc.frequency}

3
pkg/services/alerting/result_handler.go
View File

@@ -73,6 +73,9 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
// when two servers are raising. This makes sure that the server
// with the last state change always sends a notification.
evalContext.Rule.StateChanges = cmd.Result.StateChanges
// Update the last state change of the alert rule in memory
evalContext.Rule.LastStateChange = time.Now()
}
// save annotation

5
pkg/services/alerting/rule.go
View File

@@ -4,6 +4,7 @@ import (
"fmt"
"regexp"
"strconv"
"time"
"github.com/grafana/grafana/pkg/components/simplejson"
@@ -18,6 +19,8 @@ type Rule struct {
Frequency int64
Name string
Message string
LastStateChange time.Time
For time.Duration
NoDataState m.NoDataOption
ExecutionErrorState m.ExecutionErrorOption
State m.AlertStateType
@@ -100,6 +103,8 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
model.Message = ruleDef.Message
model.Frequency = ruleDef.Frequency
model.State = ruleDef.State
model.LastStateChange = ruleDef.NewStateDate
model.For = ruleDef.For
model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))
model.StateChanges = ruleDef.StateChanges

0
pkg/services/alerting/test-data/collapsed-panels.json → pkg/services/alerting/testdata/collapsed-panels.json vendored
View File

0
pkg/services/alerting/test-data/dash-without-id.json → pkg/services/alerting/testdata/dash-without-id.json vendored
View File

1
pkg/services/alerting/test-data/graphite-alert.json → pkg/services/alerting/testdata/graphite-alert.json vendored
View File

@@ -23,6 +23,7 @@
"message": "desc1",
"handler": 1,
"frequency": "60s",
"for": "2m",
"conditions": [
{
"type": "query",

0
pkg/services/alerting/test-data/influxdb-alert.json → pkg/services/alerting/testdata/influxdb-alert.json vendored
View File

0
pkg/services/alerting/test-data/panel-with-id-0.json → pkg/services/alerting/testdata/panel-with-id-0.json vendored
View File

0
pkg/services/alerting/test-data/panels-missing-id.json → pkg/services/alerting/testdata/panels-missing-id.json vendored
View File

0
pkg/services/alerting/test-data/v5-dashboard.json → pkg/services/alerting/testdata/v5-dashboard.json vendored
View File

9
pkg/services/sqlstore/alert.go
View File

@@ -193,7 +193,8 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS
if alertToUpdate.ContainsUpdates(alert) {
alert.Updated = timeNow()
alert.State = alertToUpdate.State
sess.MustCols("message")
sess.MustCols("message", "for")
_, err := sess.ID(alert.Id).Update(alert)
if err != nil {
return err
@@ -204,7 +205,7 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS
} else {
alert.Updated = timeNow()
alert.Created = timeNow()
alert.State = m.AlertStatePending
alert.State = m.AlertStateUnknown
alert.NewStateDate = timeNow()
_, err := sess.Insert(alert)
@@ -299,7 +300,7 @@ func PauseAlert(cmd *m.PauseAlertCommand) error {
params = append(params, string(m.AlertStatePaused))
params = append(params, timeNow())
} else {
params = append(params, string(m.AlertStatePending))
params = append(params, string(m.AlertStateUnknown))
params = append(params, timeNow())
}
@@ -323,7 +324,7 @@ func PauseAllAlerts(cmd *m.PauseAllAlertCommand) error {
if cmd.Paused {
newState = string(m.AlertStatePaused)
} else {
newState = string(m.AlertStatePending)
newState = string(m.AlertStateUnknown)
}
res, err := sess.Exec(`UPDATE alert SET state = ?, new_state_date = ?`, newState, timeNow())

4
pkg/services/sqlstore/alert_test.go
View File

@@ -109,7 +109,7 @@ func TestAlertingDataAccess(t *testing.T) {
So(alert.DashboardId, ShouldEqual, testDash.Id)
So(alert.PanelId, ShouldEqual, 1)
So(alert.Name, ShouldEqual, "Alerting title")
So(alert.State, ShouldEqual, "pending")
So(alert.State, ShouldEqual, m.AlertStateUnknown)
So(alert.NewStateDate, ShouldNotBeNil)
So(alert.EvalData, ShouldNotBeNil)
So(alert.EvalData.Get("test").MustString(), ShouldEqual, "test")
@@ -154,7 +154,7 @@ func TestAlertingDataAccess(t *testing.T) {
So(query.Result[0].Name, ShouldEqual, "Name")
Convey("Alert state should not be updated", func() {
So(query.Result[0].State, ShouldEqual, "pending")
So(query.Result[0].State, ShouldEqual, m.AlertStateUnknown)
})
})

4
pkg/services/sqlstore/migrations/alert_mig.go
View File

@@ -133,4 +133,8 @@ func addAlertMigrations(mg *Migrator) {
mg.AddMigration("create alert_notification_state table v1", NewAddTableMigration(alert_notification_state))
mg.AddMigration("add index alert_notification_state org_id & alert_id & notifier_id",
NewAddIndexMigration(alert_notification_state, alert_notification_state.Indices[0]))
mg.AddMigration("Add for to alert table", NewAddColumnMigration(alertV1, &Column{
Name: "for", Type: DB_BigInt, Nullable: true,
}))
}

1
public/app/core/utils/colors.ts
View File

@@ -7,6 +7,7 @@ export const DEFAULT_ANNOTATION_COLOR = 'rgba(0, 211, 255, 1)';
export const OK_COLOR = 'rgba(11, 237, 50, 1)';
export const ALERTING_COLOR = 'rgba(237, 46, 24, 1)';
export const NO_DATA_COLOR = 'rgba(150, 150, 150, 1)';
export const PENDING_COLOR = 'rgba(247, 149, 32, 1)';
export const REGION_FILL_ALPHA = 0.09;
const colors = [

1
public/app/features/alerting/AlertRuleList.tsx
View File

@@ -29,6 +29,7 @@ export class AlertRuleList extends PureComponent<Props, any> {
{ text: 'Alerting', value: 'alerting' },
{ text: 'No Data', value: 'no_data' },
{ text: 'Paused', value: 'paused' },
{ text: 'Pending', value: 'pending' },
];
componentDidMount() {

4
public/app/features/alerting/AlertTabCtrl.ts
View File

@@ -169,6 +169,7 @@ export class AlertTabCtrl {
alert.frequency = alert.frequency || '1m';
alert.handler = alert.handler || 1;
alert.notifications = alert.notifications || [];
alert.for = alert.for || '0m';
const defaultName = this.panel.title + ' alert';
alert.name = alert.name || defaultName;
@@ -217,7 +218,7 @@ export class AlertTabCtrl {
buildDefaultCondition() {
return {
type: 'query',
query: { params: ['A', '15m', 'now'] },
query: { params: ['A', '5m', 'now'] },
reducer: { type: 'avg', params: [] },
evaluator: { type: 'gt', params: [null] },
operator: { type: 'and' },
@@ -354,6 +355,7 @@ export class AlertTabCtrl {
enable() {
this.panel.alert = {};
this.initModel();
this.panel.alert.for = '5m'; //default value for new alerts. for existing alerts we use 0m to avoid breaking changes
}
evaluatorParamsChanged() {

12
public/app/features/alerting/__snapshots__/AlertRuleList.test.tsx.snap
View File

@@ -81,6 +81,12 @@ exports[`Render should render alert rules 1`] = `
>
Paused
</option>
<option
key="pending"
value="pending"
>
Pending
</option>
</select>
</div>
</div>
@@ -230,6 +236,12 @@ exports[`Render should render component 1`] = `
>
Paused
</option>
<option
key="pending"
value="pending"
>
Pending
</option>
</select>
</div>
</div>

272
public/app/features/alerting/partials/alert_tab.html
View File

@@ -1,147 +1,159 @@
<div class="edit-tab-with-sidemenu" ng-if="ctrl.alert">
<aside class="edit-sidemenu-aside">
<ul class="edit-sidemenu">
<li ng-class="{active: ctrl.subTabIndex === 0}">
<a ng-click="ctrl.changeTabIndex(0)">Alert Config</a>
</li>
<li ng-class="{active: ctrl.subTabIndex === 1}">
<a ng-click="ctrl.changeTabIndex(1)">
Notifications <span class="muted">({{ctrl.alertNotifications.length}})</span>
</a>
</li>
<li ng-class="{active: ctrl.subTabIndex === 2}">
<a ng-click="ctrl.changeTabIndex(2)">State history</a>
</li>
<aside class="edit-sidemenu-aside">
<ul class="edit-sidemenu">
<li ng-class="{active: ctrl.subTabIndex === 0}">
<a ng-click="ctrl.changeTabIndex(0)">Alert Config</a>
</li>
<li ng-class="{active: ctrl.subTabIndex === 1}">
<a ng-click="ctrl.changeTabIndex(1)">
Notifications <span class="muted">({{ctrl.alertNotifications.length}})</span>
</a>
</li>
<li ng-class="{active: ctrl.subTabIndex === 2}">
<a ng-click="ctrl.changeTabIndex(2)">State history</a>
</li>
<li>
<a ng-click="ctrl.delete()">Delete</a>
</li>
</ul>
</aside>
<a ng-click="ctrl.delete()">Delete</a>
</li>
</ul>
</aside>
<div class="edit-tab-content">
<div ng-if="ctrl.subTabIndex === 0">
<div class="alert alert-error m-b-2" ng-show="ctrl.error">
<i class="fa fa-warning"></i> {{ctrl.error}}
</div>
<div class="edit-tab-content">
<div ng-if="ctrl.subTabIndex === 0">
<div class="alert alert-error m-b-2" ng-show="ctrl.error">
<i class="fa fa-warning"></i> {{ctrl.error}}
</div>
<div class="gf-form-group">
<h5 class="section-heading">Alert Config</h5>
<div class="gf-form">
<span class="gf-form-label width-6">Name</span>
<input type="text" class="gf-form-input width-20" ng-model="ctrl.alert.name">
<span class="gf-form-label">Evaluate every</span>
<input class="gf-form-input max-width-5" type="text" ng-model="ctrl.alert.frequency"></input>
</div>
</div>
<div class="gf-form-group">
<h5 class="section-heading">Alert Config</h5>
<div class="gf-form">
<span class="gf-form-label width-6">Name</span>
<input type="text" class="gf-form-input width-20" ng-model="ctrl.alert.name">
</div>
<div class="gf-form-inline">
<div class="gf-form">
<span class="gf-form-label width-9">Evaluate every</span>
<input class="gf-form-input max-width-6" type="text" ng-model="ctrl.alert.frequency">
</div>
<div class="gf-form max-width-11">
<label class="gf-form-label width-5">For</label>
<input type="text" class="gf-form-input max-width-6" ng-model="ctrl.alert.for" spellcheck='false' placeholder="5m">
<info-popover mode="right-absolute">
If an alert rule has a configured For and the query violates the configured threshold it will first go from OK to Pending.
Going from OK to Pending Grafana will not send any notifications. Once the alert rule has been firing for more than For duration, it will change to Alerting and send alert notifications.
</info-popover>
</div>
</div>
</div>
<div class="gf-form-group">
<h5 class="section-heading">Conditions</h5>
<div class="gf-form-inline" ng-repeat="conditionModel in ctrl.conditionModels">
<div class="gf-form">
<metric-segment-model css-class="query-keyword width-5" ng-if="$index" property="conditionModel.operator.type" options="ctrl.evalOperators" custom="false"></metric-segment-model>
<span class="gf-form-label query-keyword width-5" ng-if="$index===0">WHEN</span>
</div>
<div class="gf-form">
<query-part-editor class="gf-form-label query-part width-9" part="conditionModel.reducerPart" handle-event="ctrl.handleReducerPartEvent(conditionModel, $event)">
</query-part-editor>
<span class="gf-form-label query-keyword">OF</span>
</div>
<div class="gf-form">
<query-part-editor class="gf-form-label query-part" part="conditionModel.queryPart" handle-event="ctrl.handleQueryPartEvent(conditionModel, $event)">
</query-part-editor>
</div>
<div class="gf-form">
<metric-segment-model property="conditionModel.evaluator.type" options="ctrl.evalFunctions" custom="false" css-class="query-keyword" on-change="ctrl.evaluatorTypeChanged(conditionModel.evaluator)"></metric-segment-model>
<input class="gf-form-input max-width-9" type="number" step="any" ng-hide="conditionModel.evaluator.params.length === 0" ng-model="conditionModel.evaluator.params[0]" ng-change="ctrl.evaluatorParamsChanged()"></input>
<label class="gf-form-label query-keyword" ng-show="conditionModel.evaluator.params.length === 2">TO</label>
<input class="gf-form-input max-width-9" type="number" step="any" ng-if="conditionModel.evaluator.params.length === 2" ng-model="conditionModel.evaluator.params[1]" ng-change="ctrl.evaluatorParamsChanged()"></input>
</div>
<div class="gf-form">
<label class="gf-form-label">
<a class="pointer" tabindex="1" ng-click="ctrl.removeCondition($index)">
<i class="fa fa-trash"></i>
</a>
</label>
</div>
</div>
<div class="gf-form-group">
<h5 class="section-heading">Conditions</h5>
<div class="gf-form-inline" ng-repeat="conditionModel in ctrl.conditionModels">
<div class="gf-form">
<metric-segment-model css-class="query-keyword width-5" ng-if="$index" property="conditionModel.operator.type" options="ctrl.evalOperators" custom="false"></metric-segment-model>
<span class="gf-form-label query-keyword width-5" ng-if="$index===0">WHEN</span>
</div>
<div class="gf-form">
<query-part-editor class="gf-form-label query-part width-9" part="conditionModel.reducerPart" handle-event="ctrl.handleReducerPartEvent(conditionModel, $event)">
</query-part-editor>
<span class="gf-form-label query-keyword">OF</span>
</div>
<div class="gf-form">
<query-part-editor class="gf-form-label query-part" part="conditionModel.queryPart" handle-event="ctrl.handleQueryPartEvent(conditionModel, $event)">
</query-part-editor>
</div>
<div class="gf-form">
<metric-segment-model property="conditionModel.evaluator.type" options="ctrl.evalFunctions" custom="false" css-class="query-keyword" on-change="ctrl.evaluatorTypeChanged(conditionModel.evaluator)"></metric-segment-model>
<input class="gf-form-input max-width-9" type="number" step="any" ng-hide="conditionModel.evaluator.params.length === 0" ng-model="conditionModel.evaluator.params[0]" ng-change="ctrl.evaluatorParamsChanged()"></input>
<label class="gf-form-label query-keyword" ng-show="conditionModel.evaluator.params.length === 2">TO</label>
<input class="gf-form-input max-width-9" type="number" step="any" ng-if="conditionModel.evaluator.params.length === 2" ng-model="conditionModel.evaluator.params[1]" ng-change="ctrl.evaluatorParamsChanged()"></input>
</div>
<div class="gf-form">
<label class="gf-form-label">
<a class="pointer" tabindex="1" ng-click="ctrl.removeCondition($index)">
<i class="fa fa-trash"></i>
</a>
</label>
</div>
</div>
<div class="gf-form">
<label class="gf-form-label dropdown">
<a class="pointer dropdown-toggle" data-toggle="dropdown">
<i class="fa fa-plus"></i>
</a>
<ul class="dropdown-menu" role="menu">
<li ng-repeat="ct in ctrl.conditionTypes" role="menuitem">
<a ng-click="ctrl.addCondition(ct.value);">{{ct.text}}</a>
</li>
</ul>
</label>
</div>
</div>
<div class="gf-form">
<label class="gf-form-label dropdown">
<a class="pointer dropdown-toggle" data-toggle="dropdown">
<i class="fa fa-plus"></i>
</a>
<ul class="dropdown-menu" role="menu">
<li ng-repeat="ct in ctrl.conditionTypes" role="menuitem">
<a ng-click="ctrl.addCondition(ct.value);">{{ct.text}}</a>
</li>
</ul>
</label>
</div>
</div>
<div class="gf-form-group">
<div class="gf-form">
<span class="gf-form-label width-18">If no data or all values are null</span>
<span class="gf-form-label query-keyword">SET STATE TO</span>
<div class="gf-form-select-wrapper">
<select class="gf-form-input" ng-model="ctrl.alert.noDataState" ng-options="f.value as f.text for f in ctrl.noDataModes">
</select>
</div>
</div>
<div class="gf-form-group">
<div class="gf-form">
<span class="gf-form-label width-18">If no data or all values are null</span>
<span class="gf-form-label query-keyword">SET STATE TO</span>
<div class="gf-form-select-wrapper">
<select class="gf-form-input" ng-model="ctrl.alert.noDataState" ng-options="f.value as f.text for f in ctrl.noDataModes">
</select>
</div>
</div>
<div class="gf-form">
<span class="gf-form-label width-18">If execution error or timeout</span>
<span class="gf-form-label query-keyword">SET STATE TO</span>
<div class="gf-form-select-wrapper">
<select class="gf-form-input" ng-model="ctrl.alert.executionErrorState" ng-options="f.value as f.text for f in ctrl.executionErrorModes">
</select>
</div>
</div>
<div class="gf-form">
<span class="gf-form-label width-18">If execution error or timeout</span>
<span class="gf-form-label query-keyword">SET STATE TO</span>
<div class="gf-form-select-wrapper">
<select class="gf-form-input" ng-model="ctrl.alert.executionErrorState" ng-options="f.value as f.text for f in ctrl.executionErrorModes">
</select>
</div>
</div>
<div class="gf-form-button-row">
<button class="btn btn-inverse" ng-click="ctrl.test()">
Test Rule
</button>
</div>
</div>
<div class="gf-form-button-row">
<button class="btn btn-inverse" ng-click="ctrl.test()">
Test Rule
</button>
</div>
</div>
<div class="gf-form-group" ng-if="ctrl.testing">
Evaluating rule <i class="fa fa-spinner fa-spin"></i>
</div>
<div class="gf-form-group" ng-if="ctrl.testing">
Evaluating rule <i class="fa fa-spinner fa-spin"></i>
</div>
<div class="gf-form-group" ng-if="ctrl.testResult">
<json-tree root-name="result" object="ctrl.testResult" start-expanded="true"></json-tree>
</div>
</div>
<div class="gf-form-group" ng-if="ctrl.testResult">
<json-tree root-name="result" object="ctrl.testResult" start-expanded="true"></json-tree>
</div>
</div>
<div class="gf-form-group" ng-if="ctrl.subTabIndex === 1">
<h5 class="section-heading">Notifications</h5>
<div class="gf-form-inline">
<div class="gf-form max-width-30">
<span class="gf-form-label width-8">Send to</span>
<span class="gf-form-label" ng-repeat="nc in ctrl.alertNotifications" ng-style="{'background-color': nc.bgColor }">
<i class="{{nc.iconClass}}"></i>&nbsp;{{nc.name}}&nbsp;
<i class="fa fa-remove pointer muted" ng-click="ctrl.removeNotification($index)" ng-if="nc.isDefault === false"></i>
</span>
<metric-segment segment="ctrl.addNotificationSegment" get-options="ctrl.getNotifications()" on-change="ctrl.notificationAdded()"></metric-segment>
</div>
</div>
<div class="gf-form gf-form--v-stretch">
<span class="gf-form-label width-8">Message</span>
<textarea class="gf-form-input" rows="10" ng-model="ctrl.alert.message" placeholder="Notification message details..."></textarea>
</div>
</div>
<div class="gf-form-group" ng-if="ctrl.subTabIndex === 1">
<h5 class="section-heading">Notifications</h5>
<div class="gf-form-inline">
<div class="gf-form max-width-30">
<span class="gf-form-label width-8">Send to</span>
<span class="gf-form-label" ng-repeat="nc in ctrl.alertNotifications" ng-style="{'background-color': nc.bgColor }">
<i class="{{nc.iconClass}}"></i>&nbsp;{{nc.name}}&nbsp;
<i class="fa fa-remove pointer muted" ng-click="ctrl.removeNotification($index)" ng-if="nc.isDefault === false"></i>
</span>
<metric-segment segment="ctrl.addNotificationSegment" get-options="ctrl.getNotifications()" on-change="ctrl.notificationAdded()"></metric-segment>
</div>
</div>
<div class="gf-form gf-form--v-stretch">
<span class="gf-form-label width-8">Message</span>
<textarea class="gf-form-input" rows="10" ng-model="ctrl.alert.message" placeholder="Notification message details..."></textarea>
</div>
</div>
<div class="gf-form-group" style="max-width: 720px;" ng-if="ctrl.subTabIndex === 2">
<button class="btn btn-mini btn-danger pull-right" ng-click="ctrl.clearHistory()"><i class="fa fa-trash"></i>&nbsp;Clear history</button>
<h5 class="section-heading" style="whitespace: nowrap">
State history <span class="muted small">(last 50 state changes)</span>
</h5>
<div class="gf-form-group" style="max-width: 720px;" ng-if="ctrl.subTabIndex === 2">
<button class="btn btn-mini btn-danger pull-right" ng-click="ctrl.clearHistory()"><i class="fa fa-trash"></i>&nbsp;Clear history</button>
<h5 class="section-heading" style="whitespace: nowrap">
State history <span class="muted small">(last 50 state changes)</span>
</h5>
<div ng-show="ctrl.alertHistory.length === 0">
<br>
<i>No state changes recorded</i>
</div>
<div ng-show="ctrl.alertHistory.length === 0">
<br>
<i>No state changes recorded</i>
</div>
<ol class="alert-rule-list" >
<li class="alert-rule-item" ng-repeat="al in ctrl.alertHistory">

7
public/app/features/alerting/state/alertDef.ts
View File

@@ -99,6 +99,13 @@ function getStateDisplayModel(state) {
stateClass: 'alert-state-warning',
};
}
case 'unknown': {
return {
text: 'UNKNOWN',
iconClass: 'fa fa-question',
stateClass: 'alert-state-paused',
};
}
}
throw { message: 'Unknown alert state' };

2
public/app/features/annotations/annotation_tooltip.ts
View File

@@ -32,7 +32,7 @@ export function annotationTooltipDirective($sanitize, dashboardSrv, contextSrv,
if (event.alertId) {
const stateModel = alertDef.getStateDisplayModel(event.newState);
titleStateClass = stateModel.stateClass;
title = `<i class="icon-gf ${stateModel.iconClass}"></i> ${stateModel.text}`;
title = `<i class="${stateModel.iconClass}"></i> ${stateModel.text}`;
text = alertDef.getAlertAnnotationInfo(event);
if (event.text) {
text = text + '<br />' + event.text;

6
public/app/features/annotations/event_manager.ts
View File

@@ -7,6 +7,7 @@ import {
OK_COLOR,
ALERTING_COLOR,
NO_DATA_COLOR,
PENDING_COLOR,
DEFAULT_ANNOTATION_COLOR,
REGION_FILL_ALPHA,
} from 'app/core/utils/colors';
@@ -71,6 +72,11 @@ export class EventManager {
position: 'BOTTOM',
markerSize: 5,
},
$__pending: {
color: PENDING_COLOR,
position: 'BOTTOM',
markerSize: 5,
},
$__editing: {
color: DEFAULT_ANNOTATION_COLOR,
position: 'BOTTOM',

6
public/app/features/panel/panel_directive.ts
View File

@@ -161,7 +161,11 @@ module.directive('grafanaPanel', ($rootScope, $document, $timeout) => {
panelContainer.removeClass('panel-alert-state--' + lastAlertState);
}
if (ctrl.alertState.state === 'ok' || ctrl.alertState.state === 'alerting') {
if (
ctrl.alertState.state === 'ok' ||
ctrl.alertState.state === 'alerting' ||
ctrl.alertState.state === 'pending'
) {
panelContainer.addClass('panel-alert-state--' + ctrl.alertState.state);
}

1
public/app/plugins/panel/alertlist/editor.html
View File

@@ -50,5 +50,6 @@
<gf-form-switch class="gf-form" label="No data" label-class="width-10" checked="ctrl.stateFilter['no_data']" on-change="ctrl.updateStateFilter()"></gf-form-switch>
<gf-form-switch class="gf-form" label="Execution error" label-class="width-10" checked="ctrl.stateFilter['execution_error']" on-change="ctrl.updateStateFilter()"></gf-form-switch>
<gf-form-switch class="gf-form" label="Alerting" label-class="width-10" checked="ctrl.stateFilter['alerting']" on-change="ctrl.updateStateFilter()"></gf-form-switch>
<gf-form-switch class="gf-form" label="Pending" label-class="width-10" checked="ctrl.stateFilter['pending']" on-change="ctrl.updateStateFilter()"></gf-form-switch>
</div>
</div>

7
public/sass/pages/_alerting.scss
View File

@@ -66,6 +66,13 @@
content: '\e611';
}
}
&--pending {
.panel-alert-icon:before {
color: $warn;
content: '\e611';
}
}
}
@keyframes alerting-panel {