IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

feat(alerting): progress on handling no data in alert query, #5860

Browse Source
This commit is contained in:
Torkel Ödegaard 2016-09-06 20:40:12 +02:00
parent b1ed641d73
commit fbae6abb3c
18 changed files with 93 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/metrics/metrics.go
View File

@ -33,7 +33,7 @@ var (
M_Alerting_Result_State_Warning Counter
M_Alerting_Result_State_Ok Counter
M_Alerting_Result_State_Paused Counter
M_Alerting_Result_State_Pending Counter
M_Alerting_Result_State_Unknown Counter
M_Alerting_Result_State_ExecutionError Counter
M_Alerting_Active_Alerts Counter
M_Alerting_Notification_Sent_Slack Counter
@ -81,7 +81,7 @@ func initMetricVars(settings *MetricSettings) {
M_Alerting_Result_State_Warning = RegCounter("alerting.result", "state", "warning")
M_Alerting_Result_State_Ok = RegCounter("alerting.result", "state", "ok")
M_Alerting_Result_State_Paused = RegCounter("alerting.result", "state", "paused")
M_Alerting_Result_State_Pending = RegCounter("alerting.result", "state", "pending")
M_Alerting_Result_State_Unknown = RegCounter("alerting.result", "state", "unknown")
M_Alerting_Result_State_ExecutionError = RegCounter("alerting.result", "state", "execution_error")
M_Alerting_Active_Alerts = RegCounter("alerting.active_alerts")

4
pkg/models/alert.go
View File

@ -10,7 +10,7 @@ type AlertStateType string
type AlertSeverityType string
const (
AlertStatePending AlertStateType = "pending"
AlertStateUnknown AlertStateType = "unknown"
AlertStateExeuctionError AlertStateType = "execution_error"
AlertStatePaused AlertStateType = "paused"
AlertStateCritical AlertStateType = "critical"
@ -19,7 +19,7 @@ const (
)
func (s AlertStateType) IsValid() bool {
return s == AlertStateOK || s == AlertStatePending || s == AlertStateExeuctionError || s == AlertStatePaused || s == AlertStateCritical || s == AlertStateWarning
return s == AlertStateOK || s == AlertStateUnknown || s == AlertStateExeuctionError || s == AlertStatePaused || s == AlertStateCritical || s == AlertStateWarning
}
const (

34
pkg/services/alerting/conditions/evaluator.go
View File

@ -5,25 +5,21 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/services/alerting"
"github.com/grafana/grafana/pkg/tsdb"
)
var (
defaultTypes []string = []string{"gt", "lt"}
rangedTypes []string = []string{"within_range", "outside_range"}
paramlessTypes []string = []string{"no_value"}
defaultTypes []string = []string{"gt", "lt"}
rangedTypes []string = []string{"within_range", "outside_range"}
)
type AlertEvaluator interface {
Eval(timeSeries *tsdb.TimeSeries, reducedValue float64) bool
Eval(reducedValue *float64) bool
}
type ParameterlessEvaluator struct {
Type string
}
type NoDataEvaluator struct{}
func (e *ParameterlessEvaluator) Eval(series *tsdb.TimeSeries, reducedValue float64) bool {
return len(series.Points) == 0
func (e *NoDataEvaluator) Eval(reducedValue *float64) bool {
return reducedValue == nil
}
type ThresholdEvaluator struct {
@ -47,14 +43,12 @@ func newThresholdEvaludator(typ string, model *simplejson.Json) (*ThresholdEvalu
return defaultEval, nil
}
func (e *ThresholdEvaluator) Eval(series *tsdb.TimeSeries, reducedValue float64) bool {
func (e *ThresholdEvaluator) Eval(reducedValue *float64) bool {
switch e.Type {
case "gt":
return reducedValue > e.Threshold
return *reducedValue > e.Threshold
case "lt":
return reducedValue < e.Threshold
case "no_value":
return len(series.Points) == 0
return *reducedValue < e.Threshold
}
return false
@ -88,12 +82,12 @@ func newRangedEvaluator(typ string, model *simplejson.Json) (*RangedEvaluator, e
return rangedEval, nil
}
func (e *RangedEvaluator) Eval(series *tsdb.TimeSeries, reducedValue float64) bool {
func (e *RangedEvaluator) Eval(reducedValue *float64) bool {
switch e.Type {
case "within_range":
return (e.Lower < reducedValue && e.Upper > reducedValue) || (e.Upper < reducedValue && e.Lower > reducedValue)
return (e.Lower < *reducedValue && e.Upper > *reducedValue) || (e.Upper < *reducedValue && e.Lower > *reducedValue)
case "outside_range":
return (e.Upper < reducedValue && e.Lower < reducedValue) || (e.Upper > reducedValue && e.Lower > reducedValue)
return (e.Upper < *reducedValue && e.Lower < *reducedValue) || (e.Upper > *reducedValue && e.Lower > *reducedValue)
}
return false
@ -113,8 +107,8 @@ func NewAlertEvaluator(model *simplejson.Json) (AlertEvaluator, error) {
return newRangedEvaluator(typ, model)
}
if inSlice(typ, paramlessTypes) {
return &ParameterlessEvaluator{Type: typ}, nil
if typ == "no_data" {
return &NoDataEvaluator{}, nil
}
return nil, alerting.ValidationError{Reason: "Evaludator invalid evaluator type"}

15
pkg/services/alerting/conditions/evaluator_test.go
View File

@ -4,7 +4,6 @@ import (
"testing"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/tsdb"
. "github.com/smartystreets/goconvey/convey"
)
@ -15,19 +14,7 @@ func evalutorScenario(json string, reducedValue float64, datapoints ...float64)
evaluator, err := NewAlertEvaluator(jsonModel)
So(err, ShouldBeNil)
var timeserie [][2]float64
dummieTimestamp := float64(521452145)
for _, v := range datapoints {
timeserie = append(timeserie, [2]float64{v, dummieTimestamp})
}
tsdb := &tsdb.TimeSeries{
Name: "test time serie",
Points: timeserie,
}
return evaluator.Eval(tsdb, reducedValue)
return evaluator.Eval(reducedValue)
}
func TestEvalutors(t *testing.T) {

11
pkg/services/alerting/conditions/query.go
View File

@ -40,22 +40,27 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext) {
for _, series := range seriesList {
reducedValue := c.Reducer.Reduce(series)
evalMatch := c.Evaluator.Eval(series, reducedValue)
evalMatch := c.Evaluator.Eval(reducedValue)
if context.IsTestRun {
context.Logs = append(context.Logs, &alerting.ResultLogEntry{
Message: fmt.Sprintf("Condition[%d]: Eval: %v, Metric: %s, Value: %1.3f", c.Index, evalMatch, series.Name, reducedValue),
Message: fmt.Sprintf("Condition[%d]: Eval: %v, Metric: %s, Value: %1.3f", c.Index, evalMatch, series.Name, *reducedValue),
})
}
if evalMatch {
context.EvalMatches = append(context.EvalMatches, &alerting.EvalMatch{
Metric: series.Name,
Value: reducedValue,
Value: *reducedValue,
})
}
context.Firing = evalMatch
// handle no data scenario
if reducedValue == nil {
context.NoDataFound = true
}
}
}

10
pkg/services/alerting/conditions/reducer.go
View File

@ -3,14 +3,18 @@ package conditions
import "github.com/grafana/grafana/pkg/tsdb"
type QueryReducer interface {
Reduce(timeSeries *tsdb.TimeSeries) float64
Reduce(timeSeries *tsdb.TimeSeries) *float64
}
type SimpleReducer struct {
Type string
}
func (s *SimpleReducer) Reduce(series *tsdb.TimeSeries) float64 {
func (s *SimpleReducer) Reduce(series *tsdb.TimeSeries) *float64 {
if len(series.Points) == 0 {
return nil
}
var value float64 = 0
switch s.Type {
@ -46,7 +50,7 @@ func (s *SimpleReducer) Reduce(series *tsdb.TimeSeries) float64 {
value = float64(len(series.Points))
}
return value
return &value
}
func NewSimpleReducer(typ string) *SimpleReducer {

1
pkg/services/alerting/eval_context.go
View File

@ -26,6 +26,7 @@ type EvalContext struct {
dashboardSlug string
ImagePublicUrl string
ImageOnDiskPath string
NoDataFound bool
}
type StateDescription struct {

11
pkg/services/alerting/result_handler.go
View File

@ -41,7 +41,12 @@ func (handler *DefaultResultHandler) Handle(ctx *EvalContext) {
ctx.Rule.State = m.AlertStateType(ctx.Rule.Severity)
annotationData = simplejson.NewFromAny(ctx.EvalMatches)
} else {
ctx.Rule.State = m.AlertStateOK
// handle no data case
if ctx.NoDataFound {
ctx.Rule.State = ctx.Rule.NoDataState
} else {
ctx.Rule.State = m.AlertStateOK
}
}
countStateResult(ctx.Rule.State)
@ -91,8 +96,8 @@ func countStateResult(state m.AlertStateType) {
metrics.M_Alerting_Result_State_Ok.Inc(1)
case m.AlertStatePaused:
metrics.M_Alerting_Result_State_Paused.Inc(1)
case m.AlertStatePending:
metrics.M_Alerting_Result_State_Pending.Inc(1)
case m.AlertStateUnknown:
metrics.M_Alerting_Result_State_Unknown.Inc(1)
case m.AlertStateExeuctionError:
metrics.M_Alerting_Result_State_ExecutionError.Inc(1)
}

2
pkg/services/alerting/rule.go
View File

@ -18,6 +18,7 @@ type Rule struct {
Frequency int64
Name string
Message string
NoDataState m.AlertStateType
State m.AlertStateType
Severity m.AlertSeverityType
Conditions []Condition
@ -67,6 +68,7 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
model.Frequency = ruleDef.Frequency
model.Severity = ruleDef.Severity
model.State = ruleDef.State
model.NoDataState = m.AlertStateType(ruleDef.Settings.Get("noDataState").MustString("unknown"))
for _, v := range ruleDef.Settings.Get("notifications").MustArray() {
jsonModel := simplejson.NewFromAny(v)

9
pkg/services/alerting/rule_test.go
View File

@ -4,7 +4,7 @@ import (
"testing"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
m "github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey"
)
@ -45,6 +45,7 @@ func TestAlertRuleModel(t *testing.T) {
"name": "name2",
"description": "desc2",
"handler": 0,
"noDataMode": "critical",
"enabled": true,
"frequency": "60s",
"conditions": [
@ -63,7 +64,7 @@ func TestAlertRuleModel(t *testing.T) {
alertJSON, jsonErr := simplejson.NewJson([]byte(json))
So(jsonErr, ShouldBeNil)
alert := &models.Alert{
alert := &m.Alert{
Id: 1,
OrgId: 1,
DashboardId: 1,
@ -80,6 +81,10 @@ func TestAlertRuleModel(t *testing.T) {
Convey("Can read notifications", func() {
So(len(alertRule.Notifications), ShouldEqual, 2)
})
Convey("Can read noDataMode", func() {
So(len(alertRule.NoDataMode), ShouldEqual, m.AlertStateCritical)
})
})
})
}

2
pkg/services/sqlstore/alert.go
View File

@ -159,7 +159,7 @@ func upsertAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *xor
} else {
alert.Updated = time.Now()
alert.Created = time.Now()
alert.State = m.AlertStatePending
alert.State = m.AlertStateUnknown
alert.NewStateDate = time.Now()
_, err := sess.Insert(alert)

9
pkg/tsdb/graphite/graphite.go
View File

@ -11,6 +11,7 @@ import (
"time"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb"
)
@ -47,6 +48,10 @@ func (e *GraphiteExecutor) Execute(queries tsdb.QuerySlice, context *tsdb.QueryC
formData["target"] = []string{query.Query}
}
if setting.Env == setting.DEV {
glog.Debug("Graphite request", "params", formData)
}
req, err := e.createRequest(formData)
if err != nil {
result.Error = err
@ -71,6 +76,10 @@ func (e *GraphiteExecutor) Execute(queries tsdb.QuerySlice, context *tsdb.QueryC
Name: series.Target,
Points: series.DataPoints,
})
if setting.Env == setting.DEV {
glog.Debug("Graphite response", "target", series.Target, "datapoints", len(series.DataPoints))
}
}
result.QueryResults["A"] = queryRes

12
public/app/features/alerting/alert_def.ts
View File

@ -36,6 +36,13 @@ var reducerTypes = [
{text: 'count()', value: 'count'},
];
var noDataModes = [
{text: 'OK', value: 'ok'},
{text: 'Critical', value: 'critical'},
{text: 'Warning', value: 'warning'},
{text: 'Unknown', value: 'unknown'},
];
function createReducerPart(model) {
var def = new QueryPartDef({type: model.type, defaultParams: []});
return new QueryPart(model, def);
@ -69,9 +76,9 @@ function getStateDisplayModel(state) {
stateClass: 'alert-state-warning'
};
}
case 'pending': {
case 'unknown': {
return {
text: 'PENDING',
text: 'UNKNOWN',
iconClass: "fa fa-question",
stateClass: 'alert-state-warning'
};
@ -100,6 +107,7 @@ export default {
conditionTypes: conditionTypes,
evalFunctions: evalFunctions,
severityLevels: severityLevels,
noDataModes: noDataModes,
reducerTypes: reducerTypes,
createReducerPart: createReducerPart,
};

2
public/app/features/alerting/alert_list_ctrl.ts
View File

@ -13,7 +13,7 @@ export class AlertListCtrl {
stateFilters = [
{text: 'All', value: null},
{text: 'OK', value: 'ok'},
{text: 'Pending', value: 'pending'},
{text: 'Unknown', value: 'unknown'},
{text: 'Warning', value: 'warning'},
{text: 'Critical', value: 'critical'},
{text: 'Execution Error', value: 'execution_error'},

3
public/app/features/alerting/alert_tab_ctrl.ts
View File

@ -18,6 +18,7 @@ export class AlertTabCtrl {
conditionModels: any;
evalFunctions: any;
severityLevels: any;
noDataModes: any;
addNotificationSegment;
notifications;
alertNotifications;
@ -41,6 +42,7 @@ export class AlertTabCtrl {
this.evalFunctions = alertDef.evalFunctions;
this.conditionTypes = alertDef.conditionTypes;
this.severityLevels = alertDef.severityLevels;
this.noDataModes = alertDef.noDataModes;
this.appSubUrl = config.appSubUrl;
}
@ -138,6 +140,7 @@ export class AlertTabCtrl {
alert.conditions.push(this.buildDefaultCondition());
}
alert.noDataState = alert.noDataState || 'unknown';
alert.severity = alert.severity || 'critical';
alert.frequency = alert.frequency || '60s';
alert.handler = alert.handler || 1;

21
public/app/features/alerting/partials/alert_tab.html
View File

@ -52,19 +52,20 @@
<span class="gf-form-label query-keyword width-5" ng-if="$index">AND</span>
<span class="gf-form-label query-keyword width-5" ng-if="$index===0">WHEN</span>
</div>
<div class="gf-form">
<query-part-editor class="gf-form-label query-part" part="conditionModel.queryPart" handle-event="ctrl.handleQueryPartEvent(conditionModel, $event)">
<div class="gf-form">
<query-part-editor class="gf-form-label query-part" part="conditionModel.reducerPart" handle-event="ctrl.handleReducerPartEvent(conditionModel, $event)">
</query-part-editor>
<span class="gf-form-label query-keyword">OF</span>
</div>
<div class="gf-form">
<span class="gf-form-label">Reducer</span>
<query-part-editor class="gf-form-label query-part" part="conditionModel.reducerPart" handle-event="ctrl.handleReducerPartEvent(conditionModel, $event)">
<query-part-editor class="gf-form-label query-part" part="conditionModel.queryPart" handle-event="ctrl.handleQueryPartEvent(conditionModel, $event)">
</query-part-editor>
</div>
<div class="gf-form">
<metric-segment-model property="conditionModel.evaluator.type" options="ctrl.evalFunctions" custom="false" css-class="query-keyword" on-change="ctrl.evaluatorTypeChanged(conditionModel.evaluator)"></metric-segment-model>
<input class="gf-form-input max-width-7" type="number" ng-hide="conditionModel.evaluator.params.length === 0" ng-model="conditionModel.evaluator.params[0]" ng-change="ctrl.evaluatorParamsChanged()"></input>
<label class="gf-form-label query-keyword" ng-show="conditionModel.evaluator.params.length === 2">TO</label>
<input class="gf-form-input max-width-7" type="number" ng-if="conditionModel.evaluator.params.length === 2" ng-model="conditionModel.evaluator.params[1]" ng-change="ctrl.evaluatorParamsChanged()"></input>
</div>
<div class="gf-form">
<label class="gf-form-label">
@ -88,6 +89,18 @@
</label>
</div>
</div>
<div class="gf-form-group">
<div class="gf-form">
<span class="gf-form-label">If no data points or all values are null</span>
<span class="gf-form-label query-keyword">SET STATE TO</span>
<div class="gf-form-select-wrapper">
<select class="gf-form-input" ng-model="ctrl.alert.noDataState" ng-options="f.value as f.text for f in ctrl.noDataModes">
</select>
</div>
</div>
<div class="gf-form-button-row">
<button class="btn btn-inverse" ng-click="ctrl.test()">
Test Rule

1
public/sass/_variables.dark.scss
View File

@ -39,7 +39,6 @@ $brand-primary: $orange;
$brand-success: $green;
$brand-warning: $brand-primary;
$brand-danger: $red;
$brand-text-highlight: #f7941d;
// Status colors
// -------------------------

1
public/sass/_variables.light.scss
View File

@ -44,7 +44,6 @@ $brand-primary: $orange;
$brand-success: $green;
$brand-warning: $orange;
$brand-danger: $red;
$brand-text-highlight: #f7941d;
// Status colors
// -------------------------