Merge pull request #13947 from bergquist/alerting_for

Introduce alert debouncing

pkg/api/alerting.go

@@ -295,7 +295,7 @@ func PauseAlert(c *m.ReqContext, dto dtos.PauseAlertCommand) Response {
 		return Error(500, "", err)
 	}
 
-	var response m.AlertStateType = m.AlertStatePending
+	var response m.AlertStateType = m.AlertStateUnknown
 	pausedState := "un-paused"
 	if cmd.Paused {
 		response = m.AlertStatePaused

pkg/models/alert.go

@@ -19,6 +19,7 @@ const (
 	AlertStateAlerting AlertStateType = "alerting"
 	AlertStateOK       AlertStateType = "ok"
 	AlertStatePending  AlertStateType = "pending"
+	AlertStateUnknown  AlertStateType = "unknown"
 )
 
 const (
@@ -39,7 +40,12 @@ var (
 )
 
 func (s AlertStateType) IsValid() bool {
-	return s == AlertStateOK || s == AlertStateNoData || s == AlertStatePaused || s == AlertStatePending
+	return s == AlertStateOK ||
+		s == AlertStateNoData ||
+		s == AlertStatePaused ||
+		s == AlertStatePending ||
+		s == AlertStateAlerting ||
+		s == AlertStateUnknown
 }
 
 func (s NoDataOption) IsValid() bool {
@@ -66,12 +72,13 @@ type Alert struct {
 	PanelId        int64
 	Name           string
 	Message        string
-	Severity       string
+	Severity       string //Unused
 	State          AlertStateType
-	Handler        int64
+	Handler        int64 //Unused
 	Silenced       bool
 	ExecutionError string
 	Frequency      int64
+	For            time.Duration
 
 	EvalData     *simplejson.Json
 	NewStateDate time.Time

pkg/services/alerting/eval_context.go

@@ -68,8 +68,13 @@ func (c *EvalContext) GetStateModel() *StateDescription {
 			Color: "#D63232",
 			Text:  "Alerting",
 		}
+	case m.AlertStateUnknown:
+		return &StateDescription{
+			Color: "#888888",
+			Text:  "Unknown",
+		}
 	default:
-		panic("Unknown rule state " + c.Rule.State)
+		panic("Unknown rule state for alert " + c.Rule.State)
 	}
 }
 
@@ -113,7 +118,26 @@ func (c *EvalContext) GetRuleUrl() (string, error) {
 	return fmt.Sprintf(urlFormat, m.GetFullDashboardUrl(ref.Uid, ref.Slug), c.Rule.PanelId, c.Rule.OrgId), nil
 }
 
+// GetNewState returns the new state from the alert rule evaluation
 func (c *EvalContext) GetNewState() m.AlertStateType {
+	ns := getNewStateInternal(c)
+	if ns != m.AlertStateAlerting || c.Rule.For == 0 {
+		return ns
+	}
+
+	since := time.Now().Sub(c.Rule.LastStateChange)
+	if c.PrevAlertState == m.AlertStatePending && since > c.Rule.For {
+		return m.AlertStateAlerting
+	}
+
+	if c.PrevAlertState == m.AlertStateAlerting {
+		return m.AlertStateAlerting
+	}
+
+	return m.AlertStatePending
+}
+
+func getNewStateInternal(c *EvalContext) m.AlertStateType {
 	if c.Error != nil {
 		c.log.Error("Alert Rule Result Error",
 			"ruleId", c.Rule.Id,
@@ -125,11 +149,13 @@ func (c *EvalContext) GetNewState() m.AlertStateType {
 			return c.PrevAlertState
 		}
 		return c.Rule.ExecutionErrorState.ToAlertState()
+	}
 
-	} else if c.Firing {
+	if c.Firing {
 		return m.AlertStateAlerting
+	}
 
-	} else if c.NoDataFound {
+	if c.NoDataFound {
 		c.log.Info("Alert Rule returned no data",
 			"ruleId", c.Rule.Id,
 			"name", c.Rule.Name,

pkg/services/alerting/eval_context_test.go

@@ -2,11 +2,11 @@ package alerting
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"testing"
+	"time"
 
 	"github.com/grafana/grafana/pkg/models"
-	. "github.com/smartystreets/goconvey/convey"
 )
 
 func TestStateIsUpdatedWhenNeeded(t *testing.T) {
@@ -31,71 +31,177 @@ func TestStateIsUpdatedWhenNeeded(t *testing.T) {
 	})
 }
 
-func TestAlertingEvalContext(t *testing.T) {
-	Convey("Should compute and replace properly new rule state", t, func() {
+func TestGetStateFromEvalContext(t *testing.T) {
+	tcs := []struct {
+		name     string
+		expected models.AlertStateType
+		applyFn  func(ec *EvalContext)
+		focus    bool
+	}{
+		{
+			name:     "ok -> alerting",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.Firing = true
+				ec.PrevAlertState = models.AlertStateOK
+			},
+		},
+		{
+			name:     "ok -> error(alerting)",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Error = errors.New("test error")
+				ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
+			},
+		},
+		{
+			name:     "ok -> pending. since its been firing for less than FOR",
+			expected: models.AlertStatePending,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Firing = true
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2)
+				ec.Rule.For = time.Minute * 5
+			},
+		},
+		{
+			name:     "ok -> pending. since it has to be pending longer than FOR and prev state is ok",
+			expected: models.AlertStatePending,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Firing = true
+				ec.Rule.LastStateChange = time.Now().Add(-(time.Hour * 5))
+				ec.Rule.For = time.Minute * 2
+			},
+		},
+		{
+			name:     "pending -> alerting. since its been firing for more than FOR and prev state is pending",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Firing = true
+				ec.Rule.LastStateChange = time.Now().Add(-(time.Hour * 5))
+				ec.Rule.For = time.Minute * 2
+			},
+		},
+		{
+			name:     "alerting -> alerting. should not update regardless of FOR",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateAlerting
+				ec.Firing = true
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
+				ec.Rule.For = time.Minute * 2
+			},
+		},
+		{
+			name:     "ok -> ok. should not update regardless of FOR",
+			expected: models.AlertStateOK,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
+				ec.Rule.For = time.Minute * 2
+			},
+		},
+		{
+			name:     "ok -> error(keep_last)",
+			expected: models.AlertStateOK,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Error = errors.New("test error")
+				ec.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
+			},
+		},
+		{
+			name:     "pending -> error(keep_last)",
+			expected: models.AlertStatePending,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Error = errors.New("test error")
+				ec.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
+			},
+		},
+		{
+			name:     "ok -> no_data(alerting)",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Rule.NoDataState = models.NoDataSetAlerting
+				ec.NoDataFound = true
+			},
+		},
+		{
+			name:     "ok -> no_data(keep_last)",
+			expected: models.AlertStateOK,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStateOK
+				ec.Rule.NoDataState = models.NoDataKeepState
+				ec.NoDataFound = true
+			},
+		},
+		{
+			name:     "pending -> no_data(keep_last)",
+			expected: models.AlertStatePending,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Rule.NoDataState = models.NoDataKeepState
+				ec.NoDataFound = true
+			},
+		},
+		{
+			name:     "pending -> no_data(alerting) with for duration have not passed",
+			expected: models.AlertStatePending,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Rule.NoDataState = models.NoDataSetAlerting
+				ec.NoDataFound = true
+				ec.Rule.For = time.Minute * 5
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2)
+			},
+		},
+		{
+			name:     "pending -> no_data(alerting) should set alerting since time passed FOR",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Rule.NoDataState = models.NoDataSetAlerting
+				ec.NoDataFound = true
+				ec.Rule.For = time.Minute * 2
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
+			},
+		},
+		{
+			name:     "pending -> error(alerting) with for duration have not passed ",
+			expected: models.AlertStatePending,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
+				ec.Error = errors.New("test error")
+				ec.Rule.For = time.Minute * 5
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2)
+			},
+		},
+		{
+			name:     "pending -> error(alerting) should set alerting since time passed FOR",
+			expected: models.AlertStateAlerting,
+			applyFn: func(ec *EvalContext) {
+				ec.PrevAlertState = models.AlertStatePending
+				ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
+				ec.Error = errors.New("test error")
+				ec.Rule.For = time.Minute * 2
+				ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
+			},
+		},
+	}
+
+	for _, tc := range tcs {
 		ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}})
-		dummieError := fmt.Errorf("dummie error")
 
-		Convey("ok -> alerting", func() {
-			ctx.PrevAlertState = models.AlertStateOK
-			ctx.Firing = true
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
-		})
-
-		Convey("ok -> error(alerting)", func() {
-			ctx.PrevAlertState = models.AlertStateOK
-			ctx.Error = dummieError
-			ctx.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
-		})
-
-		Convey("ok -> error(keep_last)", func() {
-			ctx.PrevAlertState = models.AlertStateOK
-			ctx.Error = dummieError
-			ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
-		})
-
-		Convey("pending -> error(keep_last)", func() {
-			ctx.PrevAlertState = models.AlertStatePending
-			ctx.Error = dummieError
-			ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
-		})
-
-		Convey("ok -> no_data(alerting)", func() {
-			ctx.PrevAlertState = models.AlertStateOK
-			ctx.Rule.NoDataState = models.NoDataSetAlerting
-			ctx.NoDataFound = true
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
-		})
-
-		Convey("ok -> no_data(keep_last)", func() {
-			ctx.PrevAlertState = models.AlertStateOK
-			ctx.Rule.NoDataState = models.NoDataKeepState
-			ctx.NoDataFound = true
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
-		})
-
-		Convey("pending -> no_data(keep_last)", func() {
-			ctx.PrevAlertState = models.AlertStatePending
-			ctx.Rule.NoDataState = models.NoDataKeepState
-			ctx.NoDataFound = true
-
-			ctx.Rule.State = ctx.GetNewState()
-			So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
-		})
-	})
+		tc.applyFn(ctx)
+		have := ctx.GetNewState()
+		if have != tc.expected {
+			t.Errorf("failed: %s \n expected '%s' have '%s'\n", tc.name, tc.expected, string(have))
+		}
+	}
 }

pkg/services/alerting/extractor.go

@@ -2,8 +2,8 @@ package alerting
 
 import (
 	"errors"
-
 	"fmt"
+	"time"
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/components/simplejson"
@@ -115,6 +115,15 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json,
 			return nil, ValidationError{Reason: "Could not parse frequency"}
 		}
 
+		rawFor := jsonAlert.Get("for").MustString()
+		var forValue time.Duration
+		if rawFor != "" {
+			forValue, err = time.ParseDuration(rawFor)
+			if err != nil {
+				return nil, ValidationError{Reason: "Could not parse for"}
+			}
+		}
+
 		alert := &m.Alert{
 			DashboardId: e.Dash.Id,
 			OrgId:       e.OrgID,
@@ -124,6 +133,7 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json,
 			Handler:     jsonAlert.Get("handler").MustInt64(),
 			Message:     jsonAlert.Get("message").MustString(),
 			Frequency:   frequency,
+			For:         forValue,
 		}
 
 		for _, condition := range jsonAlert.Get("conditions").MustArray() {

pkg/services/alerting/extractor_test.go

@@ -3,6 +3,7 @@ package alerting
 import (
 	"io/ioutil"
 	"testing"
+	"time"
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/components/simplejson"
@@ -46,7 +47,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 			return nil
 		})
 
-		json, err := ioutil.ReadFile("./test-data/graphite-alert.json")
+		json, err := ioutil.ReadFile("./testdata/graphite-alert.json")
 		So(err, ShouldBeNil)
 
 		Convey("Extractor should not modify the original json", func() {
@@ -118,6 +119,11 @@ func TestAlertRuleExtraction(t *testing.T) {
 					So(alerts[1].PanelId, ShouldEqual, 4)
 				})
 
+				Convey("should extract for param", func() {
+					So(alerts[0].For, ShouldEqual, time.Minute*2)
+					So(alerts[1].For, ShouldEqual, time.Duration(0))
+				})
+
 				Convey("should extract name and desc", func() {
 					So(alerts[0].Name, ShouldEqual, "name1")
 					So(alerts[0].Message, ShouldEqual, "desc1")
@@ -140,7 +146,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Panels missing id should return error", func() {
-			panelWithoutId, err := ioutil.ReadFile("./test-data/panels-missing-id.json")
+			panelWithoutId, err := ioutil.ReadFile("./testdata/panels-missing-id.json")
 			So(err, ShouldBeNil)
 
 			dashJson, err := simplejson.NewJson(panelWithoutId)
@@ -156,7 +162,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Panel with id set to zero should return error", func() {
-			panelWithIdZero, err := ioutil.ReadFile("./test-data/panel-with-id-0.json")
+			panelWithIdZero, err := ioutil.ReadFile("./testdata/panel-with-id-0.json")
 			So(err, ShouldBeNil)
 
 			dashJson, err := simplejson.NewJson(panelWithIdZero)
@@ -172,7 +178,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Parse alerts from dashboard without rows", func() {
-			json, err := ioutil.ReadFile("./test-data/v5-dashboard.json")
+			json, err := ioutil.ReadFile("./testdata/v5-dashboard.json")
 			So(err, ShouldBeNil)
 
 			dashJson, err := simplejson.NewJson(json)
@@ -192,7 +198,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Parse and validate dashboard containing influxdb alert", func() {
-			json, err := ioutil.ReadFile("./test-data/influxdb-alert.json")
+			json, err := ioutil.ReadFile("./testdata/influxdb-alert.json")
 			So(err, ShouldBeNil)
 
 			dashJson, err := simplejson.NewJson(json)
@@ -221,7 +227,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Should be able to extract collapsed panels", func() {
-			json, err := ioutil.ReadFile("./test-data/collapsed-panels.json")
+			json, err := ioutil.ReadFile("./testdata/collapsed-panels.json")
 			So(err, ShouldBeNil)
 
 			dashJson, err := simplejson.NewJson(json)
@@ -242,7 +248,7 @@ func TestAlertRuleExtraction(t *testing.T) {
 		})
 
 		Convey("Parse and validate dashboard without id and containing an alert", func() {
-			json, err := ioutil.ReadFile("./test-data/dash-without-id.json")
+			json, err := ioutil.ReadFile("./testdata/dash-without-id.json")
 			So(err, ShouldBeNil)
 
 			dashJSON, err := simplejson.NewJson(json)

pkg/services/alerting/notifiers/alertmanager_test.go

@@ -1,13 +1,60 @@
 package notifiers
 
 import (
+	"context"
 	"testing"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/log"
 	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/alerting"
 	. "github.com/smartystreets/goconvey/convey"
 )
 
+func TestWhenAlertManagerShouldNotify(t *testing.T) {
+	tcs := []struct {
+		prevState m.AlertStateType
+		newState  m.AlertStateType
+
+		expect bool
+	}{
+		{
+			prevState: m.AlertStatePending,
+			newState:  m.AlertStateOK,
+			expect:    false,
+		},
+		{
+			prevState: m.AlertStateAlerting,
+			newState:  m.AlertStateOK,
+			expect:    true,
+		},
+		{
+			prevState: m.AlertStateOK,
+			newState:  m.AlertStatePending,
+			expect:    false,
+		},
+		{
+			prevState: m.AlertStateUnknown,
+			newState:  m.AlertStatePending,
+			expect:    false,
+		},
+	}
+
+	for _, tc := range tcs {
+		am := &AlertmanagerNotifier{log: log.New("test.logger")}
+		evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{
+			State: tc.prevState,
+		})
+
+		evalContext.Rule.State = tc.newState
+
+		res := am.ShouldNotify(context.TODO(), evalContext, &m.AlertNotificationState{})
+		if res != tc.expect {
+			t.Errorf("got %v expected %v", res, tc.expect)
+		}
+	}
+}
+
 func TestAlertmanagerNotifier(t *testing.T) {
 	Convey("Alertmanager notifier tests", t, func() {
 

pkg/services/alerting/notifiers/base.go

@@ -67,6 +67,16 @@ func (n *NotifierBase) ShouldNotify(ctx context.Context, context *alerting.EvalC
 	}
 
 	// Do not notify when we become OK for the first time.
+	if context.PrevAlertState == models.AlertStateUnknown && context.Rule.State == models.AlertStateOK {
+		return false
+	}
+
+	// Do not notify when we become OK for the first time.
+	if context.PrevAlertState == models.AlertStateUnknown && context.Rule.State == models.AlertStatePending {
+		return false
+	}
+
+	// Do not notify when we become OK from pending
 	if context.PrevAlertState == models.AlertStatePending && context.Rule.State == models.AlertStateOK {
 		return false
 	}

pkg/services/alerting/notifiers/base_test.go

@@ -29,7 +29,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateOK,
 			prevState:    m.AlertStatePending,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: false,
 		},
@@ -38,7 +37,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateAlerting,
 			prevState:    m.AlertStateOK,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: true,
 		},
@@ -47,7 +45,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStatePending,
 			prevState:    m.AlertStateOK,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: false,
 		},
@@ -56,7 +53,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateOK,
 			prevState:    m.AlertStateOK,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: false,
 		},
@@ -65,7 +61,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateOK,
 			prevState:    m.AlertStateOK,
 			sendReminder: true,
-			state:        &m.AlertNotificationState{},
 
 			expect: false,
 		},
@@ -74,7 +69,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			newState:     m.AlertStateOK,
 			prevState:    m.AlertStateAlerting,
 			sendReminder: false,
-			state:        &m.AlertNotificationState{},
 
 			expect: true,
 		},
@@ -94,7 +88,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			prevState:    m.AlertStateAlerting,
 			frequency:    time.Minute * 10,
 			sendReminder: true,
-			state:        &m.AlertNotificationState{},
 
 			expect: true,
 		},
@@ -132,6 +125,27 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			prevState: m.AlertStateOK,
 			state:     &m.AlertNotificationState{State: m.AlertNotificationStatePending, UpdatedAt: tnow.Add(-2 * time.Minute).Unix()},
 
+			expect: true,
+		},
+		{
+			name:      "unknown -> ok",
+			prevState: m.AlertStateUnknown,
+			newState:  m.AlertStateOK,
+
+			expect: false,
+		},
+		{
+			name:      "unknown -> pending",
+			prevState: m.AlertStateUnknown,
+			newState:  m.AlertStatePending,
+
+			expect: false,
+		},
+		{
+			name:      "unknown -> alerting",
+			prevState: m.AlertStateUnknown,
+			newState:  m.AlertStateAlerting,
+
 			expect: true,
 		},
 	}
@@ -141,6 +155,10 @@ func TestShouldSendAlertNotification(t *testing.T) {
 			State: tc.prevState,
 		})
 
+		if tc.state == nil {
+			tc.state = &m.AlertNotificationState{}
+		}
+
 		evalContext.Rule.State = tc.newState
 		nb := &NotifierBase{SendReminder: tc.sendReminder, Frequency: tc.frequency}
 

pkg/services/alerting/result_handler.go

@@ -73,6 +73,9 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
 			// when two servers are raising. This makes sure that the server
 			// with the last state change always sends a notification.
 			evalContext.Rule.StateChanges = cmd.Result.StateChanges
+
+			// Update the last state change of the alert rule in memory
+			evalContext.Rule.LastStateChange = time.Now()
 		}
 
 		// save annotation

pkg/services/alerting/rule.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"regexp"
 	"strconv"
+	"time"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
 
@@ -18,6 +19,8 @@ type Rule struct {
 	Frequency           int64
 	Name                string
 	Message             string
+	LastStateChange     time.Time
+	For                 time.Duration
 	NoDataState         m.NoDataOption
 	ExecutionErrorState m.ExecutionErrorOption
 	State               m.AlertStateType
@@ -100,6 +103,8 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
 	model.Message = ruleDef.Message
 	model.Frequency = ruleDef.Frequency
 	model.State = ruleDef.State
+	model.LastStateChange = ruleDef.NewStateDate
+	model.For = ruleDef.For
 	model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
 	model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))
 	model.StateChanges = ruleDef.StateChanges

pkg/services/alerting/testdata/graphite-alert.json

@@ -23,6 +23,7 @@
           "message": "desc1",
           "handler": 1,
           "frequency": "60s",
+          "for": "2m",
           "conditions": [
           {
             "type": "query",

pkg/services/sqlstore/alert.go

@@ -193,7 +193,8 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS
 			if alertToUpdate.ContainsUpdates(alert) {
 				alert.Updated = timeNow()
 				alert.State = alertToUpdate.State
-				sess.MustCols("message")
+				sess.MustCols("message", "for")
+
 				_, err := sess.ID(alert.Id).Update(alert)
 				if err != nil {
 					return err
@@ -204,7 +205,7 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS
 		} else {
 			alert.Updated = timeNow()
 			alert.Created = timeNow()
-			alert.State = m.AlertStatePending
+			alert.State = m.AlertStateUnknown
 			alert.NewStateDate = timeNow()
 
 			_, err := sess.Insert(alert)
@@ -299,7 +300,7 @@ func PauseAlert(cmd *m.PauseAlertCommand) error {
 			params = append(params, string(m.AlertStatePaused))
 			params = append(params, timeNow())
 		} else {
-			params = append(params, string(m.AlertStatePending))
+			params = append(params, string(m.AlertStateUnknown))
 			params = append(params, timeNow())
 		}
 
@@ -323,7 +324,7 @@ func PauseAllAlerts(cmd *m.PauseAllAlertCommand) error {
 		if cmd.Paused {
 			newState = string(m.AlertStatePaused)
 		} else {
-			newState = string(m.AlertStatePending)
+			newState = string(m.AlertStateUnknown)
 		}
 
 		res, err := sess.Exec(`UPDATE alert SET state = ?, new_state_date = ?`, newState, timeNow())

pkg/services/sqlstore/alert_test.go

@@ -109,7 +109,7 @@ func TestAlertingDataAccess(t *testing.T) {
 			So(alert.DashboardId, ShouldEqual, testDash.Id)
 			So(alert.PanelId, ShouldEqual, 1)
 			So(alert.Name, ShouldEqual, "Alerting title")
-			So(alert.State, ShouldEqual, "pending")
+			So(alert.State, ShouldEqual, m.AlertStateUnknown)
 			So(alert.NewStateDate, ShouldNotBeNil)
 			So(alert.EvalData, ShouldNotBeNil)
 			So(alert.EvalData.Get("test").MustString(), ShouldEqual, "test")
@@ -154,7 +154,7 @@ func TestAlertingDataAccess(t *testing.T) {
 				So(query.Result[0].Name, ShouldEqual, "Name")
 
 				Convey("Alert state should not be updated", func() {
-					So(query.Result[0].State, ShouldEqual, "pending")
+					So(query.Result[0].State, ShouldEqual, m.AlertStateUnknown)
 				})
 			})
 

pkg/services/sqlstore/migrations/alert_mig.go

@@ -133,4 +133,8 @@ func addAlertMigrations(mg *Migrator) {
 	mg.AddMigration("create alert_notification_state table v1", NewAddTableMigration(alert_notification_state))
 	mg.AddMigration("add index alert_notification_state org_id & alert_id & notifier_id",
 		NewAddIndexMigration(alert_notification_state, alert_notification_state.Indices[0]))
+
+	mg.AddMigration("Add for to alert table", NewAddColumnMigration(alertV1, &Column{
+		Name: "for", Type: DB_BigInt, Nullable: true,
+	}))
 }