Alerting: Persist rule position in the group (#50051)

Migrations:
* add a new column alert_group_idx to alert_rule table
* add a new column alert_group_idx to alert_rule_version table
* re-index existing rules during migration

API:
* set group index on update. Use the natural order of items in  the array as group index
* sort rules in the group on GET
* update the version of all rules of all affected groups. This will make optimistic lock work in the case of multiple concurrent request touching the same groups.

UI:
* update UI to keep the order of alerts in a group

pkg/services/ngalert/models/alert_rule.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"sort"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
@@ -125,6 +126,7 @@ type AlertRule struct {
 	DashboardUID    *string `xorm:"dashboard_uid"`
 	PanelID         *int64  `xorm:"panel_id"`
 	RuleGroup       string
+	RuleGroupIndex  int `xorm:"rule_group_idx"`
 	NoDataState     NoDataState
 	ExecErrState    ExecutionErrorState
 	// ideally this field should have been apimodels.ApiDuration
@@ -140,6 +142,9 @@ type SchedulableAlertRule struct {
 	OrgID           int64  `xorm:"org_id"`
 	IntervalSeconds int64
 	Version         int64
+	NamespaceUID    string `xorm:"namespace_uid"`
+	RuleGroup       string
+	RuleGroupIndex  int `xorm:"rule_group_idx"`
 }
 
 type LabelOption func(map[string]string)
@@ -251,6 +256,7 @@ type AlertRuleVersion struct {
 	RuleUID          string `xorm:"rule_uid"`
 	RuleNamespaceUID string `xorm:"rule_namespace_uid"`
 	RuleGroup        string
+	RuleGroupIndex   int `xorm:"rule_group_idx"`
 	ParentVersion    int64
 	RestoredFrom     int64
 	Version          int64
@@ -297,7 +303,7 @@ type ListAlertRulesQuery struct {
 	DashboardUID string
 	PanelID      int64
 
-	Result []*AlertRule
+	Result RulesGroup
 }
 
 type GetAlertRulesForSchedulingQuery struct {
@@ -394,3 +400,14 @@ func ValidateRuleGroupInterval(intervalSeconds, baseIntervalSeconds int64) error
 	}
 	return nil
 }
+
+type RulesGroup []*AlertRule
+
+func (g RulesGroup) SortByGroupIndex() {
+	sort.Slice(g, func(i, j int) bool {
+		if g[i].RuleGroupIndex == g[j].RuleGroupIndex {
+			return g[i].ID < g[j].ID
+		}
+		return g[i].RuleGroupIndex < g[j].RuleGroupIndex
+	})
+}

pkg/services/ngalert/models/alert_rule_test.go

@@ -3,6 +3,7 @@ package models
 import (
 	"encoding/json"
 	"math/rand"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -354,6 +355,13 @@ func TestDiff(t *testing.T) {
 			assert.Equal(t, rule2.For, diff[0].Right.Interface())
 			difCnt++
 		}
+		if rule1.RuleGroupIndex != rule2.RuleGroupIndex {
+			diff := diffs.GetDiffsForField("RuleGroupIndex")
+			assert.Len(t, diff, 1)
+			assert.Equal(t, rule1.RuleGroupIndex, diff[0].Left.Interface())
+			assert.Equal(t, rule2.RuleGroupIndex, diff[0].Right.Interface())
+			difCnt++
+		}
 
 		require.Lenf(t, diffs, difCnt, "Got some unexpected diffs. Either add to ignore or add assert to it")
 
@@ -538,3 +546,33 @@ func TestDiff(t *testing.T) {
 		})
 	})
 }
+
+func TestSortByGroupIndex(t *testing.T) {
+	t.Run("should sort rules by GroupIndex", func(t *testing.T) {
+		rules := GenerateAlertRules(rand.Intn(5)+5, AlertRuleGen(WithUniqueGroupIndex()))
+		rand.Shuffle(len(rules), func(i, j int) {
+			rules[i], rules[j] = rules[j], rules[i]
+		})
+		require.False(t, sort.SliceIsSorted(rules, func(i, j int) bool {
+			return rules[i].RuleGroupIndex < rules[j].RuleGroupIndex
+		}))
+		RulesGroup(rules).SortByGroupIndex()
+		require.True(t, sort.SliceIsSorted(rules, func(i, j int) bool {
+			return rules[i].RuleGroupIndex < rules[j].RuleGroupIndex
+		}))
+	})
+
+	t.Run("should sort by ID if same GroupIndex", func(t *testing.T) {
+		rules := GenerateAlertRules(rand.Intn(5)+5, AlertRuleGen(WithUniqueID(), WithGroupIndex(rand.Int())))
+		rand.Shuffle(len(rules), func(i, j int) {
+			rules[i], rules[j] = rules[j], rules[i]
+		})
+		require.False(t, sort.SliceIsSorted(rules, func(i, j int) bool {
+			return rules[i].ID < rules[j].ID
+		}))
+		RulesGroup(rules).SortByGroupIndex()
+		require.True(t, sort.SliceIsSorted(rules, func(i, j int) bool {
+			return rules[i].ID < rules[j].ID
+		}))
+	})
+}

pkg/services/ngalert/models/testing.go

@@ -9,9 +9,11 @@ import (
 	"github.com/grafana/grafana/pkg/util"
 )
 
+type AlertRuleMutator func(*AlertRule)
+
 // AlertRuleGen provides a factory function that generates a random AlertRule.
 // The mutators arguments allows changing fields of the resulting structure
-func AlertRuleGen(mutators ...func(*AlertRule)) func() *AlertRule {
+func AlertRuleGen(mutators ...AlertRuleMutator) func() *AlertRule {
 	return func() *AlertRule {
 		randNoDataState := func() NoDataState {
 			s := [...]NoDataState{
@@ -74,6 +76,7 @@ func AlertRuleGen(mutators ...func(*AlertRule)) func() *AlertRule {
 			DashboardUID:    dashUID,
 			PanelID:         panelID,
 			RuleGroup:       "TEST-GROUP-" + util.GenerateShortUID(),
+			RuleGroupIndex:  rand.Int(),
 			NoDataState:     randNoDataState(),
 			ExecErrState:    randErrState(),
 			For:             forInterval,
@@ -88,6 +91,48 @@ func AlertRuleGen(mutators ...func(*AlertRule)) func() *AlertRule {
 	}
 }
 
+func WithUniqueID() AlertRuleMutator {
+	usedID := make(map[int64]struct{})
+	return func(rule *AlertRule) {
+		for {
+			id := rand.Int63()
+			if _, ok := usedID[id]; !ok {
+				usedID[id] = struct{}{}
+				rule.ID = id
+				return
+			}
+		}
+	}
+}
+
+func WithGroupIndex(groupIndex int) AlertRuleMutator {
+	return func(rule *AlertRule) {
+		rule.RuleGroupIndex = groupIndex
+	}
+}
+
+func WithUniqueGroupIndex() AlertRuleMutator {
+	usedIdx := make(map[int]struct{})
+	return func(rule *AlertRule) {
+		for {
+			idx := rand.Int()
+			if _, ok := usedIdx[idx]; !ok {
+				usedIdx[idx] = struct{}{}
+				rule.RuleGroupIndex = idx
+				return
+			}
+		}
+	}
+}
+
+func WithSequentialGroupIndex() AlertRuleMutator {
+	idx := 1
+	return func(rule *AlertRule) {
+		rule.RuleGroupIndex = idx
+		idx++
+	}
+}
+
 func GenerateAlertQuery() AlertQuery {
 	f := rand.Intn(10) + 5
 	t := rand.Intn(f)
@@ -155,6 +200,7 @@ func CopyRule(r *AlertRule) *AlertRule {
 		UID:             r.UID,
 		NamespaceUID:    r.NamespaceUID,
 		RuleGroup:       r.RuleGroup,
+		RuleGroupIndex:  r.RuleGroupIndex,
 		NoDataState:     r.NoDataState,
 		ExecErrState:    r.ExecErrState,
 		For:             r.For,