Alerting: Add type label rule_group_rules metric (#91425)

* Add group and type labels to rule_group_rules metric

* Don't include group to avoid high cardinality

* Add comments

* Reset rule_group_rules before recording new values

* Edit description for rule_group_rules

* Include ruleGroup combo key in labels

* Fix lint

pkg/services/ngalert/metrics/scheduler.go

@@ -126,9 +126,9 @@ func NewSchedulerMetrics(r prometheus.Registerer) *Scheduler {
 				Namespace: Namespace,
 				Subsystem: Subsystem,
 				Name:      "rule_group_rules",
-				Help:      "The number of alert rules that are scheduled, both active and paused.",
+				Help:      "The number of alert rules that are scheduled, by type and state.",
 			},
-			[]string{"org", "state", "rule_group"},
+			[]string{"org", "type", "state", "rule_group"},
 		),
 		Groups: promauto.With(r).NewGaugeVec(
 			prometheus.GaugeOpts{

pkg/services/ngalert/schedule/metrics.go

@@ -32,38 +32,45 @@ func sortedUIDs(alertRules []*models.AlertRule) []string {
 	return uids
 }
 
-// updateRulesMetrics updates metrics for alert rules.
-// Keeps a state in the schedule between calls to delete metrics for rules that are no longer present.
-func (sch *schedule) updateRulesMetrics(alertRules []*models.AlertRule) {
-	rulesPerOrgFolderGroup := make(map[models.AlertRuleGroupKeyWithFolderFullpath]int64)       // AlertRuleGroupKeyWithFolderFullpath -> count
-	rulesPerOrgFolderGroupPaused := make(map[models.AlertRuleGroupKeyWithFolderFullpath]int64) // AlertRuleGroupKeyWithFolderFullpath -> count
-	orgsNfSettings := make(map[int64]int64)                                                    // orgID -> count
-	groupsPerOrg := make(map[int64]map[string]struct{})                                        // orgID -> set of groups
+type ruleKey struct {
+	orgID     int64
+	ruleGroup models.AlertRuleGroupKeyWithFolderFullpath
+	ruleType  models.RuleType
+	state     string
+}
 
-	// Remember what orgs and alert groups we process in the current update metrics call,
-	// so we can delete metrics for orgs and groups that are no longer present in the new state.
-	updateMetricsForOrgsAndGroups := map[int64]map[models.AlertRuleGroupKeyWithFolderFullpath]struct{}{} // orgID -> set of AlertRuleGroupWithFolderTitle
+func (sch *schedule) updateRulesMetrics(alertRules []*models.AlertRule) {
+	// main rule_group_rules metric labels
+	buckets := make(map[ruleKey]int64)
+	// gauge for rules with notification settings per org
+	orgsNfSettings := make(map[int64]int64)
+	// gauge for groups per org
+	groupsPerOrg := make(map[int64]map[string]struct{})
 
 	for _, rule := range alertRules {
-		key := models.AlertRuleGroupKeyWithFolderFullpath{
+		// Count rules by org, type and state
+		state := metrics.AlertRuleActiveLabelValue
+		if rule.IsPaused {
+			state = metrics.AlertRulePausedLabelValue
+		}
+		ruleGroup := models.AlertRuleGroupKeyWithFolderFullpath{
 			AlertRuleGroupKey: rule.GetGroupKey(),
 			FolderFullpath:    sch.schedulableAlertRules.folderTitles[rule.GetFolderKey()],
 		}
-		rulesPerOrgFolderGroup[key]++
-
-		if _, ok := updateMetricsForOrgsAndGroups[rule.OrgID]; !ok {
-			updateMetricsForOrgsAndGroups[rule.OrgID] = make(map[models.AlertRuleGroupKeyWithFolderFullpath]struct{})
-		}
-		updateMetricsForOrgsAndGroups[rule.OrgID][key] = struct{}{}
-
-		if rule.IsPaused {
-			rulesPerOrgFolderGroupPaused[key]++
+		key := ruleKey{
+			orgID:     rule.OrgID,
+			ruleGroup: ruleGroup,
+			ruleType:  rule.Type(),
+			state:     state,
 		}
+		buckets[key]++
 
+		// Count rules with notification settings per org
 		if len(rule.NotificationSettings) > 0 {
 			orgsNfSettings[rule.OrgID]++
 		}
 
+		// Count groups per org
 		orgGroups, ok := groupsPerOrg[rule.OrgID]
 		if !ok {
 			orgGroups = make(map[string]struct{})
@@ -72,59 +79,28 @@ func (sch *schedule) updateRulesMetrics(alertRules []*models.AlertRule) {
 		orgGroups[rule.RuleGroup] = struct{}{}
 	}
 
-	for key, numRules := range rulesPerOrgFolderGroup {
-		numRulesPaused := rulesPerOrgFolderGroupPaused[key]
-		ruleGroupLabelValue := makeRuleGroupLabelValue(key)
-		sch.metrics.GroupRules.WithLabelValues(fmt.Sprint(key.OrgID), metrics.AlertRuleActiveLabelValue, ruleGroupLabelValue).Set(float64(numRules - numRulesPaused))
-		sch.metrics.GroupRules.WithLabelValues(fmt.Sprint(key.OrgID), metrics.AlertRulePausedLabelValue, ruleGroupLabelValue).Set(float64(numRulesPaused))
-	}
+	// Reset metrics to avoid stale data
+	sch.metrics.GroupRules.Reset()
+	sch.metrics.SimpleNotificationRules.Reset()
+	sch.metrics.Groups.Reset()
 
-	for orgID := range updateMetricsForOrgsAndGroups {
-		sch.metrics.SimpleNotificationRules.WithLabelValues(fmt.Sprint(orgID)).Set(float64(orgsNfSettings[orgID]))
-		sch.metrics.Groups.WithLabelValues(fmt.Sprint(orgID)).Set(float64(len(groupsPerOrg[orgID])))
+	// Set metrics
+	for key, count := range buckets {
+		sch.metrics.GroupRules.WithLabelValues(fmt.Sprint(key.orgID), key.ruleType.String(), key.state, makeRuleGroupLabelValue(key.ruleGroup)).Set(float64(count))
+	}
+	for orgID, numRulesNfSettings := range orgsNfSettings {
+		sch.metrics.SimpleNotificationRules.WithLabelValues(fmt.Sprint(orgID)).Set(float64(numRulesNfSettings))
+	}
+	for orgID, groups := range groupsPerOrg {
+		sch.metrics.Groups.WithLabelValues(fmt.Sprint(orgID)).Set(float64(len(groups)))
 	}
-
 	// While these are the rules that we iterate over, at the moment there's no 100% guarantee that they'll be
 	// scheduled as rules could be removed before we get a chance to evaluate them.
 	sch.metrics.SchedulableAlertRules.Set(float64(len(alertRules)))
 	sch.metrics.SchedulableAlertRulesHash.Set(float64(hashUIDs(alertRules)))
-
-	// Delete metrics for rule groups and orgs that are no longer present in the new state
-	for orgID, alertRuleGroupsMap := range sch.lastUpdatedMetricsForOrgsAndGroups {
-		if orgOrGroupDeleted(updateMetricsForOrgsAndGroups, orgID, nil) {
-			sch.metrics.SimpleNotificationRules.DeleteLabelValues(fmt.Sprint(orgID))
-			sch.metrics.Groups.DeleteLabelValues(fmt.Sprint(orgID))
-		}
-
-		for key := range alertRuleGroupsMap {
-			if orgOrGroupDeleted(updateMetricsForOrgsAndGroups, orgID, &key) {
-				ruleGroupLabelValue := makeRuleGroupLabelValue(key)
-				sch.metrics.GroupRules.DeleteLabelValues(fmt.Sprint(key.AlertRuleGroupKey.OrgID), metrics.AlertRuleActiveLabelValue, ruleGroupLabelValue)
-				sch.metrics.GroupRules.DeleteLabelValues(fmt.Sprint(key.AlertRuleGroupKey.OrgID), metrics.AlertRulePausedLabelValue, ruleGroupLabelValue)
-			}
-		}
-	}
-
-	// update the call state
-	sch.lastUpdatedMetricsForOrgsAndGroups = updateMetricsForOrgsAndGroups
 }
 
 // makeRuleGroupLabelValue returns a string that can be used as a label (rule_group) value for alert rule group metrics.
 func makeRuleGroupLabelValue(key models.AlertRuleGroupKeyWithFolderFullpath) string {
 	return fmt.Sprintf("%s;%s", key.FolderFullpath, key.AlertRuleGroupKey.RuleGroup)
 }
-
-// orgOrGroupDeleted returns true if the org or group is no longer present in the new update metrics state.
-func orgOrGroupDeleted(updateMetrics map[int64]map[models.AlertRuleGroupKeyWithFolderFullpath]struct{}, orgID int64, alertRuleGroupKey *models.AlertRuleGroupKeyWithFolderFullpath) bool {
-	if _, ok := updateMetrics[orgID]; !ok {
-		return true
-	}
-
-	if alertRuleGroupKey != nil {
-		if _, ok := updateMetrics[orgID][*alertRuleGroupKey]; !ok {
-			return true
-		}
-	}
-
-	return false
-}

pkg/services/ngalert/schedule/schedule.go

@@ -87,10 +87,6 @@ type schedule struct {
 	rrCfg                setting.RecordingRuleSettings
 
 	metrics *metrics.Scheduler
-	// lastUpdatedMetricsForOrgsAndGroups contains AlertRuleGroupKeyWithFolderFullpaths that
-	// were passed to updateRulesMetrics in the current tick. This is used to
-	// delete metrics for the rules/groups that are not longer present.
-	lastUpdatedMetricsForOrgsAndGroups map[int64]map[ngmodels.AlertRuleGroupKeyWithFolderFullpath]struct{} // orgID -> set of AlertRuleGroupKeyWithFolderFullpath
 
 	alertsSender    AlertsSender
 	minRuleInterval time.Duration
@@ -134,25 +130,24 @@ func NewScheduler(cfg SchedulerCfg, stateManager *state.Manager) *schedule {
 	}
 
 	sch := schedule{
-		registry:                           newRuleRegistry(),
-		maxAttempts:                        cfg.MaxAttempts,
-		clock:                              cfg.C,
-		baseInterval:                       cfg.BaseInterval,
-		log:                                cfg.Log,
-		evaluatorFactory:                   cfg.EvaluatorFactory,
-		ruleStore:                          cfg.RuleStore,
-		metrics:                            cfg.Metrics,
-		lastUpdatedMetricsForOrgsAndGroups: make(map[int64]map[ngmodels.AlertRuleGroupKeyWithFolderFullpath]struct{}),
-		appURL:                             cfg.AppURL,
-		disableGrafanaFolder:               cfg.DisableGrafanaFolder,
-		jitterEvaluations:                  cfg.JitterEvaluations,
-		rrCfg:                              cfg.RecordingRulesCfg,
-		stateManager:                       stateManager,
-		minRuleInterval:                    cfg.MinRuleInterval,
-		schedulableAlertRules:              alertRulesRegistry{rules: make(map[ngmodels.AlertRuleKey]*ngmodels.AlertRule)},
-		alertsSender:                       cfg.AlertSender,
-		tracer:                             cfg.Tracer,
-		recordingWriter:                    cfg.RecordingWriter,
+		registry:              newRuleRegistry(),
+		maxAttempts:           cfg.MaxAttempts,
+		clock:                 cfg.C,
+		baseInterval:          cfg.BaseInterval,
+		log:                   cfg.Log,
+		evaluatorFactory:      cfg.EvaluatorFactory,
+		ruleStore:             cfg.RuleStore,
+		metrics:               cfg.Metrics,
+		appURL:                cfg.AppURL,
+		disableGrafanaFolder:  cfg.DisableGrafanaFolder,
+		jitterEvaluations:     cfg.JitterEvaluations,
+		rrCfg:                 cfg.RecordingRulesCfg,
+		stateManager:          stateManager,
+		minRuleInterval:       cfg.MinRuleInterval,
+		schedulableAlertRules: alertRulesRegistry{rules: make(map[ngmodels.AlertRuleKey]*ngmodels.AlertRule)},
+		alertsSender:          cfg.AlertSender,
+		tracer:                cfg.Tracer,
+		recordingWriter:       cfg.RecordingWriter,
 	}
 
 	return &sch

pkg/services/ngalert/schedule/schedule_unit_test.go

@@ -128,11 +128,10 @@ func TestProcessTicks(t *testing.T) {
 
 	t.Run("after 1st tick rule metrics should report one active alert rule", func(t *testing.T) {
 		expectedMetric := fmt.Sprintf(
-			`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused.
+			`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state.
         	            	# TYPE grafana_alerting_rule_group_rules gauge
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 1
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 0
-			`, alertRule1.OrgID, folderWithRuleGroup1)
+        	            	grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active",type="alerting"} 1
+				`, alertRule1.OrgID, folderWithRuleGroup1)
 
 		err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules")
 		require.NoError(t, err)
@@ -158,12 +157,10 @@ func TestProcessTicks(t *testing.T) {
 
 	t.Run("after 2nd tick rule metrics should report two active alert rules in two groups", func(t *testing.T) {
 		expectedMetric := fmt.Sprintf(
-			`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused.
+			`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state.
         	            	# TYPE grafana_alerting_rule_group_rules gauge
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 1
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 0
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active"} 1
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="paused"} 0
+        	            	grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active",type="alerting"} 1
+        	            	grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active",type="alerting"} 1
 				`, alertRule1.OrgID, folderWithRuleGroup1, folderWithRuleGroup2)
 
 		err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules")
@@ -216,12 +213,10 @@ func TestProcessTicks(t *testing.T) {
 
 	t.Run("after 5th tick rule metrics should report one active and one paused alert rules in two groups", func(t *testing.T) {
 		expectedMetric := fmt.Sprintf(
-			`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused.
+			`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state.
         	            	# TYPE grafana_alerting_rule_group_rules gauge
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 0
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 1
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active"} 1
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="paused"} 0
+        	            	grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused",type="alerting"} 1
+        	            	grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active",type="alerting"} 1
 				`, alertRule1.OrgID, folderWithRuleGroup1, folderWithRuleGroup2)
 
 		err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules")
@@ -251,14 +246,12 @@ func TestProcessTicks(t *testing.T) {
 
 	t.Run("after 6th tick rule metrics should report two paused alert rules in two groups", func(t *testing.T) {
 		expectedMetric := fmt.Sprintf(
-			`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused.
+			`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state.
         	            	# TYPE grafana_alerting_rule_group_rules gauge
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 0
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 1
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active"} 0
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="paused"} 1
-				`, alertRule1.OrgID, folderWithRuleGroup1, folderWithRuleGroup2)
+        	            	grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused",type="alerting"} 1
+        	            	grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="paused",type="alerting"} 1
 
+				`, alertRule1.OrgID, folderWithRuleGroup1, folderWithRuleGroup2)
 		err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules")
 		require.NoError(t, err)
 	})
@@ -281,12 +274,10 @@ func TestProcessTicks(t *testing.T) {
 
 	t.Run("after 7th tick rule metrics should report two active alert rules in two groups", func(t *testing.T) {
 		expectedMetric := fmt.Sprintf(
-			`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused.
+			`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state.
         	            	# TYPE grafana_alerting_rule_group_rules gauge
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 1
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 0
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active"} 1
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="paused"} 0
+        	            	grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active",type="alerting"} 1
+        	            	grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active",type="alerting"} 1
 				`, alertRule1.OrgID, folderWithRuleGroup1, folderWithRuleGroup2)
 
 		err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules")
@@ -310,12 +301,10 @@ func TestProcessTicks(t *testing.T) {
 
 	t.Run("after 8th tick rule metrics should report one active alert rule", func(t *testing.T) {
 		expectedMetric := fmt.Sprintf(
-			`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused.
-							# TYPE grafana_alerting_rule_group_rules gauge
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 1
-							grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 0
-				`, alertRule2.OrgID, folderWithRuleGroup2)
-
+			`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state.
+        	            	# TYPE grafana_alerting_rule_group_rules gauge
+        	            	grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active",type="alerting"} 1
+				`, alertRule1.OrgID, folderWithRuleGroup2)
 		err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules")
 		require.NoError(t, err)
 	})
@@ -533,10 +522,9 @@ func TestSchedule_updateRulesMetrics(t *testing.T) {
 			sch.updateRulesMetrics([]*models.AlertRule{alertRule1})
 
 			expectedMetric := fmt.Sprintf(
-				`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused.
+				`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state.
 								# TYPE grafana_alerting_rule_group_rules gauge
-								grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 1
-								grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 0
+								grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active", type="alerting"} 1
 				`, alertRule1.OrgID, folderWithRuleGroup1)
 
 			err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules")
@@ -555,12 +543,10 @@ func TestSchedule_updateRulesMetrics(t *testing.T) {
 			sch.updateRulesMetrics([]*models.AlertRule{alertRule1, alertRule2})
 
 			expectedMetric := fmt.Sprintf(
-				`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused.
+				`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state.
 								# TYPE grafana_alerting_rule_group_rules gauge
-								grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 1
-								grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 0
-								grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active"} 1
-								grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="paused"} 0
+								grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active",type="alerting"} 1
+                	            grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active",type="alerting"} 1
 				`, alertRule1.OrgID, folderWithRuleGroup1, folderWithRuleGroup2)
 
 			err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules")
@@ -572,12 +558,10 @@ func TestSchedule_updateRulesMetrics(t *testing.T) {
 			sch.updateRulesMetrics([]*models.AlertRule{alertRule1, alertRule2})
 
 			expectedMetric := fmt.Sprintf(
-				`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused.
+				`# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state.
 								# TYPE grafana_alerting_rule_group_rules gauge
-								grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 1
-								grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 0
-								grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active"} 1
-								grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="paused"} 0
+								grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active",type="alerting"} 1
+                	            grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active",type="alerting"} 1
 				`, alertRule1.OrgID, folderWithRuleGroup1, folderWithRuleGroup2)
 
 			err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules")
@@ -666,15 +650,9 @@ func TestSchedule_updateRulesMetrics(t *testing.T) {
 		t.Run("it should not show metrics", func(t *testing.T) {
 			sch.updateRulesMetrics([]*models.AlertRule{alertRuleWithoutNotificationSettings})
 
-			// Because alertRuleWithoutNotificationSettings.orgID is present,
-			// the metric is also present but set to 0 because the org has no rules with NotificationSettings.
-			expectedMetric := fmt.Sprintf(
-				`# HELP grafana_alerting_simple_routing_rules The number of alert rules using simplified routing.
-								# TYPE grafana_alerting_simple_routing_rules gauge
-								grafana_alerting_simple_routing_rules{org="%[1]d"} 0
-				`, alertRuleWithoutNotificationSettings.OrgID)
+			expectedMetric := ""
 			err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_simple_routing_rules")
-			require.NoError(t, err)
+			require.ErrorContains(t, err, "expected metric name(s) not found: [grafana_alerting_simple_routing_rules]")
 		})
 
 		alertRule1 := models.RuleGen.With(
@@ -722,7 +700,6 @@ func TestSchedule_updateRulesMetrics(t *testing.T) {
 			expectedMetric := fmt.Sprintf(
 				`# HELP grafana_alerting_simple_routing_rules The number of alert rules using simplified routing.
 								# TYPE grafana_alerting_simple_routing_rules gauge
-								grafana_alerting_simple_routing_rules{org="%[1]d"} 0
 								grafana_alerting_simple_routing_rules{org="%[2]d"} 1
 				`, alertRuleWithoutNotificationSettings.OrgID, alertRule2.OrgID)