diff --git a/pkg/services/ngalert/metrics/scheduler.go b/pkg/services/ngalert/metrics/scheduler.go index 461f26871d5f..9c39727b3823 100644 --- a/pkg/services/ngalert/metrics/scheduler.go +++ b/pkg/services/ngalert/metrics/scheduler.go @@ -126,9 +126,9 @@ func NewSchedulerMetrics(r prometheus.Registerer) *Scheduler { Namespace: Namespace, Subsystem: Subsystem, Name: "rule_group_rules", - Help: "The number of alert rules that are scheduled, both active and paused.", + Help: "The number of alert rules that are scheduled, by type and state.", }, - []string{"org", "state", "rule_group"}, + []string{"org", "type", "state", "rule_group"}, ), Groups: promauto.With(r).NewGaugeVec( prometheus.GaugeOpts{ diff --git a/pkg/services/ngalert/schedule/metrics.go b/pkg/services/ngalert/schedule/metrics.go index 73a2cb327087..0d3e04f65894 100644 --- a/pkg/services/ngalert/schedule/metrics.go +++ b/pkg/services/ngalert/schedule/metrics.go @@ -32,38 +32,45 @@ func sortedUIDs(alertRules []*models.AlertRule) []string { return uids } -// updateRulesMetrics updates metrics for alert rules. -// Keeps a state in the schedule between calls to delete metrics for rules that are no longer present. -func (sch *schedule) updateRulesMetrics(alertRules []*models.AlertRule) { - rulesPerOrgFolderGroup := make(map[models.AlertRuleGroupKeyWithFolderFullpath]int64) // AlertRuleGroupKeyWithFolderFullpath -> count - rulesPerOrgFolderGroupPaused := make(map[models.AlertRuleGroupKeyWithFolderFullpath]int64) // AlertRuleGroupKeyWithFolderFullpath -> count - orgsNfSettings := make(map[int64]int64) // orgID -> count - groupsPerOrg := make(map[int64]map[string]struct{}) // orgID -> set of groups +type ruleKey struct { + orgID int64 + ruleGroup models.AlertRuleGroupKeyWithFolderFullpath + ruleType models.RuleType + state string +} - // Remember what orgs and alert groups we process in the current update metrics call, - // so we can delete metrics for orgs and groups that are no longer present in the new state. - updateMetricsForOrgsAndGroups := map[int64]map[models.AlertRuleGroupKeyWithFolderFullpath]struct{}{} // orgID -> set of AlertRuleGroupWithFolderTitle +func (sch *schedule) updateRulesMetrics(alertRules []*models.AlertRule) { + // main rule_group_rules metric labels + buckets := make(map[ruleKey]int64) + // gauge for rules with notification settings per org + orgsNfSettings := make(map[int64]int64) + // gauge for groups per org + groupsPerOrg := make(map[int64]map[string]struct{}) for _, rule := range alertRules { - key := models.AlertRuleGroupKeyWithFolderFullpath{ + // Count rules by org, type and state + state := metrics.AlertRuleActiveLabelValue + if rule.IsPaused { + state = metrics.AlertRulePausedLabelValue + } + ruleGroup := models.AlertRuleGroupKeyWithFolderFullpath{ AlertRuleGroupKey: rule.GetGroupKey(), FolderFullpath: sch.schedulableAlertRules.folderTitles[rule.GetFolderKey()], } - rulesPerOrgFolderGroup[key]++ - - if _, ok := updateMetricsForOrgsAndGroups[rule.OrgID]; !ok { - updateMetricsForOrgsAndGroups[rule.OrgID] = make(map[models.AlertRuleGroupKeyWithFolderFullpath]struct{}) - } - updateMetricsForOrgsAndGroups[rule.OrgID][key] = struct{}{} - - if rule.IsPaused { - rulesPerOrgFolderGroupPaused[key]++ + key := ruleKey{ + orgID: rule.OrgID, + ruleGroup: ruleGroup, + ruleType: rule.Type(), + state: state, } + buckets[key]++ + // Count rules with notification settings per org if len(rule.NotificationSettings) > 0 { orgsNfSettings[rule.OrgID]++ } + // Count groups per org orgGroups, ok := groupsPerOrg[rule.OrgID] if !ok { orgGroups = make(map[string]struct{}) @@ -72,59 +79,28 @@ func (sch *schedule) updateRulesMetrics(alertRules []*models.AlertRule) { orgGroups[rule.RuleGroup] = struct{}{} } - for key, numRules := range rulesPerOrgFolderGroup { - numRulesPaused := rulesPerOrgFolderGroupPaused[key] - ruleGroupLabelValue := makeRuleGroupLabelValue(key) - sch.metrics.GroupRules.WithLabelValues(fmt.Sprint(key.OrgID), metrics.AlertRuleActiveLabelValue, ruleGroupLabelValue).Set(float64(numRules - numRulesPaused)) - sch.metrics.GroupRules.WithLabelValues(fmt.Sprint(key.OrgID), metrics.AlertRulePausedLabelValue, ruleGroupLabelValue).Set(float64(numRulesPaused)) - } + // Reset metrics to avoid stale data + sch.metrics.GroupRules.Reset() + sch.metrics.SimpleNotificationRules.Reset() + sch.metrics.Groups.Reset() - for orgID := range updateMetricsForOrgsAndGroups { - sch.metrics.SimpleNotificationRules.WithLabelValues(fmt.Sprint(orgID)).Set(float64(orgsNfSettings[orgID])) - sch.metrics.Groups.WithLabelValues(fmt.Sprint(orgID)).Set(float64(len(groupsPerOrg[orgID]))) + // Set metrics + for key, count := range buckets { + sch.metrics.GroupRules.WithLabelValues(fmt.Sprint(key.orgID), key.ruleType.String(), key.state, makeRuleGroupLabelValue(key.ruleGroup)).Set(float64(count)) + } + for orgID, numRulesNfSettings := range orgsNfSettings { + sch.metrics.SimpleNotificationRules.WithLabelValues(fmt.Sprint(orgID)).Set(float64(numRulesNfSettings)) + } + for orgID, groups := range groupsPerOrg { + sch.metrics.Groups.WithLabelValues(fmt.Sprint(orgID)).Set(float64(len(groups))) } - // While these are the rules that we iterate over, at the moment there's no 100% guarantee that they'll be // scheduled as rules could be removed before we get a chance to evaluate them. sch.metrics.SchedulableAlertRules.Set(float64(len(alertRules))) sch.metrics.SchedulableAlertRulesHash.Set(float64(hashUIDs(alertRules))) - - // Delete metrics for rule groups and orgs that are no longer present in the new state - for orgID, alertRuleGroupsMap := range sch.lastUpdatedMetricsForOrgsAndGroups { - if orgOrGroupDeleted(updateMetricsForOrgsAndGroups, orgID, nil) { - sch.metrics.SimpleNotificationRules.DeleteLabelValues(fmt.Sprint(orgID)) - sch.metrics.Groups.DeleteLabelValues(fmt.Sprint(orgID)) - } - - for key := range alertRuleGroupsMap { - if orgOrGroupDeleted(updateMetricsForOrgsAndGroups, orgID, &key) { - ruleGroupLabelValue := makeRuleGroupLabelValue(key) - sch.metrics.GroupRules.DeleteLabelValues(fmt.Sprint(key.AlertRuleGroupKey.OrgID), metrics.AlertRuleActiveLabelValue, ruleGroupLabelValue) - sch.metrics.GroupRules.DeleteLabelValues(fmt.Sprint(key.AlertRuleGroupKey.OrgID), metrics.AlertRulePausedLabelValue, ruleGroupLabelValue) - } - } - } - - // update the call state - sch.lastUpdatedMetricsForOrgsAndGroups = updateMetricsForOrgsAndGroups } // makeRuleGroupLabelValue returns a string that can be used as a label (rule_group) value for alert rule group metrics. func makeRuleGroupLabelValue(key models.AlertRuleGroupKeyWithFolderFullpath) string { return fmt.Sprintf("%s;%s", key.FolderFullpath, key.AlertRuleGroupKey.RuleGroup) } - -// orgOrGroupDeleted returns true if the org or group is no longer present in the new update metrics state. -func orgOrGroupDeleted(updateMetrics map[int64]map[models.AlertRuleGroupKeyWithFolderFullpath]struct{}, orgID int64, alertRuleGroupKey *models.AlertRuleGroupKeyWithFolderFullpath) bool { - if _, ok := updateMetrics[orgID]; !ok { - return true - } - - if alertRuleGroupKey != nil { - if _, ok := updateMetrics[orgID][*alertRuleGroupKey]; !ok { - return true - } - } - - return false -} diff --git a/pkg/services/ngalert/schedule/schedule.go b/pkg/services/ngalert/schedule/schedule.go index ff171c6a1fb2..bd33a45ff1da 100644 --- a/pkg/services/ngalert/schedule/schedule.go +++ b/pkg/services/ngalert/schedule/schedule.go @@ -87,10 +87,6 @@ type schedule struct { rrCfg setting.RecordingRuleSettings metrics *metrics.Scheduler - // lastUpdatedMetricsForOrgsAndGroups contains AlertRuleGroupKeyWithFolderFullpaths that - // were passed to updateRulesMetrics in the current tick. This is used to - // delete metrics for the rules/groups that are not longer present. - lastUpdatedMetricsForOrgsAndGroups map[int64]map[ngmodels.AlertRuleGroupKeyWithFolderFullpath]struct{} // orgID -> set of AlertRuleGroupKeyWithFolderFullpath alertsSender AlertsSender minRuleInterval time.Duration @@ -134,25 +130,24 @@ func NewScheduler(cfg SchedulerCfg, stateManager *state.Manager) *schedule { } sch := schedule{ - registry: newRuleRegistry(), - maxAttempts: cfg.MaxAttempts, - clock: cfg.C, - baseInterval: cfg.BaseInterval, - log: cfg.Log, - evaluatorFactory: cfg.EvaluatorFactory, - ruleStore: cfg.RuleStore, - metrics: cfg.Metrics, - lastUpdatedMetricsForOrgsAndGroups: make(map[int64]map[ngmodels.AlertRuleGroupKeyWithFolderFullpath]struct{}), - appURL: cfg.AppURL, - disableGrafanaFolder: cfg.DisableGrafanaFolder, - jitterEvaluations: cfg.JitterEvaluations, - rrCfg: cfg.RecordingRulesCfg, - stateManager: stateManager, - minRuleInterval: cfg.MinRuleInterval, - schedulableAlertRules: alertRulesRegistry{rules: make(map[ngmodels.AlertRuleKey]*ngmodels.AlertRule)}, - alertsSender: cfg.AlertSender, - tracer: cfg.Tracer, - recordingWriter: cfg.RecordingWriter, + registry: newRuleRegistry(), + maxAttempts: cfg.MaxAttempts, + clock: cfg.C, + baseInterval: cfg.BaseInterval, + log: cfg.Log, + evaluatorFactory: cfg.EvaluatorFactory, + ruleStore: cfg.RuleStore, + metrics: cfg.Metrics, + appURL: cfg.AppURL, + disableGrafanaFolder: cfg.DisableGrafanaFolder, + jitterEvaluations: cfg.JitterEvaluations, + rrCfg: cfg.RecordingRulesCfg, + stateManager: stateManager, + minRuleInterval: cfg.MinRuleInterval, + schedulableAlertRules: alertRulesRegistry{rules: make(map[ngmodels.AlertRuleKey]*ngmodels.AlertRule)}, + alertsSender: cfg.AlertSender, + tracer: cfg.Tracer, + recordingWriter: cfg.RecordingWriter, } return &sch diff --git a/pkg/services/ngalert/schedule/schedule_unit_test.go b/pkg/services/ngalert/schedule/schedule_unit_test.go index 13d9519deb43..9f7a3a05deae 100644 --- a/pkg/services/ngalert/schedule/schedule_unit_test.go +++ b/pkg/services/ngalert/schedule/schedule_unit_test.go @@ -128,11 +128,10 @@ func TestProcessTicks(t *testing.T) { t.Run("after 1st tick rule metrics should report one active alert rule", func(t *testing.T) { expectedMetric := fmt.Sprintf( - `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused. + `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state. # TYPE grafana_alerting_rule_group_rules gauge - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 1 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 0 - `, alertRule1.OrgID, folderWithRuleGroup1) + grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active",type="alerting"} 1 + `, alertRule1.OrgID, folderWithRuleGroup1) err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules") require.NoError(t, err) @@ -158,12 +157,10 @@ func TestProcessTicks(t *testing.T) { t.Run("after 2nd tick rule metrics should report two active alert rules in two groups", func(t *testing.T) { expectedMetric := fmt.Sprintf( - `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused. + `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state. # TYPE grafana_alerting_rule_group_rules gauge - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 1 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 0 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active"} 1 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="paused"} 0 + grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active",type="alerting"} 1 + grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active",type="alerting"} 1 `, alertRule1.OrgID, folderWithRuleGroup1, folderWithRuleGroup2) err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules") @@ -216,12 +213,10 @@ func TestProcessTicks(t *testing.T) { t.Run("after 5th tick rule metrics should report one active and one paused alert rules in two groups", func(t *testing.T) { expectedMetric := fmt.Sprintf( - `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused. + `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state. # TYPE grafana_alerting_rule_group_rules gauge - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 0 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 1 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active"} 1 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="paused"} 0 + grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused",type="alerting"} 1 + grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active",type="alerting"} 1 `, alertRule1.OrgID, folderWithRuleGroup1, folderWithRuleGroup2) err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules") @@ -251,14 +246,12 @@ func TestProcessTicks(t *testing.T) { t.Run("after 6th tick rule metrics should report two paused alert rules in two groups", func(t *testing.T) { expectedMetric := fmt.Sprintf( - `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused. + `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state. # TYPE grafana_alerting_rule_group_rules gauge - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 0 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 1 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active"} 0 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="paused"} 1 - `, alertRule1.OrgID, folderWithRuleGroup1, folderWithRuleGroup2) + grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused",type="alerting"} 1 + grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="paused",type="alerting"} 1 + `, alertRule1.OrgID, folderWithRuleGroup1, folderWithRuleGroup2) err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules") require.NoError(t, err) }) @@ -281,12 +274,10 @@ func TestProcessTicks(t *testing.T) { t.Run("after 7th tick rule metrics should report two active alert rules in two groups", func(t *testing.T) { expectedMetric := fmt.Sprintf( - `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused. + `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state. # TYPE grafana_alerting_rule_group_rules gauge - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 1 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 0 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active"} 1 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="paused"} 0 + grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active",type="alerting"} 1 + grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active",type="alerting"} 1 `, alertRule1.OrgID, folderWithRuleGroup1, folderWithRuleGroup2) err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules") @@ -310,12 +301,10 @@ func TestProcessTicks(t *testing.T) { t.Run("after 8th tick rule metrics should report one active alert rule", func(t *testing.T) { expectedMetric := fmt.Sprintf( - `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused. - # TYPE grafana_alerting_rule_group_rules gauge - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 1 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 0 - `, alertRule2.OrgID, folderWithRuleGroup2) - + `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state. + # TYPE grafana_alerting_rule_group_rules gauge + grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active",type="alerting"} 1 + `, alertRule1.OrgID, folderWithRuleGroup2) err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules") require.NoError(t, err) }) @@ -533,10 +522,9 @@ func TestSchedule_updateRulesMetrics(t *testing.T) { sch.updateRulesMetrics([]*models.AlertRule{alertRule1}) expectedMetric := fmt.Sprintf( - `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused. + `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state. # TYPE grafana_alerting_rule_group_rules gauge - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 1 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 0 + grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active", type="alerting"} 1 `, alertRule1.OrgID, folderWithRuleGroup1) err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules") @@ -555,12 +543,10 @@ func TestSchedule_updateRulesMetrics(t *testing.T) { sch.updateRulesMetrics([]*models.AlertRule{alertRule1, alertRule2}) expectedMetric := fmt.Sprintf( - `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused. + `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state. # TYPE grafana_alerting_rule_group_rules gauge - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 1 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 0 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active"} 1 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="paused"} 0 + grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active",type="alerting"} 1 + grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active",type="alerting"} 1 `, alertRule1.OrgID, folderWithRuleGroup1, folderWithRuleGroup2) err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules") @@ -572,12 +558,10 @@ func TestSchedule_updateRulesMetrics(t *testing.T) { sch.updateRulesMetrics([]*models.AlertRule{alertRule1, alertRule2}) expectedMetric := fmt.Sprintf( - `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, both active and paused. + `# HELP grafana_alerting_rule_group_rules The number of alert rules that are scheduled, by type and state. # TYPE grafana_alerting_rule_group_rules gauge - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active"} 1 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="paused"} 0 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active"} 1 - grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="paused"} 0 + grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[2]s",state="active",type="alerting"} 1 + grafana_alerting_rule_group_rules{org="%[1]d",rule_group="%[3]s",state="active",type="alerting"} 1 `, alertRule1.OrgID, folderWithRuleGroup1, folderWithRuleGroup2) err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_rule_group_rules") @@ -666,15 +650,9 @@ func TestSchedule_updateRulesMetrics(t *testing.T) { t.Run("it should not show metrics", func(t *testing.T) { sch.updateRulesMetrics([]*models.AlertRule{alertRuleWithoutNotificationSettings}) - // Because alertRuleWithoutNotificationSettings.orgID is present, - // the metric is also present but set to 0 because the org has no rules with NotificationSettings. - expectedMetric := fmt.Sprintf( - `# HELP grafana_alerting_simple_routing_rules The number of alert rules using simplified routing. - # TYPE grafana_alerting_simple_routing_rules gauge - grafana_alerting_simple_routing_rules{org="%[1]d"} 0 - `, alertRuleWithoutNotificationSettings.OrgID) + expectedMetric := "" err := testutil.GatherAndCompare(reg, bytes.NewBufferString(expectedMetric), "grafana_alerting_simple_routing_rules") - require.NoError(t, err) + require.ErrorContains(t, err, "expected metric name(s) not found: [grafana_alerting_simple_routing_rules]") }) alertRule1 := models.RuleGen.With( @@ -722,7 +700,6 @@ func TestSchedule_updateRulesMetrics(t *testing.T) { expectedMetric := fmt.Sprintf( `# HELP grafana_alerting_simple_routing_rules The number of alert rules using simplified routing. # TYPE grafana_alerting_simple_routing_rules gauge - grafana_alerting_simple_routing_rules{org="%[1]d"} 0 grafana_alerting_simple_routing_rules{org="%[2]d"} 1 `, alertRuleWithoutNotificationSettings.OrgID, alertRule2.OrgID)