Alerting: Keep private annotations across evaluations (#49080)

pkg/services/ngalert/models/alert_rule.go

@@ -74,12 +74,6 @@ const (
 	OkErrState       ExecutionErrorState = "OK"
 )
 
-// InternalLabelNameSet are labels that grafana automatically include as part of the labelset.
-var InternalLabelNameSet = map[string]struct{}{
-	RuleUIDLabel:      {},
-	NamespaceUIDLabel: {},
-}
-
 const (
 	RuleUIDLabel      = "__alert_rule_uid__"
 	NamespaceUIDLabel = "__alert_rule_namespace_uid__"
@@ -89,6 +83,15 @@ const (
 	PanelIDAnnotation      = "__panelId__"
 )
 
+var (
+	// InternalLabelNameSet are labels that grafana automatically include as part of the labelset.
+	InternalLabelNameSet = map[string]struct{}{
+		RuleUIDLabel:      {},
+		NamespaceUIDLabel: {},
+	}
+	InternalAnnotationNameSet = map[string]struct{}{}
+)
+
 // AlertRule is the model for alert rules in unified alerting.
 type AlertRule struct {
 	ID              int64 `xorm:"pk autoincr 'id'"`

pkg/services/ngalert/state/cache.go

@@ -60,7 +60,17 @@ func (c *cache) getOrCreate(ctx context.Context, alertRule *ngModels.AlertRule,
 	}
 
 	if state, ok := c.states[alertRule.OrgID][alertRule.UID][id]; ok {
-		// Annotations can change over time for the same alert.
+		// Annotations can change over time, however we also want to maintain
+		// certain annotations across evaluations
+		for k, v := range state.Annotations {
+			if _, ok := ngModels.InternalAnnotationNameSet[k]; ok {
+				// If the annotation is not present then it should be copied from the
+				// previous state to the next state
+				if _, ok := annotations[k]; !ok {
+					annotations[k] = v
+				}
+			}
+		}
 		state.Annotations = annotations
 		c.states[alertRule.OrgID][alertRule.UID][id] = state
 		return state