IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Add "backend" label to state history writes metrics (#65395)

Browse Source 
* Add backend label to state history writes metrics

* Update test expectations
This commit is contained in:
Alexander Weaver
2023-03-28 08:49:51 -05:00
committed by GitHub
parent f0ddf900c0
commit dd04757fc9
5 changed files with 10 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/services/ngalert/metrics/historian.go
View File

@@ -41,13 +41,13 @@ func NewHistorianMetrics(r prometheus.Registerer) *Historian {
Subsystem: Subsystem, Subsystem: Subsystem,
Name: "state_history_writes_total", Name: "state_history_writes_total",
Help: "The total number of state history batches that were attempted to be written.", Help: "The total number of state history batches that were attempted to be written.",
}, []string{"org"}), }, []string{"org", "backend"}),
WritesFailed: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ WritesFailed: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
Namespace: Namespace, Namespace: Namespace,
Subsystem: Subsystem, Subsystem: Subsystem,
Name: "state_history_writes_failed_total", Name: "state_history_writes_failed_total",
Help: "The total number of failed writes of state history batches.", Help: "The total number of failed writes of state history batches.",
}, []string{"org"}), }, []string{"org", "backend"}),
WriteDuration: instrument.NewHistogramCollector(promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{ WriteDuration: instrument.NewHistogramCollector(promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{
Namespace: Namespace, Namespace: Namespace,
Subsystem: Subsystem, Subsystem: Subsystem,

4
pkg/services/ngalert/state/historian/annotation.go
View File

@@ -199,11 +199,11 @@ func (h *AnnotationBackend) recordAnnotations(ctx context.Context, panel *panelK
} }
org := fmt.Sprint(orgID) org := fmt.Sprint(orgID)
h.metrics.WritesTotal.WithLabelValues(org).Inc() h.metrics.WritesTotal.WithLabelValues(org, "annotations").Inc()
h.metrics.TransitionsTotal.WithLabelValues(org).Add(float64(len(annotations))) h.metrics.TransitionsTotal.WithLabelValues(org).Add(float64(len(annotations)))
if err := h.annotations.SaveMany(ctx, annotations); err != nil { if err := h.annotations.SaveMany(ctx, annotations); err != nil {
logger.Error("Error saving alert annotation batch", "error", err) logger.Error("Error saving alert annotation batch", "error", err)
h.metrics.WritesFailed.WithLabelValues(org).Inc() h.metrics.WritesFailed.WithLabelValues(org, "annotations").Inc()
h.metrics.TransitionsFailed.WithLabelValues(org).Add(float64(len(annotations))) h.metrics.TransitionsFailed.WithLabelValues(org).Add(float64(len(annotations)))
return fmt.Errorf("error saving alert annotation batch: %w", err) return fmt.Errorf("error saving alert annotation batch: %w", err)
} }

4
pkg/services/ngalert/state/historian/annotation_test.go
View File

@@ -83,10 +83,10 @@ grafana_alerting_state_history_transitions_failed_total{org="1"} 1
grafana_alerting_state_history_transitions_total{org="1"} 2 grafana_alerting_state_history_transitions_total{org="1"} 2
# HELP grafana_alerting_state_history_writes_failed_total The total number of failed writes of state history batches. # HELP grafana_alerting_state_history_writes_failed_total The total number of failed writes of state history batches.
# TYPE grafana_alerting_state_history_writes_failed_total counter # TYPE grafana_alerting_state_history_writes_failed_total counter
grafana_alerting_state_history_writes_failed_total{org="1"} 1 grafana_alerting_state_history_writes_failed_total{backend="annotations",org="1"} 1
# HELP grafana_alerting_state_history_writes_total The total number of state history batches that were attempted to be written. # HELP grafana_alerting_state_history_writes_total The total number of state history batches that were attempted to be written.
# TYPE grafana_alerting_state_history_writes_total counter # TYPE grafana_alerting_state_history_writes_total counter
grafana_alerting_state_history_writes_total{org="1"} 2 grafana_alerting_state_history_writes_total{backend="annotations",org="1"} 2
`) `)
err := testutil.GatherAndCompare(reg, exp, err := testutil.GatherAndCompare(reg, exp,
"grafana_alerting_state_history_transitions_total", "grafana_alerting_state_history_transitions_total",

4
pkg/services/ngalert/state/historian/loki.go
View File

@@ -83,7 +83,7 @@ func (h *RemoteLokiBackend) Record(ctx context.Context, rule history_model.RuleM
defer close(errCh) defer close(errCh)
org := fmt.Sprint(rule.OrgID) org := fmt.Sprint(rule.OrgID)
h.metrics.WritesTotal.WithLabelValues(org).Inc() h.metrics.WritesTotal.WithLabelValues(org, "loki").Inc()
samples := 0 samples := 0
for _, s := range streams { for _, s := range streams {
samples += len(s.Values) samples += len(s.Values)
@@ -92,7 +92,7 @@ func (h *RemoteLokiBackend) Record(ctx context.Context, rule history_model.RuleM
if err := h.recordStreams(ctx, streams, logger); err != nil { if err := h.recordStreams(ctx, streams, logger); err != nil {
logger.Error("Failed to save alert state history batch", "error", err) logger.Error("Failed to save alert state history batch", "error", err)
h.metrics.WritesFailed.WithLabelValues(org).Inc() h.metrics.WritesFailed.WithLabelValues(org, "loki").Inc()
h.metrics.TransitionsFailed.WithLabelValues(org).Add(float64(samples)) h.metrics.TransitionsFailed.WithLabelValues(org).Add(float64(samples))
errCh <- fmt.Errorf("failed to save alert state history batch: %w", err) errCh <- fmt.Errorf("failed to save alert state history batch: %w", err)
} }

4
pkg/services/ngalert/state/historian/loki_test.go
View File

@@ -298,10 +298,10 @@ grafana_alerting_state_history_transitions_failed_total{org="1"} 1
grafana_alerting_state_history_transitions_total{org="1"} 2 grafana_alerting_state_history_transitions_total{org="1"} 2
# HELP grafana_alerting_state_history_writes_failed_total The total number of failed writes of state history batches. # HELP grafana_alerting_state_history_writes_failed_total The total number of failed writes of state history batches.
# TYPE grafana_alerting_state_history_writes_failed_total counter # TYPE grafana_alerting_state_history_writes_failed_total counter
grafana_alerting_state_history_writes_failed_total{org="1"} 1 grafana_alerting_state_history_writes_failed_total{backend="loki",org="1"} 1
# HELP grafana_alerting_state_history_writes_total The total number of state history batches that were attempted to be written. # HELP grafana_alerting_state_history_writes_total The total number of state history batches that were attempted to be written.
# TYPE grafana_alerting_state_history_writes_total counter # TYPE grafana_alerting_state_history_writes_total counter
grafana_alerting_state_history_writes_total{org="1"} 2 grafana_alerting_state_history_writes_total{backend="loki",org="1"} 2
`) `)
err := testutil.GatherAndCompare(reg, exp, err := testutil.GatherAndCompare(reg, exp,
"grafana_alerting_state_history_transitions_total", "grafana_alerting_state_history_transitions_total",