Alerting: Introduce Metric Aggregation starting with Silences (#62512)

* Alerting: Introduce Metric Aggregation starting with Silences
---------

Co-authored-by: Alexander Weaver <weaver.alex.d@gmail.com>
This commit is contained in:
gotjosh 2023-01-31 19:54:38 +00:00 committed by GitHub
parent 138575cbe9
commit 55e7cf1aed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 126 additions and 16 deletions

3
go.mod
View File

@ -420,3 +420,6 @@ replace github.com/prometheus/alertmanager => github.com/grafana/prometheus-aler
replace google.golang.org/grpc => google.golang.org/grpc v1.45.0
replace google.golang.org/genproto => google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3
// Remove this once https://github.com/grafana/dskit/pull/258 is merged.
replace github.com/grafana/dskit => github.com/gotjosh/dskit v0.0.0-20230131123646-8dda768daa27

4
go.sum
View File

@ -1247,14 +1247,14 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gotjosh/dskit v0.0.0-20230131123646-8dda768daa27 h1:rWMt8wsjGjzT/6AX6/Ie0JTA0CNZzzbDfup34lSJnTw=
github.com/gotjosh/dskit v0.0.0-20230131123646-8dda768daa27/go.mod h1:ulYLLoSd71AWIjxgifLO86Lndx82Yj+IcV+fFnh8tkI=
github.com/grafana/alerting v0.0.0-20230125210216-facc6b27b9e0 h1:BzkQNnj+eevX30EMqJiUS1w3CPoGc8kp7pDf/ari/4Y=
github.com/grafana/alerting v0.0.0-20230125210216-facc6b27b9e0/go.mod h1:NoSLbfmUwE+omWFReFrLtbtOItmvTbuQERJ6XFYp9ME=
github.com/grafana/codejen v0.0.3 h1:tAWxoTUuhgmEqxJPOLtJoxlPBbMULFwKFOcRsPRPXDw=
github.com/grafana/codejen v0.0.3/go.mod h1:zmwwM/DRyQB7pfuBjTWII3CWtxcXh8LTwAYGfDfpR6s=
github.com/grafana/cuetsy v0.1.5 h1:mnFwAXdbqCsyL8r7kkdUMJ4kOAR26cxIPmrZj7JzTeY=
github.com/grafana/cuetsy v0.1.5/go.mod h1:4KWkUOslwvRTpEv7wdQG0jDFTuJmU+0L9x0h4kWxa2A=
github.com/grafana/dskit v0.0.0-20230126115530-71478074eab8 h1:5nqLvzKugVUb9sCQkKuOPecRshawSrbHsXyGxBkTBus=
github.com/grafana/dskit v0.0.0-20230126115530-71478074eab8/go.mod h1:zj+5BNZAVmQafV583uLTAOzRr963KPdEm4d6NPmtbwg=
github.com/grafana/go-mssqldb v0.0.0-20210326084033-d0ce3c521036 h1:GplhUk6Xes5JIhUUrggPcPBhOn+eT8+WsHiebvq7GgA=
github.com/grafana/go-mssqldb v0.0.0-20210326084033-d0ce3c521036/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
github.com/grafana/grafana-aws-sdk v0.12.0 h1:eUjFdFZeZE+nyu/RMRz+qFxTBew69ToLBrbRhTbjkfM=

View File

@ -1,21 +1,31 @@
package metrics
import (
"fmt"
"strconv"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/dskit/metrics"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
type MultiOrgAlertmanager struct {
Registerer prometheus.Registerer
Registerer prometheus.Registerer
registries *metrics.TenantRegistries
ActiveConfigurations prometheus.Gauge
DiscoveredConfigurations prometheus.Gauge
registries *OrgRegistries
aggregatedMetrics *AlertmanagerAggregatedMetrics
}
func NewMultiOrgAlertmanagerMetrics(r prometheus.Registerer) *MultiOrgAlertmanager {
return &MultiOrgAlertmanager{
registries := metrics.NewTenantRegistries(log.New("ngalert.multiorg.alertmanager.metrics")) //TODO: Should this be here? Probably not.
moa := &MultiOrgAlertmanager{
Registerer: r,
registries: NewOrgRegistries(),
registries: registries,
DiscoveredConfigurations: promauto.With(r).NewGauge(prometheus.GaugeOpts{
Namespace: Namespace,
Subsystem: Subsystem,
@ -28,5 +38,112 @@ func NewMultiOrgAlertmanagerMetrics(r prometheus.Registerer) *MultiOrgAlertmanag
Name: "active_configurations",
Help: "The number of active Alertmanager configurations.",
}),
aggregatedMetrics: NewAlertmanagerAggregatedMetrics(registries),
}
// These metrics use a different registration method as the struct itself represents a custom collector.
// There's no way to "auto-register" a collector.
r.MustRegister(moa.aggregatedMetrics)
return moa
}
// RemoveOrgRegistry removes the *prometheus.Registry for the specified org. It is safe to call concurrently.
func (moa *MultiOrgAlertmanager) RemoveOrgRegistry(id int64) {
moa.registries.RemoveTenantRegistry(strconv.FormatInt(id, 10), false)
}
// GetOrCreateOrgRegistry gets or creates a *prometheus.Registry for the specified org. It is safe to call concurrently.
func (moa *MultiOrgAlertmanager) GetOrCreateOrgRegistry(id int64) prometheus.Registerer {
sid := strconv.FormatInt(id, 10)
reg := moa.registries.GetRegistryForTenant(sid)
if reg != nil {
return reg
}
result := prometheus.NewRegistry()
moa.registries.AddTenantRegistry(sid, result)
return result
}
// AlertmanagerAggregatedMetrics are metrics collected directly from the registry.
// Unlike metrics.Alertmanager they are not called within this codebase hence the need for direct collection.
type AlertmanagerAggregatedMetrics struct {
registries *metrics.TenantRegistries
// exported metrics, gathered from Alertmanager Silences
silencesGCDuration *prometheus.Desc
silencesSnapshotDuration *prometheus.Desc
silencesSnapshotSize *prometheus.Desc
silencesQueriesTotal *prometheus.Desc
silencesQueryErrorsTotal *prometheus.Desc
silencesQueryDuration *prometheus.Desc
silences *prometheus.Desc
silencesPropagatedMessagesTotal *prometheus.Desc
}
func NewAlertmanagerAggregatedMetrics(registries *metrics.TenantRegistries) *AlertmanagerAggregatedMetrics {
aggregatedMetrics := &AlertmanagerAggregatedMetrics{
registries: registries,
silencesGCDuration: prometheus.NewDesc(
fmt.Sprintf("%s_%s_silences_gc_duration_seconds", Namespace, Subsystem),
"Duration of the last silence garbage collection cycle.",
nil, nil),
silencesSnapshotDuration: prometheus.NewDesc(
fmt.Sprintf("%s_%s_silences_snapshot_duration_seconds", Namespace, Subsystem),
"Duration of the last silence snapshot.",
nil, nil),
silencesSnapshotSize: prometheus.NewDesc(
fmt.Sprintf("%s_%s_silences_snapshot_size_bytes", Namespace, Subsystem),
"Size of the last silence snapshot in bytes.",
nil, nil),
silencesQueriesTotal: prometheus.NewDesc(
fmt.Sprintf("%s_%s_silences_queries_total", Namespace, Subsystem),
"How many silence queries were received.",
nil, nil),
silencesQueryErrorsTotal: prometheus.NewDesc(
fmt.Sprintf("%s_%s_silences_query_errors_total", Namespace, Subsystem),
"How many silence received queries did not succeed.",
nil, nil),
silencesQueryDuration: prometheus.NewDesc(
fmt.Sprintf("%s_%s_silences_query_duration_seconds", Namespace, Subsystem),
"Duration of silence query evaluation.",
nil, nil),
silencesPropagatedMessagesTotal: prometheus.NewDesc(
fmt.Sprintf("%s_%s_silences_gossip_messages_propagated_total", Namespace, Subsystem),
"Number of received gossip messages that have been further gossiped.",
nil, nil),
silences: prometheus.NewDesc(
fmt.Sprintf("%s_%s_silences", Namespace, Subsystem),
"How many silences by state.",
[]string{"org", "state"}, nil),
}
return aggregatedMetrics
}
func (a *AlertmanagerAggregatedMetrics) Describe(out chan<- *prometheus.Desc) {
out <- a.silencesGCDuration
out <- a.silencesSnapshotDuration
out <- a.silencesSnapshotSize
out <- a.silencesQueriesTotal
out <- a.silencesQueryErrorsTotal
out <- a.silencesQueryDuration
out <- a.silencesPropagatedMessagesTotal
out <- a.silences
}
func (a *AlertmanagerAggregatedMetrics) Collect(out chan<- prometheus.Metric) {
data := a.registries.BuildMetricFamiliesPerTenant()
data.SendSumOfSummaries(out, a.silencesGCDuration, "alertmanager_silences_gc_duration_seconds")
data.SendSumOfSummaries(out, a.silencesSnapshotDuration, "alertmanager_silences_snapshot_duration_seconds")
data.SendSumOfGauges(out, a.silencesSnapshotSize, "alertmanager_silences_snapshot_size_bytes")
data.SendSumOfCounters(out, a.silencesQueriesTotal, "alertmanager_silences_queries_total")
data.SendSumOfCounters(out, a.silencesQueryErrorsTotal, "alertmanager_silences_query_errors_total")
data.SendSumOfHistograms(out, a.silencesQueryDuration, "alertmanager_silences_query_duration_seconds")
data.SendSumOfCounters(out, a.silencesPropagatedMessagesTotal, "alertmanager_silences_gossip_messages_propagated_total")
data.SendSumOfGaugesPerTenantWithLabels(out, a.silences, "alertmanager_silences", "state")
}

View File

@ -57,13 +57,3 @@ func (ng *NGAlert) GetAPIMetrics() *API {
func (ng *NGAlert) GetMultiOrgAlertmanagerMetrics() *MultiOrgAlertmanager {
return ng.multiOrgAlertmanagerMetrics
}
// RemoveOrgRegistry removes the *prometheus.Registry for the specified org. It is safe to call concurrently.
func (moa *MultiOrgAlertmanager) RemoveOrgRegistry(id int64) {
moa.registries.RemoveOrgRegistry(id)
}
// GetOrCreateOrgRegistry gets or creates a *prometheus.Registry for the specified org. It is safe to call concurrently.
func (moa *MultiOrgAlertmanager) GetOrCreateOrgRegistry(id int64) prometheus.Registerer {
return moa.registries.GetOrCreateOrgRegistry(id)
}