Alerting: Attach hash of instance labels to state history log lines (#65968)

* Add instanceID which is hash of labels

* Rename field to fingerprint

* Move to prometheus style signature

* Appease linter
This commit is contained in:
Alexander Weaver 2023-04-19 14:22:19 -05:00 committed by GitHub
parent c96b704af3
commit 3634079b8f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 31 additions and 1 deletions

View File

@ -1,10 +1,12 @@
package historian
import (
"fmt"
"strings"
"time"
"github.com/grafana/grafana-plugin-sdk-go/data"
prometheus "github.com/prometheus/common/model"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
@ -42,6 +44,12 @@ func removePrivateLabels(labels data.Labels) data.Labels {
return result
}
// labelFingerprint calculates a stable Prometheus-style signature for a label set.
func labelFingerprint(labels data.Labels) string {
sig := prometheus.LabelsToSignature(labels)
return fmt.Sprintf("%016x", sig)
}
// panelKey uniquely identifies a panel.
type panelKey struct {
orgID int64

View File

@ -255,6 +255,7 @@ func statesToStream(rule history_model.RuleMeta, states []state.StateTransition,
continue
}
sanitizedLabels := removePrivateLabels(state.Labels)
entry := lokiEntry{
SchemaVersion: 1,
Previous: state.PreviousFormatted(),
@ -263,7 +264,8 @@ func statesToStream(rule history_model.RuleMeta, states []state.StateTransition,
Condition: rule.Condition,
DashboardUID: rule.DashboardUID,
PanelID: rule.PanelID,
InstanceLabels: removePrivateLabels(state.Labels),
Fingerprint: labelFingerprint(sanitizedLabels),
InstanceLabels: sanitizedLabels,
}
if state.State.State == eval.Error {
entry.Error = state.Error.Error()
@ -306,6 +308,7 @@ type lokiEntry struct {
Condition string `json:"condition"`
DashboardUID string `json:"dashboardUID"`
PanelID int64 `json:"panelID"`
Fingerprint string `json:"fingerprint"`
// InstanceLabels is exactly the set of labels associated with the alert instance in Alertmanager.
// These should not be conflated with labels associated with log streams.
InstanceLabels map[string]string `json:"labels"`

View File

@ -154,6 +154,25 @@ func TestRemoteLokiBackend(t *testing.T) {
entry := requireSingleEntry(t, res)
require.Equal(t, rule.Condition, entry.Condition)
})
t.Run("stores fingerprint of instance labels", func(t *testing.T) {
rule := createTestRule()
l := log.NewNopLogger()
states := singleFromNormal(&state.State{
State: eval.Alerting,
Labels: data.Labels{
"statelabel": "labelvalue",
"labeltwo": "labelvalue",
"labelthree": "labelvalue",
},
})
res := statesToStream(rule, states, nil, l)
entry := requireSingleEntry(t, res)
exp := labelFingerprint(states[0].Labels)
require.Equal(t, exp, entry.Fingerprint)
})
})
t.Run("selector string", func(t *testing.T) {