IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Instrument outbound requests for Loki Historian and Remote Alertmanager with tracing (#89185)

Browse Source 
* Add TracedClient

* Handle errors and status codes

* Wire up tracing to normal ASH and loki annotation mapping

* Add tracing to remote alertmanager

* one more spot

* and not or

* More consistency with other grafana traces, lower cardinality name
This commit is contained in:
Alexander Weaver 2024-06-14 13:24:12 -05:00 committed by GitHub
parent 6262c56132
commit 8491e02caf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 124 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/services/annotations/annotationsimpl/annotations.go
View File

@ -8,6 +8,7 @@ import (
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/annotations"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/tag"
@ -27,6 +28,7 @@ func ProvideService(
cfg *setting.Cfg,
features featuremgmt.FeatureToggles,
tagService tag.Service,
tracer tracing.Tracer,
) *RepositoryImpl {
l := log.New("annotations")
l.Debug("Initializing annotations service")
@ -35,7 +37,7 @@ func ProvideService(
write := xormStore
var read readStore
historianStore := loki.NewLokiHistorianStore(cfg.UnifiedAlerting.StateHistory, features, db, log.New("annotations.loki"))
historianStore := loki.NewLokiHistorianStore(cfg.UnifiedAlerting.StateHistory, features, db, log.New("annotations.loki"), tracer)
if historianStore != nil {
l.Debug("Using composite read store")
read = NewCompositeStore(log.New("annotations.composite"), xormStore, historianStore)

4
pkg/services/annotations/annotationsimpl/annotations_test.go
View File

@ -48,7 +48,7 @@ func TestIntegrationAnnotationListingWithRBAC(t *testing.T) {
features := featuremgmt.WithFeatures()
tagService := tagimpl.ProvideService(sql)
repo := ProvideService(sql, cfg, features, tagService)
repo := ProvideService(sql, cfg, features, tagService, tracing.InitializeTracerForTest())
dashboard1 := testutil.CreateDashboard(t, sql, cfg, features, dashboards.SaveDashboardCommand{
UserID: 1,
@ -317,7 +317,7 @@ func TestIntegrationAnnotationListingWithInheritedRBAC(t *testing.T) {
cfg := setting.NewCfg()
cfg.AnnotationMaximumTagsLength = 60
repo := ProvideService(sql, cfg, tc.features, tagimpl.ProvideService(sql))
repo := ProvideService(sql, cfg, tc.features, tagimpl.ProvideService(sql), tracing.InitializeTracerForTest())
usr.Permissions = map[int64]map[string][]string{1: tc.permissions}
testutil.SetupRBACPermission(t, sql, role, usr)

5
pkg/services/annotations/annotationsimpl/loki/historian_store.go
View File

@ -17,6 +17,7 @@ import (
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
ngmetrics "github.com/grafana/grafana/pkg/services/ngalert/metrics"
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/state"
@ -53,7 +54,7 @@ type LokiHistorianStore struct {
log log.Logger
}
func NewLokiHistorianStore(cfg setting.UnifiedAlertingStateHistorySettings, ft featuremgmt.FeatureToggles, db db.DB, log log.Logger) *LokiHistorianStore {
func NewLokiHistorianStore(cfg setting.UnifiedAlertingStateHistorySettings, ft featuremgmt.FeatureToggles, db db.DB, log log.Logger, tracer tracing.Tracer) *LokiHistorianStore {
if !useStore(cfg, ft) {
return nil
}
@ -64,7 +65,7 @@ func NewLokiHistorianStore(cfg setting.UnifiedAlertingStateHistorySettings, ft f
}
return &LokiHistorianStore{
client: historian.NewLokiClient(lokiCfg, historian.NewRequester(), ngmetrics.NewHistorianMetrics(prometheus.DefaultRegisterer, subsystem), log),
client: historian.NewLokiClient(lokiCfg, historian.NewRequester(), ngmetrics.NewHistorianMetrics(prometheus.DefaultRegisterer, subsystem), log, tracer),
db: db,
log: log,
}

55
pkg/services/ngalert/client/client.go
View File

@ -7,6 +7,11 @@ import (
"strconv"
"github.com/grafana/dskit/instrument"
"github.com/grafana/grafana/pkg/infra/tracing"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
"go.opentelemetry.io/otel/trace"
)
// Requester executes an HTTP request.
@ -14,7 +19,7 @@ type Requester interface {
Do(req *http.Request) (*http.Response, error)
}
// TimedClient instruments a request. It implements Requester.
// TimedClient instruments a request with metrics. It implements Requester.
type TimedClient struct {
client Requester
collector instrument.Collector
@ -70,3 +75,51 @@ func TimeRequest(ctx context.Context, operation string, coll instrument.Collecto
coll, toStatusCode, doRequest)
return response, err
}
// TracedClient instruments a request with tracing. It implements Requester.
type TracedClient struct {
client Requester
tracer tracing.Tracer
name string
}
func NewTracedClient(client Requester, tracer tracing.Tracer, name string) *TracedClient {
return &TracedClient{
client: client,
tracer: tracer,
name: name,
}
}
// Do executes the request.
func (c TracedClient) Do(r *http.Request) (*http.Response, error) {
ctx, span := c.tracer.Start(r.Context(), c.name, trace.WithSpanKind(trace.SpanKindClient))
defer span.End()
span.SetAttributes(semconv.HTTPURL(r.URL.String()))
span.SetAttributes(semconv.HTTPMethod(r.Method))
c.tracer.Inject(ctx, r.Header, span)
r = r.WithContext(ctx)
resp, err := c.client.Do(r)
if err != nil {
span.SetStatus(codes.Error, "request failed")
span.RecordError(err)
} else {
if resp.ContentLength > 0 {
span.SetAttributes(attribute.Int64("http.content_length", resp.ContentLength))
}
span.SetAttributes(semconv.HTTPStatusCode(resp.StatusCode))
if resp.StatusCode >= 400 && resp.StatusCode < 600 {
span.RecordError(fmt.Errorf("error with HTTP status code %d", resp.StatusCode))
}
}
return resp, err
}
// RoundTrip implements the RoundTripper interface.
func (c TracedClient) RoundTrip(r *http.Request) (*http.Response, error) {
return c.Do(r)
}

20
pkg/services/ngalert/ngalert.go
View File

@ -200,7 +200,7 @@ func (ng *AlertNG) init() error {
PromoteConfig: true,
SyncInterval: ng.Cfg.UnifiedAlerting.RemoteAlertmanager.SyncInterval,
}
remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m)
remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m, ng.tracer)
if err != nil {
moaLogger.Error("Failed to create remote Alertmanager", "err", err)
return nil, err
@ -234,7 +234,7 @@ func (ng *AlertNG) init() error {
TenantID: ng.Cfg.UnifiedAlerting.RemoteAlertmanager.TenantID,
URL: ng.Cfg.UnifiedAlerting.RemoteAlertmanager.URL,
}
remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m)
remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m, ng.tracer)
if err != nil {
moaLogger.Error("Failed to create remote Alertmanager, falling back to using only the internal one", "err", err)
return internalAM, nil
@ -270,7 +270,7 @@ func (ng *AlertNG) init() error {
URL: ng.Cfg.UnifiedAlerting.RemoteAlertmanager.URL,
SyncInterval: ng.Cfg.UnifiedAlerting.RemoteAlertmanager.SyncInterval,
}
remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m)
remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m, ng.tracer)
if err != nil {
moaLogger.Error("Failed to create remote Alertmanager, falling back to using only the internal one", "err", err)
return internalAM, nil
@ -359,7 +359,7 @@ func (ng *AlertNG) init() error {
// There are a set of feature toggles available that act as short-circuits for common configurations.
// If any are set, override the config accordingly.
ApplyStateHistoryFeatureToggles(&ng.Cfg.UnifiedAlerting.StateHistory, ng.FeatureToggles, ng.Log)
history, err := configureHistorianBackend(initCtx, ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService, ng.store, ng.Metrics.GetHistorianMetrics(), ng.Log)
history, err := configureHistorianBackend(initCtx, ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService, ng.store, ng.Metrics.GetHistorianMetrics(), ng.Log, ng.tracer)
if err != nil {
return err
}
@ -523,7 +523,7 @@ type Historian interface {
state.Historian
}
func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingStateHistorySettings, ar annotations.Repository, ds dashboards.DashboardService, rs historian.RuleStore, met *metrics.Historian, l log.Logger) (Historian, error) {
func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingStateHistorySettings, ar annotations.Repository, ds dashboards.DashboardService, rs historian.RuleStore, met *metrics.Historian, l log.Logger, tracer tracing.Tracer) (Historian, error) {
if !cfg.Enabled {
met.Info.WithLabelValues("noop").Set(0)
return historian.NewNopHistorian(), nil
@ -538,7 +538,7 @@ func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingS
if backend == historian.BackendTypeMultiple {
primaryCfg := cfg
primaryCfg.Backend = cfg.MultiPrimary
primary, err := configureHistorianBackend(ctx, primaryCfg, ar, ds, rs, met, l)
primary, err := configureHistorianBackend(ctx, primaryCfg, ar, ds, rs, met, l, tracer)
if err != nil {
return nil, fmt.Errorf("multi-backend target \"%s\" was misconfigured: %w", cfg.MultiPrimary, err)
}
@ -547,7 +547,7 @@ func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingS
for _, b := range cfg.MultiSecondaries {
secCfg := cfg
secCfg.Backend = b
sec, err := configureHistorianBackend(ctx, secCfg, ar, ds, rs, met, l)
sec, err := configureHistorianBackend(ctx, secCfg, ar, ds, rs, met, l, tracer)
if err != nil {
return nil, fmt.Errorf("multi-backend target \"%s\" was miconfigured: %w", b, err)
}
@ -569,7 +569,7 @@ func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingS
}
req := historian.NewRequester()
lokiBackendLogger := log.New("ngalert.state.historian", "backend", "loki")
backend := historian.NewRemoteLokiBackend(lokiBackendLogger, lcfg, req, met)
backend := historian.NewRemoteLokiBackend(lokiBackendLogger, lcfg, req, met, tracer)
testConnCtx, cancelFunc := context.WithTimeout(ctx, 10*time.Second)
defer cancelFunc()
@ -627,8 +627,8 @@ func ApplyStateHistoryFeatureToggles(cfg *setting.UnifiedAlertingStateHistorySet
}
}
func createRemoteAlertmanager(cfg remote.AlertmanagerConfig, kvstore kvstore.KVStore, decryptFn remote.DecryptFn, autogenFn remote.AutogenFn, m *metrics.RemoteAlertmanager) (*remote.Alertmanager, error) {
return remote.NewAlertmanager(cfg, notifier.NewFileStore(cfg.OrgID, kvstore), decryptFn, autogenFn, m)
func createRemoteAlertmanager(cfg remote.AlertmanagerConfig, kvstore kvstore.KVStore, decryptFn remote.DecryptFn, autogenFn remote.AutogenFn, m *metrics.RemoteAlertmanager, tracer tracing.Tracer) (*remote.Alertmanager, error) {
return remote.NewAlertmanager(cfg, notifier.NewFileStore(cfg.OrgID, kvstore), decryptFn, autogenFn, m, tracer)
}
func createRecordingWriter(featureToggles featuremgmt.FeatureToggles, settings setting.RecordingRuleSettings) (schedule.RecordingWriter, error) {

18
pkg/services/ngalert/ngalert_test.go
View File

@ -62,12 +62,13 @@ func TestConfigureHistorianBackend(t *testing.T) {
t.Run("fail initialization if invalid backend", func(t *testing.T) {
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
logger := log.NewNopLogger()
tracer := tracing.InitializeTracerForTest()
cfg := setting.UnifiedAlertingStateHistorySettings{
Enabled: true,
Backend: "invalid-backend",
}
_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
require.ErrorContains(t, err, "unrecognized")
})
@ -75,13 +76,14 @@ func TestConfigureHistorianBackend(t *testing.T) {
t.Run("fail initialization if invalid multi-backend primary", func(t *testing.T) {
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
logger := log.NewNopLogger()
tracer := tracing.InitializeTracerForTest()
cfg := setting.UnifiedAlertingStateHistorySettings{
Enabled: true,
Backend: "multiple",
MultiPrimary: "invalid-backend",
}
_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
require.ErrorContains(t, err, "multi-backend target")
require.ErrorContains(t, err, "unrecognized")
@ -90,6 +92,7 @@ func TestConfigureHistorianBackend(t *testing.T) {
t.Run("fail initialization if invalid multi-backend secondary", func(t *testing.T) {
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
logger := log.NewNopLogger()
tracer := tracing.InitializeTracerForTest()
cfg := setting.UnifiedAlertingStateHistorySettings{
Enabled: true,
Backend: "multiple",
@ -97,7 +100,7 @@ func TestConfigureHistorianBackend(t *testing.T) {
MultiSecondaries: []string{"annotations", "invalid-backend"},
}
_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
require.ErrorContains(t, err, "multi-backend target")
require.ErrorContains(t, err, "unrecognized")
@ -106,6 +109,7 @@ func TestConfigureHistorianBackend(t *testing.T) {
t.Run("do not fail initialization if pinging Loki fails", func(t *testing.T) {
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
logger := log.NewNopLogger()
tracer := tracing.InitializeTracerForTest()
cfg := setting.UnifiedAlertingStateHistorySettings{
Enabled: true,
Backend: "loki",
@ -114,7 +118,7 @@ func TestConfigureHistorianBackend(t *testing.T) {
LokiWriteURL: "http://gone.invalid",
}
h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
require.NotNil(t, h)
require.NoError(t, err)
@ -124,12 +128,13 @@ func TestConfigureHistorianBackend(t *testing.T) {
reg := prometheus.NewRegistry()
met := metrics.NewHistorianMetrics(reg, metrics.Subsystem)
logger := log.NewNopLogger()
tracer := tracing.InitializeTracerForTest()
cfg := setting.UnifiedAlertingStateHistorySettings{
Enabled: true,
Backend: "annotations",
}
h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
require.NotNil(t, h)
require.NoError(t, err)
@ -146,11 +151,12 @@ grafana_alerting_state_history_info{backend="annotations"} 1
reg := prometheus.NewRegistry()
met := metrics.NewHistorianMetrics(reg, metrics.Subsystem)
logger := log.NewNopLogger()
tracer := tracing.InitializeTracerForTest()
cfg := setting.UnifiedAlertingStateHistorySettings{
Enabled: false,
}
h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
require.NotNil(t, h)
require.NoError(t, err)

3
pkg/services/ngalert/notifier/multiorg_alertmanager_remote_test.go
View File

@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/services/ngalert/models"
@ -66,7 +67,7 @@ func TestMultiorgAlertmanager_RemoteSecondaryMode(t *testing.T) {
DefaultConfig: setting.GetAlertmanagerDefaultConfiguration(),
}
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
remoteAM, err := remote.NewAlertmanager(externalAMCfg, notifier.NewFileStore(orgID, kvStore), secretsService.Decrypt, remote.NoopAutogenFn, m)
remoteAM, err := remote.NewAlertmanager(externalAMCfg, notifier.NewFileStore(orgID, kvStore), secretsService.Decrypt, remote.NoopAutogenFn, m, tracing.InitializeTracerForTest())
require.NoError(t, err)
// Use both Alertmanager implementations in the forked Alertmanager.

7
pkg/services/ngalert/remote/alertmanager.go
View File

@ -25,6 +25,7 @@ import (
"gopkg.in/yaml.v3"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/services/ngalert/models"
@ -100,7 +101,7 @@ func (cfg *AlertmanagerConfig) Validate() error {
return nil
}
func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, decryptFn DecryptFn, autogenFn AutogenFn, metrics *metrics.RemoteAlertmanager) (*Alertmanager, error) {
func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, decryptFn DecryptFn, autogenFn AutogenFn, metrics *metrics.RemoteAlertmanager, tracer tracing.Tracer) (*Alertmanager, error) {
if err := cfg.Validate(); err != nil {
return nil, err
}
@ -118,7 +119,7 @@ func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, decryptFn Decrypt
URL: u,
PromoteConfig: cfg.PromoteConfig,
}
mc, err := remoteClient.New(mcCfg, metrics)
mc, err := remoteClient.New(mcCfg, metrics, tracer)
if err != nil {
return nil, err
}
@ -129,7 +130,7 @@ func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, decryptFn Decrypt
Password: cfg.BasicAuthPassword,
Logger: logger,
}
amc, err := remoteClient.NewAlertmanager(amcCfg, metrics)
amc, err := remoteClient.NewAlertmanager(amcCfg, metrics, tracer)
if err != nil {
return nil, err
}

16
pkg/services/ngalert/remote/alertmanager_test.go
View File

@ -23,6 +23,7 @@ import (
"github.com/grafana/alerting/definition"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
@ -105,7 +106,7 @@ func TestNewAlertmanager(t *testing.T) {
DefaultConfig: defaultGrafanaConfig,
}
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m)
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
if test.expErr != "" {
require.EqualError(tt, err, test.expErr)
return
@ -177,7 +178,7 @@ func TestApplyConfig(t *testing.T) {
// An error response from the remote Alertmanager should result in the readiness check failing.
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
am, err := NewAlertmanager(cfg, fstore, secretsService.Decrypt, NoopAutogenFn, m)
am, err := NewAlertmanager(cfg, fstore, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
require.NoError(t, err)
config := &ngmodels.AlertConfiguration{
@ -305,6 +306,7 @@ func TestCompareAndSendConfiguration(t *testing.T) {
decryptFn,
test.autogenFn,
m,
tracing.InitializeTracerForTest(),
)
require.NoError(t, err)
@ -364,7 +366,7 @@ func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(db.InitTestDB(t)))
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
am, err := NewAlertmanager(cfg, fstore, secretsService.Decrypt, NoopAutogenFn, m)
am, err := NewAlertmanager(cfg, fstore, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
require.NoError(t, err)
encodedFullState, err := am.getFullState(ctx)
@ -521,7 +523,7 @@ func TestIntegrationRemoteAlertmanagerGetStatus(t *testing.T) {
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m)
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
require.NoError(t, err)
// We should get the default Cloud Alertmanager configuration.
@ -555,7 +557,7 @@ func TestIntegrationRemoteAlertmanagerSilences(t *testing.T) {
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m)
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
require.NoError(t, err)
// We should have no silences at first.
@ -640,7 +642,7 @@ func TestIntegrationRemoteAlertmanagerAlerts(t *testing.T) {
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m)
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
require.NoError(t, err)
// Wait until the Alertmanager is ready to send alerts.
@ -709,7 +711,7 @@ func TestIntegrationRemoteAlertmanagerReceivers(t *testing.T) {
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m)
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
require.NoError(t, err)
// We should start with the default config.

8
pkg/services/ngalert/remote/client/alertmanager.go
View File

@ -9,6 +9,7 @@ import (
httptransport "github.com/go-openapi/runtime/client"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/ngalert/client"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
amclient "github.com/prometheus/alertmanager/api/v2/client"
@ -31,7 +32,7 @@ type Alertmanager struct {
logger log.Logger
}
func NewAlertmanager(cfg *AlertmanagerConfig, metrics *metrics.RemoteAlertmanager) (*Alertmanager, error) {
func NewAlertmanager(cfg *AlertmanagerConfig, metrics *metrics.RemoteAlertmanager, tracer tracing.Tracer) (*Alertmanager, error) {
// First, add the authentication middleware.
c := &http.Client{Transport: &MimirAuthRoundTripper{
TenantID: cfg.TenantID,
@ -40,14 +41,15 @@ func NewAlertmanager(cfg *AlertmanagerConfig, metrics *metrics.RemoteAlertmanage
}}
tc := client.NewTimedClient(c, metrics.RequestLatency)
trc := client.NewTracedClient(tc, tracer, "remote.alertmanager.client")
apiEndpoint := *cfg.URL
// Next, make sure you set the right path.
u := apiEndpoint.JoinPath(alertmanagerAPIMountPath, amclient.DefaultBasePath)
// Create an Alertmanager client using the timed client as the transport.
// Create an Alertmanager client using the instrumented client as the transport.
r := httptransport.New(u.Host, u.Path, []string{u.Scheme})
r.Transport = tc
r.Transport = trc
return &Alertmanager{
logger: cfg.Logger,

7
pkg/services/ngalert/remote/client/mimir.go
View File

@ -12,6 +12,7 @@ import (
"strings"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/client"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@ -68,7 +69,7 @@ func (e *errorResponse) Error() string {
return e.Error2
}
func New(cfg *Config, metrics *metrics.RemoteAlertmanager) (*Mimir, error) {
func New(cfg *Config, metrics *metrics.RemoteAlertmanager, tracer tracing.Tracer) (*Mimir, error) {
rt := &MimirAuthRoundTripper{
TenantID: cfg.TenantID,
Password: cfg.Password,
@ -78,10 +79,12 @@ func New(cfg *Config, metrics *metrics.RemoteAlertmanager) (*Mimir, error) {
c := &http.Client{
Transport: rt,
}
tc := client.NewTimedClient(c, metrics.RequestLatency)
trc := client.NewTracedClient(tc, tracer, "remote.alertmanager.client")
return &Mimir{
endpoint: cfg.URL,
client: client.NewTimedClient(c, metrics.RequestLatency),
client: trc,
logger: cfg.Logger,
metrics: metrics,
promoteConfig: cfg.PromoteConfig,

5
pkg/services/ngalert/state/historian/loki.go
View File

@ -14,6 +14,7 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/ngalert/client"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@ -55,9 +56,9 @@ type RemoteLokiBackend struct {
log log.Logger
}
func NewRemoteLokiBackend(logger log.Logger, cfg LokiConfig, req client.Requester, metrics *metrics.Historian) *RemoteLokiBackend {
func NewRemoteLokiBackend(logger log.Logger, cfg LokiConfig, req client.Requester, metrics *metrics.Historian, tracer tracing.Tracer) *RemoteLokiBackend {
return &RemoteLokiBackend{
client: NewLokiClient(cfg, req, metrics, logger),
client: NewLokiClient(cfg, req, metrics, logger, tracer),
externalLabels: cfg.ExternalLabels,
clock: clock.New(),
metrics: metrics,

6
pkg/services/ngalert/state/historian/loki_http.go
View File

@ -12,6 +12,7 @@ import (
"time"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/ngalert/client"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/setting"
@ -103,10 +104,11 @@ const (
NeqRegEx Operator = "!~"
)
func NewLokiClient(cfg LokiConfig, req client.Requester, metrics *metrics.Historian, logger log.Logger) *HttpLokiClient {
func NewLokiClient(cfg LokiConfig, req client.Requester, metrics *metrics.Historian, logger log.Logger, tracer tracing.Tracer) *HttpLokiClient {
tc := client.NewTimedClient(req, metrics.WriteDuration)
trc := client.NewTracedClient(tc, tracer, "ngalert.historian.client")
return &HttpLokiClient{
client: tc,
client: trc,
encoder: cfg.Encoder,
cfg: cfg,
metrics: metrics,

7
pkg/services/ngalert/state/historian/loki_http_test.go
View File

@ -18,6 +18,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
)
func TestLokiConfig(t *testing.T) {
@ -227,7 +228,7 @@ func TestLokiHTTPClient_Manual(t *testing.T) {
ReadPathURL: url,
WritePathURL: url,
Encoder: JsonEncoder{},
}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem), log.NewNopLogger())
}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem), log.NewNopLogger(), tracing.InitializeTracerForTest())
// Unauthorized request should fail against Grafana Cloud.
err = client.Ping(context.Background())
@ -255,7 +256,7 @@ func TestLokiHTTPClient_Manual(t *testing.T) {
BasicAuthUser: "<your_username>",
BasicAuthPassword: "<your_password>",
Encoder: JsonEncoder{},
}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem), log.NewNopLogger())
}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem), log.NewNopLogger(), tracing.InitializeTracerForTest())
// When running on prem, you might need to set the tenant id,
// so the x-scope-orgid header is set.
@ -389,7 +390,7 @@ func createTestLokiClient(req client.Requester) *HttpLokiClient {
Encoder: JsonEncoder{},
}
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
return NewLokiClient(cfg, req, met, log.NewNopLogger())
return NewLokiClient(cfg, req, met, log.NewNopLogger(), tracing.InitializeTracerForTest())
}
func reqBody(t *testing.T, req *http.Request) string {

3
pkg/services/ngalert/state/historian/loki_test.go
View File

@ -13,6 +13,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/ngalert/client"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@ -514,7 +515,7 @@ func createTestLokiBackend(req client.Requester, met *metrics.Historian) *Remote
ExternalLabels: map[string]string{"externalLabelKey": "externalLabelValue"},
}
lokiBackendLogger := log.New("ngalert.state.historian", "backend", "loki")
return NewRemoteLokiBackend(lokiBackendLogger, cfg, req, met)
return NewRemoteLokiBackend(lokiBackendLogger, cfg, req, met, tracing.InitializeTracerForTest())
}
func singleFromNormal(st *state.State) []state.StateTransition {

3
pkg/services/publicdashboards/service/common_test.go
View File

@ -5,6 +5,7 @@ import (
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/annotations"
"github.com/grafana/grafana/pkg/services/annotations/annotationsimpl"
"github.com/grafana/grafana/pkg/services/dashboards"
@ -29,7 +30,7 @@ func newPublicDashboardServiceImpl(
db, cfg := db.InitTestDBWithCfg(t)
tagService := tagimpl.ProvideService(db)
if annotationsRepo == nil {
annotationsRepo = annotationsimpl.ProvideService(db, cfg, featuremgmt.WithFeatures(), tagService)
annotationsRepo = annotationsimpl.ProvideService(db, cfg, featuremgmt.WithFeatures(), tagService, tracing.InitializeTracerForTest())
}
if publicDashboardStore == nil {