IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Tracing: Standardize on otel tracing (#75528)

Browse Source
This commit is contained in:
Marcus Efraimsson 2023-10-03 14:54:20 +02:00 committed by GitHub
parent 4432c4c75c
commit e4c1a7a141
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
46 changed files with 321 additions and 439 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
contribute/backend/instrumentation.md
View File

@ -156,7 +156,7 @@ A distributed trace is data that tracks an application request as it flows throu
### Usage
Grafana currently supports two tracing implementations, [OpenTelemetry](https://opentelemetry.io/) and [OpenTracing](https://opentracing.io/). OpenTracing is deprecated, but still supported until we remove it. The two different implementations implements the `Tracer` and `Span` interfaces, defined in the _pkg/infra/tracing_ package, which you can use to create traces and spans. To get a hold of a `Tracer` you would need to get it injected as dependency into your service, see [Services](services.md) for more details.
Grafana uses [OpenTelemetry](https://opentelemetry.io/) for distributed tracing. There's an interface `Tracer` in the _pkg/infra/tracing_ package that implements the [OpenTelemetry Tracer interface](go.opentelemetry.io/otel/trace), which you can use to create traces and spans. To get a hold of a `Tracer` you would need to get it injected as dependency into your service, see [Services](services.md) for more details. For more information, see https://opentelemetry.io/docs/instrumentation/go/manual/.
Example:
@ -166,6 +166,7 @@ import (
"github.com/grafana/grafana/pkg/infra/tracing"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
type MyService struct {
@ -179,36 +180,36 @@ func ProvideService(tracer tracing.Tracer) *MyService {
}
func (s *MyService) Hello(ctx context.Context, name string) (string, error) {
ctx, span := s.tracer.Start(ctx, "MyService.Hello")
ctx, span := s.tracer.Start(ctx, "MyService.Hello", trace.WithAttributes(
attribute.String("my_attribute", "val"),
))
// this make sure the span is marked as finished when this
// method ends to allow the span to be flushed and sent to
// storage backend.
defer span.End()
// Add some event to show Events usage
span.AddEvents(
[]string{"message"},
[]tracing.EventValue{
{Str: "checking name..."},
})
span.AddEvent("checking name...")
if name == "" {
err := fmt.Errorf("name cannot be empty")
// sets the span’s status to Error to make the span tracking
// a failed operation as an error span.
span.SetStatus(codes.Error, "failed to check name")
// record err as an exception span event for this span
span.RecordError(err)
return "", err
}
// Add some other event to show Events usage
span.AddEvents(
[]string{"message"},
[]tracing.EventValue{
{Str: "name checked"},
})
span.AddEvent("name checked")
// Add attribute to show Attributes usage
span.SetAttributes("my_service.name", name, attribute.Key("my_service.name").String(name))
span.SetAttributes(
attribute.String("my_service.name", name),
attribute.Int64("my_service.some_other", int64(1337)),
)
return fmt.Sprintf("Hello %s", name), nil
}
@ -243,6 +244,22 @@ If span names, attribute or event values originates from user input they **shoul
Be **careful** to not expose any sensitive information in span names, attribute or event values, e.g. secrets, credentials etc.
### Span attributes
Consider using `attributes.<Type>("<key>", <value>)` in favor of `attributes.Key("<key>").<Type>(<value>)` since it requires less characters and thereby reads easier.
Example:
```go
attribute.String("datasource_name", proxy.ds.Name)
// vs
attribute.Key("datasource_name").String(proxy.ds.Name)
attribute.Int64("org_id", proxy.ctx.SignedInUser.OrgID)
// vs
attribute.Key("org_id").Int64(proxy.ctx.SignedInUser.OrgID)
```
### How to collect, visualize and query traces (and correlate logs with traces) locally
#### 1. Start Jaeger
@ -255,20 +272,11 @@ make devenv sources=jaeger
To enable tracing in Grafana, you must set the address in your config.ini file
opentelemetry tracing (recommended):
```ini
[tracing.opentelemetry.jaeger]
address = http://localhost:14268/api/traces
```
opentracing tracing (deprecated/not recommended):
```ini
[tracing.jaeger]
address = localhost:6831
```
#### 3. Search/browse collected logs and traces in Grafana Explore
You need provisioned gdev-jaeger and gdev-loki datasources, see [developer dashboard and data sources](https://github.com/grafana/grafana/tree/main/devenv#developer-dashboards-and-data-sources) for setup instructions.

4
go.mod
View File

@ -121,7 +121,7 @@ require (
gopkg.in/mail.v2 v2.3.1 // @grafana/backend-platform
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // @grafana/alerting-squad-backend
xorm.io/builder v0.3.6 // indirect; @grafana/backend-platform
xorm.io/builder v0.3.6 // @grafana/backend-platform
xorm.io/core v0.7.3 // @grafana/backend-platform
xorm.io/xorm v0.8.2 // @grafana/alerting-squad-backend
)
@ -174,7 +174,7 @@ require (
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-msgpack v0.5.5 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect; @grafana/grafana-as-code
github.com/hashicorp/go-multierror v1.1.1 // @grafana/grafana-as-code
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/golang-lru v0.6.0 // indirect
github.com/hashicorp/yamux v0.1.1 // indirect

15
pkg/api/pluginproxy/ds_proxy.go
View File

@ -12,6 +12,7 @@ import (
"time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/api/datasource"
"github.com/grafana/grafana/pkg/infra/httpclient"
@ -142,10 +143,12 @@ func (proxy *DataSourceProxy) HandleRequest() {
proxy.ctx.Req = proxy.ctx.Req.WithContext(ctx)
span.SetAttributes("datasource_name", proxy.ds.Name, attribute.Key("datasource_name").String(proxy.ds.Name))
span.SetAttributes("datasource_type", proxy.ds.Type, attribute.Key("datasource_type").String(proxy.ds.Type))
span.SetAttributes("user", proxy.ctx.SignedInUser.Login, attribute.Key("user").String(proxy.ctx.SignedInUser.Login))
span.SetAttributes("org_id", proxy.ctx.SignedInUser.OrgID, attribute.Key("org_id").Int64(proxy.ctx.SignedInUser.OrgID))
span.SetAttributes(
attribute.String("datasource_name", proxy.ds.Name),
attribute.String("datasource_type", proxy.ds.Type),
attribute.String("user", proxy.ctx.SignedInUser.Login),
attribute.Int64("org_id", proxy.ctx.SignedInUser.OrgID),
)
proxy.addTraceFromHeaderValue(span, "X-Panel-Id", "panel_id")
proxy.addTraceFromHeaderValue(span, "X-Dashboard-Id", "dashboard_id")
@ -155,11 +158,11 @@ func (proxy *DataSourceProxy) HandleRequest() {
reverseProxy.ServeHTTP(proxy.ctx.Resp, proxy.ctx.Req)
}
func (proxy *DataSourceProxy) addTraceFromHeaderValue(span tracing.Span, headerName string, tagName string) {
func (proxy *DataSourceProxy) addTraceFromHeaderValue(span trace.Span, headerName string, tagName string) {
panelId := proxy.ctx.Req.Header.Get(headerName)
dashId, err := strconv.Atoi(panelId)
if err == nil {
span.SetAttributes(tagName, dashId, attribute.Key(tagName).Int(dashId))
span.SetAttributes(attribute.Int(tagName, dashId))
}
}

6
pkg/api/pluginproxy/pluginproxy.go
View File

@ -109,8 +109,10 @@ func (proxy *PluginProxy) HandleRequest() {
proxy.ctx.Req = proxy.ctx.Req.WithContext(ctx)
span.SetAttributes("user", proxy.ctx.SignedInUser.Login, attribute.Key("user").String(proxy.ctx.SignedInUser.Login))
span.SetAttributes("org_id", proxy.ctx.SignedInUser.OrgID, attribute.Key("org_id").Int64(proxy.ctx.SignedInUser.OrgID))
span.SetAttributes(
attribute.String("user", proxy.ctx.SignedInUser.Login),
attribute.Int64("org_id", proxy.ctx.SignedInUser.OrgID),
)
proxy.tracer.Inject(ctx, proxy.ctx.Req.Header, span)

2
pkg/bus/bus.go
View File

@ -55,7 +55,7 @@ func (b *InProcBus) Publish(ctx context.Context, msg Msg) error {
_, span := b.tracer.Start(ctx, "bus - "+msgName)
defer span.End()
span.SetAttributes("msg", msgName, attribute.Key("msg").String(msgName))
span.SetAttributes(attribute.String("msg", msgName))
return nil
}

4
pkg/expr/commands.go
View File

@ -70,7 +70,7 @@ func (gm *MathCommand) NeedsVars() []string {
// failed to execute.
func (gm *MathCommand) Execute(ctx context.Context, _ time.Time, vars mathexp.Vars, tracer tracing.Tracer) (mathexp.Results, error) {
_, span := tracer.Start(ctx, "SSE.ExecuteMath")
span.SetAttributes("expression", gm.RawExpression, attribute.Key("expression").String(gm.RawExpression))
span.SetAttributes(attribute.String("expression", gm.RawExpression))
defer span.End()
return gm.Expression.Execute(gm.refID, vars, tracer)
}
@ -163,7 +163,7 @@ func (gr *ReduceCommand) Execute(ctx context.Context, _ time.Time, vars mathexp.
_, span := tracer.Start(ctx, "SSE.ExecuteReduce")
defer span.End()
span.SetAttributes("reducer", gr.Reducer, attribute.Key("reducer").String(gr.Reducer))
span.SetAttributes(attribute.String("reducer", gr.Reducer))
newRes := mathexp.Results{}
for i, val := range vars[gr.VarToReduce].Values {

2
pkg/expr/dataplane.go
View File

@ -61,7 +61,7 @@ func shouldUseDataplane(frames data.Frames, logger log.Logger, disable bool) (dt
func handleDataplaneFrames(ctx context.Context, tracer tracing.Tracer, t data.FrameType, frames data.Frames) (mathexp.Results, error) {
_, span := tracer.Start(ctx, "SSE.HandleDataPlaneData")
defer span.End()
span.SetAttributes("dataplane.type", t, attribute.Key("dataplane.type").String(string(t)))
span.SetAttributes(attribute.String("dataplane.type", string(t)))
switch t.Kind() {
case data.KindUnknown:

4
pkg/expr/graph.go
View File

@ -99,10 +99,10 @@ func (dp *DataPipeline) execute(c context.Context, now time.Time, s *Service) (m
}
c, span := s.tracer.Start(c, "SSE.ExecuteNode")
span.SetAttributes("node.refId", node.RefID(), attribute.Key("node.refId").String(node.RefID()))
span.SetAttributes(attribute.String("node.refId", node.RefID()))
if len(node.NeedsVars()) > 0 {
inputRefIDs := node.NeedsVars()
span.SetAttributes("node.inputRefIDs", inputRefIDs, attribute.Key("node.inputRefIDs").StringSlice(inputRefIDs))
span.SetAttributes(attribute.StringSlice("node.inputRefIDs", inputRefIDs))
}
defer span.End()

28
pkg/expr/nodes.go
View File

@ -10,12 +10,12 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"gonum.org/v1/gonum/graph/simple"
"github.com/grafana/grafana/pkg/expr/classic"
"github.com/grafana/grafana/pkg/expr/mathexp"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
)
@ -236,8 +236,10 @@ func executeDSNodesGrouped(ctx context.Context, now time.Time, vars mathexp.Vars
"datasourceVersion", firstNode.datasource.Version,
)
span.SetAttributes("datasource.type", firstNode.datasource.Type, attribute.Key("datasource.type").String(firstNode.datasource.Type))
span.SetAttributes("datasource.uid", firstNode.datasource.UID, attribute.Key("datasource.uid").String(firstNode.datasource.UID))
span.SetAttributes(
attribute.String("datasource.type", firstNode.datasource.Type),
attribute.String("datasource.uid", firstNode.datasource.UID),
)
req := &backend.QueryDataRequest{
PluginContext: pCtx,
@ -261,11 +263,8 @@ func executeDSNodesGrouped(ctx context.Context, now time.Time, vars mathexp.Vars
if e != nil {
responseType = "error"
respStatus = "failure"
span.AddEvents([]string{"error", "message"},
[]tracing.EventValue{
{Str: fmt.Sprintf("%v", err)},
{Str: "failed to query data source"},
})
span.SetStatus(codes.Error, "failed to query data source")
span.RecordError(e)
}
logger.Debug("Data source queried", "responseType", responseType)
useDataplane := strings.HasPrefix(responseType, "dataplane-")
@ -313,8 +312,10 @@ func (dn *DSNode) Execute(ctx context.Context, now time.Time, _ mathexp.Vars, s
if err != nil {
return mathexp.Results{}, err
}
span.SetAttributes("datasource.type", dn.datasource.Type, attribute.Key("datasource.type").String(dn.datasource.Type))
span.SetAttributes("datasource.uid", dn.datasource.UID, attribute.Key("datasource.uid").String(dn.datasource.UID))
span.SetAttributes(
attribute.String("datasource.type", dn.datasource.Type),
attribute.String("datasource.uid", dn.datasource.UID),
)
req := &backend.QueryDataRequest{
PluginContext: pCtx,
@ -337,11 +338,8 @@ func (dn *DSNode) Execute(ctx context.Context, now time.Time, _ mathexp.Vars, s
if e != nil {
responseType = "error"
respStatus = "failure"
span.AddEvents([]string{"error", "message"},
[]tracing.EventValue{
{Str: fmt.Sprintf("%v", err)},
{Str: "failed to query data source"},
})
span.SetStatus(codes.Error, "failed to query data source")
span.RecordError(e)
}
logger.Debug("Data source queried", "responseType", responseType)
useDataplane := strings.HasPrefix(responseType, "dataplane-")

12
pkg/infra/httpclient/httpclientprovider/tracing_middleware.go
View File

@ -10,6 +10,7 @@ import (
"go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/log"
@ -30,17 +31,18 @@ func TracingMiddleware(logger log.Logger, tracer tracing.Tracer) httpclient.Midd
ctx = httptrace.WithClientTrace(ctx, otelhttptrace.NewClientTrace(ctx, otelhttptrace.WithoutSubSpans(), otelhttptrace.WithoutHeaders()))
req = req.WithContext(ctx)
for k, v := range opts.Labels {
span.SetAttributes(k, v, attribute.Key(k).String(v))
span.SetAttributes(attribute.String(k, v))
}
tracer.Inject(ctx, req.Header, span)
res, err := next.RoundTrip(req)
span.SetAttributes("http.url", req.URL.String(), attribute.String("http.url", req.URL.String()))
span.SetAttributes("http.method", req.Method, attribute.String("http.method", req.Method))
span.SetAttributes(semconv.HTTPURL(req.URL.String()))
span.SetAttributes(semconv.HTTPMethod(req.Method))
// ext.SpanKind.Set(span, ext.SpanKindRPCClientEnum)
if err != nil {
span.SetStatus(codes.Error, "request failed")
span.RecordError(err)
return res, err
}
@ -49,10 +51,10 @@ func TracingMiddleware(logger log.Logger, tracer tracing.Tracer) httpclient.Midd
// we avoid measuring contentlength less than zero because it indicates
// that the content size is unknown. https://godoc.org/github.com/badu/http#Response
if res.ContentLength > 0 {
span.SetAttributes(httpContentLengthTagKey, res.ContentLength, attribute.Key(httpContentLengthTagKey).Int64(res.ContentLength))
span.SetAttributes(attribute.Int64(httpContentLengthTagKey, res.ContentLength))
}
span.SetAttributes("http.status_code", res.StatusCode, attribute.Int("http.status_code", res.StatusCode))
span.SetAttributes(semconv.HTTPStatusCode(res.StatusCode))
if res.StatusCode >= 400 {
span.SetStatus(codes.Error, fmt.Sprintf("error with HTTP status code %s", strconv.Itoa(res.StatusCode)))
}

4
pkg/infra/serverlock/serverlock.go
View File

@ -34,7 +34,7 @@ type ServerLockService struct {
func (sl *ServerLockService) LockAndExecute(ctx context.Context, actionName string, maxInterval time.Duration, fn func(ctx context.Context)) error {
start := time.Now()
ctx, span := sl.tracer.Start(ctx, "ServerLockService.LockAndExecute")
span.SetAttributes("serverlock.actionName", actionName, attribute.Key("serverlock.actionName").String(actionName))
span.SetAttributes(attribute.String("serverlock.actionName", actionName))
defer span.End()
ctxLogger := sl.log.FromContext(ctx)
@ -138,7 +138,7 @@ func (sl *ServerLockService) getOrCreate(ctx context.Context, actionName string)
func (sl *ServerLockService) LockExecuteAndRelease(ctx context.Context, actionName string, maxInterval time.Duration, fn func(ctx context.Context)) error {
start := time.Now()
ctx, span := sl.tracer.Start(ctx, "ServerLockService.LockExecuteAndRelease")
span.SetAttributes("serverlock.actionName", actionName, attribute.Key("serverlock.actionName").String(actionName))
span.SetAttributes(attribute.String("serverlock.actionName", actionName))
defer span.End()
ctxLogger := sl.log.FromContext(ctx)

2
pkg/infra/tracing/test_helper.go
View File

@ -24,7 +24,7 @@ func InitializeTracerForTest(opts ...TracerForTestOption) Tracer {
otel.SetTracerProvider(tp)
ots := &Opentelemetry{Propagation: "jaeger,w3c", tracerProvider: tp}
ots := &TracingService{Propagation: "jaeger,w3c", tracerProvider: tp}
_ = ots.initOpentelemetryTracer()
return ots
}

185
pkg/infra/tracing/tracing.go
View File

@ -15,14 +15,13 @@ import (
"go.opentelemetry.io/contrib/samplers/jaegerremote"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/exporters/jaeger"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
tracesdk "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
trace "go.opentelemetry.io/otel/trace"
"github.com/go-kit/log/level"
@ -44,7 +43,7 @@ const (
w3cPropagator string = "w3c"
)
type Opentelemetry struct {
type TracingService struct {
enabled string
Address string
Propagation string
@ -57,7 +56,7 @@ type Opentelemetry struct {
log log.Logger
tracerProvider tracerProvider
tracer trace.Tracer
trace.Tracer
Cfg *setting.Cfg
}
@ -68,24 +67,10 @@ type tracerProvider interface {
Shutdown(ctx context.Context) error
}
type OpentelemetrySpan struct {
span trace.Span
}
type EventValue struct {
Str string
Num int64
}
// Tracer defines the service used to create new spans.
type Tracer interface {
// Run implements registry.BackgroundService.
Run(context.Context) error
// Start creates a new [Span] and places trace metadata on the
// [context.Context] passed to the method.
// Chose a low cardinality spanName and use [Span.SetAttributes]
// or [Span.AddEvents] for high cardinality data.
Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span)
trace.Tracer
// Inject adds identifying information for the span to the
// headers defined in [http.Header] map (this mutates http.Header).
//
@ -94,47 +79,10 @@ type Tracer interface {
// information passed as [Span] is preferred.
// Both the context and span must be derived from the same call to
// [Tracer.Start].
Inject(context.Context, http.Header, Span)
// OtelTracer returns the trace.Tracer if available or nil.
OtelTracer() trace.Tracer
Inject(context.Context, http.Header, trace.Span)
}
// Span defines a time range for an operation. This is equivalent to a
// single line in a flame graph.
type Span interface {
// End finalizes the Span and adds its end timestamp.
// Any further operations on the Span are not permitted after
// End has been called.
End()
// SetAttributes adds additional data to a span.
// SetAttributes repeats the key value pair with [string] and [any]
// used for OpenTracing and [attribute.KeyValue] used for
// OpenTelemetry.
SetAttributes(key string, value any, kv attribute.KeyValue)
// SetName renames the span.
SetName(name string)
// SetStatus can be used to indicate whether the span was
// successfully or unsuccessfully executed.
//
// Only useful for OpenTelemetry.
SetStatus(code codes.Code, description string)
// RecordError adds an error to the span.
//
// Only useful for OpenTelemetry.
RecordError(err error, options ...trace.EventOption)
// AddEvents adds additional data with a temporal dimension to the
// span.
//
// Panics if the length of keys is shorter than the length of values.
AddEvents(keys []string, values []EventValue)
// contextWithSpan returns a context.Context that holds the parent
// context plus a reference to this span.
ContextWithSpan(ctx context.Context) context.Context
}
func ProvideService(cfg *setting.Cfg) (Tracer, error) {
func ProvideService(cfg *setting.Cfg) (*TracingService, error) {
ots, err := ParseSettings(cfg)
if err != nil {
return nil, err
@ -153,8 +101,8 @@ func ProvideService(cfg *setting.Cfg) (Tracer, error) {
return ots, nil
}
func ParseSettings(cfg *setting.Cfg) (*Opentelemetry, error) {
ots := &Opentelemetry{
func ParseSettings(cfg *setting.Cfg) (*TracingService, error) {
ots := &TracingService{
Cfg: cfg,
log: log.New("tracing"),
}
@ -162,38 +110,13 @@ func ParseSettings(cfg *setting.Cfg) (*Opentelemetry, error) {
return ots, err
}
type traceKey struct{}
type traceValue struct {
ID string
IsSampled bool
}
func TraceIDFromContext(c context.Context, requireSampled bool) string {
v := c.Value(traceKey{})
// Return traceID if a) it is present and b) it is sampled when requireSampled param is true
if trace, ok := v.(traceValue); ok && (!requireSampled || trace.IsSampled) {
return trace.ID
func TraceIDFromContext(ctx context.Context, requireSampled bool) string {
spanCtx := trace.SpanContextFromContext(ctx)
if !spanCtx.HasTraceID() || !spanCtx.IsValid() || (requireSampled && !spanCtx.IsSampled()) {
return ""
}
return ""
}
// SpanFromContext returns the Span previously associated with ctx, or nil, if no such span could be found.
// It is the equivalent of opentracing.SpanFromContext and trace.SpanFromContext.
func SpanFromContext(ctx context.Context) Span {
if span := trace.SpanFromContext(ctx); span != nil {
return OpentelemetrySpan{span: span}
}
return nil
}
// ContextWithSpan returns a new context.Context that holds a reference to the given span.
// If span is nil, a new context without an active span is returned.
// It is the equivalent of opentracing.ContextWithSpan and trace.ContextWithSpan.
func ContextWithSpan(ctx context.Context, span Span) context.Context {
if span != nil {
return span.ContextWithSpan(ctx)
}
return ctx
return spanCtx.TraceID().String()
}
type noopTracerProvider struct {
@ -204,7 +127,7 @@ func (noopTracerProvider) Shutdown(ctx context.Context) error {
return nil
}
func (ots *Opentelemetry) parseSettings() error {
func (ots *TracingService) parseSettings() error {
legacyAddress, legacyTags := "", ""
if section, err := ots.Cfg.Raw.GetSection("tracing.jaeger"); err == nil {
legacyAddress = section.Key("address").MustString("")
@ -263,7 +186,7 @@ func (ots *Opentelemetry) parseSettings() error {
return nil
}
func (ots *Opentelemetry) OTelExporterEnabled() bool {
func (ots *TracingService) OTelExporterEnabled() bool {
return ots.enabled == otlpExporter
}
@ -283,7 +206,7 @@ func splitCustomAttribs(s string) ([]attribute.KeyValue, error) {
return res, nil
}
func (ots *Opentelemetry) initJaegerTracerProvider() (*tracesdk.TracerProvider, error) {
func (ots *TracingService) initJaegerTracerProvider() (*tracesdk.TracerProvider, error) {
var ep jaeger.EndpointOption
// Create the Jaeger exporter: address can be either agent address (host:port) or collector URL
if strings.HasPrefix(ots.Address, "http://") || strings.HasPrefix(ots.Address, "https://") {
@ -328,7 +251,7 @@ func (ots *Opentelemetry) initJaegerTracerProvider() (*tracesdk.TracerProvider,
return tp, nil
}
func (ots *Opentelemetry) initOTLPTracerProvider() (*tracesdk.TracerProvider, error) {
func (ots *TracingService) initOTLPTracerProvider() (*tracesdk.TracerProvider, error) {
client := otlptracegrpc.NewClient(otlptracegrpc.WithEndpoint(ots.Address), otlptracegrpc.WithInsecure())
exp, err := otlptrace.New(context.Background(), client)
if err != nil {
@ -343,7 +266,7 @@ func (ots *Opentelemetry) initOTLPTracerProvider() (*tracesdk.TracerProvider, er
return initTracerProvider(exp, ots.Cfg.BuildVersion, sampler, ots.customAttribs...)
}
func (ots *Opentelemetry) initSampler() (tracesdk.Sampler, error) {
func (ots *TracingService) initSampler() (tracesdk.Sampler, error) {
switch ots.sampler {
case "const", "":
if ots.samplerParam >= 1 {
@ -390,11 +313,11 @@ func initTracerProvider(exp tracesdk.SpanExporter, version string, sampler trace
return tp, nil
}
func (ots *Opentelemetry) initNoopTracerProvider() (tracerProvider, error) {
func (ots *TracingService) initNoopTracerProvider() (tracerProvider, error) {
return &noopTracerProvider{TracerProvider: trace.NewNoopTracerProvider()}, nil
}
func (ots *Opentelemetry) initOpentelemetryTracer() error {
func (ots *TracingService) initOpentelemetryTracer() error {
var tp tracerProvider
var err error
switch ots.enabled {
@ -450,12 +373,12 @@ func (ots *Opentelemetry) initOpentelemetryTracer() error {
ots.tracerProvider = tp
}
ots.tracer = otel.GetTracerProvider().Tracer("component-main")
ots.Tracer = otel.GetTracerProvider().Tracer("component-main")
return nil
}
func (ots *Opentelemetry) Run(ctx context.Context) error {
func (ots *TracingService) Run(ctx context.Context) error {
otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) {
err = level.Error(ots.log).Log("msg", "OpenTelemetry handler returned an error", "err", err)
if err != nil {
@ -478,68 +401,12 @@ func (ots *Opentelemetry) Run(ctx context.Context) error {
return nil
}
func (ots *Opentelemetry) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) {
ctx, span := ots.tracer.Start(ctx, spanName, opts...)
opentelemetrySpan := OpentelemetrySpan{
span: span,
}
if traceID := span.SpanContext().TraceID(); traceID.IsValid() {
ctx = context.WithValue(ctx, traceKey{}, traceValue{traceID.String(), span.SpanContext().IsSampled()})
}
return ctx, opentelemetrySpan
}
func (ots *Opentelemetry) Inject(ctx context.Context, header http.Header, _ Span) {
func (ots *TracingService) Inject(ctx context.Context, header http.Header, _ trace.Span) {
otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(header))
}
func (ots *Opentelemetry) OtelTracer() trace.Tracer {
return ots.tracer
}
func (s OpentelemetrySpan) End() {
s.span.End()
}
func (s OpentelemetrySpan) SetAttributes(key string, value any, kv attribute.KeyValue) {
s.span.SetAttributes(kv)
}
func (s OpentelemetrySpan) SetName(name string) {
s.span.SetName(name)
}
func (s OpentelemetrySpan) SetStatus(code codes.Code, description string) {
s.span.SetStatus(code, description)
}
func (s OpentelemetrySpan) RecordError(err error, options ...trace.EventOption) {
s.span.RecordError(err, options...)
}
func (s OpentelemetrySpan) AddEvents(keys []string, values []EventValue) {
for i, v := range values {
if v.Str != "" {
s.span.AddEvent(keys[i], trace.WithAttributes(attribute.Key(keys[i]).String(v.Str)))
}
if v.Num != 0 {
s.span.AddEvent(keys[i], trace.WithAttributes(attribute.Key(keys[i]).Int64(v.Num)))
}
}
}
func (s OpentelemetrySpan) ContextWithSpan(ctx context.Context) context.Context {
if s.span != nil {
ctx = trace.ContextWithSpan(ctx, s.span)
// Grafana also manages its own separate traceID in the context in addition to what opentracing handles.
// It's derived from the span. Ensure that we propagate this too.
if traceID := s.span.SpanContext().TraceID(); traceID.IsValid() {
ctx = context.WithValue(ctx, traceKey{}, traceValue{traceID.String(), s.span.SpanContext().IsSampled()})
}
}
return ctx
func (ots *TracingService) OtelTracer() trace.Tracer {
return ots
}
type rateLimiter struct {

17
pkg/infra/tracing/tracing_test.go
View File

@ -176,23 +176,22 @@ func TestTracingConfig(t *testing.T) {
tracer, err := ProvideService(cfg)
assert.NoError(t, err)
// make sure tracker is properly configured
otel := tracer.(*Opentelemetry)
assert.Equal(t, test.ExpectedExporter, otel.enabled)
assert.Equal(t, test.ExpectedAddress, otel.Address)
assert.Equal(t, test.ExpectedPropagator, otel.Propagation)
assert.Equal(t, test.ExpectedAttrs, otel.customAttribs)
assert.Equal(t, test.ExpectedExporter, tracer.enabled)
assert.Equal(t, test.ExpectedAddress, tracer.Address)
assert.Equal(t, test.ExpectedPropagator, tracer.Propagation)
assert.Equal(t, test.ExpectedAttrs, tracer.customAttribs)
if test.ExpectedSampler != "" {
assert.Equal(t, test.ExpectedSampler, otel.sampler)
assert.Equal(t, test.ExpectedSamplerParam, otel.samplerParam)
assert.Equal(t, test.ExpectedSamplingServerURL, otel.samplerRemoteURL)
assert.Equal(t, test.ExpectedSampler, tracer.sampler)
assert.Equal(t, test.ExpectedSamplerParam, tracer.samplerParam)
assert.Equal(t, test.ExpectedSamplingServerURL, tracer.samplerRemoteURL)
}
})
}
}
func TestInitSampler(t *testing.T) {
otel := &Opentelemetry{}
otel := &TracingService{}
sampler, err := otel.initSampler()
require.NoError(t, err)
assert.Equal(t, "AlwaysOffSampler", sampler.Description())

2
pkg/infra/usagestats/service/usage_stats.go
View File

@ -77,7 +77,7 @@ func (uss *UsageStats) runMetricsFunc(ctx context.Context, fn usagestats.Metrics
start := time.Now()
ctx, span := uss.tracer.Start(ctx, "UsageStats.Gather")
fnName := runtime.FuncForPC(reflect.ValueOf(fn).Pointer()).Name()
span.SetAttributes("usageStats.function", fnName, attribute.Key("usageStats.function").String(fnName))
span.SetAttributes(attribute.String("usageStats.function", fnName))
defer span.End()
fnMetrics, err := fn(ctx)

8
pkg/middleware/request_tracing.go
View File

@ -9,9 +9,9 @@ import (
"strings"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/propagation"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/tracing"
@ -97,9 +97,9 @@ func RequestTracing(tracer tracing.Tracer) web.Middleware {
status := rw.Status()
span.SetAttributes("http.status_code", status, attribute.Int("http.status_code", status))
span.SetAttributes("http.url", req.RequestURI, attribute.String("http.url", req.RequestURI))
span.SetAttributes("http.method", req.Method, attribute.String("http.method", req.Method))
span.SetAttributes(semconv.HTTPStatusCode(status))
span.SetAttributes(semconv.HTTPURL(req.RequestURI))
span.SetAttributes(semconv.HTTPMethod(req.Method))
if status >= 400 {
span.SetStatus(codes.Error, fmt.Sprintf("error with HTTP status code %s", strconv.Itoa(status)))
}

5
pkg/plugins/backendplugin/coreplugin/registry.go
View File

@ -87,9 +87,8 @@ func ProvideCoreRegistry(tracer tracing.Tracer, am *azuremonitor.Service, cw *cl
es *elasticsearch.Service, grap *graphite.Service, idb *influxdb.Service, lk *loki.Service, otsdb *opentsdb.Service,
pr *prometheus.Service, t *tempo.Service, td *testdatasource.Service, pg *postgres.Service, my *mysql.Service,
ms *mssql.Service, graf *grafanads.Service, pyroscope *pyroscope.Service, parca *parca.Service) *Registry {
if otelTracer := tracer.OtelTracer(); otelTracer != nil {
sdktracing.InitDefaultTracer(otelTracer)
}
// Non-optimal global solution to replace plugin SDK default tracer for core plugins.
sdktracing.InitDefaultTracer(tracer)
return NewRegistry(map[string]backendplugin.PluginFactoryFunc{
CloudWatch: asBackendPlugin(cw.Executor),

2
pkg/registry/backgroundsvcs/background_services.go
View File

@ -46,7 +46,7 @@ import (
func ProvideBackgroundServiceRegistry(
httpServer *api.HTTPServer, ng *ngalert.AlertNG, cleanup *cleanup.CleanUpService, live *live.GrafanaLive,
pushGateway *pushhttp.Gateway, notifications *notifications.NotificationService, pluginStore *pluginStore.Service,
rendering *rendering.RenderingService, tokenService auth.UserTokenBackgroundService, tracing tracing.Tracer,
rendering *rendering.RenderingService, tokenService auth.UserTokenBackgroundService, tracing *tracing.TracingService,
provisioning *provisioning.ProvisioningServiceImpl, alerting *alerting.AlertEngine, usageStats *uss.UsageStats,
statsCollector *statscollector.Service, grafanaUpdateChecker *updatechecker.GrafanaService,
pluginsUpdateChecker *updatechecker.PluginsService, metrics *metrics.InternalMetricsService,

1
pkg/server/wire.go
View File

@ -244,6 +244,7 @@ var wireBasicSet = wire.NewSet(
notifications.ProvideService,
notifications.ProvideSmtpService,
tracing.ProvideService,
wire.Bind(new(tracing.Tracer), new(*tracing.TracingService)),
metrics.ProvideService,
testdatasource.ProvideService,
ldapapi.ProvideService,

27
pkg/services/alerting/engine.go
View File

@ -9,6 +9,7 @@ import (
"github.com/benbjohnson/clock"
"github.com/prometheus/client_golang/prometheus"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"golang.org/x/sync/errgroup"
"github.com/grafana/grafana/pkg/infra/localcache"
@ -206,13 +207,8 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan
defer func() {
if err := recover(); err != nil {
e.log.Error("Alert Panic", "error", err, "stack", log.Stack(1))
span.SetStatus(codes.Error, "failed to execute alert rule. panic was recovered.")
span.RecordError(fmt.Errorf("%v", err))
span.AddEvents(
[]string{"error", "message"},
[]tracing.EventValue{
{Str: fmt.Sprintf("%v", err)},
{Str: "failed to execute alert rule. panic was recovered."},
})
span.End()
close(attemptChan)
}
@ -220,20 +216,17 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan
e.evalHandler.Eval(evalContext)
span.SetAttributes("alertId", evalContext.Rule.ID, attribute.Key("alertId").Int64(evalContext.Rule.ID))
span.SetAttributes("dashboardId", evalContext.Rule.DashboardID, attribute.Key("dashboardId").Int64(evalContext.Rule.DashboardID))
span.SetAttributes("firing", evalContext.Firing, attribute.Key("firing").Bool(evalContext.Firing))
span.SetAttributes("nodatapoints", evalContext.NoDataFound, attribute.Key("nodatapoints").Bool(evalContext.NoDataFound))
span.SetAttributes("attemptID", attemptID, attribute.Key("attemptID").Int(attemptID))
span.SetAttributes(
attribute.Int64("alertId", evalContext.Rule.ID),
attribute.Int64("dashboardId", evalContext.Rule.DashboardID),
attribute.Bool("firing", evalContext.Firing),
attribute.Bool("nodatapoints", evalContext.NoDataFound),
attribute.Int("attemptID", attemptID),
)
if evalContext.Error != nil {
span.SetStatus(codes.Error, "alerting execution attempt failed")
span.RecordError(evalContext.Error)
span.AddEvents(
[]string{"error", "message"},
[]tracing.EventValue{
{Str: fmt.Sprintf("%v", evalContext.Error)},
{Str: "alerting execution attempt failed"},
})
if attemptID < setting.AlertingMaxAttempts {
span.End()

11
pkg/services/authn/authnimpl/service.go
View File

@ -8,6 +8,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/network"
@ -262,9 +263,10 @@ func (s *Service) RegisterPostAuthHook(hook authn.PostAuthHookFn, priority uint)
}
func (s *Service) Login(ctx context.Context, client string, r *authn.Request) (identity *authn.Identity, err error) {
ctx, span := s.tracer.Start(ctx, "authn.Login")
ctx, span := s.tracer.Start(ctx, "authn.Login", trace.WithAttributes(
attribute.String(attributeKeyClient, client),
))
defer span.End()
span.SetAttributes(attributeKeyClient, client, attribute.Key(attributeKeyClient).String(client))
defer func() {
for _, hook := range s.postLoginHooks.items {
@ -316,9 +318,10 @@ func (s *Service) RegisterPostLoginHook(hook authn.PostLoginHookFn, priority uin
}
func (s *Service) RedirectURL(ctx context.Context, client string, r *authn.Request) (*authn.Redirect, error) {
ctx, span := s.tracer.Start(ctx, "authn.RedirectURL")
ctx, span := s.tracer.Start(ctx, "authn.RedirectURL", trace.WithAttributes(
attribute.String(attributeKeyClient, client),
))
defer span.End()
span.SetAttributes(attributeKeyClient, client, attribute.Key(attributeKeyClient).String(client))
c, ok := s.clients[client]
if !ok {

2
pkg/services/cleanup/cleanup.go
View File

@ -138,7 +138,7 @@ func (srv *CleanUpService) cleanUpTmpFiles(ctx context.Context) {
for _, f := range folders {
ctx, span := srv.tracer.Start(ctx, "delete stale files in temporary directory")
span.SetAttributes("directory", f, attribute.Key("directory").String(f))
span.SetAttributes(attribute.String("directory", f))
srv.cleanUpTmpFolder(ctx, f)
span.End()
}

14
pkg/services/contexthandler/contexthandler.go
View File

@ -18,6 +18,8 @@ import (
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/web"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
func ProvideService(cfg *setting.Cfg, tracer tracing.Tracer, features *featuremgmt.FeatureManager, authnService authn.Service,
@ -125,13 +127,11 @@ func (h *ContextHandler) Middleware(next http.Handler) http.Handler {
}
reqContext.Logger = reqContext.Logger.New("userId", reqContext.UserID, "orgId", reqContext.OrgID, "uname", reqContext.Login)
span.AddEvents(
[]string{"uname", "orgId", "userId"},
[]tracing.EventValue{
{Str: reqContext.Login},
{Num: reqContext.OrgID},
{Num: reqContext.UserID}},
)
span.AddEvent("user", trace.WithAttributes(
attribute.String("uname", reqContext.Login),
attribute.Int64("orgId", reqContext.OrgID),
attribute.Int64("userId", reqContext.UserID),
))
next.ServeHTTP(w, r)
})

2
pkg/services/live/pipeline/pipeline.go
View File

@ -14,7 +14,7 @@ import (
"go.opentelemetry.io/otel/exporters/jaeger"
"go.opentelemetry.io/otel/sdk/resource"
tracesdk "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/services/auth/identity"

46
pkg/services/ngalert/schedule/schedule.go
View File

@ -9,6 +9,8 @@ import (
"github.com/benbjohnson/clock"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"golang.org/x/sync/errgroup"
"github.com/grafana/grafana/pkg/infra/log"
@ -372,7 +374,7 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
notify(states)
}
evaluate := func(ctx context.Context, f fingerprint, attempt int64, e *evaluation, span tracing.Span) {
evaluate := func(ctx context.Context, f fingerprint, attempt int64, e *evaluation, span trace.Span) {
logger := logger.New("version", e.rule.Version, "fingerprint", f, "attempt", attempt, "now", e.scheduledAt).FromContext(ctx)
start := sch.clock.Now()
@ -406,21 +408,13 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
}
}
}
span.SetStatus(codes.Error, "rule evaluation failed")
span.RecordError(err)
span.AddEvents(
[]string{"error", "message"},
[]tracing.EventValue{
{Str: fmt.Sprintf("%v", err)},
{Str: "rule evaluation failed"},
})
} else {
logger.Debug("Alert rule evaluated", "results", results, "duration", dur)
span.AddEvents(
[]string{"message", "results"},
[]tracing.EventValue{
{Str: "rule evaluated"},
{Num: int64(len(results))},
})
span.AddEvent("rule evaluated", trace.WithAttributes(
attribute.Int64("results", int64(len(results))),
))
}
if ctx.Err() != nil { // check if the context is not cancelled. The evaluation can be a long-running task.
logger.Debug("Skip updating the state because the context has been cancelled")
@ -438,13 +432,10 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
start = sch.clock.Now()
alerts := state.FromStateTransitionToPostableAlerts(processedStates, sch.stateManager, sch.appURL)
span.AddEvents(
[]string{"message", "state_transitions", "alerts_to_send"},
[]tracing.EventValue{
{Str: "results processed"},
{Num: int64(len(processedStates))},
{Num: int64(len(alerts.PostableAlerts))},
})
span.AddEvent("results processed", trace.WithAttributes(
attribute.Int64("state_transitions", int64(len(processedStates))),
attribute.Int64("alerts_to_send", int64(len(alerts.PostableAlerts))),
))
if len(alerts.PostableAlerts) > 0 {
sch.alertsSender.Send(key, alerts)
}
@ -517,16 +508,17 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
logger.Debug("Skip rule evaluation because it is paused")
return nil
}
tracingCtx, span := sch.tracer.Start(grafanaCtx, "alert rule execution")
defer span.End()
span.SetAttributes("rule_uid", ctx.rule.UID, attribute.String("rule_uid", ctx.rule.UID))
span.SetAttributes("org_id", ctx.rule.OrgID, attribute.Int64("org_id", ctx.rule.OrgID))
span.SetAttributes("rule_version", ctx.rule.Version, attribute.Int64("rule_version", ctx.rule.Version))
fpStr := currentFingerprint.String()
span.SetAttributes("rule_fingerprint", fpStr, attribute.String("rule_fingerprint", fpStr))
utcTick := ctx.scheduledAt.UTC().Format(time.RFC3339Nano)
span.SetAttributes("tick", utcTick, attribute.String("tick", utcTick))
tracingCtx, span := sch.tracer.Start(grafanaCtx, "alert rule execution", trace.WithAttributes(
attribute.String("rule_uid", ctx.rule.UID),
attribute.Int64("org_id", ctx.rule.OrgID),
attribute.Int64("rule_version", ctx.rule.Version),
attribute.String("rule_fingerprint", fpStr),
attribute.String("tick", utcTick),
))
defer span.End()
evaluate(tracingCtx, f, attempt, ctx, span)
return nil

4
pkg/services/ngalert/state/historian/annotation.go
View File

@ -11,10 +11,10 @@ import (
"github.com/benbjohnson/clock"
"github.com/grafana/grafana-plugin-sdk-go/data"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/annotations"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@ -73,7 +73,7 @@ func (h *AnnotationBackend) Record(ctx context.Context, rule history_model.RuleM
writeCtx := context.Background()
writeCtx, cancel := context.WithTimeout(writeCtx, StateHistoryWriteTimeout)
writeCtx = history_model.WithRuleData(writeCtx, rule)
writeCtx = tracing.ContextWithSpan(writeCtx, tracing.SpanFromContext(ctx))
writeCtx = trace.ContextWithSpan(writeCtx, trace.SpanFromContext(ctx))
go func(ctx context.Context) {
defer cancel()

4
pkg/services/ngalert/state/historian/loki.go
View File

@ -11,10 +11,10 @@ import (
"github.com/benbjohnson/clock"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/weaveworks/common/http/client"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/services/ngalert/models"
@ -89,7 +89,7 @@ func (h *RemoteLokiBackend) Record(ctx context.Context, rule history_model.RuleM
writeCtx := context.Background()
writeCtx, cancel := context.WithTimeout(writeCtx, StateHistoryWriteTimeout)
writeCtx = history_model.WithRuleData(writeCtx, rule)
writeCtx = tracing.ContextWithSpan(writeCtx, tracing.SpanFromContext(ctx))
writeCtx = trace.ContextWithSpan(writeCtx, trace.SpanFromContext(ctx))
go func(ctx context.Context) {
defer cancel()

38
pkg/services/ngalert/state/manager.go
View File

@ -9,6 +9,7 @@ import (
"github.com/grafana/dskit/concurrency"
"github.com/grafana/grafana-plugin-sdk-go/data"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
@ -251,42 +252,33 @@ func (st *Manager) ResetStateByRuleUID(ctx context.Context, rule *ngModels.Alert
// ProcessEvalResults updates the current states that belong to a rule with the evaluation results.
// if extraLabels is not empty, those labels will be added to every state. The extraLabels take precedence over rule labels and result labels
func (st *Manager) ProcessEvalResults(ctx context.Context, evaluatedAt time.Time, alertRule *ngModels.AlertRule, results eval.Results, extraLabels data.Labels) []StateTransition {
tracingCtx, span := st.tracer.Start(ctx, "alert rule state calculation")
defer span.End()
span.SetAttributes("rule_uid", alertRule.UID, attribute.String("rule_uid", alertRule.UID))
span.SetAttributes("org_id", alertRule.OrgID, attribute.Int64("org_id", alertRule.OrgID))
span.SetAttributes("rule_version", alertRule.Version, attribute.Int64("rule_version", alertRule.Version))
utcTick := evaluatedAt.UTC().Format(time.RFC3339Nano)
span.SetAttributes("tick", utcTick, attribute.String("tick", utcTick))
span.SetAttributes("results", len(results), attribute.Int("tick", len(results)))
tracingCtx, span := st.tracer.Start(ctx, "alert rule state calculation", trace.WithAttributes(
attribute.String("rule_uid", alertRule.UID),
attribute.Int64("org_id", alertRule.OrgID),
attribute.Int64("rule_version", alertRule.Version),
attribute.String("tick", utcTick),
attribute.Int("results", len(results))))
defer span.End()
logger := st.log.FromContext(tracingCtx)
logger.Debug("State manager processing evaluation results", "resultCount", len(results))
states := st.setNextStateForRule(tracingCtx, alertRule, results, extraLabels, logger)
span.AddEvents([]string{"message", "state_transitions"},
[]tracing.EventValue{
{Str: "results processed"},
{Num: int64(len(states))},
})
span.AddEvent("results processed", trace.WithAttributes(
attribute.Int64("state_transitions", int64(len(states))),
))
staleStates := st.deleteStaleStatesFromCache(ctx, logger, evaluatedAt, alertRule)
st.deleteAlertStates(tracingCtx, logger, staleStates)
if len(staleStates) > 0 {
span.AddEvents([]string{"message", "state_transitions"},
[]tracing.EventValue{
{Str: "deleted stale states"},
{Num: int64(len(staleStates))},
})
span.AddEvent("deleted stale states", trace.WithAttributes(
attribute.Int64("state_transitions", int64(len(staleStates))),
))
}
st.saveAlertStates(tracingCtx, logger, states...)
span.AddEvents([]string{"message"},
[]tracing.EventValue{
{Str: "updated database"},
})
span.AddEvent("updated database")
allChanges := append(states, staleStates...)
if st.historian != nil {

22
pkg/services/pluginsintegration/clientmiddleware/tracing_middleware.go
View File

@ -8,6 +8,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/plugins"
@ -32,10 +33,10 @@ type TracingMiddleware struct {
// setSpanAttributeFromHTTPHeader takes a ReqContext and a span, and adds the specified HTTP header as a span attribute
// (string value), if the header is present.
func setSpanAttributeFromHTTPHeader(headers http.Header, span tracing.Span, attributeName, headerName string) {
func setSpanAttributeFromHTTPHeader(headers http.Header, span trace.Span, attributeName, headerName string) {
// Set the attribute as string
if v := headers.Get(headerName); v != "" {
span.SetAttributes(attributeName, v, attribute.Key(attributeName).String(v))
span.SetAttributes(attribute.String(attributeName, v))
}
}
@ -46,23 +47,24 @@ func (m *TracingMiddleware) traceWrap(
ctx context.Context, pluginContext backend.PluginContext, opName string,
) (context.Context, func(error)) {
// Start span
ctx, span := m.tracer.Start(ctx, "PluginClient."+opName)
ctx, span := m.tracer.Start(ctx, "PluginClient."+opName, trace.WithAttributes(
// Attach some plugin context information to span
attribute.String("plugin_id", pluginContext.PluginID),
attribute.Int64("org_id", pluginContext.OrgID),
))
// Attach some plugin context information to span
span.SetAttributes("plugin_id", pluginContext.PluginID, attribute.String("plugin_id", pluginContext.PluginID))
span.SetAttributes("org_id", pluginContext.OrgID, attribute.Int64("org_id", pluginContext.OrgID))
if settings := pluginContext.DataSourceInstanceSettings; settings != nil {
span.SetAttributes("datasource_name", settings.Name, attribute.Key("datasource_name").String(settings.Name))
span.SetAttributes("datasource_uid", settings.UID, attribute.Key("datasource_uid").String(settings.UID))
span.SetAttributes(attribute.String("datasource_name", settings.Name))
span.SetAttributes(attribute.String("datasource_uid", settings.UID))
}
if u := pluginContext.User; u != nil {
span.SetAttributes("user", u.Login, attribute.String("user", u.Login))
span.SetAttributes(attribute.String("user", u.Login))
}
// Additional attributes from http headers
if reqCtx := contexthandler.FromContext(ctx); reqCtx != nil && reqCtx.Req != nil && len(reqCtx.Req.Header) > 0 {
if v, err := strconv.Atoi(reqCtx.Req.Header.Get(query.HeaderPanelID)); err == nil {
span.SetAttributes("panel_id", v, attribute.Key("panel_id").Int(v))
span.SetAttributes(attribute.Int("panel_id", v))
}
setSpanAttributeFromHTTPHeader(reqCtx.Req.Header, span, "query_group_id", query.HeaderQueryGroupID)
setSpanAttributeFromHTTPHeader(reqCtx.Req.Header, span, "dashboard_uid", query.HeaderDashboardUID)

40
pkg/services/searchV2/index.go
View File

@ -15,6 +15,7 @@ import (
"github.com/blugelabs/bluge"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
@ -466,8 +467,9 @@ func (i *searchIndex) reportSizeOfIndexDiskBackup(orgID int64) {
}
func (i *searchIndex) buildOrgIndex(ctx context.Context, orgID int64) (int, error) {
spanCtx, span := i.tracer.Start(ctx, "searchV2 buildOrgIndex")
span.SetAttributes("org_id", orgID, attribute.Key("org_id").Int64(orgID))
spanCtx, span := i.tracer.Start(ctx, "searchV2 buildOrgIndex", trace.WithAttributes(
attribute.Int64("org_id", orgID),
))
started := time.Now()
ctx, cancel := context.WithTimeout(spanCtx, time.Minute)
@ -489,9 +491,10 @@ func (i *searchIndex) buildOrgIndex(ctx context.Context, orgID int64) (int, erro
dashboardExtender := i.extender.GetDashboardExtender(orgID)
_, initOrgIndexSpan := i.tracer.Start(ctx, "searchV2 buildOrgIndex init org index")
initOrgIndexSpan.SetAttributes("org_id", orgID, attribute.Key("org_id").Int64(orgID))
initOrgIndexSpan.SetAttributes("dashboardCount", len(dashboards), attribute.Key("dashboardCount").Int(len(dashboards)))
_, initOrgIndexSpan := i.tracer.Start(ctx, "searchV2 buildOrgIndex init org index", trace.WithAttributes(
attribute.Int64("org_id", orgID),
attribute.Int("dashboardCount", len(dashboards)),
))
index, err := initOrgIndex(dashboards, i.logger, dashboardExtender)
@ -837,10 +840,11 @@ func (l sqlDashboardLoader) loadAllDashboards(ctx context.Context, limit int, or
default:
}
dashboardQueryCtx, dashboardQuerySpan := l.tracer.Start(ctx, "sqlDashboardLoader dashboardQuery")
dashboardQuerySpan.SetAttributes("orgID", orgID, attribute.Key("orgID").Int64(orgID))
dashboardQuerySpan.SetAttributes("dashboardUID", dashboardUID, attribute.Key("dashboardUID").String(dashboardUID))
dashboardQuerySpan.SetAttributes("lastID", lastID, attribute.Key("lastID").Int64(lastID))
dashboardQueryCtx, dashboardQuerySpan := l.tracer.Start(ctx, "sqlDashboardLoader dashboardQuery", trace.WithAttributes(
attribute.Int64("orgID", orgID),
attribute.String("dashboardUID", dashboardUID),
attribute.Int64("lastID", lastID),
))
rows := make([]*dashboardQueryResult, 0)
err := l.sql.WithDbSession(dashboardQueryCtx, func(sess *db.Session) error {
@ -887,9 +891,9 @@ func (l sqlDashboardLoader) loadAllDashboards(ctx context.Context, limit int, or
}
func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, dashboardUID string) ([]dashboard, error) {
ctx, span := l.tracer.Start(ctx, "sqlDashboardLoader LoadDashboards")
span.SetAttributes("orgID", orgID, attribute.Key("orgID").Int64(orgID))
ctx, span := l.tracer.Start(ctx, "sqlDashboardLoader LoadDashboards", trace.WithAttributes(
attribute.Int64("orgID", orgID),
))
defer span.End()
var dashboards []dashboard
@ -901,8 +905,9 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
dashboards = make([]dashboard, 0, limit)
}
loadDatasourceCtx, loadDatasourceSpan := l.tracer.Start(ctx, "sqlDashboardLoader LoadDatasourceLookup")
loadDatasourceSpan.SetAttributes("orgID", orgID, attribute.Key("orgID").Int64(orgID))
loadDatasourceCtx, loadDatasourceSpan := l.tracer.Start(ctx, "sqlDashboardLoader LoadDatasourceLookup", trace.WithAttributes(
attribute.Int64("orgID", orgID),
))
// key will allow name or uid
lookup, err := kdash.LoadDatasourceLookup(loadDatasourceCtx, orgID, l.sql)
@ -930,9 +935,10 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
rows := res.dashboards
_, readDashboardSpan := l.tracer.Start(ctx, "sqlDashboardLoader readDashboard")
readDashboardSpan.SetAttributes("orgID", orgID, attribute.Key("orgID").Int64(orgID))
readDashboardSpan.SetAttributes("dashboardCount", len(rows), attribute.Key("dashboardCount").Int(len(rows)))
_, readDashboardSpan := l.tracer.Start(ctx, "sqlDashboardLoader readDashboard", trace.WithAttributes(
attribute.Int64("orgID", orgID),
attribute.Int("dashboardCount", len(rows)),
))
reader := kdash.NewStaticDashboardSummaryBuilder(lookup, false)

6
pkg/services/sqlstore/database_wrapper.go
View File

@ -13,6 +13,7 @@ import (
"github.com/lib/pq"
"github.com/mattn/go-sqlite3"
"github.com/prometheus/client_golang/prometheus"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"xorm.io/core"
@ -100,10 +101,11 @@ func (h *databaseQueryWrapper) instrument(ctx context.Context, status string, qu
_, span := h.tracer.Start(ctx, "database query", trace.WithTimestamp(begin))
defer span.End()
span.AddEvents([]string{"query", "status"}, []tracing.EventValue{{Str: query}, {Str: status}})
span.AddEvent("query", trace.WithAttributes(attribute.String("query", query)))
span.AddEvent("status", trace.WithAttributes(attribute.String("status", status)))
if err != nil {
span.AddEvents([]string{"error"}, []tracing.EventValue{{Str: err.Error()}})
span.RecordError(err)
}
ctxLogger := h.log.FromContext(ctx)

5
pkg/services/sqlstore/session.go
View File

@ -9,6 +9,7 @@ import (
"github.com/mattn/go-sqlite3"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"xorm.io/xorm"
"github.com/grafana/grafana/pkg/infra/log"
@ -37,7 +38,7 @@ func (sess *DBSession) PublishAfterCommit(msg any) {
sess.events = append(sess.events, msg)
}
func startSessionOrUseExisting(ctx context.Context, engine *xorm.Engine, beginTran bool, tracer tracing.Tracer) (*DBSession, bool, tracing.Span, error) {
func startSessionOrUseExisting(ctx context.Context, engine *xorm.Engine, beginTran bool, tracer tracing.Tracer) (*DBSession, bool, trace.Span, error) {
value := ctx.Value(ContextSessionKey{})
var sess *DBSession
sess, ok := value.(*DBSession)
@ -50,7 +51,7 @@ func startSessionOrUseExisting(ctx context.Context, engine *xorm.Engine, beginTr
}
tctx, span := tracer.Start(ctx, "open session")
span.SetAttributes("transaction", beginTran, attribute.Key("transaction").Bool(beginTran))
span.SetAttributes(attribute.Bool("transaction", beginTran))
newSess := &DBSession{Session: engine.NewSession(), transactionOpen: beginTran}

25
pkg/tsdb/azuremonitor/loganalytics/azure-log-analytics-datasource.go
View File

@ -19,6 +19,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"k8s.io/utils/strings/slices"
"github.com/grafana/grafana/pkg/infra/tracing"
@ -283,13 +284,13 @@ func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *A
return nil, err
}
ctx, span := tracer.Start(ctx, "azure log analytics query")
span.SetAttributes("target", query.Query, attribute.Key("target").String(query.Query))
span.SetAttributes("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond), attribute.Key("from").Int64(query.TimeRange.From.UnixNano()/int64(time.Millisecond)))
span.SetAttributes("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond), attribute.Key("until").Int64(query.TimeRange.To.UnixNano()/int64(time.Millisecond)))
span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID))
span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID))
ctx, span := tracer.Start(ctx, "azure log analytics query", trace.WithAttributes(
attribute.String("target", query.Query),
attribute.Int64("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond)),
attribute.Int64("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond)),
attribute.Int64("datasource_id", dsInfo.DatasourceID),
attribute.Int64("org_id", dsInfo.OrgID),
))
defer span.End()
tracer.Inject(ctx, req.Header, span)
@ -559,11 +560,11 @@ func getCorrelationWorkspaces(ctx context.Context, baseResource string, resource
req.URL.RawQuery = values.Encode()
req.Method = "GET"
ctx, span := tracer.Start(ctx, "azure traces correlation request")
span.SetAttributes("target", req.URL, attribute.Key("target").String(req.URL.String()))
span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID))
span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID))
ctx, span := tracer.Start(ctx, "azure traces correlation request", trace.WithAttributes(
attribute.String("target", req.URL.String()),
attribute.Int64("datasource_id", dsInfo.DatasourceID),
attribute.Int64("org_id", dsInfo.OrgID),
))
defer span.End()
tracer.Inject(ctx, req.Header, span)

27
pkg/tsdb/azuremonitor/metrics/azuremonitor-datasource.go
View File

@ -16,6 +16,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/featuremgmt"
@ -252,12 +253,13 @@ func (e *AzureMonitorDatasource) retrieveSubscriptionDetails(cli *http.Client, c
values.Add("api-version", "2022-12-01")
req.URL.RawQuery = values.Encode()
ctx, span := tracer.Start(ctx, "azuremonitor query")
span.SetAttributes("subscription", subscriptionId, attribute.Key("subscription").String(subscriptionId))
span.SetAttributes("datasource_id", dsId, attribute.Key("datasource_id").Int64(dsId))
span.SetAttributes("org_id", orgId, attribute.Key("org_id").Int64(orgId))
ctx, span := tracer.Start(ctx, "azuremonitor query", trace.WithAttributes(
attribute.String("subscription", subscriptionId),
attribute.Int64("datasource_id", dsId),
attribute.Int64("org_id", orgId),
))
defer span.End()
tracer.Inject(ctx, req.Header, span)
res, err := cli.Do(req)
if err != nil {
@ -302,14 +304,15 @@ func (e *AzureMonitorDatasource) executeQuery(ctx context.Context, query *types.
req.Body = io.NopCloser(strings.NewReader(fmt.Sprintf(`{"filter": "%s"}`, query.BodyFilter)))
}
ctx, span := tracer.Start(ctx, "azuremonitor query")
span.SetAttributes("target", query.Target, attribute.Key("target").String(query.Target))
span.SetAttributes("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond), attribute.Key("from").Int64(query.TimeRange.From.UnixNano()/int64(time.Millisecond)))
span.SetAttributes("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond), attribute.Key("until").Int64(query.TimeRange.To.UnixNano()/int64(time.Millisecond)))
span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID))
span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID))
ctx, span := tracer.Start(ctx, "azuremonitor query", trace.WithAttributes(
attribute.String("target", query.Target),
attribute.Int64("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond)),
attribute.Int64("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond)),
attribute.Int64("datasource_id", dsInfo.DatasourceID),
attribute.Int64("org_id", dsInfo.OrgID),
))
defer span.End()
tracer.Inject(ctx, req.Header, span)
res, err := cli.Do(req)

15
pkg/tsdb/azuremonitor/resourcegraph/azure-resource-graph-datasource.go
View File

@ -14,6 +14,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/tsdb/azuremonitor/kinds/dataquery"
@ -150,13 +151,13 @@ func (e *AzureResourceGraphDatasource) executeQuery(ctx context.Context, query *
req.URL.Path = path.Join(req.URL.Path, argQueryProviderName)
req.URL.RawQuery = params.Encode()
ctx, span := tracer.Start(ctx, "azure resource graph query")
span.SetAttributes("interpolated_query", query.InterpolatedQuery, attribute.Key("interpolated_query").String(query.InterpolatedQuery))
span.SetAttributes("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond), attribute.Key("from").Int64(query.TimeRange.From.UnixNano()/int64(time.Millisecond)))
span.SetAttributes("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond), attribute.Key("until").Int64(query.TimeRange.To.UnixNano()/int64(time.Millisecond)))
span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID))
span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID))
ctx, span := tracer.Start(ctx, "azure resource graph query", trace.WithAttributes(
attribute.String("interpolated_query", query.InterpolatedQuery),
attribute.Int64("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond)),
attribute.Int64("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond)),
attribute.Int64("datasource_id", dsInfo.DatasourceID),
attribute.Int64("org_id", dsInfo.OrgID),
))
defer span.End()
tracer.Inject(ctx, req.Header, span)

16
pkg/tsdb/cloud-monitoring/utils.go
View File

@ -16,6 +16,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
@ -124,13 +125,14 @@ func doRequestWithPagination(ctx context.Context, logger log.Logger, r *http.Req
return d, nil
}
func traceReq(ctx context.Context, tracer tracing.Tracer, req *backend.QueryDataRequest, dsInfo datasourceInfo, r *http.Request, target string) tracing.Span {
ctx, span := tracer.Start(ctx, "cloudMonitoring query")
span.SetAttributes("target", target, attribute.Key("target").String(target))
span.SetAttributes("from", req.Queries[0].TimeRange.From, attribute.Key("from").String(req.Queries[0].TimeRange.From.String()))
span.SetAttributes("until", req.Queries[0].TimeRange.To, attribute.Key("until").String(req.Queries[0].TimeRange.To.String()))
span.SetAttributes("datasource_id", dsInfo.id, attribute.Key("datasource_id").Int64(dsInfo.id))
span.SetAttributes("org_id", req.PluginContext.OrgID, attribute.Key("org_id").Int64(req.PluginContext.OrgID))
func traceReq(ctx context.Context, tracer tracing.Tracer, req *backend.QueryDataRequest, dsInfo datasourceInfo, r *http.Request, target string) trace.Span {
ctx, span := tracer.Start(ctx, "cloudMonitoring query", trace.WithAttributes(
attribute.String("target", target),
attribute.String("from", req.Queries[0].TimeRange.From.String()),
attribute.String("until", req.Queries[0].TimeRange.To.String()),
attribute.Int64("datasource_id", dsInfo.id),
attribute.Int64("org_id", req.PluginContext.OrgID),
))
tracer.Inject(ctx, r.Header, span)
return span
}

8
pkg/tsdb/elasticsearch/client/client.go
View File

@ -16,6 +16,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
@ -171,9 +172,10 @@ func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearch
var err error
multiRequests := c.createMultiSearchRequests(r.Requests)
queryParams := c.getMultiSearchQueryParameters()
_, span := c.tracer.Start(c.ctx, "datasource.elasticsearch.queryData.executeMultisearch")
span.SetAttributes("queryParams", queryParams, attribute.Key("queryParams").String(queryParams))
span.SetAttributes("url", c.ds.URL, attribute.Key("url").String(c.ds.URL))
_, span := c.tracer.Start(c.ctx, "datasource.elasticsearch.queryData.executeMultisearch", trace.WithAttributes(
attribute.String("queryParams", queryParams),
attribute.String("url", c.ds.URL),
))
defer func() {
if err != nil {
span.RecordError(err)

11
pkg/tsdb/elasticsearch/response_parser.go
View File

@ -15,6 +15,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
@ -52,13 +53,15 @@ func parseResponse(ctx context.Context, responses []*es.SearchResponse, targets
if responses == nil {
return &result, nil
}
ctx, span := tracer.Start(ctx, "datasource.elastic.parseResponse")
span.SetAttributes("responseLength", len(responses), attribute.Key("responseLength").Int(len(responses)))
ctx, span := tracer.Start(ctx, "datasource.elastic.parseResponse", trace.WithAttributes(
attribute.Int("responseLength", len(responses)),
))
defer span.End()
for i, res := range responses {
_, resSpan := tracer.Start(ctx, "datasource.elastic.parseResponse.response")
resSpan.SetAttributes("queryMetricType", targets[i].Metrics[0].Type, attribute.Key("queryMetricType").String(targets[i].Metrics[0].Type))
_, resSpan := tracer.Start(ctx, "datasource.elastic.parseResponse.response", trace.WithAttributes(
attribute.String("queryMetricType", targets[i].Metrics[0].Type),
))
start := time.Now()
target := targets[i]

15
pkg/tsdb/graphite/graphite.go
View File

@ -143,17 +143,18 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
defer span.End()
targetStr := strings.Join(formData["target"], ",")
span.SetAttributes("target", targetStr, attribute.Key("target").String(targetStr))
span.SetAttributes("from", from, attribute.Key("from").String(from))
span.SetAttributes("until", until, attribute.Key("until").String(until))
span.SetAttributes("datasource_id", dsInfo.Id, attribute.Key("datasource_id").Int64(dsInfo.Id))
span.SetAttributes("org_id", req.PluginContext.OrgID, attribute.Key("org_id").Int64(req.PluginContext.OrgID))
span.SetAttributes(
attribute.String("target", targetStr),
attribute.String("from", from),
attribute.String("until", until),
attribute.Int64("datasource_id", dsInfo.Id),
attribute.Int64("org_id", req.PluginContext.OrgID),
)
s.tracer.Inject(ctx, graphiteReq.Header, span)
res, err := dsInfo.HTTPClient.Do(graphiteReq)
if res != nil {
span.SetAttributes("graphite.response.code", res.StatusCode, attribute.Key("graphite.response.code").Int(res.StatusCode))
span.SetAttributes(attribute.Int("graphite.response.code", res.StatusCode))
}
if err != nil {
span.RecordError(err)

6
pkg/tsdb/loki/api.go
View File

@ -17,6 +17,7 @@ import (
jsoniter "github.com/json-iterator/go"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
@ -197,8 +198,9 @@ func (api *LokiAPI) DataQuery(ctx context.Context, query lokiQuery, responseOpts
}
start = time.Now()
_, span := api.tracer.Start(ctx, "datasource.loki.parseResponse")
span.SetAttributes("metricDataplane", responseOpts.metricDataplane, attribute.Key("metricDataplane").Bool(responseOpts.metricDataplane))
_, span := api.tracer.Start(ctx, "datasource.loki.parseResponse", trace.WithAttributes(
attribute.Bool("metricDataplane", responseOpts.metricDataplane),
))
defer span.End()
iter := jsoniter.Parse(jsoniter.ConfigDefault, resp.Body, 1024)

28
pkg/tsdb/loki/loki.go
View File

@ -17,6 +17,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
@ -134,8 +135,9 @@ func callResource(ctx context.Context, req *backend.CallResourceRequest, sender
}
lokiURL := fmt.Sprintf("/loki/api/v1/%s", url)
ctx, span := tracer.Start(ctx, "datasource.loki.CallResource")
span.SetAttributes("url", lokiURL, attribute.Key("url").String(lokiURL))
ctx, span := tracer.Start(ctx, "datasource.loki.CallResource", trace.WithAttributes(
attribute.String("url", lokiURL),
))
defer span.End()
api := newLokiAPI(dsInfo.HTTPClient, dsInfo.URL, plog, tracer)
@ -192,11 +194,12 @@ func queryData(ctx context.Context, req *backend.QueryDataRequest, dsInfo *datas
plog.Info("Prepared request to Loki", "duration", time.Since(start), "queriesLength", len(queries), "stage", stagePrepareRequest, "runInParallel", runInParallel)
ctx, span := tracer.Start(ctx, "datasource.loki.queryData.runQueries")
span.SetAttributes("runInParallel", runInParallel, attribute.Key("runInParallel").Bool(runInParallel))
span.SetAttributes("queriesLength", len(queries), attribute.Key("queriesLength").Int((len(queries))))
ctx, span := tracer.Start(ctx, "datasource.loki.queryData.runQueries", trace.WithAttributes(
attribute.Bool("runInParallel", runInParallel),
attribute.Int("queriesLength", len(queries)),
))
if req.GetHTTPHeader("X-Query-Group-Id") != "" {
span.SetAttributes("query_group_id", req.GetHTTPHeader("X-Query-Group-Id"), attribute.Key("query_group_id").String(req.GetHTTPHeader("X-Query-Group-Id")))
span.SetAttributes(attribute.String("query_group_id", req.GetHTTPHeader("X-Query-Group-Id")))
}
defer span.End()
start = time.Now()
@ -224,13 +227,14 @@ func queryData(ctx context.Context, req *backend.QueryDataRequest, dsInfo *datas
}
func executeQuery(ctx context.Context, query *lokiQuery, req *backend.QueryDataRequest, runInParallel bool, api *LokiAPI, responseOpts ResponseOpts, tracer tracing.Tracer, plog log.Logger) backend.DataResponse {
ctx, span := tracer.Start(ctx, "datasource.loki.queryData.runQueries.runQuery")
span.SetAttributes("runInParallel", runInParallel, attribute.Key("runInParallel").Bool(runInParallel))
span.SetAttributes("expr", query.Expr, attribute.Key("expr").String(query.Expr))
span.SetAttributes("start_unixnano", query.Start, attribute.Key("start_unixnano").Int64(query.Start.UnixNano()))
span.SetAttributes("stop_unixnano", query.End, attribute.Key("stop_unixnano").Int64(query.End.UnixNano()))
ctx, span := tracer.Start(ctx, "datasource.loki.queryData.runQueries.runQuery", trace.WithAttributes(
attribute.Bool("runInParallel", runInParallel),
attribute.String("expr", query.Expr),
attribute.Int64("start_unixnano", query.Start.UnixNano()),
attribute.Int64("stop_unixnano", query.End.UnixNano()),
))
if req.GetHTTPHeader("X-Query-Group-Id") != "" {
span.SetAttributes("query_group_id", req.GetHTTPHeader("X-Query-Group-Id"), attribute.Key("query_group_id").String(req.GetHTTPHeader("X-Query-Group-Id")))
span.SetAttributes(attribute.String("query_group_id", req.GetHTTPHeader("X-Query-Group-Id")))
}
defer span.End()

10
pkg/tsdb/prometheus/querydata/request.go
View File

@ -212,9 +212,9 @@ func (s *QueryData) exemplarQuery(ctx context.Context, c *client.Client, q *mode
}
func (s *QueryData) trace(ctx context.Context, q *models.Query) (context.Context, func()) {
return utils.StartTrace(ctx, s.tracer, "datasource.prometheus", []utils.Attribute{
{Key: "expr", Value: q.Expr, Kv: attribute.Key("expr").String(q.Expr)},
{Key: "start_unixnano", Value: q.Start, Kv: attribute.Key("start_unixnano").Int64(q.Start.UnixNano())},
{Key: "stop_unixnano", Value: q.End, Kv: attribute.Key("stop_unixnano").Int64(q.End.UnixNano())},
})
return utils.StartTrace(ctx, s.tracer, "datasource.prometheus",
attribute.String("expr", q.Expr),
attribute.Int64("start_unixnano", q.Start.UnixNano()),
attribute.Int64("stop_unixnano", q.End.UnixNano()),
)
}

4
pkg/tsdb/prometheus/querydata/response.go
View File

@ -25,7 +25,7 @@ func (s *QueryData) parseResponse(ctx context.Context, q *models.Query, res *htt
}
}()
ctx, endSpan := utils.StartTrace(ctx, s.tracer, "datasource.prometheus.parseResponse", []utils.Attribute{})
ctx, endSpan := utils.StartTrace(ctx, s.tracer, "datasource.prometheus.parseResponse")
defer endSpan()
iter := jsoniter.Parse(jsoniter.ConfigDefault, res.Body, 1024)
@ -54,7 +54,7 @@ func (s *QueryData) parseResponse(ctx context.Context, q *models.Query, res *htt
}
func (s *QueryData) processExemplars(ctx context.Context, q *models.Query, dr backend.DataResponse) backend.DataResponse {
_, endSpan := utils.StartTrace(ctx, s.tracer, "datasource.prometheus.processExemplars", []utils.Attribute{})
_, endSpan := utils.StartTrace(ctx, s.tracer, "datasource.prometheus.processExemplars")
defer endSpan()
sampler := s.exemplarSampler()
labelTracker := exemplar.NewLabelTracker()

14
pkg/tsdb/prometheus/utils/utils.go
View File

@ -7,6 +7,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/tracing"
)
@ -22,21 +23,12 @@ func GetJsonData(settings backend.DataSourceInstanceSettings) (map[string]any, e
return jsonData, nil
}
type Attribute struct {
Key string
Value any
Kv attribute.KeyValue
}
// StartTrace setups a trace but does not panic if tracer is nil which helps with testing
func StartTrace(ctx context.Context, tracer tracing.Tracer, name string, attributes []Attribute) (context.Context, func()) {
func StartTrace(ctx context.Context, tracer tracing.Tracer, name string, attributes ...attribute.KeyValue) (context.Context, func()) {
if tracer == nil {
return ctx, func() {}
}
ctx, span := tracer.Start(ctx, name)
for _, attr := range attributes {
span.SetAttributes(attr.Key, attr.Value, attr.Kv)
}
ctx, span := tracer.Start(ctx, name, trace.WithAttributes(attributes...))
return ctx, func() {
span.End()
}

2
pkg/tsdb/tempo/trace_transform.go
View File

@ -10,7 +10,7 @@ import (
"go.opentelemetry.io/collector/pdata/pcommon"
"go.opentelemetry.io/collector/pdata/ptrace"
"go.opentelemetry.io/otel/attribute"
semconv "go.opentelemetry.io/otel/semconv/v1.15.0"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
)
type KeyValue struct {

3
pkg/util/proxyutil/reverse_proxy_test.go
View File

@ -180,7 +180,8 @@ func TestReverseProxy(t *testing.T) {
{status: 599, expectedSource: requestmeta.StatusSourceDownstream},
}
for _, tc := range testCases {
for _, testCase := range testCases {
tc := testCase
t.Run(fmt.Sprintf("status %d => source %s ", tc.status, tc.expectedSource), func(t *testing.T) {
upstream := newUpstreamServer(t, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(tc.status)