IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2026-08-13 06:34:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

Tracing: Standardize on otel tracing (#75528)

Browse Source
This commit is contained in:
Marcus Efraimsson
2023-10-03 14:54:20 +02:00
committed by GitHub
parent 4432c4c75c
commit e4c1a7a141
46 changed files with 321 additions and 439 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/tsdb/azuremonitor/loganalytics/azure-log-analytics-datasource.go
+13 -12
View File
@@ -19,6 +19,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"k8s.io/utils/strings/slices"
"github.com/grafana/grafana/pkg/infra/tracing"
@@ -283,13 +284,13 @@ func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *A
return nil, err
}
ctx, span := tracer.Start(ctx, "azure log analytics query")
span.SetAttributes("target", query.Query, attribute.Key("target").String(query.Query))
span.SetAttributes("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond), attribute.Key("from").Int64(query.TimeRange.From.UnixNano()/int64(time.Millisecond)))
span.SetAttributes("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond), attribute.Key("until").Int64(query.TimeRange.To.UnixNano()/int64(time.Millisecond)))
span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID))
span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID))
ctx, span := tracer.Start(ctx, "azure log analytics query", trace.WithAttributes(
attribute.String("target", query.Query),
attribute.Int64("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond)),
attribute.Int64("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond)),
attribute.Int64("datasource_id", dsInfo.DatasourceID),
attribute.Int64("org_id", dsInfo.OrgID),
))
defer span.End()
tracer.Inject(ctx, req.Header, span)
@@ -559,11 +560,11 @@ func getCorrelationWorkspaces(ctx context.Context, baseResource string, resource
req.URL.RawQuery = values.Encode()
req.Method = "GET"
ctx, span := tracer.Start(ctx, "azure traces correlation request")
span.SetAttributes("target", req.URL, attribute.Key("target").String(req.URL.String()))
span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID))
span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID))
ctx, span := tracer.Start(ctx, "azure traces correlation request", trace.WithAttributes(
attribute.String("target", req.URL.String()),
attribute.Int64("datasource_id", dsInfo.DatasourceID),
attribute.Int64("org_id", dsInfo.OrgID),
))
defer span.End()
tracer.Inject(ctx, req.Header, span)
pkg/tsdb/azuremonitor/metrics/azuremonitor-datasource.go
+15 -12
View File
@@ -16,6 +16,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/featuremgmt"
@@ -252,12 +253,13 @@ func (e *AzureMonitorDatasource) retrieveSubscriptionDetails(cli *http.Client, c
values.Add("api-version", "2022-12-01")
req.URL.RawQuery = values.Encode()
ctx, span := tracer.Start(ctx, "azuremonitor query")
span.SetAttributes("subscription", subscriptionId, attribute.Key("subscription").String(subscriptionId))
span.SetAttributes("datasource_id", dsId, attribute.Key("datasource_id").Int64(dsId))
span.SetAttributes("org_id", orgId, attribute.Key("org_id").Int64(orgId))
ctx, span := tracer.Start(ctx, "azuremonitor query", trace.WithAttributes(
attribute.String("subscription", subscriptionId),
attribute.Int64("datasource_id", dsId),
attribute.Int64("org_id", orgId),
))
defer span.End()
tracer.Inject(ctx, req.Header, span)
res, err := cli.Do(req)
if err != nil {
@@ -302,14 +304,15 @@ func (e *AzureMonitorDatasource) executeQuery(ctx context.Context, query *types.
req.Body = io.NopCloser(strings.NewReader(fmt.Sprintf(`{"filter": "%s"}`, query.BodyFilter)))
}
ctx, span := tracer.Start(ctx, "azuremonitor query")
span.SetAttributes("target", query.Target, attribute.Key("target").String(query.Target))
span.SetAttributes("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond), attribute.Key("from").Int64(query.TimeRange.From.UnixNano()/int64(time.Millisecond)))
span.SetAttributes("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond), attribute.Key("until").Int64(query.TimeRange.To.UnixNano()/int64(time.Millisecond)))
span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID))
span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID))
ctx, span := tracer.Start(ctx, "azuremonitor query", trace.WithAttributes(
attribute.String("target", query.Target),
attribute.Int64("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond)),
attribute.Int64("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond)),
attribute.Int64("datasource_id", dsInfo.DatasourceID),
attribute.Int64("org_id", dsInfo.OrgID),
))
defer span.End()
tracer.Inject(ctx, req.Header, span)
res, err := cli.Do(req)
pkg/tsdb/azuremonitor/resourcegraph/azure-resource-graph-datasource.go
+8 -7
View File
@@ -14,6 +14,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/tsdb/azuremonitor/kinds/dataquery"
@@ -150,13 +151,13 @@ func (e *AzureResourceGraphDatasource) executeQuery(ctx context.Context, query *
req.URL.Path = path.Join(req.URL.Path, argQueryProviderName)
req.URL.RawQuery = params.Encode()
ctx, span := tracer.Start(ctx, "azure resource graph query")
span.SetAttributes("interpolated_query", query.InterpolatedQuery, attribute.Key("interpolated_query").String(query.InterpolatedQuery))
span.SetAttributes("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond), attribute.Key("from").Int64(query.TimeRange.From.UnixNano()/int64(time.Millisecond)))
span.SetAttributes("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond), attribute.Key("until").Int64(query.TimeRange.To.UnixNano()/int64(time.Millisecond)))
span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID))
span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID))
ctx, span := tracer.Start(ctx, "azure resource graph query", trace.WithAttributes(
attribute.String("interpolated_query", query.InterpolatedQuery),
attribute.Int64("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond)),
attribute.Int64("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond)),
attribute.Int64("datasource_id", dsInfo.DatasourceID),
attribute.Int64("org_id", dsInfo.OrgID),
))
defer span.End()
tracer.Inject(ctx, req.Header, span)
pkg/tsdb/cloud-monitoring/utils.go
+9 -7
View File
@@ -16,6 +16,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
@@ -124,13 +125,14 @@ func doRequestWithPagination(ctx context.Context, logger log.Logger, r *http.Req
return d, nil
}
func traceReq(ctx context.Context, tracer tracing.Tracer, req *backend.QueryDataRequest, dsInfo datasourceInfo, r *http.Request, target string) tracing.Span {
ctx, span := tracer.Start(ctx, "cloudMonitoring query")
span.SetAttributes("target", target, attribute.Key("target").String(target))
span.SetAttributes("from", req.Queries[0].TimeRange.From, attribute.Key("from").String(req.Queries[0].TimeRange.From.String()))
span.SetAttributes("until", req.Queries[0].TimeRange.To, attribute.Key("until").String(req.Queries[0].TimeRange.To.String()))
span.SetAttributes("datasource_id", dsInfo.id, attribute.Key("datasource_id").Int64(dsInfo.id))
span.SetAttributes("org_id", req.PluginContext.OrgID, attribute.Key("org_id").Int64(req.PluginContext.OrgID))
func traceReq(ctx context.Context, tracer tracing.Tracer, req *backend.QueryDataRequest, dsInfo datasourceInfo, r *http.Request, target string) trace.Span {
ctx, span := tracer.Start(ctx, "cloudMonitoring query", trace.WithAttributes(
attribute.String("target", target),
attribute.String("from", req.Queries[0].TimeRange.From.String()),
attribute.String("until", req.Queries[0].TimeRange.To.String()),
attribute.Int64("datasource_id", dsInfo.id),
attribute.Int64("org_id", req.PluginContext.OrgID),
))
tracer.Inject(ctx, r.Header, span)
return span
}
pkg/tsdb/elasticsearch/client/client.go
+5 -3
View File
@@ -16,6 +16,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
@@ -171,9 +172,10 @@ func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearch
var err error
multiRequests := c.createMultiSearchRequests(r.Requests)
queryParams := c.getMultiSearchQueryParameters()
_, span := c.tracer.Start(c.ctx, "datasource.elasticsearch.queryData.executeMultisearch")
span.SetAttributes("queryParams", queryParams, attribute.Key("queryParams").String(queryParams))
span.SetAttributes("url", c.ds.URL, attribute.Key("url").String(c.ds.URL))
_, span := c.tracer.Start(c.ctx, "datasource.elasticsearch.queryData.executeMultisearch", trace.WithAttributes(
attribute.String("queryParams", queryParams),
attribute.String("url", c.ds.URL),
))
defer func() {
if err != nil {
span.RecordError(err)
pkg/tsdb/elasticsearch/response_parser.go
+7 -4
View File
@@ -15,6 +15,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
@@ -52,13 +53,15 @@ func parseResponse(ctx context.Context, responses []*es.SearchResponse, targets
if responses == nil {
return &result, nil
}
ctx, span := tracer.Start(ctx, "datasource.elastic.parseResponse")
span.SetAttributes("responseLength", len(responses), attribute.Key("responseLength").Int(len(responses)))
ctx, span := tracer.Start(ctx, "datasource.elastic.parseResponse", trace.WithAttributes(
attribute.Int("responseLength", len(responses)),
))
defer span.End()
for i, res := range responses {
_, resSpan := tracer.Start(ctx, "datasource.elastic.parseResponse.response")
resSpan.SetAttributes("queryMetricType", targets[i].Metrics[0].Type, attribute.Key("queryMetricType").String(targets[i].Metrics[0].Type))
_, resSpan := tracer.Start(ctx, "datasource.elastic.parseResponse.response", trace.WithAttributes(
attribute.String("queryMetricType", targets[i].Metrics[0].Type),
))
start := time.Now()
target := targets[i]
pkg/tsdb/graphite/graphite.go
+8 -7
View File
@@ -143,17 +143,18 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
defer span.End()
targetStr := strings.Join(formData["target"], ",")
span.SetAttributes("target", targetStr, attribute.Key("target").String(targetStr))
span.SetAttributes("from", from, attribute.Key("from").String(from))
span.SetAttributes("until", until, attribute.Key("until").String(until))
span.SetAttributes("datasource_id", dsInfo.Id, attribute.Key("datasource_id").Int64(dsInfo.Id))
span.SetAttributes("org_id", req.PluginContext.OrgID, attribute.Key("org_id").Int64(req.PluginContext.OrgID))
span.SetAttributes(
attribute.String("target", targetStr),
attribute.String("from", from),
attribute.String("until", until),
attribute.Int64("datasource_id", dsInfo.Id),
attribute.Int64("org_id", req.PluginContext.OrgID),
)
s.tracer.Inject(ctx, graphiteReq.Header, span)
res, err := dsInfo.HTTPClient.Do(graphiteReq)
if res != nil {
span.SetAttributes("graphite.response.code", res.StatusCode, attribute.Key("graphite.response.code").Int(res.StatusCode))
span.SetAttributes(attribute.Int("graphite.response.code", res.StatusCode))
}
if err != nil {
span.RecordError(err)
pkg/tsdb/loki/api.go
+4 -2
View File
@@ -17,6 +17,7 @@ import (
jsoniter "github.com/json-iterator/go"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
@@ -197,8 +198,9 @@ func (api *LokiAPI) DataQuery(ctx context.Context, query lokiQuery, responseOpts
}
start = time.Now()
_, span := api.tracer.Start(ctx, "datasource.loki.parseResponse")
span.SetAttributes("metricDataplane", responseOpts.metricDataplane, attribute.Key("metricDataplane").Bool(responseOpts.metricDataplane))
_, span := api.tracer.Start(ctx, "datasource.loki.parseResponse", trace.WithAttributes(
attribute.Bool("metricDataplane", responseOpts.metricDataplane),
))
defer span.End()
iter := jsoniter.Parse(jsoniter.ConfigDefault, resp.Body, 1024)
pkg/tsdb/loki/loki.go
+16 -12
View File
@@ -17,6 +17,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
@@ -134,8 +135,9 @@ func callResource(ctx context.Context, req *backend.CallResourceRequest, sender
}
lokiURL := fmt.Sprintf("/loki/api/v1/%s", url)
ctx, span := tracer.Start(ctx, "datasource.loki.CallResource")
span.SetAttributes("url", lokiURL, attribute.Key("url").String(lokiURL))
ctx, span := tracer.Start(ctx, "datasource.loki.CallResource", trace.WithAttributes(
attribute.String("url", lokiURL),
))
defer span.End()
api := newLokiAPI(dsInfo.HTTPClient, dsInfo.URL, plog, tracer)
@@ -192,11 +194,12 @@ func queryData(ctx context.Context, req *backend.QueryDataRequest, dsInfo *datas
plog.Info("Prepared request to Loki", "duration", time.Since(start), "queriesLength", len(queries), "stage", stagePrepareRequest, "runInParallel", runInParallel)
ctx, span := tracer.Start(ctx, "datasource.loki.queryData.runQueries")
span.SetAttributes("runInParallel", runInParallel, attribute.Key("runInParallel").Bool(runInParallel))
span.SetAttributes("queriesLength", len(queries), attribute.Key("queriesLength").Int((len(queries))))
ctx, span := tracer.Start(ctx, "datasource.loki.queryData.runQueries", trace.WithAttributes(
attribute.Bool("runInParallel", runInParallel),
attribute.Int("queriesLength", len(queries)),
))
if req.GetHTTPHeader("X-Query-Group-Id") != "" {
span.SetAttributes("query_group_id", req.GetHTTPHeader("X-Query-Group-Id"), attribute.Key("query_group_id").String(req.GetHTTPHeader("X-Query-Group-Id")))
span.SetAttributes(attribute.String("query_group_id", req.GetHTTPHeader("X-Query-Group-Id")))
}
defer span.End()
start = time.Now()
@@ -224,13 +227,14 @@ func queryData(ctx context.Context, req *backend.QueryDataRequest, dsInfo *datas
}
func executeQuery(ctx context.Context, query *lokiQuery, req *backend.QueryDataRequest, runInParallel bool, api *LokiAPI, responseOpts ResponseOpts, tracer tracing.Tracer, plog log.Logger) backend.DataResponse {
ctx, span := tracer.Start(ctx, "datasource.loki.queryData.runQueries.runQuery")
span.SetAttributes("runInParallel", runInParallel, attribute.Key("runInParallel").Bool(runInParallel))
span.SetAttributes("expr", query.Expr, attribute.Key("expr").String(query.Expr))
span.SetAttributes("start_unixnano", query.Start, attribute.Key("start_unixnano").Int64(query.Start.UnixNano()))
span.SetAttributes("stop_unixnano", query.End, attribute.Key("stop_unixnano").Int64(query.End.UnixNano()))
ctx, span := tracer.Start(ctx, "datasource.loki.queryData.runQueries.runQuery", trace.WithAttributes(
attribute.Bool("runInParallel", runInParallel),
attribute.String("expr", query.Expr),
attribute.Int64("start_unixnano", query.Start.UnixNano()),
attribute.Int64("stop_unixnano", query.End.UnixNano()),
))
if req.GetHTTPHeader("X-Query-Group-Id") != "" {
span.SetAttributes("query_group_id", req.GetHTTPHeader("X-Query-Group-Id"), attribute.Key("query_group_id").String(req.GetHTTPHeader("X-Query-Group-Id")))
span.SetAttributes(attribute.String("query_group_id", req.GetHTTPHeader("X-Query-Group-Id")))
}
defer span.End()
pkg/tsdb/prometheus/querydata/request.go
+5 -5
View File
@@ -212,9 +212,9 @@ func (s *QueryData) exemplarQuery(ctx context.Context, c *client.Client, q *mode
}
func (s *QueryData) trace(ctx context.Context, q *models.Query) (context.Context, func()) {
return utils.StartTrace(ctx, s.tracer, "datasource.prometheus", []utils.Attribute{
{Key: "expr", Value: q.Expr, Kv: attribute.Key("expr").String(q.Expr)},
{Key: "start_unixnano", Value: q.Start, Kv: attribute.Key("start_unixnano").Int64(q.Start.UnixNano())},
{Key: "stop_unixnano", Value: q.End, Kv: attribute.Key("stop_unixnano").Int64(q.End.UnixNano())},
})
return utils.StartTrace(ctx, s.tracer, "datasource.prometheus",
attribute.String("expr", q.Expr),
attribute.Int64("start_unixnano", q.Start.UnixNano()),
attribute.Int64("stop_unixnano", q.End.UnixNano()),
)
}
pkg/tsdb/prometheus/querydata/response.go
+2 -2
View File
@@ -25,7 +25,7 @@ func (s *QueryData) parseResponse(ctx context.Context, q *models.Query, res *htt
}
}()
ctx, endSpan := utils.StartTrace(ctx, s.tracer, "datasource.prometheus.parseResponse", []utils.Attribute{})
ctx, endSpan := utils.StartTrace(ctx, s.tracer, "datasource.prometheus.parseResponse")
defer endSpan()
iter := jsoniter.Parse(jsoniter.ConfigDefault, res.Body, 1024)
@@ -54,7 +54,7 @@ func (s *QueryData) parseResponse(ctx context.Context, q *models.Query, res *htt
}
func (s *QueryData) processExemplars(ctx context.Context, q *models.Query, dr backend.DataResponse) backend.DataResponse {
_, endSpan := utils.StartTrace(ctx, s.tracer, "datasource.prometheus.processExemplars", []utils.Attribute{})
_, endSpan := utils.StartTrace(ctx, s.tracer, "datasource.prometheus.processExemplars")
defer endSpan()
sampler := s.exemplarSampler()
labelTracker := exemplar.NewLabelTracker()
pkg/tsdb/prometheus/utils/utils.go
+3 -11
View File
@@ -7,6 +7,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/grafana/grafana/pkg/infra/tracing"
)
@@ -22,21 +23,12 @@ func GetJsonData(settings backend.DataSourceInstanceSettings) (map[string]any, e
return jsonData, nil
}
type Attribute struct {
Key string
Value any
Kv attribute.KeyValue
}
// StartTrace setups a trace but does not panic if tracer is nil which helps with testing
func StartTrace(ctx context.Context, tracer tracing.Tracer, name string, attributes []Attribute) (context.Context, func()) {
func StartTrace(ctx context.Context, tracer tracing.Tracer, name string, attributes ...attribute.KeyValue) (context.Context, func()) {
if tracer == nil {
return ctx, func() {}
}
ctx, span := tracer.Start(ctx, name)
for _, attr := range attributes {
span.SetAttributes(attr.Key, attr.Value, attr.Kv)
}
ctx, span := tracer.Start(ctx, name, trace.WithAttributes(attributes...))
return ctx, func() {
span.End()
}
pkg/tsdb/tempo/trace_transform.go
+1 -1
View File
@@ -10,7 +10,7 @@ import (
"go.opentelemetry.io/collector/pdata/pcommon"
"go.opentelemetry.io/collector/pdata/ptrace"
"go.opentelemetry.io/otel/attribute"
semconv "go.opentelemetry.io/otel/semconv/v1.15.0"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
)
type KeyValue struct {