From c7b6822a5e9cba703e521536947682aaf801203e Mon Sep 17 00:00:00 2001 From: Carl Bergquist Date: Mon, 11 Nov 2024 12:53:24 +0100 Subject: [PATCH] Loki: Apply scopes to expression if feature toggle is enabled. (#95917) Signed-off-by: bergquist Co-authored-by: Kyle Brandt --- .../src/types/featureToggles.gen.ts | 1 + pkg/services/featuremgmt/registry.go | 7 ++++ pkg/services/featuremgmt/toggles_gen.csv | 1 + pkg/services/featuremgmt/toggles_gen.go | 4 ++ pkg/services/featuremgmt/toggles_gen.json | 13 +++++++ pkg/tsdb/loki/loki.go | 12 +++--- pkg/tsdb/loki/parse_query.go | 10 ++++- pkg/tsdb/loki/parse_query_test.go | 39 +++++++++++++++++-- pkg/tsdb/loki/types.go | 2 + 9 files changed, 79 insertions(+), 10 deletions(-) diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 1403f805ffa..a6c2360c2f6 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -162,6 +162,7 @@ export interface FeatureToggles { onPremToCloudMigrationsAlerts?: boolean; alertingSaveStatePeriodic?: boolean; promQLScope?: boolean; + logQLScope?: boolean; sqlExpressions?: boolean; nodeGraphDotLayout?: boolean; groupToNestedTableTransformation?: boolean; diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 52b74b826b2..0de68f4b330 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -1086,6 +1086,13 @@ var ( Owner: grafanaObservabilityMetricsSquad, Expression: "true", }, + { + Name: "logQLScope", + Description: "In-development feature that will allow injection of labels into loki queries.", + Stage: FeatureStagePrivatePreview, + Owner: grafanaObservabilityLogsSquad, + Expression: "false", + }, { Name: "sqlExpressions", Description: "Enables using SQL and DuckDB functions as Expressions.", diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 34708eea3fc..4c8c3cb9857 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -143,6 +143,7 @@ onPremToCloudMigrations,preview,@grafana/grafana-operator-experience-squad,false onPremToCloudMigrationsAlerts,experimental,@grafana/grafana-operator-experience-squad,false,false,false alertingSaveStatePeriodic,privatePreview,@grafana/alerting-squad,false,false,false promQLScope,GA,@grafana/observability-metrics,false,false,false +logQLScope,privatePreview,@grafana/observability-logs,false,false,false sqlExpressions,experimental,@grafana/grafana-app-platform-squad,false,false,false nodeGraphDotLayout,experimental,@grafana/observability-traces-and-profiling,false,false,true groupToNestedTableTransformation,GA,@grafana/dataviz-squad,false,false,true diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 5b6dd231dc6..ef2b4e9e01c 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -583,6 +583,10 @@ const ( // In-development feature that will allow injection of labels into prometheus queries. FlagPromQLScope = "promQLScope" + // FlagLogQLScope + // In-development feature that will allow injection of labels into loki queries. + FlagLogQLScope = "logQLScope" + // FlagSqlExpressions // Enables using SQL and DuckDB functions as Expressions. FlagSqlExpressions = "sqlExpressions" diff --git a/pkg/services/featuremgmt/toggles_gen.json b/pkg/services/featuremgmt/toggles_gen.json index fa9f0263bab..613800f5641 100644 --- a/pkg/services/featuremgmt/toggles_gen.json +++ b/pkg/services/featuremgmt/toggles_gen.json @@ -1865,6 +1865,19 @@ "frontend": true } }, + { + "metadata": { + "name": "logQLScope", + "resourceVersion": "1730842404843", + "creationTimestamp": "2024-11-05T21:33:24Z" + }, + "spec": { + "description": "In-development feature that will allow injection of labels into loki queries.", + "stage": "privatePreview", + "codeowner": "@grafana/observability-logs", + "expression": "false" + } + }, { "metadata": { "name": "logRequestsInstrumentedAsUnknown", diff --git a/pkg/tsdb/loki/loki.go b/pkg/tsdb/loki/loki.go index d23985d7b54..a290c73ce1b 100644 --- a/pkg/tsdb/loki/loki.go +++ b/pkg/tsdb/loki/loki.go @@ -22,6 +22,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient" "github.com/grafana/grafana/pkg/infra/tracing" + "github.com/grafana/grafana/pkg/promlib/models" "github.com/grafana/grafana/pkg/services/contexthandler" "github.com/grafana/grafana/pkg/services/featuremgmt" ngalertmodels "github.com/grafana/grafana/pkg/services/ngalert/models" @@ -70,8 +71,9 @@ type datasourceInfo struct { type QueryJSONModel struct { dataquery.LokiDataQuery - Direction *string `json:"direction,omitempty"` - SupportingQueryType *string `json:"supportingQueryType"` + Direction *string `json:"direction,omitempty"` + SupportingQueryType *string `json:"supportingQueryType"` + Scopes []models.ScopeFilter `json:"scopes"` } type ResponseOpts struct { @@ -168,7 +170,7 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) s.applyHeaders(ctx, req) } - return queryData(ctx, req, dsInfo, responseOpts, s.tracer, logger, isFeatureEnabled(ctx, featuremgmt.FlagLokiRunQueriesInParallel), isFeatureEnabled(ctx, featuremgmt.FlagLokiStructuredMetadata)) + return queryData(ctx, req, dsInfo, responseOpts, s.tracer, logger, isFeatureEnabled(ctx, featuremgmt.FlagLokiRunQueriesInParallel), isFeatureEnabled(ctx, featuremgmt.FlagLokiStructuredMetadata), isFeatureEnabled(ctx, featuremgmt.FlagLogQLScope)) } func (s *Service) applyHeaders(ctx context.Context, req backend.ForwardHTTPHeaders) { @@ -188,13 +190,13 @@ func (s *Service) applyHeaders(ctx context.Context, req backend.ForwardHTTPHeade } } -func queryData(ctx context.Context, req *backend.QueryDataRequest, dsInfo *datasourceInfo, responseOpts ResponseOpts, tracer tracing.Tracer, plog log.Logger, runInParallel bool, requestStructuredMetadata bool) (*backend.QueryDataResponse, error) { +func queryData(ctx context.Context, req *backend.QueryDataRequest, dsInfo *datasourceInfo, responseOpts ResponseOpts, tracer tracing.Tracer, plog log.Logger, runInParallel bool, requestStructuredMetadata, logQLScopes bool) (*backend.QueryDataResponse, error) { result := backend.NewQueryDataResponse() api := newLokiAPI(dsInfo.HTTPClient, dsInfo.URL, plog, tracer, requestStructuredMetadata) start := time.Now() - queries, err := parseQuery(req) + queries, err := parseQuery(req, logQLScopes) if err != nil { plog.Error("Failed to prepare request to Loki", "error", err, "duration", time.Since(start), "queriesLength", len(queries), "stage", stagePrepareRequest) return result, err diff --git a/pkg/tsdb/loki/parse_query.go b/pkg/tsdb/loki/parse_query.go index d824bf596dc..7efed0cab83 100644 --- a/pkg/tsdb/loki/parse_query.go +++ b/pkg/tsdb/loki/parse_query.go @@ -125,7 +125,7 @@ func parseSupportingQueryType(jsonPointerValue *string) SupportingQueryType { } } -func parseQuery(queryContext *backend.QueryDataRequest) ([]*lokiQuery, error) { +func parseQuery(queryContext *backend.QueryDataRequest, logqlScopesEnabled bool) ([]*lokiQuery, error) { qs := []*lokiQuery{} for _, query := range queryContext.Queries { model, err := parseQueryModel(query.JSON) @@ -171,6 +171,13 @@ func parseQuery(queryContext *backend.QueryDataRequest) ([]*lokiQuery, error) { legendFormat = *model.LegendFormat } + if logqlScopesEnabled { + rewrittenExpr, err := ApplyScopes(expr, model.Scopes) + if err == nil { + expr = rewrittenExpr + } + } + supportingQueryType := parseSupportingQueryType(model.SupportingQueryType) qs = append(qs, &lokiQuery{ @@ -184,6 +191,7 @@ func parseQuery(queryContext *backend.QueryDataRequest) ([]*lokiQuery, error) { End: end, RefID: query.RefID, SupportingQueryType: supportingQueryType, + Scopes: model.Scopes, }) } diff --git a/pkg/tsdb/loki/parse_query_test.go b/pkg/tsdb/loki/parse_query_test.go index 50a111c489a..497f1c880c8 100644 --- a/pkg/tsdb/loki/parse_query_test.go +++ b/pkg/tsdb/loki/parse_query_test.go @@ -30,7 +30,7 @@ func TestParseQuery(t *testing.T) { }, }, } - models, err := parseQuery(queryContext) + models, err := parseQuery(queryContext, false) require.NoError(t, err) require.Equal(t, time.Second*15, models[0].Step) require.Equal(t, "go_goroutines 15s 15000 3000s 3000 3000000", models[0].Expr) @@ -57,7 +57,7 @@ func TestParseQuery(t *testing.T) { }, }, } - models, err := parseQuery(queryContext) + models, err := parseQuery(queryContext, false) require.NoError(t, err) require.Equal(t, SupportingQueryLogsVolume, models[0].SupportingQueryType) }) @@ -83,7 +83,7 @@ func TestParseQuery(t *testing.T) { }, }, } - models, err := parseQuery(queryContext) + models, err := parseQuery(queryContext, false) require.NoError(t, err) require.Equal(t, SupportingQueryType("foo"), models[0].SupportingQueryType) }) @@ -109,11 +109,42 @@ func TestParseQuery(t *testing.T) { }, }, } - models, err := parseQuery(queryContext) + models, err := parseQuery(queryContext, false) require.NoError(t, err) require.Equal(t, SupportingQueryNone, models[0].SupportingQueryType) }) + t.Run("parsing query model with scopes", func(t *testing.T) { + queryContext := &backend.QueryDataRequest{ + Queries: []backend.DataQuery{ + { + JSON: []byte(` + { + "expr": "{} |= \"problems\"", + "format": "time_series", + "refId": "A", + "scopes": [{"key": "namespace", "value": "logish", "operator": "equals"}] + }`, + ), + TimeRange: backend.TimeRange{ + From: time.Now().Add(-3000 * time.Second), + To: time.Now(), + }, + Interval: time.Second * 15, + MaxDataPoints: 200, + }, + }, + } + models, err := parseQuery(queryContext, true) + require.NoError(t, err) + require.Equal(t, time.Second*15, models[0].Step) + require.Equal(t, 1, len(models[0].Scopes)) + require.Equal(t, "namespace", models[0].Scopes[0].Key) + require.Equal(t, "logish", models[0].Scopes[0].Value) + require.Equal(t, "equals", string(models[0].Scopes[0].Operator)) + require.Equal(t, `{namespace="logish"} |= "problems"`, models[0].Expr) + }) + t.Run("interpolate variables, range between 1s and 0.5s", func(t *testing.T) { expr := "go_goroutines $__interval $__interval_ms $__range $__range_s $__range_ms" queryType := dataquery.LokiQueryTypeRange diff --git a/pkg/tsdb/loki/types.go b/pkg/tsdb/loki/types.go index 5ac5ee514c3..d6d5d7ce1ca 100644 --- a/pkg/tsdb/loki/types.go +++ b/pkg/tsdb/loki/types.go @@ -3,6 +3,7 @@ package loki import ( "time" + "github.com/grafana/grafana/pkg/promlib/models" "github.com/grafana/grafana/pkg/tsdb/loki/kinds/dataquery" ) @@ -39,4 +40,5 @@ type lokiQuery struct { End time.Time RefID string SupportingQueryType SupportingQueryType + Scopes []models.ScopeFilter }