diff --git a/packages/grafana-data/src/types/scopes.ts b/packages/grafana-data/src/types/scopes.ts index f0128c59cb7..2f9b785ca27 100644 --- a/packages/grafana-data/src/types/scopes.ts +++ b/packages/grafana-data/src/types/scopes.ts @@ -3,10 +3,19 @@ export interface ScopeDashboardBindingSpec { scope: string; } +export type ScopeFilterOperator = 'equals' | 'not-equals' | 'regex-match' | 'regex-not-match'; + +export const scopeFilterOperatorMap: Record = { + '=': 'equals', + '!=': 'not-equals', + '=~': 'regex-match', + '!~': 'regex-not-match', +}; + export interface ScopeSpecFilter { key: string; value: string; - operator: string; + operator: ScopeFilterOperator; } export interface ScopeSpec { diff --git a/packages/grafana-prometheus/src/dataquery.ts b/packages/grafana-prometheus/src/dataquery.ts index 6601b7ac8ab..c56a5f91307 100644 --- a/packages/grafana-prometheus/src/dataquery.ts +++ b/packages/grafana-prometheus/src/dataquery.ts @@ -1,5 +1,5 @@ // Core Grafana history https://github.com/grafana/grafana/blob/v11.0.0-preview/public/app/plugins/datasource/prometheus/dataquery.ts -import { ScopeSpec } from '@grafana/data'; +import { ScopeSpec, ScopeSpecFilter } from '@grafana/data'; import * as common from '@grafana/schema'; export enum QueryEditorMode { @@ -44,4 +44,5 @@ export interface Prometheus extends common.DataQuery { */ range?: boolean; scope?: ScopeSpec; + adhocFilters?: ScopeSpecFilter[]; } diff --git a/packages/grafana-prometheus/src/datasource.test.ts b/packages/grafana-prometheus/src/datasource.test.ts index a8ae4298ef1..ca64b11b4e1 100644 --- a/packages/grafana-prometheus/src/datasource.test.ts +++ b/packages/grafana-prometheus/src/datasource.test.ts @@ -3,6 +3,7 @@ import { cloneDeep } from 'lodash'; import { lastValueFrom, of } from 'rxjs'; import { + AdHocVariableFilter, AnnotationEvent, AnnotationQueryRequest, CoreApp, @@ -12,6 +13,7 @@ import { dateTime, LoadingState, rangeUtil, + ScopeSpecFilter, TimeRange, VariableHide, } from '@grafana/data'; @@ -1168,6 +1170,46 @@ describe('modifyQuery', () => { expect(result.expr).toEqual('go_goroutines{cluster="us-cluster", pod!="pod-123"}'); }); }); + + describe('scope filters', () => { + const instanceSettings = { + access: 'proxy', + id: 1, + jsonData: {}, + name: 'scoped-prom', + readOnly: false, + type: 'prometheus', + uid: 'scoped-prom', + } as unknown as DataSourceInstanceSettings; + const ds = new PrometheusDatasource(instanceSettings, templateSrvStub); + + it('should convert each adhoc operator to scope operator properly', () => { + const adhocFilter: AdHocVariableFilter[] = [ + { key: 'eq', value: 'eqv', operator: '=' }, + { + key: 'neq', + value: 'neqv', + operator: '!=', + }, + { key: 'reg', value: 'regv', operator: '=~' }, + { key: 'nreg', value: 'nregv', operator: '!~' }, + ]; + const expectedScopeFilter: ScopeSpecFilter[] = [ + { key: 'eq', value: 'eqv', operator: 'equals' }, + { + key: 'neq', + value: 'neqv', + operator: 'not-equals', + }, + { key: 'reg', value: 'regv', operator: 'regex-match' }, + { key: 'nreg', value: 'nregv', operator: 'regex-not-match' }, + ]; + const result = ds.generateScopeFilters(adhocFilter); + result.forEach((r, i) => { + expect(r).toEqual(expectedScopeFilter[i]); + }); + }); + }); }); }); diff --git a/packages/grafana-prometheus/src/datasource.ts b/packages/grafana-prometheus/src/datasource.ts index ea36501aea5..aa921678724 100644 --- a/packages/grafana-prometheus/src/datasource.ts +++ b/packages/grafana-prometheus/src/datasource.ts @@ -26,6 +26,8 @@ import { rangeUtil, renderLegendFormat, ScopedVars, + scopeFilterOperatorMap, + ScopeSpecFilter, TimeRange, } from '@grafana/data'; import { @@ -478,8 +480,13 @@ export class PrometheusDatasource let expr = target.expr; - // Apply adhoc filters - expr = this.enhanceExprWithAdHocFilters(options.filters, expr); + if (config.featureToggles.promQLScope) { + // Apply scope filters + query.adhocFilters = this.generateScopeFilters(options.filters); + } else { + // Apply adhoc filters + expr = this.enhanceExprWithAdHocFilters(options.filters, expr); + } // Only replace vars in expression after having (possibly) updated interval vars query.expr = this.templateSrv.replace(expr, scopedVars, this.interpolateQueryExpr); @@ -494,6 +501,18 @@ export class PrometheusDatasource return query; } + /** + * This converts the adhocVariableFilter array and converts it to scopeFilter array + * @param filters + */ + generateScopeFilters(filters?: AdHocVariableFilter[]): ScopeSpecFilter[] { + if (!filters) { + return []; + } + + return filters.map((f) => ({ ...f, operator: scopeFilterOperatorMap[f.operator] })); + } + getRateIntervalScopedVariable(interval: number, scrapeInterval: number) { // Fall back to the default scrape interval of 15s if scrapeInterval is 0 for some reason. if (scrapeInterval === 0) { @@ -736,6 +755,7 @@ export class PrometheusDatasource const expandedQuery = { ...query, + ...(config.featureToggles.promQLScope ? { adhocFilters: this.generateScopeFilters(filters) } : {}), datasource: this.getRef(), expr: withAdhocFilters, interval: this.templateSrv.replace(query.interval, scopedVars), @@ -906,6 +926,7 @@ export class PrometheusDatasource return { ...target, + ...(config.featureToggles.promQLScope ? { adhocFilters: this.generateScopeFilters(filters) } : {}), expr: exprWithAdHocFilters, interval: this.templateSrv.replace(target.interval, variables), legendFormat: this.templateSrv.replace(target.legendFormat, variables), diff --git a/pkg/promlib/models/query.go b/pkg/promlib/models/query.go index bb6c996cbab..4f657183cec 100644 --- a/pkg/promlib/models/query.go +++ b/pkg/promlib/models/query.go @@ -63,8 +63,11 @@ type PrometheusQueryProperties struct { // Series name override or template. Ex. {{hostname}} will be replaced with label value for hostname LegendFormat string `json:"legendFormat,omitempty"` - // ??? + // A set of filters applied to apply to the query Scope *ScopeSpec `json:"scope,omitempty"` + + // Additional Ad-hoc filters that take precedence over Scope on conflict. + AdhocFilters []ScopeFilter `json:"adhocFilters,omitempty"` } // ScopeSpec is a hand copy of the ScopeSpec struct from pkg/apis/scope/v0alpha1/types.go @@ -188,12 +191,19 @@ func Parse(query backend.DataQuery, dsScrapeInterval string, intervalCalculator dsScrapeInterval, timeRange, ) - if enableScope && model.Scope != nil && len(model.Scope.Filters) > 0 { - expr, err = ApplyQueryScope(expr, *model.Scope) + + if enableScope { + var scopeFilters []ScopeFilter + if model.Scope != nil { + scopeFilters = model.Scope.Filters + } + + expr, err = ApplyQueryFilters(expr, scopeFilters, model.AdhocFilters) if err != nil { return nil, err } } + if !model.Instant && !model.Range { // In older dashboards, we were not setting range query param and !range && !instant was run as range query model.Range = true diff --git a/pkg/promlib/models/query.panel.schema.json b/pkg/promlib/models/query.panel.schema.json index ffd2328f1de..f52bb24a968 100644 --- a/pkg/promlib/models/query.panel.schema.json +++ b/pkg/promlib/models/query.panel.schema.json @@ -14,6 +14,31 @@ "expr" ], "properties": { + "adhocFilters": { + "description": "Additional Ad-hoc filters that take precedence over Scope on conflict.", + "type": "array", + "items": { + "description": "ScopeFilter is a hand copy of the ScopeFilter struct from pkg/apis/scope/v0alpha1/types.go to avoid import (temp fix)", + "type": "object", + "required": [ + "key", + "value", + "operator" + ], + "properties": { + "key": { + "type": "string" + }, + "operator": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "additionalProperties": false + } + }, "datasource": { "description": "The datasource", "type": "object", @@ -138,7 +163,7 @@ "additionalProperties": false }, "scope": { - "description": "???", + "description": "A set of filters applied to apply to the query", "type": "object", "required": [ "title", diff --git a/pkg/promlib/models/query.request.schema.json b/pkg/promlib/models/query.request.schema.json index c8e797d1f38..979f69a6cb0 100644 --- a/pkg/promlib/models/query.request.schema.json +++ b/pkg/promlib/models/query.request.schema.json @@ -24,6 +24,31 @@ "expr" ], "properties": { + "adhocFilters": { + "description": "Additional Ad-hoc filters that take precedence over Scope on conflict.", + "type": "array", + "items": { + "description": "ScopeFilter is a hand copy of the ScopeFilter struct from pkg/apis/scope/v0alpha1/types.go to avoid import (temp fix)", + "type": "object", + "required": [ + "key", + "value", + "operator" + ], + "properties": { + "key": { + "type": "string" + }, + "operator": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "additionalProperties": false + } + }, "datasource": { "description": "The datasource", "type": "object", @@ -148,7 +173,7 @@ "additionalProperties": false }, "scope": { - "description": "???", + "description": "A set of filters applied to apply to the query", "type": "object", "required": [ "title", diff --git a/pkg/promlib/models/query.types.json b/pkg/promlib/models/query.types.json index 7ae0e10374e..95f7ab499d8 100644 --- a/pkg/promlib/models/query.types.json +++ b/pkg/promlib/models/query.types.json @@ -8,7 +8,7 @@ { "metadata": { "name": "default", - "resourceVersion": "1711374012365", + "resourceVersion": "1713187448137", "creationTimestamp": "2024-03-25T13:19:04Z" }, "spec": { @@ -17,6 +17,31 @@ "additionalProperties": false, "description": "PrometheusQueryProperties defines the specific properties used for prometheus", "properties": { + "adhocFilters": { + "description": "Additional Ad-hoc filters that take precedence over Scope on conflict.", + "items": { + "additionalProperties": false, + "description": "ScopeFilter is a hand copy of the ScopeFilter struct from pkg/apis/scope/v0alpha1/types.go to avoid import (temp fix)", + "properties": { + "key": { + "type": "string" + }, + "operator": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "key", + "value", + "operator" + ], + "type": "object" + }, + "type": "array" + }, "editorMode": { "description": "what we should show in the editor\n\n\nPossible enum values:\n - `\"builder\"` \n - `\"code\"` ", "enum": [ @@ -62,7 +87,7 @@ }, "scope": { "additionalProperties": false, - "description": "???", + "description": "A set of filters applied to apply to the query", "properties": { "category": { "type": "string" diff --git a/pkg/promlib/models/scope.go b/pkg/promlib/models/scope.go index 9207aa747a8..04c6895ae9f 100644 --- a/pkg/promlib/models/scope.go +++ b/pkg/promlib/models/scope.go @@ -7,13 +7,13 @@ import ( "github.com/prometheus/prometheus/promql/parser" ) -func ApplyQueryScope(rawExpr string, scope ScopeSpec) (string, error) { +func ApplyQueryFilters(rawExpr string, scopeFilters, adHocFilters []ScopeFilter) (string, error) { expr, err := parser.ParseExpr(rawExpr) if err != nil { return "", err } - matchers, err := scopeFiltersToMatchers(scope.Filters) + matchers, err := filtersToMatchers(scopeFilters, adHocFilters) if err != nil { return "", err } @@ -58,27 +58,38 @@ func ApplyQueryScope(rawExpr string, scope ScopeSpec) (string, error) { return expr.String(), nil } -func scopeFiltersToMatchers(filters []ScopeFilter) ([]*labels.Matcher, error) { - matchers := make([]*labels.Matcher, 0, len(filters)) - for _, f := range filters { - var mt labels.MatchType - switch f.Operator { - case FilterOperatorEquals: - mt = labels.MatchEqual - case FilterOperatorNotEquals: - mt = labels.MatchNotEqual - case FilterOperatorRegexMatch: - mt = labels.MatchRegexp - case FilterOperatorRegexNotMatch: - mt = labels.MatchNotRegexp - default: - return nil, fmt.Errorf("unknown operator %q", f.Operator) - } - m, err := labels.NewMatcher(mt, f.Key, f.Value) +func filtersToMatchers(scopeFilters, adhocFilters []ScopeFilter) ([]*labels.Matcher, error) { + filterMap := make(map[string]*labels.Matcher) + + for _, filter := range append(scopeFilters, adhocFilters...) { + matcher, err := filterToMatcher(filter) if err != nil { return nil, err } - matchers = append(matchers, m) + filterMap[filter.Key] = matcher } + + matchers := make([]*labels.Matcher, 0, len(filterMap)) + for _, matcher := range filterMap { + matchers = append(matchers, matcher) + } + return matchers, nil } + +func filterToMatcher(f ScopeFilter) (*labels.Matcher, error) { + var mt labels.MatchType + switch f.Operator { + case FilterOperatorEquals: + mt = labels.MatchEqual + case FilterOperatorNotEquals: + mt = labels.MatchNotEqual + case FilterOperatorRegexMatch: + mt = labels.MatchRegexp + case FilterOperatorRegexNotMatch: + mt = labels.MatchNotRegexp + default: + return nil, fmt.Errorf("unknown operator %q", f.Operator) + } + return labels.NewMatcher(mt, f.Key, f.Value) +} diff --git a/pkg/promlib/models/scope_test.go b/pkg/promlib/models/scope_test.go new file mode 100644 index 00000000000..a517633a292 --- /dev/null +++ b/pkg/promlib/models/scope_test.go @@ -0,0 +1,105 @@ +package models + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestApplyQueryFilters(t *testing.T) { + tests := []struct { + name string + query string + adhocFilters []ScopeFilter + scopeFilters []ScopeFilter + expected string + expectErr bool + }{ + { + name: "No filters with no existing filter", + query: `http_requests_total`, + expected: `http_requests_total`, + expectErr: false, + }, + { + name: "No filters with existing filter", + query: `http_requests_total{job="prometheus"}`, + expected: `http_requests_total{job="prometheus"}`, + expectErr: false, + }, + { + name: "Adhoc filter with existing filter", + query: `http_requests_total{job="prometheus"}`, + adhocFilters: []ScopeFilter{ + {Key: "method", Value: "get", Operator: FilterOperatorEquals}, + }, + expected: `http_requests_total{job="prometheus",method="get"}`, + expectErr: false, + }, + { + name: "Adhoc filter with no existing filter", + query: `http_requests_total`, + adhocFilters: []ScopeFilter{ + {Key: "method", Value: "get", Operator: FilterOperatorEquals}, + {Key: "job", Value: "prometheus", Operator: FilterOperatorEquals}, + }, + expected: `http_requests_total{job="prometheus",method="get"}`, + expectErr: false, + }, + { + name: "Scope filter", + query: `http_requests_total{job="prometheus"}`, + scopeFilters: []ScopeFilter{ + {Key: "status", Value: "200", Operator: FilterOperatorEquals}, + }, + expected: `http_requests_total{job="prometheus",status="200"}`, + expectErr: false, + }, + { + name: "Adhoc and Scope filter no existing filter", + query: `http_requests_total`, + scopeFilters: []ScopeFilter{ + {Key: "status", Value: "200", Operator: FilterOperatorEquals}, + }, + adhocFilters: []ScopeFilter{ + {Key: "job", Value: "prometheus", Operator: FilterOperatorEquals}, + }, + expected: `http_requests_total{job="prometheus",status="200"}`, + expectErr: false, + }, + { + name: "Adhoc and Scope filter conflict - adhoc wins", + query: `http_requests_total{job="prometheus"}`, + scopeFilters: []ScopeFilter{ + {Key: "status", Value: "404", Operator: FilterOperatorEquals}, + }, + adhocFilters: []ScopeFilter{ + {Key: "status", Value: "200", Operator: FilterOperatorEquals}, + }, + expected: `http_requests_total{job="prometheus",status="200"}`, + expectErr: false, + }, + { + name: "Adhoc filters with more complex expression", + query: `capacity_bytes{job="prometheus"} + available_bytes{job="grafana"} / 1024`, + adhocFilters: []ScopeFilter{ + {Key: "job", Value: "alloy", Operator: FilterOperatorEquals}, + }, + expected: `capacity_bytes{job="alloy"} + available_bytes{job="alloy"} / 1024`, + expectErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + expr, err := ApplyQueryFilters(tt.query, tt.scopeFilters, tt.adhocFilters) + + if tt.expectErr { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, expr, tt.expected) + } + }) + } +} diff --git a/public/app/features/dashboard-scene/scene/ScopesScene.test.tsx b/public/app/features/dashboard-scene/scene/ScopesScene.test.tsx index 72f9c2d9ee6..1a056ae88d7 100644 --- a/public/app/features/dashboard-scene/scene/ScopesScene.test.tsx +++ b/public/app/features/dashboard-scene/scene/ScopesScene.test.tsx @@ -52,8 +52,8 @@ const scopesMocks: Record< description: 'Description 1', category: 'Category 1', filters: [ - { key: 'a-key', operator: '=', value: 'a-value' }, - { key: 'b-key', operator: '!=', value: 'b-value' }, + { key: 'a-key', operator: 'equals', value: 'a-value' }, + { key: 'b-key', operator: 'not-equals', value: 'b-value' }, ], }, dashboards: [dashboardsMocks.dashboard1, dashboardsMocks.dashboard2, dashboardsMocks.dashboard3], @@ -67,7 +67,7 @@ const scopesMocks: Record< type: 'Type 2', description: 'Description 2', category: 'Category 2', - filters: [{ key: 'c-key', operator: '!=', value: 'c-value' }], + filters: [{ key: 'c-key', operator: 'not-equals', value: 'c-value' }], }, dashboards: [dashboardsMocks.dashboard3], }, @@ -80,7 +80,7 @@ const scopesMocks: Record< type: 'Type 1', description: 'Description 3', category: 'Category 1', - filters: [{ key: 'd-key', operator: '=', value: 'd-value' }], + filters: [{ key: 'd-key', operator: 'equals', value: 'd-value' }], }, dashboards: [dashboardsMocks.dashboard1, dashboardsMocks.dashboard2], },