IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Prometheus: Add BE support for Adhoc Filters (#85969)

Browse Source 
---------

Co-authored-by: ismail simsek <ismailsimsek09@gmail.com>
This commit is contained in:
Kyle Brandt 2024-04-16 09:56:50 -04:00 committed by GitHub
parent 8520892923
commit a12669951b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 309 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
packages/grafana-data/src/types/scopes.ts
View File

@ -3,10 +3,19 @@ export interface ScopeDashboardBindingSpec {
scope: string;
}
export type ScopeFilterOperator = 'equals' | 'not-equals' | 'regex-match' | 'regex-not-match';
export const scopeFilterOperatorMap: Record<string, ScopeFilterOperator> = {
'=': 'equals',
'!=': 'not-equals',
'=~': 'regex-match',
'!~': 'regex-not-match',
};
export interface ScopeSpecFilter {
key: string;
value: string;
operator: string;
operator: ScopeFilterOperator;
}
export interface ScopeSpec {

3
packages/grafana-prometheus/src/dataquery.ts
View File

@ -1,5 +1,5 @@
// Core Grafana history https://github.com/grafana/grafana/blob/v11.0.0-preview/public/app/plugins/datasource/prometheus/dataquery.ts
import { ScopeSpec } from '@grafana/data';
import { ScopeSpec, ScopeSpecFilter } from '@grafana/data';
import * as common from '@grafana/schema';
export enum QueryEditorMode {
@ -44,4 +44,5 @@ export interface Prometheus extends common.DataQuery {
*/
range?: boolean;
scope?: ScopeSpec;
adhocFilters?: ScopeSpecFilter[];
}

42
packages/grafana-prometheus/src/datasource.test.ts
View File

@ -3,6 +3,7 @@ import { cloneDeep } from 'lodash';
import { lastValueFrom, of } from 'rxjs';
import {
AdHocVariableFilter,
AnnotationEvent,
AnnotationQueryRequest,
CoreApp,
@ -12,6 +13,7 @@ import {
dateTime,
LoadingState,
rangeUtil,
ScopeSpecFilter,
TimeRange,
VariableHide,
} from '@grafana/data';
@ -1168,6 +1170,46 @@ describe('modifyQuery', () => {
expect(result.expr).toEqual('go_goroutines{cluster="us-cluster", pod!="pod-123"}');
});
});
describe('scope filters', () => {
const instanceSettings = {
access: 'proxy',
id: 1,
jsonData: {},
name: 'scoped-prom',
readOnly: false,
type: 'prometheus',
uid: 'scoped-prom',
} as unknown as DataSourceInstanceSettings<PromOptions>;
const ds = new PrometheusDatasource(instanceSettings, templateSrvStub);
it('should convert each adhoc operator to scope operator properly', () => {
const adhocFilter: AdHocVariableFilter[] = [
{ key: 'eq', value: 'eqv', operator: '=' },
{
key: 'neq',
value: 'neqv',
operator: '!=',
},
{ key: 'reg', value: 'regv', operator: '=~' },
{ key: 'nreg', value: 'nregv', operator: '!~' },
];
const expectedScopeFilter: ScopeSpecFilter[] = [
{ key: 'eq', value: 'eqv', operator: 'equals' },
{
key: 'neq',
value: 'neqv',
operator: 'not-equals',
},
{ key: 'reg', value: 'regv', operator: 'regex-match' },
{ key: 'nreg', value: 'nregv', operator: 'regex-not-match' },
];
const result = ds.generateScopeFilters(adhocFilter);
result.forEach((r, i) => {
expect(r).toEqual(expectedScopeFilter[i]);
});
});
});
});
});

21
packages/grafana-prometheus/src/datasource.ts
View File

@ -26,6 +26,8 @@ import {
rangeUtil,
renderLegendFormat,
ScopedVars,
scopeFilterOperatorMap,
ScopeSpecFilter,
TimeRange,
} from '@grafana/data';
import {
@ -478,8 +480,13 @@ export class PrometheusDatasource
let expr = target.expr;
if (config.featureToggles.promQLScope) {
// Apply scope filters
query.adhocFilters = this.generateScopeFilters(options.filters);
} else {
// Apply adhoc filters
expr = this.enhanceExprWithAdHocFilters(options.filters, expr);
}
// Only replace vars in expression after having (possibly) updated interval vars
query.expr = this.templateSrv.replace(expr, scopedVars, this.interpolateQueryExpr);
@ -494,6 +501,18 @@ export class PrometheusDatasource
return query;
}
/**
* This converts the adhocVariableFilter array and converts it to scopeFilter array
* @param filters
*/
generateScopeFilters(filters?: AdHocVariableFilter[]): ScopeSpecFilter[] {
if (!filters) {
return [];
}
return filters.map((f) => ({ ...f, operator: scopeFilterOperatorMap[f.operator] }));
}
getRateIntervalScopedVariable(interval: number, scrapeInterval: number) {
// Fall back to the default scrape interval of 15s if scrapeInterval is 0 for some reason.
if (scrapeInterval === 0) {
@ -736,6 +755,7 @@ export class PrometheusDatasource
const expandedQuery = {
...query,
...(config.featureToggles.promQLScope ? { adhocFilters: this.generateScopeFilters(filters) } : {}),
datasource: this.getRef(),
expr: withAdhocFilters,
interval: this.templateSrv.replace(query.interval, scopedVars),
@ -906,6 +926,7 @@ export class PrometheusDatasource
return {
...target,
...(config.featureToggles.promQLScope ? { adhocFilters: this.generateScopeFilters(filters) } : {}),
expr: exprWithAdHocFilters,
interval: this.templateSrv.replace(target.interval, variables),
legendFormat: this.templateSrv.replace(target.legendFormat, variables),

16
pkg/promlib/models/query.go
View File

@ -63,8 +63,11 @@ type PrometheusQueryProperties struct {
// Series name override or template. Ex. {{hostname}} will be replaced with label value for hostname
LegendFormat string `json:"legendFormat,omitempty"`
// ???
// A set of filters applied to apply to the query
Scope *ScopeSpec `json:"scope,omitempty"`
// Additional Ad-hoc filters that take precedence over Scope on conflict.
AdhocFilters []ScopeFilter `json:"adhocFilters,omitempty"`
}
// ScopeSpec is a hand copy of the ScopeSpec struct from pkg/apis/scope/v0alpha1/types.go
@ -188,12 +191,19 @@ func Parse(query backend.DataQuery, dsScrapeInterval string, intervalCalculator
dsScrapeInterval,
timeRange,
)
if enableScope && model.Scope != nil && len(model.Scope.Filters) > 0 {
expr, err = ApplyQueryScope(expr, *model.Scope)
if enableScope {
var scopeFilters []ScopeFilter
if model.Scope != nil {
scopeFilters = model.Scope.Filters
}
expr, err = ApplyQueryFilters(expr, scopeFilters, model.AdhocFilters)
if err != nil {
return nil, err
}
}
if !model.Instant && !model.Range {
// In older dashboards, we were not setting range query param and !range && !instant was run as range query
model.Range = true

27
pkg/promlib/models/query.panel.schema.json
View File

@ -14,6 +14,31 @@
"expr"
],
"properties": {
"adhocFilters": {
"description": "Additional Ad-hoc filters that take precedence over Scope on conflict.",
"type": "array",
"items": {
"description": "ScopeFilter is a hand copy of the ScopeFilter struct from pkg/apis/scope/v0alpha1/types.go to avoid import (temp fix)",
"type": "object",
"required": [
"key",
"value",
"operator"
],
"properties": {
"key": {
"type": "string"
},
"operator": {
"type": "string"
},
"value": {
"type": "string"
}
},
"additionalProperties": false
}
},
"datasource": {
"description": "The datasource",
"type": "object",
@ -138,7 +163,7 @@
"additionalProperties": false
},
"scope": {
"description": "???",
"description": "A set of filters applied to apply to the query",
"type": "object",
"required": [
"title",

27
pkg/promlib/models/query.request.schema.json
View File

@ -24,6 +24,31 @@
"expr"
],
"properties": {
"adhocFilters": {
"description": "Additional Ad-hoc filters that take precedence over Scope on conflict.",
"type": "array",
"items": {
"description": "ScopeFilter is a hand copy of the ScopeFilter struct from pkg/apis/scope/v0alpha1/types.go to avoid import (temp fix)",
"type": "object",
"required": [
"key",
"value",
"operator"
],
"properties": {
"key": {
"type": "string"
},
"operator": {
"type": "string"
},
"value": {
"type": "string"
}
},
"additionalProperties": false
}
},
"datasource": {
"description": "The datasource",
"type": "object",
@ -148,7 +173,7 @@
"additionalProperties": false
},
"scope": {
"description": "???",
"description": "A set of filters applied to apply to the query",
"type": "object",
"required": [
"title",

29
pkg/promlib/models/query.types.json
View File

@ -8,7 +8,7 @@
{
"metadata": {
"name": "default",
"resourceVersion": "1711374012365",
"resourceVersion": "1713187448137",
"creationTimestamp": "2024-03-25T13:19:04Z"
},
"spec": {
@ -17,6 +17,31 @@
"additionalProperties": false,
"description": "PrometheusQueryProperties defines the specific properties used for prometheus",
"properties": {
"adhocFilters": {
"description": "Additional Ad-hoc filters that take precedence over Scope on conflict.",
"items": {
"additionalProperties": false,
"description": "ScopeFilter is a hand copy of the ScopeFilter struct from pkg/apis/scope/v0alpha1/types.go to avoid import (temp fix)",
"properties": {
"key": {
"type": "string"
},
"operator": {
"type": "string"
},
"value": {
"type": "string"
}
},
"required": [
"key",
"value",
"operator"
],
"type": "object"
},
"type": "array"
},
"editorMode": {
"description": "what we should show in the editor\n\n\nPossible enum values:\n - `\"builder\"` \n - `\"code\"` ",
"enum": [
@ -62,7 +87,7 @@
},
"scope": {
"additionalProperties": false,
"description": "???",
"description": "A set of filters applied to apply to the query",
"properties": {
"category": {
"type": "string"

35
pkg/promlib/models/scope.go
View File

@ -7,13 +7,13 @@ import (
"github.com/prometheus/prometheus/promql/parser"
)
func ApplyQueryScope(rawExpr string, scope ScopeSpec) (string, error) {
func ApplyQueryFilters(rawExpr string, scopeFilters, adHocFilters []ScopeFilter) (string, error) {
expr, err := parser.ParseExpr(rawExpr)
if err != nil {
return "", err
}
matchers, err := scopeFiltersToMatchers(scope.Filters)
matchers, err := filtersToMatchers(scopeFilters, adHocFilters)
if err != nil {
return "", err
}
@ -58,9 +58,26 @@ func ApplyQueryScope(rawExpr string, scope ScopeSpec) (string, error) {
return expr.String(), nil
}
func scopeFiltersToMatchers(filters []ScopeFilter) ([]*labels.Matcher, error) {
matchers := make([]*labels.Matcher, 0, len(filters))
for _, f := range filters {
func filtersToMatchers(scopeFilters, adhocFilters []ScopeFilter) ([]*labels.Matcher, error) {
filterMap := make(map[string]*labels.Matcher)
for _, filter := range append(scopeFilters, adhocFilters...) {
matcher, err := filterToMatcher(filter)
if err != nil {
return nil, err
}
filterMap[filter.Key] = matcher
}
matchers := make([]*labels.Matcher, 0, len(filterMap))
for _, matcher := range filterMap {
matchers = append(matchers, matcher)
}
return matchers, nil
}
func filterToMatcher(f ScopeFilter) (*labels.Matcher, error) {
var mt labels.MatchType
switch f.Operator {
case FilterOperatorEquals:
@ -74,11 +91,5 @@ func scopeFiltersToMatchers(filters []ScopeFilter) ([]*labels.Matcher, error) {
default:
return nil, fmt.Errorf("unknown operator %q", f.Operator)
}
m, err := labels.NewMatcher(mt, f.Key, f.Value)
if err != nil {
return nil, err
}
matchers = append(matchers, m)
}
return matchers, nil
return labels.NewMatcher(mt, f.Key, f.Value)
}

105
pkg/promlib/models/scope_test.go Normal file
View File

@ -0,0 +1,105 @@
package models
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestApplyQueryFilters(t *testing.T) {
tests := []struct {
name string
query string
adhocFilters []ScopeFilter
scopeFilters []ScopeFilter
expected string
expectErr bool
}{
{
name: "No filters with no existing filter",
query: `http_requests_total`,
expected: `http_requests_total`,
expectErr: false,
},
{
name: "No filters with existing filter",
query: `http_requests_total{job="prometheus"}`,
expected: `http_requests_total{job="prometheus"}`,
expectErr: false,
},
{
name: "Adhoc filter with existing filter",
query: `http_requests_total{job="prometheus"}`,
adhocFilters: []ScopeFilter{
{Key: "method", Value: "get", Operator: FilterOperatorEquals},
},
expected: `http_requests_total{job="prometheus",method="get"}`,
expectErr: false,
},
{
name: "Adhoc filter with no existing filter",
query: `http_requests_total`,
adhocFilters: []ScopeFilter{
{Key: "method", Value: "get", Operator: FilterOperatorEquals},
{Key: "job", Value: "prometheus", Operator: FilterOperatorEquals},
},
expected: `http_requests_total{job="prometheus",method="get"}`,
expectErr: false,
},
{
name: "Scope filter",
query: `http_requests_total{job="prometheus"}`,
scopeFilters: []ScopeFilter{
{Key: "status", Value: "200", Operator: FilterOperatorEquals},
},
expected: `http_requests_total{job="prometheus",status="200"}`,
expectErr: false,
},
{
name: "Adhoc and Scope filter no existing filter",
query: `http_requests_total`,
scopeFilters: []ScopeFilter{
{Key: "status", Value: "200", Operator: FilterOperatorEquals},
},
adhocFilters: []ScopeFilter{
{Key: "job", Value: "prometheus", Operator: FilterOperatorEquals},
},
expected: `http_requests_total{job="prometheus",status="200"}`,
expectErr: false,
},
{
name: "Adhoc and Scope filter conflict - adhoc wins",
query: `http_requests_total{job="prometheus"}`,
scopeFilters: []ScopeFilter{
{Key: "status", Value: "404", Operator: FilterOperatorEquals},
},
adhocFilters: []ScopeFilter{
{Key: "status", Value: "200", Operator: FilterOperatorEquals},
},
expected: `http_requests_total{job="prometheus",status="200"}`,
expectErr: false,
},
{
name: "Adhoc filters with more complex expression",
query: `capacity_bytes{job="prometheus"} + available_bytes{job="grafana"} / 1024`,
adhocFilters: []ScopeFilter{
{Key: "job", Value: "alloy", Operator: FilterOperatorEquals},
},
expected: `capacity_bytes{job="alloy"} + available_bytes{job="alloy"} / 1024`,
expectErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
expr, err := ApplyQueryFilters(tt.query, tt.scopeFilters, tt.adhocFilters)
if tt.expectErr {
require.Error(t, err)
} else {
require.NoError(t, err)
require.Equal(t, expr, tt.expected)
}
})
}
}

8
public/app/features/dashboard-scene/scene/ScopesScene.test.tsx
View File

@ -52,8 +52,8 @@ const scopesMocks: Record<
description: 'Description 1',
category: 'Category 1',
filters: [
{ key: 'a-key', operator: '=', value: 'a-value' },
{ key: 'b-key', operator: '!=', value: 'b-value' },
{ key: 'a-key', operator: 'equals', value: 'a-value' },
{ key: 'b-key', operator: 'not-equals', value: 'b-value' },
],
},
dashboards: [dashboardsMocks.dashboard1, dashboardsMocks.dashboard2, dashboardsMocks.dashboard3],
@ -67,7 +67,7 @@ const scopesMocks: Record<
type: 'Type 2',
description: 'Description 2',
category: 'Category 2',
filters: [{ key: 'c-key', operator: '!=', value: 'c-value' }],
filters: [{ key: 'c-key', operator: 'not-equals', value: 'c-value' }],
},
dashboards: [dashboardsMocks.dashboard3],
},
@ -80,7 +80,7 @@ const scopesMocks: Record<
type: 'Type 1',
description: 'Description 3',
category: 'Category 1',
filters: [{ key: 'd-key', operator: '=', value: 'd-value' }],
filters: [{ key: 'd-key', operator: 'equals', value: 'd-value' }],
},
dashboards: [dashboardsMocks.dashboard1, dashboardsMocks.dashboard2],
},