Prometheus: Add BE support for Adhoc Filters (#85969)

--------- Co-authored-by: ismail simsek <ismailsimsek09@gmail.com>
2025-02-25 18:55:37 -06:00 · 2024-04-16 09:56:50 -04:00
parent 8520892923
commit a12669951b
11 changed files with 309 additions and 35 deletions
--- a/pkg/promlib/models/query.go
+++ b/pkg/promlib/models/query.go
@@ -63,8 +63,11 @@ type PrometheusQueryProperties struct {
 	// Series name override or template. Ex. {{hostname}} will be replaced with label value for hostname
 	LegendFormat string `json:"legendFormat,omitempty"`

-	// ???
+	// A set of filters applied to apply to the query
 	Scope *ScopeSpec `json:"scope,omitempty"`
+
+	// Additional Ad-hoc filters that take precedence over Scope on conflict.
+	AdhocFilters []ScopeFilter `json:"adhocFilters,omitempty"`
 }

 // ScopeSpec is a hand copy of the ScopeSpec struct from pkg/apis/scope/v0alpha1/types.go
@@ -188,12 +191,19 @@ func Parse(query backend.DataQuery, dsScrapeInterval string, intervalCalculator
 		dsScrapeInterval,
 		timeRange,
 	)
-	if enableScope && model.Scope != nil && len(model.Scope.Filters) > 0 {
-		expr, err = ApplyQueryScope(expr, *model.Scope)
+
+	if enableScope {
+		var scopeFilters []ScopeFilter
+		if model.Scope != nil {
+			scopeFilters = model.Scope.Filters
+		}
+
+		expr, err = ApplyQueryFilters(expr, scopeFilters, model.AdhocFilters)
 		if err != nil {
 			return nil, err
 		}
 	}
+
 	if !model.Instant && !model.Range {
 		// In older dashboards, we were not setting range query param and !range && !instant was run as range query
 		model.Range = true
--- a/pkg/promlib/models/query.panel.schema.json
+++ b/pkg/promlib/models/query.panel.schema.json
@@ -14,6 +14,31 @@
          "expr"
        ],
        "properties": {
+          "adhocFilters": {
+            "description": "Additional Ad-hoc filters that take precedence over Scope on conflict.",
+            "type": "array",
+            "items": {
+              "description": "ScopeFilter is a hand copy of the ScopeFilter struct from pkg/apis/scope/v0alpha1/types.go to avoid import (temp fix)",
+              "type": "object",
+              "required": [
+                "key",
+                "value",
+                "operator"
+              ],
+              "properties": {
+                "key": {
+                  "type": "string"
+                },
+                "operator": {
+                  "type": "string"
+                },
+                "value": {
+                  "type": "string"
+                }
+              },
+              "additionalProperties": false
+            }
+          },
          "datasource": {
            "description": "The datasource",
            "type": "object",
@@ -138,7 +163,7 @@
            "additionalProperties": false
          },
          "scope": {
-            "description": "???",
+            "description": "A set of filters applied to apply to the query",
            "type": "object",
            "required": [
              "title",
--- a/pkg/promlib/models/query.request.schema.json
+++ b/pkg/promlib/models/query.request.schema.json
@@ -24,6 +24,31 @@
          "expr"
        ],
        "properties": {
+          "adhocFilters": {
+            "description": "Additional Ad-hoc filters that take precedence over Scope on conflict.",
+            "type": "array",
+            "items": {
+              "description": "ScopeFilter is a hand copy of the ScopeFilter struct from pkg/apis/scope/v0alpha1/types.go to avoid import (temp fix)",
+              "type": "object",
+              "required": [
+                "key",
+                "value",
+                "operator"
+              ],
+              "properties": {
+                "key": {
+                  "type": "string"
+                },
+                "operator": {
+                  "type": "string"
+                },
+                "value": {
+                  "type": "string"
+                }
+              },
+              "additionalProperties": false
+            }
+          },
          "datasource": {
            "description": "The datasource",
            "type": "object",
@@ -148,7 +173,7 @@
            "additionalProperties": false
          },
          "scope": {
-            "description": "???",
+            "description": "A set of filters applied to apply to the query",
            "type": "object",
            "required": [
              "title",
--- a/pkg/promlib/models/query.types.json
+++ b/pkg/promlib/models/query.types.json
@@ -8,7 +8,7 @@
    {
      "metadata": {
        "name": "default",
-        "resourceVersion": "1711374012365",
+        "resourceVersion": "1713187448137",
        "creationTimestamp": "2024-03-25T13:19:04Z"
      },
      "spec": {
@@ -17,6 +17,31 @@
          "additionalProperties": false,
          "description": "PrometheusQueryProperties defines the specific properties used for prometheus",
          "properties": {
+            "adhocFilters": {
+              "description": "Additional Ad-hoc filters that take precedence over Scope on conflict.",
+              "items": {
+                "additionalProperties": false,
+                "description": "ScopeFilter is a hand copy of the ScopeFilter struct from pkg/apis/scope/v0alpha1/types.go to avoid import (temp fix)",
+                "properties": {
+                  "key": {
+                    "type": "string"
+                  },
+                  "operator": {
+                    "type": "string"
+                  },
+                  "value": {
+                    "type": "string"
+                  }
+                },
+                "required": [
+                  "key",
+                  "value",
+                  "operator"
+                ],
+                "type": "object"
+              },
+              "type": "array"
+            },
            "editorMode": {
              "description": "what we should show in the editor\n\n\nPossible enum values:\n - `\"builder\"` \n - `\"code\"` ",
              "enum": [
@@ -62,7 +87,7 @@
            },
            "scope": {
              "additionalProperties": false,
-              "description": "???",
+              "description": "A set of filters applied to apply to the query",
              "properties": {
                "category": {
                  "type": "string"
--- a/pkg/promlib/models/scope.go
+++ b/pkg/promlib/models/scope.go
@@ -7,13 +7,13 @@ import (
 	"github.com/prometheus/prometheus/promql/parser"
 )

-func ApplyQueryScope(rawExpr string, scope ScopeSpec) (string, error) {
+func ApplyQueryFilters(rawExpr string, scopeFilters, adHocFilters []ScopeFilter) (string, error) {
 	expr, err := parser.ParseExpr(rawExpr)
 	if err != nil {
 		return "", err
 	}

-	matchers, err := scopeFiltersToMatchers(scope.Filters)
+	matchers, err := filtersToMatchers(scopeFilters, adHocFilters)
 	if err != nil {
 		return "", err
 	}
@@ -58,27 +58,38 @@ func ApplyQueryScope(rawExpr string, scope ScopeSpec) (string, error) {
 	return expr.String(), nil
 }

-func scopeFiltersToMatchers(filters []ScopeFilter) ([]*labels.Matcher, error) {
-	matchers := make([]*labels.Matcher, 0, len(filters))
-	for _, f := range filters {
-		var mt labels.MatchType
-		switch f.Operator {
-		case FilterOperatorEquals:
-			mt = labels.MatchEqual
-		case FilterOperatorNotEquals:
-			mt = labels.MatchNotEqual
-		case FilterOperatorRegexMatch:
-			mt = labels.MatchRegexp
-		case FilterOperatorRegexNotMatch:
-			mt = labels.MatchNotRegexp
-		default:
-			return nil, fmt.Errorf("unknown operator %q", f.Operator)
-		}
-		m, err := labels.NewMatcher(mt, f.Key, f.Value)
+func filtersToMatchers(scopeFilters, adhocFilters []ScopeFilter) ([]*labels.Matcher, error) {
+	filterMap := make(map[string]*labels.Matcher)
+
+	for _, filter := range append(scopeFilters, adhocFilters...) {
+		matcher, err := filterToMatcher(filter)
 		if err != nil {
 			return nil, err
 		}
-		matchers = append(matchers, m)
+		filterMap[filter.Key] = matcher
 	}
+
+	matchers := make([]*labels.Matcher, 0, len(filterMap))
+	for _, matcher := range filterMap {
+		matchers = append(matchers, matcher)
+	}
+
 	return matchers, nil
 }
+
+func filterToMatcher(f ScopeFilter) (*labels.Matcher, error) {
+	var mt labels.MatchType
+	switch f.Operator {
+	case FilterOperatorEquals:
+		mt = labels.MatchEqual
+	case FilterOperatorNotEquals:
+		mt = labels.MatchNotEqual
+	case FilterOperatorRegexMatch:
+		mt = labels.MatchRegexp
+	case FilterOperatorRegexNotMatch:
+		mt = labels.MatchNotRegexp
+	default:
+		return nil, fmt.Errorf("unknown operator %q", f.Operator)
+	}
+	return labels.NewMatcher(mt, f.Key, f.Value)
+}
--- a/pkg/promlib/models/scope_test.go
+++ b/pkg/promlib/models/scope_test.go
@@ -0,0 +1,105 @@
+package models
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestApplyQueryFilters(t *testing.T) {
+	tests := []struct {
+		name         string
+		query        string
+		adhocFilters []ScopeFilter
+		scopeFilters []ScopeFilter
+		expected     string
+		expectErr    bool
+	}{
+		{
+			name:      "No filters with no existing filter",
+			query:     `http_requests_total`,
+			expected:  `http_requests_total`,
+			expectErr: false,
+		},
+		{
+			name:      "No filters with existing filter",
+			query:     `http_requests_total{job="prometheus"}`,
+			expected:  `http_requests_total{job="prometheus"}`,
+			expectErr: false,
+		},
+		{
+			name:  "Adhoc filter with existing filter",
+			query: `http_requests_total{job="prometheus"}`,
+			adhocFilters: []ScopeFilter{
+				{Key: "method", Value: "get", Operator: FilterOperatorEquals},
+			},
+			expected:  `http_requests_total{job="prometheus",method="get"}`,
+			expectErr: false,
+		},
+		{
+			name:  "Adhoc filter with no existing filter",
+			query: `http_requests_total`,
+			adhocFilters: []ScopeFilter{
+				{Key: "method", Value: "get", Operator: FilterOperatorEquals},
+				{Key: "job", Value: "prometheus", Operator: FilterOperatorEquals},
+			},
+			expected:  `http_requests_total{job="prometheus",method="get"}`,
+			expectErr: false,
+		},
+		{
+			name:  "Scope filter",
+			query: `http_requests_total{job="prometheus"}`,
+			scopeFilters: []ScopeFilter{
+				{Key: "status", Value: "200", Operator: FilterOperatorEquals},
+			},
+			expected:  `http_requests_total{job="prometheus",status="200"}`,
+			expectErr: false,
+		},
+		{
+			name:  "Adhoc and Scope filter no existing filter",
+			query: `http_requests_total`,
+			scopeFilters: []ScopeFilter{
+				{Key: "status", Value: "200", Operator: FilterOperatorEquals},
+			},
+			adhocFilters: []ScopeFilter{
+				{Key: "job", Value: "prometheus", Operator: FilterOperatorEquals},
+			},
+			expected:  `http_requests_total{job="prometheus",status="200"}`,
+			expectErr: false,
+		},
+		{
+			name:  "Adhoc and Scope filter conflict - adhoc wins",
+			query: `http_requests_total{job="prometheus"}`,
+			scopeFilters: []ScopeFilter{
+				{Key: "status", Value: "404", Operator: FilterOperatorEquals},
+			},
+			adhocFilters: []ScopeFilter{
+				{Key: "status", Value: "200", Operator: FilterOperatorEquals},
+			},
+			expected:  `http_requests_total{job="prometheus",status="200"}`,
+			expectErr: false,
+		},
+		{
+			name:  "Adhoc filters with more complex expression",
+			query: `capacity_bytes{job="prometheus"} + available_bytes{job="grafana"} / 1024`,
+			adhocFilters: []ScopeFilter{
+				{Key: "job", Value: "alloy", Operator: FilterOperatorEquals},
+			},
+			expected:  `capacity_bytes{job="alloy"} + available_bytes{job="alloy"} / 1024`,
+			expectErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			expr, err := ApplyQueryFilters(tt.query, tt.scopeFilters, tt.adhocFilters)
+
+			if tt.expectErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, expr, tt.expected)
+			}
+		})
+	}
+}