Prometheus: Add resource for suggestions that include scopes/adhoc filters (#94001)

Co-authored-by: Dominik Prokop <dominik.prokop@grafana.com>
Co-authored-by: Bogdan Matei <bogdan.matei@grafana.com>

e2e/plugin-e2e/plugin-e2e-api-tests/as-admin-user/variableEditPage.spec.ts

@@ -5,6 +5,7 @@ import { prometheusLabels } from '../mocks/resources';
 
 test('variable query with mocked response', async ({ variableEditPage, page }) => {
   variableEditPage.mockResourceResponse('api/v1/labels?*', prometheusLabels);
+  variableEditPage.mockResourceResponse('suggestions*', prometheusLabels);
   await variableEditPage.datasource.set('gdev-prometheus');
   await variableEditPage.getByGrafanaSelector('Query type').fill('Label names');
   await page.keyboard.press('Tab');

packages/grafana-prometheus/src/datasource.ts

@@ -74,7 +74,13 @@ import {
 import { PrometheusVariableSupport } from './variables';
 
 const ANNOTATION_QUERY_STEP_DEFAULT = '60s';
-const GET_AND_POST_METADATA_ENDPOINTS = ['api/v1/query', 'api/v1/query_range', 'api/v1/series', 'api/v1/labels'];
+const GET_AND_POST_METADATA_ENDPOINTS = [
+  'api/v1/query',
+  'api/v1/query_range',
+  'api/v1/series',
+  'api/v1/labels',
+  'suggestions',
+];
 
 export const InstantQueryRefIdIndex = '-Instant';
 
@@ -263,7 +269,9 @@ export class PrometheusDatasource
             .join('&');
       }
     } else {
-      options.headers!['Content-Type'] = 'application/x-www-form-urlencoded';
+      if (!options.headers!['Content-Type']) {
+        options.headers!['Content-Type'] = 'application/x-www-form-urlencoded';
+      }
       options.data = data;
     }
 
@@ -599,6 +607,20 @@ export class PrometheusDatasource
   // it is used in metric_find_query.ts
   // and in Tempo here grafana/public/app/plugins/datasource/tempo/QueryEditor/ServiceGraphSection.tsx
   async getTagKeys(options: DataSourceGetTagKeysOptions<PromQuery>): Promise<MetricFindValue[]> {
+    if (config.featureToggles.promQLScope && !!options) {
+      const suggestions = await this.languageProvider.fetchSuggestions(
+        options.timeRange,
+        options.queries,
+        options.scopes,
+        options.filters
+      );
+
+      // filter out already used labels and empty labels
+      return suggestions
+        .filter((labelName) => !!labelName && !options.filters.find((filter) => filter.key === labelName))
+        .map((k) => ({ value: k, text: k }));
+    }
+
     if (!options || options.filters.length === 0) {
       await this.languageProvider.fetchLabels(options.timeRange, options.queries);
       return this.languageProvider.getLabelKeys().map((k) => ({ value: k, text: k }));
@@ -621,6 +643,21 @@ export class PrometheusDatasource
 
   // By implementing getTagKeys and getTagValues we add ad-hoc filters functionality
   async getTagValues(options: DataSourceGetTagValuesOptions<PromQuery>) {
+    const requestId = `[${this.uid}][${options.key}]`;
+    if (config.featureToggles.promQLScope) {
+      return (
+        await this.languageProvider.fetchSuggestions(
+          options.timeRange,
+          options.queries,
+          options.scopes,
+          options.filters,
+          options.key,
+          undefined,
+          requestId
+        )
+      ).map((v) => ({ value: v, text: v }));
+    }
+
     const labelFilters: QueryBuilderLabelFilter[] = options.filters.map((f) => ({
       label: f.key,
       value: f.value,
@@ -630,7 +667,6 @@ export class PrometheusDatasource
     const expr = promQueryModeller.renderLabels(labelFilters);
 
     if (this.hasLabelsMatchAPISupport()) {
-      const requestId = `[${this.uid}][${options.key}]`;
       return (
         await this.languageProvider.fetchSeriesValuesWithMatch(options.key, expr, requestId, options.timeRange)
       ).map((v) => ({

packages/grafana-prometheus/src/language_provider.ts

@@ -6,8 +6,12 @@ import {
   AbstractLabelMatcher,
   AbstractLabelOperator,
   AbstractQuery,
+  AdHocVariableFilter,
   getDefaultTimeRange,
   LanguageProvider,
+  Scope,
+  scopeFilterOperatorMap,
+  ScopeSpecFilter,
   TimeRange,
 } from '@grafana/data';
 import { BackendSrvRequest } from '@grafana/runtime';
@@ -407,6 +411,64 @@ export default class PromQlLanguageProvider extends LanguageProvider {
     const values = await Promise.all(DEFAULT_KEYS.map((key) => this.fetchLabelValues(key)));
     return DEFAULT_KEYS.reduce((acc, key, i) => ({ ...acc, [key]: values[i] }), {});
   });
+
+  /**
+   * Fetch labels or values for a label based on the queries, scopes, filters and time range
+   * @param timeRange
+   * @param queries
+   * @param scopes
+   * @param adhocFilters
+   * @param labelName
+   * @param limit
+   * @param requestId
+   */
+  fetchSuggestions = async (
+    timeRange?: TimeRange,
+    queries?: PromQuery[],
+    scopes?: Scope[],
+    adhocFilters?: AdHocVariableFilter[],
+    labelName?: string,
+    limit?: number,
+    requestId?: string
+  ): Promise<string[]> => {
+    if (timeRange) {
+      this.timeRange = timeRange;
+    }
+
+    const url = '/suggestions';
+    const timeParams = this.datasource.getAdjustedInterval(this.timeRange);
+    const value = await this.request(
+      url,
+      [],
+      {
+        labelName,
+        queries: queries?.map((q) => q.expr),
+        scopes: scopes?.reduce<ScopeSpecFilter[]>((acc, scope) => {
+          acc.push(...scope.spec.filters);
+
+          return acc;
+        }, []),
+        adhocFilters: adhocFilters?.map((filter) => ({
+          key: filter.key,
+          operator: scopeFilterOperatorMap[filter.operator],
+          value: filter.value,
+          values: filter.values,
+        })),
+        limit,
+        ...timeParams,
+      },
+      {
+        ...(requestId && { requestId }),
+        headers: {
+          ...this.getDefaultCacheHeaders()?.headers,
+          'Content-Type': 'application/json',
+        },
+        method: 'POST',
+      }
+    );
+
+    return value ?? [];
+  };
 }
 
 function getNameLabelValue(promQuery: string, tokens: Array<string | Prism.Token>): string {

pkg/promlib/library.go

@@ -109,7 +109,8 @@ func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceReq
 		return err
 	}
 
-	if strings.EqualFold(req.Path, "version-detect") {
+	switch {
+	case strings.EqualFold(req.Path, "version-detect"):
 		versionObj, found := i.versionCache.Get("version")
 		if found {
 			return sender.Send(versionObj.(*backend.CallResourceResponse))
@@ -121,6 +122,13 @@ func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceReq
 		}
 		i.versionCache.Set("version", vResp, cache.DefaultExpiration)
 		return sender.Send(vResp)
+
+	case strings.EqualFold(req.Path, "suggestions"):
+		resp, err := i.resource.GetSuggestions(ctx, req)
+		if err != nil {
+			return err
+		}
+		return sender.Send(resp)
 	}
 
 	resp, err := i.resource.Execute(ctx, req)

pkg/promlib/library_test.go

@@ -117,6 +117,19 @@ func TestService(t *testing.T) {
 		err := service.CallResource(context.Background(), req, sender)
 		require.NoError(t, err)
 	})
+
+	t.Run("suggest resource", func(t *testing.T) {
+		f := &fakeHTTPClientProvider{}
+		httpProvider := getMockPromTestSDKProvider(f)
+		l := backend.NewLoggerWith("logger", "test")
+		service := NewService(httpProvider, l, mockExtendTransportOptions)
+
+		req := mockSuggestResource()
+		sender := &fakeSender{}
+		err := service.CallResource(context.Background(), req, sender)
+		require.NoError(t, err)
+		require.Equal(t, `http://localhost:9090/api/v1/labels?end=2022-06-01T12%3A00%3A00Z&limit=10&match%5B%5D=go_cgo_go_to_c_calls_calls_total%7Bjob%3D~%22.%2B%22%7D&match%5B%5D=up%7Bjob%3D~%22.%2B%22%7D&start=2022-06-01T00%3A00%3A00Z`, f.Roundtripper.Req.URL.String())
+	})
 }
 
 func mockRequest() *backend.CallResourceRequest {
@@ -146,3 +159,42 @@ func mockRequest() *backend.CallResourceRequest {
 		Body:   []byte("match%5B%5D: ALERTS\nstart: 1655271408\nend: 1655293008"),
 	}
 }
+
+func mockSuggestResource() *backend.CallResourceRequest {
+	return &backend.CallResourceRequest{
+		PluginContext: backend.PluginContext{
+			OrgID:               0,
+			PluginID:            "prometheus",
+			User:                nil,
+			AppInstanceSettings: nil,
+			DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{
+				ID:               0,
+				UID:              "",
+				Type:             "prometheus",
+				Name:             "test-prom",
+				URL:              "http://localhost:9090",
+				User:             "",
+				Database:         "",
+				BasicAuthEnabled: true,
+				BasicAuthUser:    "admin",
+				Updated:          time.Time{},
+				JSONData:         []byte("{}"),
+			},
+		},
+		Path:   "suggestions",
+		URL:    "suggestions",
+		Method: http.MethodPost,
+		Body: []byte(`
+			{
+				"queries": ["up + 1", "go_cgo_go_to_c_calls_calls_total + 2"],
+				"scopes": [{
+					"key": "job",
+					"value": ".+",
+					"operator": "regex-match"
+				}],
+				"start": "2022-06-01T00:00:00Z",
+				"end": "2022-06-01T12:00:00Z",
+				"limit": 10
+			}`),
+	}
+}

pkg/promlib/models/scope.go

@@ -15,7 +15,7 @@ func ApplyFiltersAndGroupBy(rawExpr string, scopeFilters, adHocFilters []ScopeFi
 		return "", err
 	}
 
-	matchers, err := filtersToMatchers(scopeFilters, adHocFilters)
+	matchers, err := FiltersToMatchers(scopeFilters, adHocFilters)
 	if err != nil {
 		return "", err
 	}
@@ -70,7 +70,7 @@ func ApplyFiltersAndGroupBy(rawExpr string, scopeFilters, adHocFilters []ScopeFi
 	return expr.String(), nil
 }
 
-func filtersToMatchers(scopeFilters, adhocFilters []ScopeFilter) ([]*labels.Matcher, error) {
+func FiltersToMatchers(scopeFilters, adhocFilters []ScopeFilter) ([]*labels.Matcher, error) {
 	filterMap := make(map[string]*labels.Matcher)
 
 	for _, filter := range append(scopeFilters, adhocFilters...) {

pkg/promlib/resource/resource.go

@@ -3,14 +3,19 @@ package resource
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
+	"net/url"
+	"slices"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/backend/log"
 	"github.com/grafana/grafana-plugin-sdk-go/data/utils/maputil"
+	"github.com/prometheus/prometheus/promql/parser"
 
 	"github.com/grafana/grafana/pkg/promlib/client"
+	"github.com/grafana/grafana/pkg/promlib/models"
 	"github.com/grafana/grafana/pkg/promlib/utils"
 )
 
@@ -84,3 +89,110 @@ func (r *Resource) DetectVersion(ctx context.Context, req *backend.CallResourceR
 
 	return r.Execute(ctx, newReq)
 }
+
+func getSelectors(expr string) ([]string, error) {
+	parsed, err := parser.ParseExpr(expr)
+	if err != nil {
+		return nil, err
+	}
+
+	selectors := make([]string, 0)
+
+	parser.Inspect(parsed, func(node parser.Node, nodes []parser.Node) error {
+		switch v := node.(type) {
+		case *parser.VectorSelector:
+			for _, matcher := range v.LabelMatchers {
+				if matcher == nil {
+					continue
+				}
+				if matcher.Name == "__name__" {
+					selectors = append(selectors, matcher.Value)
+				}
+			}
+		}
+		return nil
+	})
+
+	return selectors, nil
+}
+
+// SuggestionRequest is the request body for the GetSuggestions resource.
+type SuggestionRequest struct {
+	// LabelName, if provided, will result in label values being returned for the given label name.
+	LabelName string `json:"labelName"`
+
+	Queries []string `json:"queries"`
+
+	Scopes       []models.ScopeFilter `json:"scopes"`
+	AdhocFilters []models.ScopeFilter `json:"adhocFilters"`
+
+	// Start and End are proxied directly to the prometheus endpoint (which is rfc3339 | unix_timestamp)
+	Start string `json:"start"`
+	End   string `json:"end"`
+
+	// Limit is the maximum number of suggestions to return and is proxied directly to the prometheus endpoint.
+	Limit int64 `json:"limit"`
+}
+
+// GetSuggestions takes a Suggestion Request in the body of the resource request.
+// It builds a to call prometheus' labels endpoint (or label values endpoint if labelName is provided)
+// The match parameters for the endpoints are built from metrics extracted from the queries
+// combined with the scopes and adhoc filters provided in the request.
+// Queries must be valid raw promql.
+func (r *Resource) GetSuggestions(ctx context.Context, req *backend.CallResourceRequest) (*backend.CallResourceResponse, error) {
+	sugReq := SuggestionRequest{}
+	err := json.Unmarshal(req.Body, &sugReq)
+	if err != nil {
+		return nil, fmt.Errorf("error unmarshalling suggestion request: %v", err)
+	}
+
+	selectorList := []string{}
+	for _, query := range sugReq.Queries {
+		s, err := getSelectors(query)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing selectors: %v", err)
+		}
+		selectorList = append(selectorList, s...)
+	}
+
+	slices.Sort(selectorList)
+	selectorList = slices.Compact(selectorList)
+
+	matchers, err := models.FiltersToMatchers(sugReq.Scopes, sugReq.AdhocFilters)
+	if err != nil {
+		return nil, fmt.Errorf("error converting filters to matchers: %v", err)
+	}
+
+	values := url.Values{}
+
+	for _, s := range selectorList {
+		vs := parser.VectorSelector{Name: s, LabelMatchers: matchers}
+		values.Add("match[]", vs.String())
+	}
+
+	if sugReq.Start != "" {
+		values.Add("start", sugReq.Start)
+	}
+	if sugReq.End != "" {
+		values.Add("end", sugReq.End)
+	}
+	if sugReq.Limit > 0 {
+		values.Add("limit", fmt.Sprintf("%d", sugReq.Limit))
+	}
+
+	newReq := &backend.CallResourceRequest{
+		PluginContext: req.PluginContext,
+	}
+
+	if sugReq.LabelName != "" {
+		// Get label values for the given name (key)
+		newReq.Path = "/api/v1/label/" + sugReq.LabelName + "/values"
+		newReq.URL = "/api/v1/label/" + sugReq.LabelName + "/values?" + values.Encode()
+	} else {
+		// Get Label names (keys)
+		newReq.Path = "/api/v1/labels"
+		newReq.URL = "/api/v1/labels?" + values.Encode()
+	}
+
+	return r.Execute(ctx, newReq)
+}