Loki: add detected_field/.../values support to language provider (#95204)

* add detected_field values to language provider

public/app/plugins/datasource/loki/LanguageProvider.test.ts

@@ -1,7 +1,8 @@
-import { AbstractLabelOperator, DataFrame, TimeRange, dateTime, getDefaultTimeRange } from '@grafana/data';
+import { AbstractLabelOperator, DataFrame, TimeRange, dateTime, getDefaultTimeRange, ScopedVars } from '@grafana/data';
 import { config } from '@grafana/runtime';
 
 import LanguageProvider from './LanguageProvider';
+import { createDetectedFieldsMetadataRequest } from './__mocks__/createDetectedFieldsMetadataRequest';
 import { createLokiDatasource } from './__mocks__/datasource';
 import { createMetadataRequest } from './__mocks__/metadataRequest';
 import { DEFAULT_MAX_LINES_SAMPLE, LokiDatasource } from './datasource';
@@ -291,6 +292,94 @@ describe('Language completion provider', () => {
     });
   });
 
+  describe('fetchDetectedLabelValues', () => {
+    const expectedOptions = {
+      start: 1546372800000,
+      end: 1546380000000,
+      limit: 999,
+    };
+
+    const options: {
+      expr?: string;
+      timeRange?: TimeRange;
+      limit?: number;
+      scopedVars?: ScopedVars;
+      throwError?: boolean;
+    } = {
+      expr: '',
+      timeRange: mockTimeRange,
+      limit: 999,
+      throwError: true,
+    };
+
+    const labelName = 'labelName';
+
+    const datasource = detectedLabelValuesSetup(['label1_val1', 'label1_val2'], {
+      end: mockTimeRange.to.valueOf(),
+      start: mockTimeRange.from.valueOf(),
+    });
+
+    const expectedResponse = ['label1_val1', 'label1_val2'];
+
+    it('should fetch detected label values if not cached', async () => {
+      const provider = await getLanguageProvider(datasource, false);
+      const requestSpy = jest.spyOn(provider, 'request');
+      const labelValues = await provider.fetchDetectedLabelValues(labelName, options);
+
+      expect(requestSpy).toHaveBeenCalledWith(`detected_field/${labelName}/values`, expectedOptions, true);
+      expect(labelValues).toEqual(expectedResponse);
+    });
+
+    it('should return cached values', async () => {
+      const provider = await getLanguageProvider(datasource, false);
+      const requestSpy = jest.spyOn(provider, 'request');
+      const labelValues = await provider.fetchDetectedLabelValues(labelName, options);
+
+      expect(requestSpy).toHaveBeenCalledTimes(1);
+      expect(labelValues).toEqual(expectedResponse);
+
+      const nextLabelValues = await provider.fetchDetectedLabelValues(labelName, options);
+      expect(requestSpy).toHaveBeenCalledTimes(1);
+      expect(requestSpy).toHaveBeenCalledWith(`detected_field/${labelName}/values`, expectedOptions, true);
+      expect(nextLabelValues).toEqual(expectedResponse);
+    });
+
+    it('should encode special characters', async () => {
+      const provider = await getLanguageProvider(datasource, false);
+      const requestSpy = jest.spyOn(provider, 'request');
+      await provider.fetchDetectedLabelValues('`\\"testkey', options);
+
+      expect(requestSpy).toHaveBeenCalledWith('detected_field/%60%5C%22testkey/values', expectedOptions, true);
+    });
+
+    it('should cache by label name', async () => {
+      const provider = await getLanguageProvider(datasource, false);
+      const requestSpy = jest.spyOn(provider, 'request');
+
+      const promise1 = provider.fetchDetectedLabelValues('testkey', options);
+      const promise2 = provider.fetchDetectedLabelValues('testkey', options);
+      const datasource2 = detectedLabelValuesSetup(['label2_val1', 'label2_val2'], {
+        end: mockTimeRange.to.valueOf(),
+        start: mockTimeRange.from.valueOf(),
+      });
+      const provider2 = await getLanguageProvider(datasource2, false);
+      const requestSpy2 = jest.spyOn(provider2, 'request');
+      const promise3 = provider2.fetchDetectedLabelValues('testkeyNew', { ...options, expr: 'new' });
+
+      expect(requestSpy).toHaveBeenCalledTimes(1);
+      expect(requestSpy2).toHaveBeenCalledTimes(1);
+
+      const values1 = await promise1;
+      const values2 = await promise2;
+      const values3 = await promise3;
+
+      expect(values1).toStrictEqual(values2);
+      expect(values2).not.toStrictEqual(values3);
+      expect(requestSpy).toHaveBeenCalledTimes(1);
+      expect(requestSpy2).toHaveBeenCalledTimes(1);
+    });
+  });
+
   describe('fetchLabels', () => {
     it('should return labels', async () => {
       const datasourceWithLabels = setup({ other: [] });
@@ -588,9 +677,11 @@ describe('Query imports', () => {
   });
 });
 
-async function getLanguageProvider(datasource: LokiDatasource) {
+async function getLanguageProvider(datasource: LokiDatasource, start = true) {
   const instance = new LanguageProvider(datasource);
-  await instance.start();
+  if (start) {
+    await instance.start();
+  }
   return instance;
 }
 
@@ -611,3 +702,13 @@ function setup(
 
   return datasource;
 }
+
+function detectedLabelValuesSetup(response: string[], rangeMock: { start: number; end: number }): LokiDatasource {
+  const datasource = createLokiDatasource();
+
+  jest.spyOn(datasource, 'getTimeRangeParams').mockReturnValue(rangeMock);
+  jest.spyOn(datasource, 'metadataRequest').mockImplementation(createDetectedFieldsMetadataRequest(response));
+  jest.spyOn(datasource, 'interpolateString').mockImplementation((string: string) => string);
+
+  return datasource;
+}

public/app/plugins/datasource/loki/LanguageProvider.ts

@@ -1,7 +1,7 @@
 import { flatten } from 'lodash';
 import { LRUCache } from 'lru-cache';
 
-import { LanguageProvider, AbstractQuery, KeyValue, getDefaultTimeRange, TimeRange } from '@grafana/data';
+import { LanguageProvider, AbstractQuery, KeyValue, getDefaultTimeRange, TimeRange, ScopedVars } from '@grafana/data';
 import { config } from '@grafana/runtime';
 
 import { DEFAULT_MAX_LINES_SAMPLE, LokiDatasource } from './datasource';
@@ -31,7 +31,9 @@ export default class LokiLanguageProvider extends LanguageProvider {
    */
   private seriesCache = new LRUCache<string, Record<string, string[]>>({ max: 10 });
   private labelsCache = new LRUCache<string, string[]>({ max: 10 });
+  private detectedFieldValuesCache = new LRUCache<string, string[]>({ max: 10 });
   private labelsPromisesCache = new LRUCache<string, Promise<string[]>>({ max: 10 });
+  private detectedLabelValuesPromisesCache = new LRUCache<string, Promise<string[]>>({ max: 10 });
 
   constructor(datasource: LokiDatasource, initialValues?: any) {
     super();
@@ -42,11 +44,14 @@ export default class LokiLanguageProvider extends LanguageProvider {
     Object.assign(this, initialValues);
   }
 
-  request = async (url: string, params?: Record<string, string | number>) => {
+  request = async (url: string, params?: Record<string, string | number>, throwError?: boolean) => {
     try {
       return await this.datasource.metadataRequest(url, params);
     } catch (error) {
       console.error(error);
+      if (throwError) {
+        throw error;
+      }
     }
 
     return undefined;
@@ -238,6 +243,65 @@ export default class LokiLanguageProvider extends LanguageProvider {
     return nanoseconds ? Math.floor(nanoseconds / NS_IN_MS / 1000 / 60 / 5) : 0;
   }
 
+  async fetchDetectedLabelValues(
+    labelName: string,
+    options?: { expr?: string; timeRange?: TimeRange; limit?: number; scopedVars?: ScopedVars; throwError?: boolean }
+  ): Promise<string[]> {
+    const label = encodeURIComponent(this.datasource.interpolateString(labelName));
+
+    const interpolatedExpr =
+      options?.expr && options.expr !== EMPTY_SELECTOR
+        ? this.datasource.interpolateString(options.expr, options.scopedVars)
+        : undefined;
+
+    const url = `detected_field/${label}/values`;
+    const range = options?.timeRange ?? this.getDefaultTimeRange();
+    const rangeParams = this.datasource.getTimeRangeParams(range);
+    const { start, end } = rangeParams;
+    const params: KeyValue<string | number> = { start, end, limit: options?.limit ?? 1000 };
+    let paramCacheKey = label;
+
+    if (interpolatedExpr) {
+      params.query = interpolatedExpr;
+      paramCacheKey += interpolatedExpr;
+    }
+
+    const cacheKey = this.generateCacheKey(url, start, end, paramCacheKey);
+
+    // Values in cache, return
+    const labelValues = this.detectedFieldValuesCache.get(cacheKey);
+    if (labelValues) {
+      return labelValues;
+    }
+
+    // Promise in cache, return
+    let labelValuesPromise = this.detectedLabelValuesPromisesCache.get(cacheKey);
+    if (labelValuesPromise) {
+      return labelValuesPromise;
+    }
+
+    labelValuesPromise = new Promise(async (resolve) => {
+      try {
+        const data = await this.request(url, params, options?.throwError);
+        if (Array.isArray(data)) {
+          const labelValues = data.slice().sort();
+          this.detectedFieldValuesCache.set(cacheKey, labelValues);
+          this.detectedLabelValuesPromisesCache.delete(cacheKey);
+          resolve(labelValues);
+        }
+      } catch (error) {
+        console.error(error);
+        resolve([]);
+        if (options?.throwError) {
+          throw error;
+        }
+      }
+    });
+    this.detectedLabelValuesPromisesCache.set(cacheKey, labelValuesPromise);
+
+    return labelValuesPromise;
+  }
+
   /**
    * Fetch label values
    *

public/app/plugins/datasource/loki/__mocks__/createDetectedFieldsMetadataRequest.ts

@@ -0,0 +1,12 @@
+export function createDetectedFieldsMetadataRequest(labelsAndValues: string[]) {
+  const lokiLabelsAndValuesEndpointRegex = /^detected_field\/([%\w]*)\/values/;
+
+  return async function metadataRequestMock(url: string) {
+    const labelsMatch = url.match(lokiLabelsAndValuesEndpointRegex);
+    if (labelsMatch) {
+      return labelsAndValues ?? [];
+    } else {
+      throw new Error(`Unexpected url error, ${url}`);
+    }
+  };
+}

public/app/plugins/datasource/loki/datasource.ts

@@ -541,7 +541,12 @@ export class LokiDatasource
     }
 
     const res = await this.getResource(url, params, options);
-    return res.data || [];
+
+    // detected_field/${label}/values has different structure then other metadata responses
+    if (!res.data && res.values) {
+      return res.values ?? [];
+    }
+    return res.data ?? [];
   }
 
   /**