Loki: Add fetchDetectedFields to LanguageProvider (#99394)

* feat: add fetchDetectedFields to loki LanguageProvider

public/app/plugins/datasource/loki/LanguageProvider.test.ts

@@ -2,6 +2,7 @@ import { AbstractLabelOperator, DataFrame, TimeRange, dateTime, getDefaultTimeRa
 import { config } from '@grafana/runtime';
 
 import LanguageProvider from './LanguageProvider';
+import { createDetectedFieldValuesMetadataRequest } from './__mocks__/createDetectedFieldValuesMetadataRequest';
 import { createDetectedFieldsMetadataRequest } from './__mocks__/createDetectedFieldsMetadataRequest';
 import { createLokiDatasource } from './__mocks__/datasource';
 import { createMetadataRequest } from './__mocks__/metadataRequest';
@@ -11,7 +12,7 @@ import {
   extractLabelKeysFromDataFrame,
   extractUnwrapLabelKeysFromDataFrame,
 } from './responseUtils';
-import { LabelType, LokiQueryType } from './types';
+import { DetectedFieldsResult, LabelType, LokiQueryType } from './types';
 
 jest.mock('./responseUtils');
 
@@ -384,6 +385,85 @@ describe('Language completion provider', () => {
       expect(requestSpy2).toHaveBeenCalledTimes(1);
     });
   });
+  describe('fetchDetectedFields', () => {
+    const expectedOptions = {
+      start: 1546372800000,
+      end: 1546380000000,
+      query: '{cluster=~".+"}',
+      limit: 999,
+    };
+
+    const options: {
+      expr: string;
+      timeRange?: TimeRange;
+      limit?: number;
+      scopedVars?: ScopedVars;
+      throwError?: boolean;
+    } = {
+      expr: '{cluster=~".+"}',
+      timeRange: mockTimeRange,
+      limit: 999,
+      throwError: true,
+    };
+
+    const expectedResponse: DetectedFieldsResult = {
+      fields: [
+        {
+          label: 'bytes',
+          type: 'bytes',
+          cardinality: 6,
+          parsers: ['logfmt'],
+        },
+        {
+          label: 'traceID',
+          type: 'string',
+          cardinality: 50,
+          parsers: null,
+        },
+        {
+          label: 'active_series',
+          type: 'int',
+          cardinality: 8,
+          parsers: ['logfmt'],
+        },
+      ],
+      limit: 999,
+    };
+
+    const datasource = detectedFieldsSetup(expectedResponse, {
+      end: mockTimeRange.to.valueOf(),
+      start: mockTimeRange.from.valueOf(),
+    });
+
+    it('should fetch detected label values', async () => {
+      const provider = await getLanguageProvider(datasource, false);
+      const requestSpy = jest.spyOn(provider, 'request');
+      const labelValues = await provider.fetchDetectedFields(options);
+
+      expect(requestSpy).toHaveBeenCalledWith(`detected_fields`, expectedOptions, true, undefined);
+      expect(labelValues).toEqual(expectedResponse);
+    });
+    it('should return values', async () => {
+      const provider = await getLanguageProvider(datasource, false);
+      const requestSpy = jest.spyOn(provider, 'request');
+      const labelValues = await provider.fetchDetectedFields(options);
+
+      expect(requestSpy).toHaveBeenCalledTimes(1);
+      expect(labelValues).toEqual(expectedResponse);
+
+      const nextLabelValues = await provider.fetchDetectedFields(options);
+      expect(requestSpy).toHaveBeenCalledTimes(2);
+      expect(requestSpy).toHaveBeenCalledWith(`detected_fields`, expectedOptions, true, undefined);
+      expect(nextLabelValues).toEqual(expectedResponse);
+    });
+    it('should encode special characters', async () => {
+      const provider = await getLanguageProvider(datasource, false);
+      const requestSpy = jest.spyOn(provider, 'request');
+      await provider.fetchDetectedFields(options);
+
+      expect(requestSpy).toHaveBeenCalledWith('detected_fields', expectedOptions, true, undefined);
+    });
+  });
 
   describe('fetchLabels', () => {
     it('should return labels', async () => {
@@ -762,6 +842,18 @@ function setup(
 function detectedLabelValuesSetup(response: string[], rangeMock: { start: number; end: number }): LokiDatasource {
   const datasource = createLokiDatasource();
 
+  jest.spyOn(datasource, 'getTimeRangeParams').mockReturnValue(rangeMock);
+  jest.spyOn(datasource, 'metadataRequest').mockImplementation(createDetectedFieldValuesMetadataRequest(response));
+  jest.spyOn(datasource, 'interpolateString').mockImplementation((string: string) => string);
+
+  return datasource;
+}
+
+function detectedFieldsSetup(
+  response: DetectedFieldsResult,
+  rangeMock: { start: number; end: number }
+): LokiDatasource {
+  const datasource = createLokiDatasource();
   jest.spyOn(datasource, 'getTimeRangeParams').mockReturnValue(rangeMock);
   jest.spyOn(datasource, 'metadataRequest').mockImplementation(createDetectedFieldsMetadataRequest(response));
   jest.spyOn(datasource, 'interpolateString').mockImplementation((string: string) => string);

public/app/plugins/datasource/loki/LanguageProvider.ts

@@ -1,7 +1,7 @@
 import { flatten } from 'lodash';
 import { LRUCache } from 'lru-cache';
 
-import { LanguageProvider, AbstractQuery, KeyValue, getDefaultTimeRange, TimeRange, ScopedVars } from '@grafana/data';
+import { AbstractQuery, getDefaultTimeRange, KeyValue, LanguageProvider, ScopedVars, TimeRange } from '@grafana/data';
 import { BackendSrvRequest, config } from '@grafana/runtime';
 
 import { DEFAULT_MAX_LINES_SAMPLE, LokiDatasource } from './datasource';
@@ -13,7 +13,7 @@ import {
   extractLogParserFromDataFrame,
   extractUnwrapLabelKeysFromDataFrame,
 } from './responseUtils';
-import { ParserAndLabelKeysResult, LokiQuery, LokiQueryType, LabelType } from './types';
+import { DetectedFieldsResult, LabelType, LokiQuery, LokiQueryType, ParserAndLabelKeysResult } from './types';
 
 const NS_IN_MS = 1000000;
 const EMPTY_SELECTOR = '{}';
@@ -256,6 +256,60 @@ export default class LokiLanguageProvider extends LanguageProvider {
     return nanoseconds ? Math.floor(nanoseconds / NS_IN_MS / 1000 / 60 / 5) : 0;
   }
 
+  async fetchDetectedFields(
+    queryOptions: {
+      expr: string;
+      timeRange?: TimeRange;
+      limit?: number;
+      scopedVars?: ScopedVars;
+    },
+    requestOptions?: Partial<BackendSrvRequest>
+  ): Promise<DetectedFieldsResult | Error> {
+    const interpolatedExpr =
+      queryOptions.expr && queryOptions.expr !== EMPTY_SELECTOR
+        ? this.datasource.interpolateString(queryOptions.expr, queryOptions.scopedVars)
+        : undefined;
+
+    if (!interpolatedExpr) {
+      throw new Error('fetchDetectedFields requires query expression');
+    }
+
+    const url = `detected_fields`;
+    const range = queryOptions?.timeRange ?? this.getDefaultTimeRange();
+    const rangeParams = this.datasource.getTimeRangeParams(range);
+    const { start, end } = rangeParams;
+    const params: KeyValue<string | number> = { start, end, limit: queryOptions?.limit ?? 1000 };
+    params.query = interpolatedExpr;
+
+    return new Promise(async (resolve, reject) => {
+      try {
+        const data = await this.request(url, params, true, requestOptions);
+        resolve(data);
+      } catch (error) {
+        console.error('error', error);
+        reject(error);
+      }
+    });
+  }
+
+  async fetchDetectedFieldValues(
+    labelName: string,
+    queryOptions?: {
+      expr?: string;
+      timeRange?: TimeRange;
+      limit?: number;
+      scopedVars?: ScopedVars;
+      throwError?: boolean;
+    },
+    requestOptions?: Partial<BackendSrvRequest>
+  ): Promise<string[] | Error> {
+    // This function was named poorly, it's not detected label values, it's detected field values! :facepalm
+    return this.fetchDetectedLabelValues(labelName, queryOptions, requestOptions);
+  }
+
+  /**
+   * @deprecated: use fetchDetectedFieldValues instead
+   */
   async fetchDetectedLabelValues(
     labelName: string,
     queryOptions?: {

public/app/plugins/datasource/loki/__mocks__/createDetectedFieldValuesMetadataRequest.ts

@@ -0,0 +1,12 @@
+export function createDetectedFieldValuesMetadataRequest(labelsAndValues: string[]) {
+  const lokiLabelsAndValuesEndpointRegex = /^detected_field\/([%\w]*)\/values/;
+
+  return async function metadataRequestMock(url: string) {
+    const labelsMatch = url.match(lokiLabelsAndValuesEndpointRegex);
+    if (labelsMatch) {
+      return labelsAndValues ?? [];
+    } else {
+      throw new Error(`Unexpected url error, ${url}`);
+    }
+  };
+}

public/app/plugins/datasource/loki/__mocks__/createDetectedFieldsMetadataRequest.ts

@@ -1,10 +1,12 @@
-export function createDetectedFieldsMetadataRequest(labelsAndValues: string[]) {
+import { DetectedFieldsResult } from '../types';
-  const lokiLabelsAndValuesEndpointRegex = /^detected_field\/([%\w]*)\/values/;
+
+export function createDetectedFieldsMetadataRequest(response: DetectedFieldsResult) {
+  const lokiLabelsAndValuesEndpointRegex = /^detected_fields/;
 
   return async function metadataRequestMock(url: string) {
     const labelsMatch = url.match(lokiLabelsAndValuesEndpointRegex);
     if (labelsMatch) {
-      return labelsAndValues ?? [];
+      return response ?? {};
     } else {
       throw new Error(`Unexpected url error, ${url}`);
     }

public/app/plugins/datasource/loki/datasource.ts

@@ -546,6 +546,11 @@ export class LokiDatasource
     if (!res.data && res.values) {
       return res.values ?? [];
     }
+
+    // detected_fields has a different return structure then other metadata responses
+    if (!res.data && res.fields) {
+      return res.fields ?? [];
+    }
     return res.data ?? [];
   }
 

public/app/plugins/datasource/loki/types.ts

@@ -105,4 +105,14 @@ export interface ParserAndLabelKeysResult {
   unwrapLabelKeys: string[];
 }
 
+export interface DetectedFieldsResult {
+  fields: Array<{
+    label: string;
+    type: 'bytes' | 'float' | 'int' | 'string' | 'duration';
+    cardinality: number;
+    parsers: Array<'logfmt' | 'json'> | null;
+  }>;
+  limit: number;
+}
+
 export type LokiGroupedRequest = { request: DataQueryRequest<LokiQuery>; partition: TimeRange[] };