From be40f531e6f2891339aa2ac71ca2a53aa4f39b19 Mon Sep 17 00:00:00 2001 From: Galen Kistler <109082771+gtk-grafana@users.noreply.github.com> Date: Thu, 23 Jan 2025 12:56:37 -0600 Subject: [PATCH] Loki: Add fetchDetectedFields to LanguageProvider (#99394) * feat: add fetchDetectedFields to loki LanguageProvider --- .../datasource/loki/LanguageProvider.test.ts | 94 ++++++++++++++++++- .../datasource/loki/LanguageProvider.ts | 58 +++++++++++- ...reateDetectedFieldValuesMetadataRequest.ts | 12 +++ .../createDetectedFieldsMetadataRequest.ts | 8 +- .../app/plugins/datasource/loki/datasource.ts | 5 + public/app/plugins/datasource/loki/types.ts | 10 ++ 6 files changed, 181 insertions(+), 6 deletions(-) create mode 100644 public/app/plugins/datasource/loki/__mocks__/createDetectedFieldValuesMetadataRequest.ts diff --git a/public/app/plugins/datasource/loki/LanguageProvider.test.ts b/public/app/plugins/datasource/loki/LanguageProvider.test.ts index 4a35e193cb6..f62277ca30d 100644 --- a/public/app/plugins/datasource/loki/LanguageProvider.test.ts +++ b/public/app/plugins/datasource/loki/LanguageProvider.test.ts @@ -2,6 +2,7 @@ import { AbstractLabelOperator, DataFrame, TimeRange, dateTime, getDefaultTimeRa import { config } from '@grafana/runtime'; import LanguageProvider from './LanguageProvider'; +import { createDetectedFieldValuesMetadataRequest } from './__mocks__/createDetectedFieldValuesMetadataRequest'; import { createDetectedFieldsMetadataRequest } from './__mocks__/createDetectedFieldsMetadataRequest'; import { createLokiDatasource } from './__mocks__/datasource'; import { createMetadataRequest } from './__mocks__/metadataRequest'; @@ -11,7 +12,7 @@ import { extractLabelKeysFromDataFrame, extractUnwrapLabelKeysFromDataFrame, } from './responseUtils'; -import { LabelType, LokiQueryType } from './types'; +import { DetectedFieldsResult, LabelType, LokiQueryType } from './types'; jest.mock('./responseUtils'); @@ -384,6 +385,85 @@ describe('Language completion provider', () => { expect(requestSpy2).toHaveBeenCalledTimes(1); }); }); + describe('fetchDetectedFields', () => { + const expectedOptions = { + start: 1546372800000, + end: 1546380000000, + query: '{cluster=~".+"}', + limit: 999, + }; + + const options: { + expr: string; + timeRange?: TimeRange; + limit?: number; + scopedVars?: ScopedVars; + throwError?: boolean; + } = { + expr: '{cluster=~".+"}', + timeRange: mockTimeRange, + limit: 999, + throwError: true, + }; + + const expectedResponse: DetectedFieldsResult = { + fields: [ + { + label: 'bytes', + type: 'bytes', + cardinality: 6, + parsers: ['logfmt'], + }, + { + label: 'traceID', + type: 'string', + cardinality: 50, + parsers: null, + }, + { + label: 'active_series', + type: 'int', + cardinality: 8, + parsers: ['logfmt'], + }, + ], + limit: 999, + }; + + const datasource = detectedFieldsSetup(expectedResponse, { + end: mockTimeRange.to.valueOf(), + start: mockTimeRange.from.valueOf(), + }); + + it('should fetch detected label values', async () => { + const provider = await getLanguageProvider(datasource, false); + const requestSpy = jest.spyOn(provider, 'request'); + const labelValues = await provider.fetchDetectedFields(options); + + expect(requestSpy).toHaveBeenCalledWith(`detected_fields`, expectedOptions, true, undefined); + expect(labelValues).toEqual(expectedResponse); + }); + it('should return values', async () => { + const provider = await getLanguageProvider(datasource, false); + const requestSpy = jest.spyOn(provider, 'request'); + const labelValues = await provider.fetchDetectedFields(options); + + expect(requestSpy).toHaveBeenCalledTimes(1); + expect(labelValues).toEqual(expectedResponse); + + const nextLabelValues = await provider.fetchDetectedFields(options); + expect(requestSpy).toHaveBeenCalledTimes(2); + expect(requestSpy).toHaveBeenCalledWith(`detected_fields`, expectedOptions, true, undefined); + expect(nextLabelValues).toEqual(expectedResponse); + }); + it('should encode special characters', async () => { + const provider = await getLanguageProvider(datasource, false); + const requestSpy = jest.spyOn(provider, 'request'); + await provider.fetchDetectedFields(options); + + expect(requestSpy).toHaveBeenCalledWith('detected_fields', expectedOptions, true, undefined); + }); + }); describe('fetchLabels', () => { it('should return labels', async () => { @@ -762,6 +842,18 @@ function setup( function detectedLabelValuesSetup(response: string[], rangeMock: { start: number; end: number }): LokiDatasource { const datasource = createLokiDatasource(); + jest.spyOn(datasource, 'getTimeRangeParams').mockReturnValue(rangeMock); + jest.spyOn(datasource, 'metadataRequest').mockImplementation(createDetectedFieldValuesMetadataRequest(response)); + jest.spyOn(datasource, 'interpolateString').mockImplementation((string: string) => string); + + return datasource; +} + +function detectedFieldsSetup( + response: DetectedFieldsResult, + rangeMock: { start: number; end: number } +): LokiDatasource { + const datasource = createLokiDatasource(); jest.spyOn(datasource, 'getTimeRangeParams').mockReturnValue(rangeMock); jest.spyOn(datasource, 'metadataRequest').mockImplementation(createDetectedFieldsMetadataRequest(response)); jest.spyOn(datasource, 'interpolateString').mockImplementation((string: string) => string); diff --git a/public/app/plugins/datasource/loki/LanguageProvider.ts b/public/app/plugins/datasource/loki/LanguageProvider.ts index 4d6af9a10b7..1982ec65e36 100644 --- a/public/app/plugins/datasource/loki/LanguageProvider.ts +++ b/public/app/plugins/datasource/loki/LanguageProvider.ts @@ -1,7 +1,7 @@ import { flatten } from 'lodash'; import { LRUCache } from 'lru-cache'; -import { LanguageProvider, AbstractQuery, KeyValue, getDefaultTimeRange, TimeRange, ScopedVars } from '@grafana/data'; +import { AbstractQuery, getDefaultTimeRange, KeyValue, LanguageProvider, ScopedVars, TimeRange } from '@grafana/data'; import { BackendSrvRequest, config } from '@grafana/runtime'; import { DEFAULT_MAX_LINES_SAMPLE, LokiDatasource } from './datasource'; @@ -13,7 +13,7 @@ import { extractLogParserFromDataFrame, extractUnwrapLabelKeysFromDataFrame, } from './responseUtils'; -import { ParserAndLabelKeysResult, LokiQuery, LokiQueryType, LabelType } from './types'; +import { DetectedFieldsResult, LabelType, LokiQuery, LokiQueryType, ParserAndLabelKeysResult } from './types'; const NS_IN_MS = 1000000; const EMPTY_SELECTOR = '{}'; @@ -256,6 +256,60 @@ export default class LokiLanguageProvider extends LanguageProvider { return nanoseconds ? Math.floor(nanoseconds / NS_IN_MS / 1000 / 60 / 5) : 0; } + async fetchDetectedFields( + queryOptions: { + expr: string; + timeRange?: TimeRange; + limit?: number; + scopedVars?: ScopedVars; + }, + requestOptions?: Partial + ): Promise { + const interpolatedExpr = + queryOptions.expr && queryOptions.expr !== EMPTY_SELECTOR + ? this.datasource.interpolateString(queryOptions.expr, queryOptions.scopedVars) + : undefined; + + if (!interpolatedExpr) { + throw new Error('fetchDetectedFields requires query expression'); + } + + const url = `detected_fields`; + const range = queryOptions?.timeRange ?? this.getDefaultTimeRange(); + const rangeParams = this.datasource.getTimeRangeParams(range); + const { start, end } = rangeParams; + const params: KeyValue = { start, end, limit: queryOptions?.limit ?? 1000 }; + params.query = interpolatedExpr; + + return new Promise(async (resolve, reject) => { + try { + const data = await this.request(url, params, true, requestOptions); + resolve(data); + } catch (error) { + console.error('error', error); + reject(error); + } + }); + } + + async fetchDetectedFieldValues( + labelName: string, + queryOptions?: { + expr?: string; + timeRange?: TimeRange; + limit?: number; + scopedVars?: ScopedVars; + throwError?: boolean; + }, + requestOptions?: Partial + ): Promise { + // This function was named poorly, it's not detected label values, it's detected field values! :facepalm + return this.fetchDetectedLabelValues(labelName, queryOptions, requestOptions); + } + + /** + * @deprecated: use fetchDetectedFieldValues instead + */ async fetchDetectedLabelValues( labelName: string, queryOptions?: { diff --git a/public/app/plugins/datasource/loki/__mocks__/createDetectedFieldValuesMetadataRequest.ts b/public/app/plugins/datasource/loki/__mocks__/createDetectedFieldValuesMetadataRequest.ts new file mode 100644 index 00000000000..b7511aa0397 --- /dev/null +++ b/public/app/plugins/datasource/loki/__mocks__/createDetectedFieldValuesMetadataRequest.ts @@ -0,0 +1,12 @@ +export function createDetectedFieldValuesMetadataRequest(labelsAndValues: string[]) { + const lokiLabelsAndValuesEndpointRegex = /^detected_field\/([%\w]*)\/values/; + + return async function metadataRequestMock(url: string) { + const labelsMatch = url.match(lokiLabelsAndValuesEndpointRegex); + if (labelsMatch) { + return labelsAndValues ?? []; + } else { + throw new Error(`Unexpected url error, ${url}`); + } + }; +} diff --git a/public/app/plugins/datasource/loki/__mocks__/createDetectedFieldsMetadataRequest.ts b/public/app/plugins/datasource/loki/__mocks__/createDetectedFieldsMetadataRequest.ts index 73d0388dfb2..fd2f87d921f 100644 --- a/public/app/plugins/datasource/loki/__mocks__/createDetectedFieldsMetadataRequest.ts +++ b/public/app/plugins/datasource/loki/__mocks__/createDetectedFieldsMetadataRequest.ts @@ -1,10 +1,12 @@ -export function createDetectedFieldsMetadataRequest(labelsAndValues: string[]) { - const lokiLabelsAndValuesEndpointRegex = /^detected_field\/([%\w]*)\/values/; +import { DetectedFieldsResult } from '../types'; + +export function createDetectedFieldsMetadataRequest(response: DetectedFieldsResult) { + const lokiLabelsAndValuesEndpointRegex = /^detected_fields/; return async function metadataRequestMock(url: string) { const labelsMatch = url.match(lokiLabelsAndValuesEndpointRegex); if (labelsMatch) { - return labelsAndValues ?? []; + return response ?? {}; } else { throw new Error(`Unexpected url error, ${url}`); } diff --git a/public/app/plugins/datasource/loki/datasource.ts b/public/app/plugins/datasource/loki/datasource.ts index 985784f4421..3ceade42594 100644 --- a/public/app/plugins/datasource/loki/datasource.ts +++ b/public/app/plugins/datasource/loki/datasource.ts @@ -546,6 +546,11 @@ export class LokiDatasource if (!res.data && res.values) { return res.values ?? []; } + + // detected_fields has a different return structure then other metadata responses + if (!res.data && res.fields) { + return res.fields ?? []; + } return res.data ?? []; } diff --git a/public/app/plugins/datasource/loki/types.ts b/public/app/plugins/datasource/loki/types.ts index a0622c27184..50fb235f267 100644 --- a/public/app/plugins/datasource/loki/types.ts +++ b/public/app/plugins/datasource/loki/types.ts @@ -105,4 +105,14 @@ export interface ParserAndLabelKeysResult { unwrapLabelKeys: string[]; } +export interface DetectedFieldsResult { + fields: Array<{ + label: string; + type: 'bytes' | 'float' | 'int' | 'string' | 'duration'; + cardinality: number; + parsers: Array<'logfmt' | 'json'> | null; + }>; + limit: number; +} + export type LokiGroupedRequest = { request: DataQueryRequest; partition: TimeRange[] };