diff --git a/public/app/plugins/datasource/loki/components/__snapshots__/LokiExploreQueryEditor.test.tsx.snap b/public/app/plugins/datasource/loki/components/__snapshots__/LokiExploreQueryEditor.test.tsx.snap index c493b7cb444..ac55777fa65 100644 --- a/public/app/plugins/datasource/loki/components/__snapshots__/LokiExploreQueryEditor.test.tsx.snap +++ b/public/app/plugins/datasource/loki/components/__snapshots__/LokiExploreQueryEditor.test.tsx.snap @@ -56,11 +56,43 @@ exports[`LokiExploreQueryEditor should render component 1`] = ` "languageProvider": LokiLanguageProvider { "cleanText": [Function], "datasource": [Circular], + "fetchSeriesLabels": [Function], "getBeginningCompletionItems": [Function], "getTermCompletionItems": [Function], - "labelKeys": Object {}, - "labelValues": Object {}, + "labelKeys": Array [], + "labelsCache": LRUCache { + Symbol(max): 10, + Symbol(lengthCalculator): [Function], + Symbol(allowStale): false, + Symbol(maxAge): 0, + Symbol(dispose): undefined, + Symbol(noDisposeOnSet): false, + Symbol(updateAgeOnGet): false, + Symbol(cache): Map {}, + Symbol(lruList): Yallist { + "head": null, + "length": 0, + "tail": null, + }, + Symbol(length): 0, + }, "request": [Function], + "seriesCache": LRUCache { + Symbol(max): 10, + Symbol(lengthCalculator): [Function], + Symbol(allowStale): false, + Symbol(maxAge): 0, + Symbol(dispose): undefined, + Symbol(noDisposeOnSet): false, + Symbol(updateAgeOnGet): false, + Symbol(cache): Map {}, + Symbol(lruList): Yallist { + "head": null, + "length": 0, + "tail": null, + }, + Symbol(length): 0, + }, "start": [Function], }, "metadataRequest": [Function], diff --git a/public/app/plugins/datasource/loki/components/useLokiSyntaxAndLabels.test.ts b/public/app/plugins/datasource/loki/components/useLokiSyntaxAndLabels.test.ts index 22b4078b58d..0b7db81e575 100644 --- a/public/app/plugins/datasource/loki/components/useLokiSyntaxAndLabels.test.ts +++ b/public/app/plugins/datasource/loki/components/useLokiSyntaxAndLabels.test.ts @@ -59,9 +59,9 @@ describe('useLokiSyntax hook', () => { await waitForNextUpdate(); expect(result.current.logLabelOptions).toEqual(logLabelOptionsMock2); - languageProvider.fetchLabelValues = (key: string) => { + languageProvider.fetchLabelValues = (key: string, absoluteRange: AbsoluteTimeRange) => { languageProvider.logLabelOptions = logLabelOptionsMock3; - return Promise.resolve(); + return Promise.resolve([]); }; act(() => result.current.setActiveOption([activeOptionMock])); diff --git a/public/app/plugins/datasource/loki/datasource.ts b/public/app/plugins/datasource/loki/datasource.ts index 6b2e02e7a3e..0934a06b0b2 100644 --- a/public/app/plugins/datasource/loki/datasource.ts +++ b/public/app/plugins/datasource/loki/datasource.ts @@ -390,9 +390,7 @@ export class LokiDatasource extends DataSourceApi { async metadataRequest(url: string, params?: Record) { const res = await this._request(url, params, { silent: true }).toPromise(); - return { - data: { data: res.data.data || res.data.values || [] }, - }; + return res.data.data || res.data.values || []; } async metricFindQuery(query: string) { @@ -423,7 +421,7 @@ export class LokiDatasource extends DataSourceApi { async labelNamesQuery() { const url = (await this.getVersion()) === 'v0' ? `${LEGACY_LOKI_ENDPOINT}/label` : `${LOKI_ENDPOINT}/label`; const result = await this.metadataRequest(url); - return result.data.data.map((value: string) => ({ text: value })); + return result.map((value: string) => ({ text: value })); } async labelValuesQuery(label: string) { @@ -432,7 +430,7 @@ export class LokiDatasource extends DataSourceApi { ? `${LEGACY_LOKI_ENDPOINT}/label/${label}/values` : `${LOKI_ENDPOINT}/label/${label}/values`; const result = await this.metadataRequest(url); - return result.data.data.map((value: string) => ({ text: value })); + return result.map((value: string) => ({ text: value })); } interpolateQueryExpr(value: any, variable: any) { diff --git a/public/app/plugins/datasource/loki/language_provider.test.ts b/public/app/plugins/datasource/loki/language_provider.test.ts index 0a3a9d9f57e..52791e326d3 100644 --- a/public/app/plugins/datasource/loki/language_provider.test.ts +++ b/public/app/plugins/datasource/loki/language_provider.test.ts @@ -1,5 +1,4 @@ import Plain from 'slate-plain-serializer'; -import { Editor as SlateEditor } from 'slate'; import LanguageProvider, { LABEL_REFRESH_INTERVAL, LokiHistoryItem, rangeToParams } from './language_provider'; import { AbsoluteTimeRange } from '@grafana/data'; @@ -85,34 +84,53 @@ describe('Language completion provider', () => { }); }); - describe('label suggestions', () => { - it('returns default label suggestions on label context', async () => { - const instance = new LanguageProvider(datasource); - const value = Plain.deserialize('{}'); - const ed = new SlateEditor({ value }); - const valueWithSelection = ed.moveForward(1).value; - const result = await instance.provideCompletionItems( - { - text: '', - prefix: '', - wrapperClasses: ['context-labels'], - value: valueWithSelection, - }, - { absoluteRange: rangeMock } - ); - expect(result.context).toBe('context-labels'); - expect(result.suggestions).toEqual([{ items: [{ label: 'job' }, { label: 'namespace' }], label: 'Labels' }]); - }); - - it('returns label suggestions from Loki', async () => { + describe('label key suggestions', () => { + it('returns all label suggestions on empty selector', async () => { const datasource = makeMockLokiDatasource({ label1: [], label2: [] }); const provider = await getLanguageProvider(datasource); - const input = createTypeaheadInput('{}', ''); + const input = createTypeaheadInput('{}', '', '', 1); const result = await provider.provideCompletionItems(input, { absoluteRange: rangeMock }); expect(result.context).toBe('context-labels'); expect(result.suggestions).toEqual([{ items: [{ label: 'label1' }, { label: 'label2' }], label: 'Labels' }]); }); + it('returns all label suggestions on selector when starting to type', async () => { + const datasource = makeMockLokiDatasource({ label1: [], label2: [] }); + const provider = await getLanguageProvider(datasource); + const input = createTypeaheadInput('{l}', '', '', 2); + const result = await provider.provideCompletionItems(input, { absoluteRange: rangeMock }); + expect(result.context).toBe('context-labels'); + expect(result.suggestions).toEqual([{ items: [{ label: 'label1' }, { label: 'label2' }], label: 'Labels' }]); + }); + }); + + describe('label suggestions facetted', () => { + it('returns facetted label suggestions based on selector', async () => { + const datasource = makeMockLokiDatasource( + { label1: [], label2: [] }, + { '{foo="bar"}': [{ label1: 'label_val1' }] } + ); + const provider = await getLanguageProvider(datasource); + const input = createTypeaheadInput('{foo="bar",}', '', '', 11); + const result = await provider.provideCompletionItems(input, { absoluteRange: rangeMock }); + expect(result.context).toBe('context-labels'); + expect(result.suggestions).toEqual([{ items: [{ label: 'label1' }], label: 'Labels' }]); + }); + + it('returns facetted label suggestions for multipule selectors', async () => { + const datasource = makeMockLokiDatasource( + { label1: [], label2: [] }, + { '{baz="42",foo="bar"}': [{ label2: 'label_val2' }] } + ); + const provider = await getLanguageProvider(datasource); + const input = createTypeaheadInput('{baz="42",foo="bar",}', '', '', 20); + const result = await provider.provideCompletionItems(input, { absoluteRange: rangeMock }); + expect(result.context).toBe('context-labels'); + expect(result.suggestions).toEqual([{ items: [{ label: 'label2' }], label: 'Labels' }]); + }); + }); + + describe('label suggestions', () => { it('returns label values suggestions from Loki', async () => { const datasource = makeMockLokiDatasource({ label1: ['label1_val1', 'label1_val2'], label2: [] }); const provider = await getLanguageProvider(datasource); diff --git a/public/app/plugins/datasource/loki/language_provider.ts b/public/app/plugins/datasource/loki/language_provider.ts index 3f8af4ac124..92dc2bb5755 100644 --- a/public/app/plugins/datasource/loki/language_provider.ts +++ b/public/app/plugins/datasource/loki/language_provider.ts @@ -1,8 +1,14 @@ // Libraries import _ from 'lodash'; +import LRU from 'lru-cache'; // Services & Utils -import { parseSelector, labelRegexp, selectorRegexp } from 'app/plugins/datasource/prometheus/language_utils'; +import { + parseSelector, + labelRegexp, + selectorRegexp, + processLabels, +} from 'app/plugins/datasource/prometheus/language_utils'; import syntax, { FUNCTIONS } from './syntax'; // Types @@ -12,7 +18,7 @@ import { PromQuery } from '../prometheus/types'; import { RATE_RANGES } from '../prometheus/promql'; import LokiDatasource from './datasource'; -import { CompletionItem, TypeaheadInput, TypeaheadOutput } from '@grafana/ui'; +import { CompletionItem, TypeaheadInput, TypeaheadOutput, CompletionItemGroup } from '@grafana/ui'; import { Grammar } from 'prismjs'; const DEFAULT_KEYS = ['job', 'namespace']; @@ -50,20 +56,27 @@ export function addHistoryMetadata(item: CompletionItem, history: LokiHistoryIte } export default class LokiLanguageProvider extends LanguageProvider { - labelKeys?: { [index: string]: string[] }; // metric -> [labelKey,...] - labelValues?: { [index: string]: { [index: string]: string[] } }; // metric -> labelKey -> [labelValue,...] + labelKeys?: string[]; logLabelOptions: any[]; logLabelFetchTs?: number; started: boolean; initialRange: AbsoluteTimeRange; datasource: LokiDatasource; + lookupsDisabled: boolean; // Dynamically set to true for big/slow instances + + /** + * Cache for labels of series. This is bit simplistic in the sense that it just counts responses each as a 1 and does + * not account for different size of a response. If that is needed a `length` function can be added in the options. + * 10 as a max size is totally arbitrary right now. + */ + private seriesCache = new LRU>(10); + private labelsCache = new LRU(10); constructor(datasource: LokiDatasource, initialValues?: any) { super(); this.datasource = datasource; - this.labelKeys = {}; - this.labelValues = {}; + this.labelKeys = []; Object.assign(this, initialValues); } @@ -75,8 +88,14 @@ export default class LokiLanguageProvider extends LanguageProvider { return syntax; } - request = (url: string, params?: any): Promise<{ data: { data: string[] } }> => { - return this.datasource.metadataRequest(url, params); + request = async (url: string, params?: any): Promise => { + try { + return await this.datasource.metadataRequest(url, params); + } catch (error) { + console.error(error); + } + + return undefined; }; /** @@ -95,12 +114,7 @@ export default class LokiLanguageProvider extends LanguageProvider { }; getLabelKeys(): string[] { - return this.labelKeys[EMPTY_SELECTOR]; - } - - async getLabelValues(key: string): Promise { - await this.fetchLabelValues(key, this.initialRange); - return this.labelValues[EMPTY_SELECTOR][key]; + return this.labelKeys; } /** @@ -219,42 +233,66 @@ export default class LokiLanguageProvider extends LanguageProvider { { text, wrapperClasses, labelKey, value }: TypeaheadInput, { absoluteRange }: any ): Promise { - let context: string; - const suggestions = []; + let context = 'context-labels'; + const suggestions: CompletionItemGroup[] = []; const line = value.anchorBlock.getText(); - const cursorOffset: number = value.selection.anchor.offset; + const cursorOffset = value.selection.anchor.offset; + const isValueStart = text.match(/^(=|=~|!=|!~)/); - // Use EMPTY_SELECTOR until series API is implemented for facetting - const selector = EMPTY_SELECTOR; + // Get normalized selector + let selector; let parsedSelector; try { parsedSelector = parseSelector(line, cursorOffset); - } catch {} + selector = parsedSelector.selector; + } catch { + selector = EMPTY_SELECTOR; + } + + if (!isValueStart && selector === EMPTY_SELECTOR) { + // start task gets all labels + await this.start(); + const allLabels = this.getLabelKeys(); + return { context, suggestions: [{ label: `Labels`, items: allLabels.map(wrapLabel) }] }; + } + const existingKeys = parsedSelector ? parsedSelector.labelKeys : []; - if ((text && text.match(/^!?=~?/)) || wrapperClasses.includes('attr-value')) { - // Label values - if (labelKey && this.labelValues[selector]) { - let labelValues = this.labelValues[selector][labelKey]; - if (!labelValues) { - await this.fetchLabelValues(labelKey, absoluteRange); - labelValues = this.labelValues[selector][labelKey]; - } + let labelValues; + // Query labels for selector + if (selector) { + if (selector === EMPTY_SELECTOR && labelKey) { + const labelValuesForKey = await this.getLabelValues(labelKey); + labelValues = { [labelKey]: labelValuesForKey }; + } else { + labelValues = await this.getSeriesLabels(selector, absoluteRange); + } + } + if (!labelValues) { + console.warn(`Server did not return any values for selector = ${selector}`); + return { context, suggestions }; + } + + if ((text && isValueStart) || wrapperClasses.includes('attr-value')) { + // Label values + if (labelKey && labelValues[labelKey]) { context = 'context-label-values'; suggestions.push({ label: `Label values for "${labelKey}"`, - items: labelValues.map(wrapLabel), + items: labelValues[labelKey].map(wrapLabel), }); } } else { // Label keys - const labelKeys = this.labelKeys[selector] || DEFAULT_KEYS; + const labelKeys = labelValues ? Object.keys(labelValues) : DEFAULT_KEYS; + if (labelKeys) { const possibleKeys = _.difference(labelKeys, existingKeys); if (possibleKeys.length) { - context = 'context-labels'; - suggestions.push({ label: `Labels`, items: possibleKeys.map(wrapLabel) }); + const newItems = possibleKeys.map(key => ({ label: key })); + const newSuggestion: CompletionItemGroup = { label: `Labels`, items: newItems }; + suggestions.push(newSuggestion); } } } @@ -302,7 +340,7 @@ export default class LokiLanguageProvider extends LanguageProvider { // Keep only labels that exist on origin and target datasource await this.start(); // fetches all existing label keys - const existingKeys = this.labelKeys[EMPTY_SELECTOR]; + const existingKeys = this.labelKeys; let labelsToKeep: { [key: string]: { value: any; operator: any } } = {}; if (existingKeys && existingKeys.length) { // Check for common labels @@ -325,22 +363,31 @@ export default class LokiLanguageProvider extends LanguageProvider { return ['{', cleanSelector, '}'].join(''); } + async getSeriesLabels(selector: string, absoluteRange: AbsoluteTimeRange) { + if (this.lookupsDisabled) { + return undefined; + } + try { + return await this.fetchSeriesLabels(selector, absoluteRange); + } catch (error) { + // TODO: better error handling + console.error(error); + return undefined; + } + } + + /** + * Fetches all label keys + * @param absoluteRange Fetches + */ async fetchLogLabels(absoluteRange: AbsoluteTimeRange): Promise { const url = '/api/prom/label'; try { this.logLabelFetchTs = Date.now(); const rangeParams = absoluteRange ? rangeToParams(absoluteRange) : {}; const res = await this.request(url, rangeParams); - const labelKeys = res.data.data.slice().sort(); - - this.labelKeys = { - ...this.labelKeys, - [EMPTY_SELECTOR]: labelKeys, - }; - this.labelValues = { - [EMPTY_SELECTOR]: {}, - }; - this.logLabelOptions = labelKeys.map((key: string) => ({ label: key, value: key, isLeaf: false })); + this.labelKeys = res.slice().sort(); + this.logLabelOptions = this.labelKeys.map((key: string) => ({ label: key, value: key, isLeaf: false })); } catch (e) { console.error(e); } @@ -353,36 +400,79 @@ export default class LokiLanguageProvider extends LanguageProvider { } } - async fetchLabelValues(key: string, absoluteRange: AbsoluteTimeRange) { - const url = `/api/prom/label/${key}/values`; - try { - const rangeParams = absoluteRange ? rangeToParams(absoluteRange) : {}; - const res = await this.request(url, rangeParams); - const values = res.data.data.slice().sort(); + /** + * Fetch labels for a selector. This is cached by it's args but also by the global timeRange currently selected as + * they can change over requested time. + * @param name + */ + fetchSeriesLabels = async (match: string, absoluteRange: AbsoluteTimeRange): Promise> => { + const rangeParams: { start?: number; end?: number } = absoluteRange ? rangeToParams(absoluteRange) : {}; + const url = '/loki/api/v1/series'; + const { start, end } = rangeParams; - // Add to label options - this.logLabelOptions = this.logLabelOptions.map(keyOption => { - if (keyOption.value === key) { - return { - ...keyOption, - children: values.map(value => ({ label: value, value })), - }; - } - return keyOption; - }); - - // Add to key map - const exisingValues = this.labelValues[EMPTY_SELECTOR]; - const nextValues = { - ...exisingValues, - [key]: values, - }; - this.labelValues = { - ...this.labelValues, - [EMPTY_SELECTOR]: nextValues, - }; - } catch (e) { - console.error(e); + const cacheKey = this.generateCacheKey(url, start, end, match); + const params = { match, start, end }; + let value = this.seriesCache.get(cacheKey); + if (!value) { + // Clear value when requesting new one. Empty object being truthy also makes sure we don't request twice. + this.seriesCache.set(cacheKey, {}); + const data = await this.request(url, params); + const { values } = processLabels(data); + value = values; + this.seriesCache.set(cacheKey, value); } + return value; + }; + + // Cache key is a bit different here. We round up to a minute the intervals. + // The rounding may seem strange but makes relative intervals like now-1h less prone to need separate request every + // millisecond while still actually getting all the keys for the correct interval. This still can create problems + // when user does not the newest values for a minute if already cached. + generateCacheKey(url: string, start: number, end: number, param: string): string { + return [url, this.roundTime(start), this.roundTime(end), param].join(); + } + + // Round nanos epoch to nearest 5 minute interval + roundTime(nanos: number): number { + return nanos ? Math.floor(nanos / NS_IN_MS / 1000 / 60 / 5) : 0; + } + + async getLabelValues(key: string): Promise { + return await this.fetchLabelValues(key, this.initialRange); + } + + async fetchLabelValues(key: string, absoluteRange: AbsoluteTimeRange): Promise { + const url = `/api/prom/label/${key}/values`; + let values: string[] = []; + const rangeParams: { start?: number; end?: number } = absoluteRange ? rangeToParams(absoluteRange) : {}; + const { start, end } = rangeParams; + + const cacheKey = this.generateCacheKey(url, start, end, key); + const params = { start, end }; + let value = this.labelsCache.get(cacheKey); + if (!value) { + try { + // Clear value when requesting new one. Empty object being truthy also makes sure we don't request twice. + this.labelsCache.set(cacheKey, []); + const res = await this.request(url, params); + values = res.slice().sort(); + value = values; + this.labelsCache.set(cacheKey, value); + + // Add to label options + this.logLabelOptions = this.logLabelOptions.map(keyOption => { + if (keyOption.value === key) { + return { + ...keyOption, + children: values.map(value => ({ label: value, value })), + }; + } + return keyOption; + }); + } catch (e) { + console.error(e); + } + } + return value; } } diff --git a/public/app/plugins/datasource/loki/mocks.ts b/public/app/plugins/datasource/loki/mocks.ts index d3d09d00951..cb38d3ef098 100644 --- a/public/app/plugins/datasource/loki/mocks.ts +++ b/public/app/plugins/datasource/loki/mocks.ts @@ -3,34 +3,45 @@ import { DataSourceSettings } from '@grafana/data'; import { LokiOptions } from './types'; import { createDatasourceSettings } from '../../../features/datasources/mocks'; -export function makeMockLokiDatasource(labelsAndValues: { [label: string]: string[] }): LokiDatasource { +interface Labels { + [label: string]: string[]; +} + +interface Series { + [label: string]: string; +} + +interface SeriesForSelector { + [selector: string]: Series[]; +} + +export function makeMockLokiDatasource(labelsAndValues: Labels, series?: SeriesForSelector): LokiDatasource { const legacyLokiLabelsAndValuesEndpointRegex = /^\/api\/prom\/label\/(\w*)\/values/; const lokiLabelsAndValuesEndpointRegex = /^\/loki\/api\/v1\/label\/(\w*)\/values/; + const lokiSeriesEndpointRegex = /^\/loki\/api\/v1\/series/; const legacyLokiLabelsEndpoint = `${LEGACY_LOKI_ENDPOINT}/label`; const lokiLabelsEndpoint = `${LOKI_ENDPOINT}/label`; const labels = Object.keys(labelsAndValues); return { - metadataRequest: (url: string) => { - let responseData; + metadataRequest: (url: string, params?: { [key: string]: string }) => { if (url === legacyLokiLabelsEndpoint || url === lokiLabelsEndpoint) { - responseData = labels; + return labels; } else { - const match = url.match(legacyLokiLabelsAndValuesEndpointRegex) || url.match(lokiLabelsAndValuesEndpointRegex); - if (match) { - responseData = labelsAndValues[match[1]]; + const legacyLabelsMatch = url.match(legacyLokiLabelsAndValuesEndpointRegex); + const labelsMatch = url.match(lokiLabelsAndValuesEndpointRegex); + const seriesMatch = url.match(lokiSeriesEndpointRegex); + if (legacyLabelsMatch) { + return labelsAndValues[legacyLabelsMatch[1]] || []; + } else if (labelsMatch) { + return labelsAndValues[labelsMatch[1]] || []; + } else if (seriesMatch) { + return series[params.match] || []; + } else { + throw new Error(`Unexpected url error, ${url}`); } } - if (responseData) { - return { - data: { - data: responseData, - }, - }; - } else { - throw new Error(`Unexpected url error, ${url}`); - } }, } as any; }