Loki: Clean up Language provider and remove unused code (#77315)

* Loki: Clean up Language provider and remove unused stuff

* Update

* Update comment

public/app/plugins/datasource/loki/LanguageProvider.test.ts

@@ -1,9 +1,6 @@
-import Plain from 'slate-plain-serializer';
-
 import { AbstractLabelOperator, DataFrame } from '@grafana/data';
-import { TypeaheadInput } from '@grafana/ui';
 
-import LanguageProvider, { LokiHistoryItem } from './LanguageProvider';
+import LanguageProvider from './LanguageProvider';
 import { LokiDatasource } from './datasource';
 import { createLokiDatasource, createMetadataRequest } from './mocks';
 import {
@@ -28,73 +25,6 @@ jest.mock('app/store/store', () => ({
 }));
 
 describe('Language completion provider', () => {
-  const datasource = setup({});
-
-  describe('query suggestions', () => {
-    it('returns no suggestions on empty context', async () => {
-      const instance = new LanguageProvider(datasource);
-      const value = Plain.deserialize('');
-      const result = await instance.provideCompletionItems({ text: '', prefix: '', value, wrapperClasses: [] });
-      expect(result.context).toBeUndefined();
-
-      expect(result.suggestions.length).toEqual(0);
-    });
-
-    it('returns history on empty context when history was provided', async () => {
-      const instance = new LanguageProvider(datasource);
-      const value = Plain.deserialize('');
-      const history: LokiHistoryItem[] = [
-        {
-          query: { refId: '1', expr: '{app="foo"}' },
-          ts: 1,
-        },
-      ];
-      const result = await instance.provideCompletionItems(
-        { text: '', prefix: '', value, wrapperClasses: [] },
-        { history }
-      );
-      expect(result.context).toBeUndefined();
-
-      expect(result.suggestions).toMatchObject([
-        {
-          label: 'History',
-          items: [
-            {
-              label: '{app="foo"}',
-            },
-          ],
-        },
-      ]);
-    });
-
-    it('returns function and history suggestions', async () => {
-      const instance = new LanguageProvider(datasource);
-      const input = createTypeaheadInput('m', 'm', undefined, 1, [], instance);
-      // Historic expressions don't have to match input, filtering is done in field
-      const history: LokiHistoryItem[] = [
-        {
-          query: { refId: '1', expr: '{app="foo"}' },
-          ts: 1,
-        },
-      ];
-      const result = await instance.provideCompletionItems(input, { history });
-      expect(result.context).toBeUndefined();
-      expect(result.suggestions.length).toEqual(2);
-      expect(result.suggestions[0].label).toEqual('History');
-      expect(result.suggestions[1].label).toEqual('Functions');
-    });
-
-    it('returns pipe operations on pipe context', async () => {
-      const instance = new LanguageProvider(datasource);
-      const input = createTypeaheadInput('{app="test"} | ', ' ', '', 15, ['context-pipe']);
-      const result = await instance.provideCompletionItems(input);
-      expect(result.context).toBeUndefined();
-      expect(result.suggestions.length).toEqual(2);
-      expect(result.suggestions[0].label).toEqual('Operators');
-      expect(result.suggestions[1].label).toEqual('Parsers');
-    });
-  });
-
   describe('fetchSeries', () => {
     it('should use match[] parameter', () => {
       const datasource = setup({}, { '{foo="bar"}': [{ label1: 'label_val1' }] });
@@ -131,102 +61,6 @@ describe('Language completion provider', () => {
     });
   });
 
-  describe('label key suggestions', () => {
-    it('returns all label suggestions on empty selector', async () => {
-      const datasource = setup({ label1: [], label2: [] });
-      const provider = await getLanguageProvider(datasource);
-      const input = createTypeaheadInput('{}', '', '', 1);
-      const result = await provider.provideCompletionItems(input);
-      expect(result.context).toBe('context-labels');
-      expect(result.suggestions).toEqual([
-        {
-          items: [
-            { label: 'label1', filterText: '"label1"' },
-            { label: 'label2', filterText: '"label2"' },
-          ],
-          label: 'Labels',
-        },
-      ]);
-    });
-
-    it('returns all label suggestions on selector when starting to type', async () => {
-      const datasource = setup({ label1: [], label2: [] });
-      const provider = await getLanguageProvider(datasource);
-      const input = createTypeaheadInput('{l}', '', '', 2);
-      const result = await provider.provideCompletionItems(input);
-      expect(result.context).toBe('context-labels');
-      expect(result.suggestions).toEqual([
-        {
-          items: [
-            { label: 'label1', filterText: '"label1"' },
-            { label: 'label2', filterText: '"label2"' },
-          ],
-          label: 'Labels',
-        },
-      ]);
-    });
-  });
-
-  describe('label suggestions facetted', () => {
-    it('returns facetted label suggestions based on selector', async () => {
-      const datasource = setup({ label1: [], label2: [] }, { '{foo="bar"}': [{ label1: 'label_val1' }] });
-      const provider = await getLanguageProvider(datasource);
-      const input = createTypeaheadInput('{foo="bar",}', '', '', 11);
-      const result = await provider.provideCompletionItems(input);
-      expect(result.context).toBe('context-labels');
-      expect(result.suggestions).toEqual([{ items: [{ label: 'label1' }], label: 'Labels' }]);
-    });
-
-    it('returns facetted label suggestions for multipule selectors', async () => {
-      const datasource = setup({ label1: [], label2: [] }, { '{baz="42",foo="bar"}': [{ label2: 'label_val2' }] });
-      const provider = await getLanguageProvider(datasource);
-      const input = createTypeaheadInput('{baz="42",foo="bar",}', '', '', 20);
-      const result = await provider.provideCompletionItems(input);
-      expect(result.context).toBe('context-labels');
-      expect(result.suggestions).toEqual([{ items: [{ label: 'label2' }], label: 'Labels' }]);
-    });
-  });
-
-  describe('label suggestions', () => {
-    it('returns label values suggestions from Loki', async () => {
-      const datasource = setup({ label1: ['label1_val1', 'label1_val2'], label2: [] });
-      const provider = await getLanguageProvider(datasource);
-      const input = createTypeaheadInput('{label1=}', '=', 'label1');
-      let result = await provider.provideCompletionItems(input);
-
-      result = await provider.provideCompletionItems(input);
-      expect(result.context).toBe('context-label-values');
-      expect(result.suggestions).toEqual([
-        {
-          items: [
-            { label: 'label1_val1', filterText: '"label1_val1"' },
-            { label: 'label1_val2', filterText: '"label1_val2"' },
-          ],
-          label: 'Label values for "label1"',
-        },
-      ]);
-    });
-    it('returns label values suggestions from Loki when re-editing', async () => {
-      const datasource = setup({ label1: ['label1_val1', 'label1_val2'], label2: [] });
-      const provider = await getLanguageProvider(datasource);
-      const input = createTypeaheadInput('{label1="label1_v"}', 'label1_v', 'label1', 17, [
-        'attr-value',
-        'context-labels',
-      ]);
-      let result = await provider.provideCompletionItems(input);
-      expect(result.context).toBe('context-label-values');
-      expect(result.suggestions).toEqual([
-        {
-          items: [
-            { label: 'label1_val1', filterText: '"label1_val1"' },
-            { label: 'label1_val2', filterText: '"label1_val2"' },
-          ],
-          label: 'Label values for "label1"',
-        },
-      ]);
-    });
-  });
-
   describe('label values', () => {
     it('should fetch label values if not cached', async () => {
       const datasource = setup({ testkey: ['label1_val1', 'label1_val2'], label2: [] });
@@ -450,31 +284,6 @@ async function getLanguageProvider(datasource: LokiDatasource) {
   return instance;
 }
 
-/**
- * @param value Value of the full input
- * @param text Last piece of text (not sure but in case of {label=} this would be just '=')
- * @param labelKey Label by which to search for values. Cutting corners a bit here as this should be inferred from value
- */
-function createTypeaheadInput(
-  value: string,
-  text: string,
-  labelKey?: string,
-  anchorOffset?: number,
-  wrapperClasses?: string[],
-  instance?: LanguageProvider
-): TypeaheadInput {
-  const deserialized = Plain.deserialize(value);
-  const range = deserialized.selection.setAnchor(deserialized.selection.anchor.setOffset(anchorOffset || 1));
-  const valueWithSelection = deserialized.setSelection(range);
-  return {
-    text,
-    prefix: instance ? instance.cleanText(text) : '',
-    wrapperClasses: wrapperClasses || ['context-labels'],
-    value: valueWithSelection,
-    labelKey,
-  };
-}
-
 function setup(
   labelsAndValues: Record<string, string[]>,
   series?: Record<string, Array<Record<string, string>>>

public/app/plugins/datasource/loki/LanguageProvider.ts

@@ -1,15 +1,8 @@
-import { chain, difference } from 'lodash';
 import { LRUCache } from 'lru-cache';
-import Prism, { Grammar } from 'prismjs';
+import Prism from 'prismjs';
 
-import { dateTime, AbsoluteTimeRange, LanguageProvider, HistoryItem, AbstractQuery, KeyValue } from '@grafana/data';
-import { CompletionItem, TypeaheadInput, TypeaheadOutput, CompletionItemGroup } from '@grafana/ui';
-import {
-  extractLabelMatchers,
-  parseSelector,
-  processLabels,
-  toPromLikeExpr,
-} from 'app/plugins/datasource/prometheus/language_utils';
+import { LanguageProvider, AbstractQuery, KeyValue } from '@grafana/data';
+import { extractLabelMatchers, processLabels, toPromLikeExpr } from 'app/plugins/datasource/prometheus/language_utils';
 
 import { LokiDatasource } from './datasource';
 import {
@@ -17,61 +10,15 @@ import {
   extractLogParserFromDataFrame,
   extractUnwrapLabelKeysFromDataFrame,
 } from './responseUtils';
-import syntax, { FUNCTIONS, PIPE_PARSERS, PIPE_OPERATORS } from './syntax';
+import syntax from './syntax';
 import { ParserAndLabelKeysResult, LokiQuery, LokiQueryType } from './types';
 
-const DEFAULT_KEYS = ['job', 'namespace'];
-const EMPTY_SELECTOR = '{}';
-const HISTORY_ITEM_COUNT = 10;
-const HISTORY_COUNT_CUTOFF = 1000 * 60 * 60 * 24; // 24h
 const NS_IN_MS = 1000000;
 
-// When changing RATE_RANGES, check if Prometheus/PromQL ranges should be changed too
-// @see public/app/plugins/datasource/prometheus/promql.ts
-const RATE_RANGES: CompletionItem[] = [
-  { label: '$__auto', sortValue: '$__auto' },
-  { label: '1m', sortValue: '00:01:00' },
-  { label: '5m', sortValue: '00:05:00' },
-  { label: '10m', sortValue: '00:10:00' },
-  { label: '30m', sortValue: '00:30:00' },
-  { label: '1h', sortValue: '01:00:00' },
-  { label: '1d', sortValue: '24:00:00' },
-];
-
-export const LABEL_REFRESH_INTERVAL = 1000 * 30; // 30sec
-
-const wrapLabel = (label: string) => ({ label, filterText: `\"${label}\"` });
-
-export type LokiHistoryItem = HistoryItem<LokiQuery>;
-
-type TypeaheadContext = {
-  history?: LokiHistoryItem[];
-  absoluteRange?: AbsoluteTimeRange;
-};
-
-export function addHistoryMetadata(item: CompletionItem, history: LokiHistoryItem[]): CompletionItem {
-  const cutoffTs = Date.now() - HISTORY_COUNT_CUTOFF;
-  const historyForItem = history.filter((h) => h.ts > cutoffTs && h.query.expr === item.label);
-  let hint = `Queried ${historyForItem.length} times in the last 24h.`;
-  const recent = historyForItem[0];
-
-  if (recent) {
-    const lastQueried = dateTime(recent.ts).fromNow();
-    hint = `${hint} Last queried ${lastQueried}.`;
-  }
-
-  return {
-    ...item,
-    documentation: hint,
-  };
-}
-
 export default class LokiLanguageProvider extends LanguageProvider {
   labelKeys: string[];
-  labelFetchTs: number;
   started = false;
   datasource: LokiDatasource;
-  lookupsDisabled = false; // Dynamically set to true for big/slow instances
 
   /**
    *  Cache for labels of series. This is bit simplistic in the sense that it just counts responses each as a 1 and does
@@ -86,18 +33,10 @@ export default class LokiLanguageProvider extends LanguageProvider {
 
     this.datasource = datasource;
     this.labelKeys = [];
-    this.labelFetchTs = 0;
 
     Object.assign(this, initialValues);
   }
 
-  // Strip syntax chars
-  cleanText = (s: string) => s.replace(/[{}[\]="(),!~+\-*/^%\|]/g, '').trim();
-
-  getSyntax(): Grammar {
-    return syntax;
-  }
-
   request = async (url: string, params?: any): Promise<any> => {
     try {
       return await this.datasource.metadataRequest(url, params);
@@ -109,8 +48,7 @@ export default class LokiLanguageProvider extends LanguageProvider {
   };
 
   /**
-   * Initialise the language provider by fetching set of labels. Without this initialisation the provider would return
-   * just a set of hardcoded default labels on provideCompletionItems or a recent queries from history.
+   * Initialize the language provider by fetching set of labels.
    */
   start = () => {
     if (!this.startTask) {
@@ -123,214 +61,21 @@ export default class LokiLanguageProvider extends LanguageProvider {
     return this.startTask;
   };
 
+  /**
+   * Returns the label keys that have been fetched.
+   * If labels have not been fetched yet, it will return an empty array.
+   * For updated labels (which should not happen often), use fetchLabels.
+   * @todo It is quite complicated to know when to use fetchLabels and when to use getLabelKeys.
+   * We should consider simplifying this and use caching in the same way as with seriesCache and labelsCache
+   * and just always use fetchLabels.
+   * Caching should be thought out properly, so we are not fetching this often, as labelKeys should not be changing often.
+   *
+   * @returns {string[]} An array of label keys or an empty array if labels have not been fetched.
+   */
   getLabelKeys(): string[] {
     return this.labelKeys;
   }
 
-  /**
-   * Return suggestions based on input that can be then plugged into a typeahead dropdown.
-   * Keep this DOM-free for testing
-   * @param input
-   * @param context Is optional in types but is required in case we are doing getLabelCompletionItems
-   * @param context.absoluteRange Required in case we are doing getLabelCompletionItems
-   * @param context.history Optional used only in getEmptyCompletionItems
-   */
-  async provideCompletionItems(input: TypeaheadInput, context?: TypeaheadContext): Promise<TypeaheadOutput> {
-    const { wrapperClasses, value, prefix, text } = input;
-    const emptyResult: TypeaheadOutput = { suggestions: [] };
-
-    if (!value) {
-      return emptyResult;
-    }
-
-    // Local text properties
-    const empty = value?.document.text.length === 0;
-    const selectedLines = value.document.getTextsAtRange(value.selection);
-    const currentLine = selectedLines.size === 1 ? selectedLines.first().getText() : null;
-
-    const nextCharacter = currentLine ? currentLine[value.selection.anchor.offset] : null;
-
-    // Syntax spans have 3 classes by default. More indicate a recognized token
-    const tokenRecognized = wrapperClasses.length > 3;
-
-    // Non-empty prefix, but not inside known token
-    const prefixUnrecognized = prefix && !tokenRecognized;
-
-    // Prevent suggestions in `function(|suffix)`
-    const noSuffix = !nextCharacter || nextCharacter === ')';
-
-    // Prefix is safe if it does not immediately follow a complete expression and has no text after it
-    const safePrefix = prefix && !text.match(/^['"~=\]})\s]+$/) && noSuffix;
-
-    // About to type next operand if preceded by binary operator
-    const operatorsPattern = /[+\-*/^%]/;
-    const isNextOperand = text.match(operatorsPattern);
-
-    // Determine candidates by CSS context
-    if (wrapperClasses.includes('context-range')) {
-      // Suggestions for metric[|]
-      return this.getRangeCompletionItems();
-    } else if (wrapperClasses.includes('context-labels')) {
-      // Suggestions for {|} and {foo=|}
-      return await this.getLabelCompletionItems(input);
-    } else if (wrapperClasses.includes('context-pipe')) {
-      return this.getPipeCompletionItem();
-    } else if (empty) {
-      // Suggestions for empty query field
-      return this.getEmptyCompletionItems(context);
-    } else if (prefixUnrecognized && noSuffix && !isNextOperand) {
-      // Show term suggestions in a couple of scenarios
-      return this.getBeginningCompletionItems(context);
-    } else if (prefixUnrecognized && safePrefix) {
-      // Show term suggestions in a couple of scenarios
-      return this.getTermCompletionItems();
-    }
-
-    return emptyResult;
-  }
-
-  getBeginningCompletionItems = (context?: TypeaheadContext): TypeaheadOutput => {
-    return {
-      suggestions: [...this.getEmptyCompletionItems(context).suggestions, ...this.getTermCompletionItems().suggestions],
-    };
-  };
-
-  getEmptyCompletionItems(context?: TypeaheadContext): TypeaheadOutput {
-    const history = context?.history;
-    const suggestions = [];
-
-    if (history?.length) {
-      const historyItems = chain(history)
-        .map((h) => h.query.expr)
-        .filter()
-        .uniq()
-        .take(HISTORY_ITEM_COUNT)
-        .map(wrapLabel)
-        .map((item) => addHistoryMetadata(item, history))
-        .value();
-
-      suggestions.push({
-        prefixMatch: true,
-        skipSort: true,
-        label: 'History',
-        items: historyItems,
-      });
-    }
-
-    return { suggestions };
-  }
-
-  getTermCompletionItems = (): TypeaheadOutput => {
-    const suggestions = [];
-
-    suggestions.push({
-      prefixMatch: true,
-      label: 'Functions',
-      items: FUNCTIONS.map((suggestion) => ({ ...suggestion, kind: 'function' })),
-    });
-
-    return { suggestions };
-  };
-
-  getPipeCompletionItem = (): TypeaheadOutput => {
-    const suggestions = [];
-
-    suggestions.push({
-      label: 'Operators',
-      items: PIPE_OPERATORS.map((suggestion) => ({ ...suggestion, kind: 'operators' })),
-    });
-
-    suggestions.push({
-      label: 'Parsers',
-      items: PIPE_PARSERS.map((suggestion) => ({ ...suggestion, kind: 'parsers' })),
-    });
-
-    return { suggestions };
-  };
-
-  getRangeCompletionItems(): TypeaheadOutput {
-    return {
-      context: 'context-range',
-      suggestions: [
-        {
-          label: 'Range vector',
-          items: [...RATE_RANGES],
-        },
-      ],
-    };
-  }
-
-  async getLabelCompletionItems({ text, wrapperClasses, labelKey, value }: TypeaheadInput): Promise<TypeaheadOutput> {
-    let context = 'context-labels';
-    const suggestions: CompletionItemGroup[] = [];
-    if (!value) {
-      return { context, suggestions: [] };
-    }
-    const line = value.anchorBlock.getText();
-    const cursorOffset = value.selection.anchor.offset;
-    const isValueStart = text.match(/^(=|=~|!=|!~)/);
-
-    // Get normalized selector
-    let selector;
-    let parsedSelector;
-    try {
-      parsedSelector = parseSelector(line, cursorOffset);
-      selector = parsedSelector.selector;
-    } catch {
-      selector = EMPTY_SELECTOR;
-    }
-
-    if (!labelKey && selector === EMPTY_SELECTOR) {
-      // start task gets all labels
-      await this.start();
-      const allLabels = this.getLabelKeys();
-      return { context, suggestions: [{ label: `Labels`, items: allLabels.map(wrapLabel) }] };
-    }
-
-    const existingKeys = parsedSelector ? parsedSelector.labelKeys : [];
-
-    let labelValues;
-    // Query labels for selector
-    if (selector) {
-      if (selector === EMPTY_SELECTOR && labelKey) {
-        const labelValuesForKey = await this.getLabelValues(labelKey);
-        labelValues = { [labelKey]: labelValuesForKey };
-      } else {
-        labelValues = await this.getSeriesLabels(selector);
-      }
-    }
-
-    if (!labelValues) {
-      console.warn(`Server did not return any values for selector = ${selector}`);
-      return { context, suggestions };
-    }
-
-    if ((text && isValueStart) || wrapperClasses.includes('attr-value')) {
-      // Label values
-      if (labelKey && labelValues[labelKey]) {
-        context = 'context-label-values';
-        suggestions.push({
-          label: `Label values for "${labelKey}"`,
-          // Filter to prevent previously selected values from being repeatedly suggested
-          items: labelValues[labelKey].map(wrapLabel).filter(({ filterText }) => filterText !== text),
-        });
-      }
-    } else {
-      // Label keys
-      const labelKeys = labelValues ? Object.keys(labelValues) : DEFAULT_KEYS;
-      if (labelKeys) {
-        const possibleKeys = difference(labelKeys, existingKeys);
-        if (possibleKeys.length) {
-          const newItems = possibleKeys.map((key) => ({ label: key }));
-          const newSuggestion: CompletionItemGroup = { label: `Labels`, items: newItems };
-          suggestions.push(newSuggestion);
-        }
-      }
-    }
-
-    return { context, suggestions };
-  }
-
   importFromAbstractQuery(labelBasedQuery: AbstractQuery): LokiQuery {
     return {
       refId: labelBasedQuery.refId,
@@ -351,10 +96,11 @@ export default class LokiLanguageProvider extends LanguageProvider {
     };
   }
 
+  /**
+   * Wrapper method over fetchSeriesLabels to retrieve series labels and handle errors.
+   * @todo remove this in favor of fetchSeriesLabels as we already in this.request do the same thing
+   */
   async getSeriesLabels(selector: string) {
-    if (this.lookupsDisabled) {
-      return undefined;
-    }
     try {
       return await this.fetchSeriesLabels(selector);
     } catch (error) {
@@ -375,7 +121,6 @@ export default class LokiLanguageProvider extends LanguageProvider {
   async fetchLabels(): Promise<string[]> {
     const url = 'labels';
     const timeRange = this.datasource.getTimeRangeParams();
-    this.labelFetchTs = Date.now().valueOf();
 
     const res = await this.request(url, timeRange);
     if (Array.isArray(res)) {
@@ -432,15 +177,18 @@ export default class LokiLanguageProvider extends LanguageProvider {
   // The rounding may seem strange but makes relative intervals like now-1h less prone to need separate request every
   // millisecond while still actually getting all the keys for the correct interval. This still can create problems
   // when user does not the newest values for a minute if already cached.
-  generateCacheKey(url: string, start: number, end: number, param: string): string {
+  private generateCacheKey(url: string, start: number, end: number, param: string): string {
     return [url, this.roundTime(start), this.roundTime(end), param].join();
   }
 
-  // Round nanos epoch to nearest 5 minute interval
-  roundTime(nanos: number): number {
-    return nanos ? Math.floor(nanos / NS_IN_MS / 1000 / 60 / 5) : 0;
+  // Round nanoseconds epoch to nearest 5 minute interval
+  private roundTime(nanoseconds: number): number {
+    return nanoseconds ? Math.floor(nanoseconds / NS_IN_MS / 1000 / 60 / 5) : 0;
   }
 
+  /**
+   * @todo remove this in favor of fetchLabelValues as it is the same thing
+   */
   async getLabelValues(key: string): Promise<string[]> {
     return await this.fetchLabelValues(key);
   }

public/app/plugins/datasource/loki/syntax.ts

@@ -191,8 +191,8 @@ export const BUILT_IN_FUNCTIONS = [
 ];
 
 export const FUNCTIONS = [...AGGREGATION_OPERATORS, ...RANGE_VEC_FUNCTIONS, ...BUILT_IN_FUNCTIONS];
-export const LOKI_KEYWORDS = [...FUNCTIONS, ...PIPE_OPERATORS, ...PIPE_PARSERS].map((keyword) => keyword.label);
 
+// Loki grammar is used for query highlight in query previews outside of code editor
 export const lokiGrammar: Grammar = {
   comment: {
     pattern: /#.*/,