Prometheus: disable dynamic label lookup on big datasources (#20936)

* Prometheus: disable dynamic label lookup on big datasources

- when a prometheus datasource has more than 10000 metrics, label lookup
for the query field is disabled
- installations of that size have slow typehead lookup times and make
the editor sluggish

* Raise dynamic lookup threshold to 10000 metrics

* Run start tasks again

public/app/plugins/datasource/prometheus/components/PromQueryField.tsx

@@ -23,7 +23,6 @@ import { PrometheusDatasource } from '../datasource';
 import PromQlLanguageProvider from '../language_provider';
 
 const HISTOGRAM_GROUP = '__histograms__';
-const METRIC_MARK = 'metric';
 const PRISM_SYNTAX = 'promql';
 export const RECORDING_RULES_GROUP = '__recording_rules__';
 
@@ -138,6 +137,7 @@ class PromQueryField extends React.PureComponent<PromQueryFieldProps, PromQueryF
 
   componentDidMount() {
     if (this.languageProvider) {
+      Prism.languages[PRISM_SYNTAX] = this.languageProvider.syntax;
       this.refreshMetrics(makePromiseCancelable(this.languageProvider.start()));
     }
     this.refreshHint();
@@ -228,17 +228,11 @@ class PromQueryField extends React.PureComponent<PromQueryFieldProps, PromQueryF
   };
 
   onUpdateLanguage = () => {
-    const { histogramMetrics, metrics } = this.languageProvider;
+    const { histogramMetrics, metrics, lookupsDisabled, lookupMetricsThreshold } = this.languageProvider;
     if (!metrics) {
       return;
     }
 
-    Prism.languages[PRISM_SYNTAX] = this.languageProvider.syntax;
-    Prism.languages[PRISM_SYNTAX][METRIC_MARK] = {
-      alias: 'variable',
-      pattern: new RegExp(`(?:^|\\s)(${metrics.join('|')})(?:$|\\s)`),
-    };
-
     // Build metrics tree
     const metricsByPrefix = groupMetricsByPrefix(metrics);
     const histogramOptions = histogramMetrics.map((hm: any) => ({ label: hm, value: hm }));
@@ -250,7 +244,16 @@ class PromQueryField extends React.PureComponent<PromQueryFieldProps, PromQueryF
           ]
         : metricsByPrefix;
 
-    this.setState({ metricsOptions, syntaxLoaded: true });
+    // Hint for big disabled lookups
+    let hint: QueryHint;
+    if (lookupsDisabled) {
+      hint = {
+        label: `Dynamic label lookup is disabled for datasources with more than ${lookupMetricsThreshold} metrics.`,
+        type: 'INFO',
+      };
+    }
+
+    this.setState({ hint, metricsOptions, syntaxLoaded: true });
   };
 
   onTypeahead = async (typeahead: TypeaheadInput): Promise<TypeaheadOutput> => {

public/app/plugins/datasource/prometheus/language_provider.test.ts

@@ -154,6 +154,7 @@ describe('Language completion provider', () => {
   describe('label suggestions', () => {
     it('returns default label suggestions on label context and no metric', async () => {
       const instance = new LanguageProvider(datasource);
+      instance.lookupsDisabled = false;
       const value = Plain.deserialize('{}');
       const ed = new SlateEditor({ value });
       const valueWithSelection = ed.moveForward(1).value;
@@ -173,6 +174,7 @@ describe('Language completion provider', () => {
         getTimeRange: () => ({ start: 0, end: 1 }),
       } as any) as PrometheusDatasource;
       const instance = new LanguageProvider(datasources);
+      instance.lookupsDisabled = false;
       const value = Plain.deserialize('metric{}');
       const ed = new SlateEditor({ value });
       const valueWithSelection = ed.moveForward(7).value;
@@ -204,6 +206,7 @@ describe('Language completion provider', () => {
         getTimeRange: () => ({ start: 0, end: 1 }),
       } as any) as PrometheusDatasource;
       const instance = new LanguageProvider(datasource);
+      instance.lookupsDisabled = false;
       const value = Plain.deserialize('{job1="foo",job2!="foo",job3=~"foo",__name__="metric",}');
       const ed = new SlateEditor({ value });
       const valueWithSelection = ed.moveForward(54).value;
@@ -224,6 +227,7 @@ describe('Language completion provider', () => {
           return { data: { data: ['value1', 'value2'] } };
         },
       } as any) as PrometheusDatasource);
+      instance.lookupsDisabled = false;
       const value = Plain.deserialize('{job!=}');
       const ed = new SlateEditor({ value });
       const valueWithSelection = ed.moveForward(8).value;
@@ -245,6 +249,7 @@ describe('Language completion provider', () => {
 
     it('returns a refresher on label context and unavailable metric', async () => {
       const instance = new LanguageProvider(datasource);
+      instance.lookupsDisabled = false;
       const value = Plain.deserialize('metric{}');
       const ed = new SlateEditor({ value });
       const valueWithSelection = ed.moveForward(7).value;
@@ -263,6 +268,7 @@ describe('Language completion provider', () => {
         ...datasource,
         metadataRequest: () => simpleMetricLabelsResponse,
       } as any) as PrometheusDatasource);
+      instance.lookupsDisabled = false;
       const value = Plain.deserialize('metric{bar=ba}');
       const ed = new SlateEditor({ value });
       const valueWithSelection = ed.moveForward(13).value;
@@ -282,6 +288,7 @@ describe('Language completion provider', () => {
         ...datasource,
         metadataRequest: () => simpleMetricLabelsResponse,
       } as any) as PrometheusDatasource);
+      instance.lookupsDisabled = false;
       const value = Plain.deserialize('sum(metric{foo="xx"}) by ()');
       const ed = new SlateEditor({ value });
       const valueWithSelection = ed.moveForward(26).value;
@@ -300,6 +307,7 @@ describe('Language completion provider', () => {
         ...datasource,
         metadataRequest: () => simpleMetricLabelsResponse,
       } as any) as PrometheusDatasource);
+      instance.lookupsDisabled = false;
       const value = Plain.deserialize('sum(metric) by ()');
       const ed = new SlateEditor({ value });
       const valueWithSelection = ed.moveForward(16).value;
@@ -318,6 +326,7 @@ describe('Language completion provider', () => {
         ...datasource,
         metadataRequest: () => simpleMetricLabelsResponse,
       } as any) as PrometheusDatasource);
+      instance.lookupsDisabled = false;
       const value = Plain.deserialize('sum(\nmetric\n)\nby ()');
       const aggregationTextBlock = value.document.getBlocks().get(3);
       const ed = new SlateEditor({ value });
@@ -343,6 +352,7 @@ describe('Language completion provider', () => {
         ...datasource,
         metadataRequest: () => simpleMetricLabelsResponse,
       } as any) as PrometheusDatasource);
+      instance.lookupsDisabled = false;
       const value = Plain.deserialize('sum(rate(metric[1h])) by ()');
       const ed = new SlateEditor({ value });
       const valueWithSelection = ed.moveForward(26).value;
@@ -366,6 +376,7 @@ describe('Language completion provider', () => {
         ...datasource,
         metadataRequest: () => simpleMetricLabelsResponse,
       } as any) as PrometheusDatasource);
+      instance.lookupsDisabled = false;
       const value = Plain.deserialize('sum(rate(metric{label1="value"}[1h])) by ()');
       const ed = new SlateEditor({ value });
       const valueWithSelection = ed.moveForward(42).value;
@@ -386,6 +397,7 @@ describe('Language completion provider', () => {
 
     it('returns no suggestions inside an unclear aggregation context using alternate syntax', async () => {
       const instance = new LanguageProvider(datasource);
+      instance.lookupsDisabled = false;
       const value = Plain.deserialize('sum by ()');
       const ed = new SlateEditor({ value });
       const valueWithSelection = ed.moveForward(8).value;
@@ -404,6 +416,7 @@ describe('Language completion provider', () => {
         ...datasource,
         metadataRequest: () => simpleMetricLabelsResponse,
       } as any) as PrometheusDatasource);
+      instance.lookupsDisabled = false;
       const value = Plain.deserialize('sum by () (metric)');
       const ed = new SlateEditor({ value });
       const valueWithSelection = ed.moveForward(8).value;
@@ -429,6 +442,7 @@ describe('Language completion provider', () => {
       } as any) as PrometheusDatasource;
 
       const instance = new LanguageProvider(datasource);
+      instance.lookupsDisabled = false;
       const value = Plain.deserialize('{}');
       const ed = new SlateEditor({ value });
       const valueWithSelection = ed.moveForward(1).value;
@@ -447,6 +461,57 @@ describe('Language completion provider', () => {
       expect((datasource.metadataRequest as Mock).mock.calls.length).toBe(2);
     });
   });
+
+  describe('dynamic lookup protection for big installations', () => {
+    it('dynamic lookup is enabled if number of metrics is reasonably low', async () => {
+      const datasource: PrometheusDatasource = ({
+        metadataRequest: () => ({ data: { data: ['foo'] as string[] } }),
+        getTimeRange: () => ({ start: 0, end: 1 }),
+      } as any) as PrometheusDatasource;
+
+      const instance = new LanguageProvider(datasource, { lookupMetricsThreshold: 1 });
+      expect(instance.lookupsDisabled).toBeTruthy();
+      await instance.start();
+      expect(instance.lookupsDisabled).toBeFalsy();
+    });
+
+    it('dynamic lookup is disabled if number of metrics is higher than threshold', async () => {
+      const datasource: PrometheusDatasource = ({
+        metadataRequest: () => ({ data: { data: ['foo', 'bar'] as string[] } }),
+        getTimeRange: () => ({ start: 0, end: 1 }),
+      } as any) as PrometheusDatasource;
+
+      const instance = new LanguageProvider(datasource, { lookupMetricsThreshold: 1 });
+      expect(instance.lookupsDisabled).toBeTruthy();
+      await instance.start();
+      expect(instance.lookupsDisabled).toBeTruthy();
+    });
+
+    it('does not issue label-based metadata requests when lookup is disabled', async () => {
+      const datasource: PrometheusDatasource = ({
+        metadataRequest: jest.fn(() => ({ data: { data: ['foo', 'bar'] as string[] } })),
+        getTimeRange: jest.fn(() => ({ start: 0, end: 1 })),
+      } as any) as PrometheusDatasource;
+
+      const instance = new LanguageProvider(datasource, { lookupMetricsThreshold: 1 });
+      const value = Plain.deserialize('{}');
+      const ed = new SlateEditor({ value });
+      const valueWithSelection = ed.moveForward(1).value;
+      const args = {
+        text: '',
+        prefix: '',
+        wrapperClasses: ['context-labels'],
+        value: valueWithSelection,
+      };
+      expect(instance.lookupsDisabled).toBeTruthy();
+      expect((datasource.metadataRequest as Mock).mock.calls.length).toBe(0);
+      await instance.start();
+      expect(instance.lookupsDisabled).toBeTruthy();
+      expect((datasource.metadataRequest as Mock).mock.calls.length).toBe(1);
+      await instance.provideCompletionItems(args);
+      expect((datasource.metadataRequest as Mock).mock.calls.length).toBe(1);
+    });
+  });
 });
 
 const simpleMetricLabelsResponse = {

public/app/plugins/datasource/prometheus/language_provider.ts

@@ -14,6 +14,7 @@ const DEFAULT_KEYS = ['job', 'instance'];
 const EMPTY_SELECTOR = '{}';
 const HISTORY_ITEM_COUNT = 5;
 const HISTORY_COUNT_CUTOFF = 1000 * 60 * 60 * 24; // 24h
+export const DEFAULT_LOOKUP_METRICS_THRESHOLD = 10000; // number of metrics defining an installation that's too big
 
 const wrapLabel = (label: string): CompletionItem => ({ label });
 
@@ -46,6 +47,8 @@ export default class PromQlLanguageProvider extends LanguageProvider {
   metrics?: string[];
   startTask: Promise<any>;
   datasource: PrometheusDatasource;
+  lookupMetricsThreshold: number;
+  lookupsDisabled: boolean; // Dynamically set to true for big/slow instances
 
   /**
    *  Cache for labels of series. This is bit simplistic in the sense that it just counts responses each as a 1 and does
@@ -54,13 +57,18 @@ export default class PromQlLanguageProvider extends LanguageProvider {
    */
   private labelsCache = new LRU<string, Record<string, string[]>>(10);
 
-  constructor(datasource: PrometheusDatasource) {
+  constructor(datasource: PrometheusDatasource, initialValues?: Partial<PromQlLanguageProvider>) {
     super();
 
     this.datasource = datasource;
     this.histogramMetrics = [];
     this.timeRange = { start: 0, end: 0 };
     this.metrics = [];
+    // Disable lookups until we know the instance is small enough
+    this.lookupMetricsThreshold = DEFAULT_LOOKUP_METRICS_THRESHOLD;
+    this.lookupsDisabled = true;
+
+    Object.assign(this, initialValues);
   }
 
   // Strip syntax chars
@@ -83,22 +91,11 @@ export default class PromQlLanguageProvider extends LanguageProvider {
     return [];
   };
 
-  start = () => {
-    if (!this.startTask) {
-      this.startTask = this.fetchMetrics();
-    }
-    return this.startTask;
-  };
-
-  fetchMetrics = async () => {
-    this.metrics = await this.fetchMetricNames();
+  start = async (): Promise<any[]> => {
+    this.metrics = await this.request('/api/v1/label/__name__/values');
+    this.lookupsDisabled = this.metrics.length > this.lookupMetricsThreshold;
     this.processHistogramMetrics(this.metrics);
-
-    return Promise.resolve([]);
-  };
-
-  fetchMetricNames = async (): Promise<string[]> => {
-    return this.request('/api/v1/label/__name__/values');
+    return [];
   };
 
   processHistogramMetrics = (data: string[]) => {
@@ -333,6 +330,9 @@ export default class PromQlLanguageProvider extends LanguageProvider {
   };
 
   async getLabelValues(selector: string, withName?: boolean) {
+    if (this.lookupsDisabled) {
+      return undefined;
+    }
     try {
       if (selector === EMPTY_SELECTOR) {
         return await this.fetchDefaultLabels();