Loki: Add support for ad-hoc filtering in dashboard (#36393)

* Add support for ad-hoc filtering in dashboards

* Add tests

* Add documentation

* Update docs/sources/variables/variable-types/add-ad-hoc-filters.md

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

* Remove unused import

* Refactor and simplify

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

public/app/plugins/datasource/loki/datasource.test.ts

@@ -34,6 +34,7 @@ const timeSrvStub = ({
 } as unknown) as TimeSrv;
 
 const templateSrvStub = {
+  getAdhocFilters: jest.fn(() => [] as any[]),
   replace: jest.fn((a: string, ...rest: any) => a),
 };
 
@@ -334,6 +335,68 @@ describe('LokiDatasource', () => {
       });
     });
 
+    describe('When using adhoc filters', () => {
+      const DEFAULT_EXPR = 'rate({bar="baz", job="foo"} |= "bar" [5m])';
+      const options = {
+        targets: [{ expr: DEFAULT_EXPR }],
+      };
+      const originalAdhocFiltersMock = templateSrvStub.getAdhocFilters();
+      const ds = new LokiDatasource({} as any, templateSrvStub as any, timeSrvStub as any);
+      ds.runRangeQuery = jest.fn(() => of({ data: [] }));
+
+      afterAll(() => {
+        templateSrvStub.getAdhocFilters.mockReturnValue(originalAdhocFiltersMock);
+      });
+
+      it('should not modify expression with no filters', async () => {
+        await ds.query(options as any).toPromise();
+        expect(ds.runRangeQuery).toBeCalledWith({ expr: DEFAULT_EXPR }, expect.anything(), expect.anything());
+      });
+
+      it('should add filters to expression', async () => {
+        templateSrvStub.getAdhocFilters.mockReturnValue([
+          {
+            key: 'k1',
+            operator: '=',
+            value: 'v1',
+          },
+          {
+            key: 'k2',
+            operator: '!=',
+            value: 'v2',
+          },
+        ]);
+
+        await ds.query(options as any).toPromise();
+        expect(ds.runRangeQuery).toBeCalledWith(
+          { expr: 'rate({bar="baz",job="foo",k1="v1",k2!="v2"} |= "bar" [5m])' },
+          expect.anything(),
+          expect.anything()
+        );
+      });
+
+      it('should add escaping if needed to regex filter expressions', async () => {
+        templateSrvStub.getAdhocFilters.mockReturnValue([
+          {
+            key: 'k1',
+            operator: '=~',
+            value: 'v.*',
+          },
+          {
+            key: 'k2',
+            operator: '=~',
+            value: `v'.*`,
+          },
+        ]);
+        await ds.query(options as any).toPromise();
+        expect(ds.runRangeQuery).toBeCalledWith(
+          { expr: 'rate({bar="baz",job="foo",k1=~"v.*",k2=~"v\\\\\'.*"} |= "bar" [5m])' },
+          expect.anything(),
+          expect.anything()
+        );
+      });
+    });
+
     describe('__range, __range_s and __range_ms variables', () => {
       const options = {
         targets: [{ expr: 'rate(process_cpu_seconds_total[$__range])', refId: 'A' }],

public/app/plugins/datasource/loki/datasource.ts

@@ -25,7 +25,8 @@ import {
   ScopedVars,
   TimeRange,
 } from '@grafana/data';
-import { getTemplateSrv, TemplateSrv, BackendSrvRequest, FetchError, getBackendSrv } from '@grafana/runtime';
+import { BackendSrvRequest, FetchError, getBackendSrv } from '@grafana/runtime';
+import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv';
 import { addLabelToQuery } from 'app/plugins/datasource/prometheus/add_label_to_query';
 import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv';
 import { convertToWebSocketUrl } from 'app/core/utils/explore';
@@ -102,10 +103,13 @@ export class LokiDatasource extends DataSourceApi<LokiQuery, LokiOptions> {
     };
     const filteredTargets = options.targets
       .filter((target) => target.expr && !target.hide)
-      .map((target) => ({
-        ...target,
-        expr: this.templateSrv.replace(target.expr, scopedVars, this.interpolateQueryExpr),
-      }));
+      .map((target) => {
+        const expr = this.addAdHocFilters(target.expr);
+        return {
+          ...target,
+          expr: this.templateSrv.replace(expr, scopedVars, this.interpolateQueryExpr),
+        };
+      });
 
     for (const target of filteredTargets) {
       if (target.instant) {
@@ -379,6 +383,15 @@ export class LokiDatasource extends DataSourceApi<LokiQuery, LokiOptions> {
     return Array.from(streams);
   }
 
+  // By implementing getTagKeys and getTagValues we add ad-hoc filtters functionality
+  async getTagKeys() {
+    return await this.labelNamesQuery();
+  }
+
+  async getTagValues(options: any = {}) {
+    return await this.labelValuesQuery(options.key);
+  }
+
   interpolateQueryExpr(value: any, variable: any) {
     // if no multi or include all do not regexEscape
     if (!variable.multi && !variable.includeAll) {
@@ -628,6 +641,22 @@ export class LokiDatasource extends DataSourceApi<LokiQuery, LokiOptions> {
     // The min interval is set to 1ms
     return Math.max(interval, 1);
   }
+
+  addAdHocFilters(queryExpr: string) {
+    const adhocFilters = this.templateSrv.getAdhocFilters(this.name);
+    let expr = queryExpr;
+
+    expr = adhocFilters.reduce((acc: string, filter: { key?: any; operator?: any; value?: any }) => {
+      const { key, operator } = filter;
+      let { value } = filter;
+      if (operator === '=~' || operator === '!~') {
+        value = lokiRegularEscape(value);
+      }
+      return addLabelToQuery(acc, key, value, operator);
+    }, expr);
+
+    return expr;
+  }
 }
 
 export function renderTemplate(aliasPattern: string, aliasData: { [key: string]: string }) {