Elasticsearch: Fix adding of adhoc filters when jumping to explore (#60691)

* Elasticsearch: Fix adding of adhoc filters when jumping to explore

* Change NOT operator to - as it is preffered solution

public/app/plugins/datasource/elasticsearch/LanguageProvider.test.ts

@@ -37,7 +37,7 @@ describe('transform abstract query to elasticsearch query', () => {
 
     expect(result).toEqual({
       ...baseLogsQuery,
-      query: 'label1:"value1" AND NOT label2:"value2" AND label3:/value3/ AND NOT label4:/value4/',
+      query: 'label1:"value1" AND -label2:"value2" AND label3:/value3/ AND -label4:/value4/',
       refId: abstractQuery.refId,
     });
   });

public/app/plugins/datasource/elasticsearch/LanguageProvider.ts

@@ -39,13 +39,13 @@ export default class ElasticsearchLanguageProvider extends LanguageProvider {
             return label.name + ':"' + label.value + '"';
           }
           case AbstractLabelOperator.NotEqual: {
-            return 'NOT ' + label.name + ':"' + label.value + '"';
+            return '-' + label.name + ':"' + label.value + '"';
           }
           case AbstractLabelOperator.EqualRegEx: {
             return label.name + ':/' + label.value + '/';
           }
           case AbstractLabelOperator.NotEqualRegEx: {
-            return 'NOT ' + label.name + ':/' + label.value + '/';
+            return '-' + label.name + ':/' + label.value + '/';
           }
         }
       })

public/app/plugins/datasource/elasticsearch/datasource.test.ts

@@ -81,6 +81,7 @@ interface TestContext {
   jsonData?: Partial<ElasticsearchOptions>;
   database?: string;
   fetchMockImplementation?: (options: BackendSrvRequest) => Observable<FetchResponse>;
+  templateSrvMock?: TemplateSrv;
 }
 
 interface Data {
@@ -92,7 +93,8 @@ function getTestContext({
   from = 'now-5m',
   jsonData,
   database = '[test-]YYYY.MM.DD',
-  fetchMockImplementation = undefined,
+  fetchMockImplementation,
+  templateSrvMock,
 }: TestContext = {}) {
   const defaultMock = (options: BackendSrvRequest) => of(createFetchResponse(data));
 
@@ -113,16 +115,18 @@ function getTestContext({
   const settings: Partial<DataSourceInstanceSettings<ElasticsearchOptions>> = { url: ELASTICSEARCH_MOCK_URL };
   settings.jsonData = jsonData as ElasticsearchOptions;
 
-  const templateSrv = {
+  const templateSrv =
-    replace: (text?: string) => {
+    templateSrvMock ??
-      if (text?.startsWith('$')) {
+    ({
-        return `resolvedVariable`;
+      replace: (text?: string) => {
-      } else {
+        if (text?.startsWith('$')) {
-        return text;
+          return `resolvedVariable`;
-      }
+        } else {
-    },
+          return text;
-    getAdhocFilters: () => [],
+        }
-  } as unknown as TemplateSrv;
+      },
+      getAdhocFilters: () => [],
+    } as unknown as TemplateSrv);
 
   const ds = createElasticDatasource(settings, templateSrv);
 
@@ -889,6 +893,24 @@ describe('ElasticDatasource', () => {
     expect((interpolatedQuery.bucketAggs![0] as Filters).settings!.filters![0].query).toBe('resolvedVariable');
   });
 
+  it('should correctly add ad hoc filters when interpolating variables in query', () => {
+    const templateSrvMock = {
+      replace: (text?: string) => text,
+      getAdhocFilters: () => [{ key: 'bar', operator: '=', value: 'test' }],
+    } as unknown as TemplateSrv;
+    const { ds } = getTestContext({ templateSrvMock });
+    const query: ElasticsearchQuery = {
+      refId: 'A',
+      bucketAggs: [{ type: 'filters', settings: { filters: [{ query: '$var', label: '' }] }, id: '1' }],
+      metrics: [{ type: 'count', id: '1' }],
+      query: 'foo:"bar"',
+    };
+
+    const interpolatedQuery = ds.interpolateVariablesInQueries([query], {})[0];
+
+    expect(interpolatedQuery.query).toBe('foo:"bar" AND bar:"test"');
+  });
+
   it('should correctly handle empty query strings in filters bucket aggregation', () => {
     const { ds } = getTestContext();
     const query: ElasticsearchQuery = {
@@ -1062,6 +1084,81 @@ describe('modifyQuery', () => {
   });
 });
 
+describe('addAdhocFilters', () => {
+  describe('with invalid filters', () => {
+    it('should filter out ad hoc filter without key', () => {
+      const templateSrvMock = {
+        getAdhocFilters: () => [{ key: '', operator: '=', value: 'a' }],
+      } as unknown as TemplateSrv;
+      const { ds } = getTestContext({ templateSrvMock });
+
+      const query = ds.addAdHocFilters('foo:"bar"');
+      expect(query).toBe('foo:"bar"');
+    });
+
+    it('should filter out ad hoc filter without value', () => {
+      const templateSrvMock = {
+        getAdhocFilters: () => [{ key: 'a', operator: '=', value: '' }],
+      } as unknown as TemplateSrv;
+      const { ds } = getTestContext({ templateSrvMock });
+
+      const query = ds.addAdHocFilters('foo:"bar"');
+      expect(query).toBe('foo:"bar"');
+    });
+
+    it('should filter out filter ad hoc filter with invalid operator', () => {
+      const templateSrvMock = {
+        getAdhocFilters: () => [{ key: 'a', operator: 'A', value: '' }],
+      } as unknown as TemplateSrv;
+      const { ds } = getTestContext({ templateSrvMock });
+
+      const query = ds.addAdHocFilters('foo:"bar"');
+      expect(query).toBe('foo:"bar"');
+    });
+  });
+
+  describe('with 1 ad hoc filter', () => {
+    const templateSrvMock = {
+      getAdhocFilters: () => [{ key: 'test', operator: '=', value: 'test1' }],
+    } as unknown as TemplateSrv;
+    const { ds } = getTestContext({ templateSrvMock });
+
+    it('should correctly add 1 ad hoc filter when query is not empty', () => {
+      const query = ds.addAdHocFilters('foo:"bar"');
+      expect(query).toBe('foo:"bar" AND test:"test1"');
+    });
+
+    it('should correctly add 1 ad hoc filter when query is empty', () => {
+      const query = ds.addAdHocFilters('');
+      expect(query).toBe('test:"test1"');
+    });
+  });
+
+  describe('with multiple ad hoc filters', () => {
+    const templateSrvMock = {
+      getAdhocFilters: () => [
+        { key: 'bar', operator: '=', value: 'baz' },
+        { key: 'job', operator: '!=', value: 'grafana' },
+        { key: 'service', operator: '=~', value: 'service' },
+        { key: 'count', operator: '>', value: '1' },
+      ],
+    } as unknown as TemplateSrv;
+    const { ds } = getTestContext({ templateSrvMock });
+
+    it('should correctly add ad hoc filters when query is not empty', () => {
+      const query = ds.addAdHocFilters('foo:"bar" AND test:"test1"');
+      expect(query).toBe(
+        'foo:"bar" AND test:"test1" AND bar:"baz" AND -job:"grafana" AND service:/service/ AND count:>1'
+      );
+    });
+
+    it('should correctly add ad hoc filters when query is  empty', () => {
+      const query = ds.addAdHocFilters('');
+      expect(query).toBe('bar:"baz" AND -job:"grafana" AND service:/service/ AND count:>1');
+    });
+  });
+});
+
 const createElasticQuery = (): DataQueryRequest<ElasticsearchQuery> => {
   return {
     requestId: '',

public/app/plugins/datasource/elasticsearch/datasource.ts

@@ -418,7 +418,7 @@ export class ElasticDatasource
       (query): ElasticsearchQuery => ({
         ...query,
         datasource: this.getRef(),
-        query: this.interpolateLuceneQuery(query.query || '', scopedVars),
+        query: this.addAdHocFilters(this.interpolateLuceneQuery(query.query || '', scopedVars)),
         bucketAggs: query.bucketAggs?.map(interpolateBucketAgg),
       })
     );
@@ -970,6 +970,37 @@ export class ElasticDatasource
     }
     return { ...query, query: expression };
   }
+
+  addAdHocFilters(query: string) {
+    const adhocFilters = this.templateSrv.getAdhocFilters(this.name);
+    if (adhocFilters.length === 0) {
+      return query;
+    }
+    const esFilters = adhocFilters.map((filter) => {
+      const { key, operator, value } = filter;
+      if (!key || !value) {
+        return;
+      }
+      switch (operator) {
+        case '=':
+          return `${key}:"${value}"`;
+        case '!=':
+          return `-${key}:"${value}"`;
+        case '=~':
+          return `${key}:/${value}/`;
+        case '!~':
+          return `-${key}:/${value}/`;
+        case '>':
+          return `${key}:>${value}`;
+        case '<':
+          return `${key}:<${value}`;
+      }
+      return;
+    });
+
+    const finalQuery = [query, ...esFilters].filter((f) => f).join(' AND ');
+    return finalQuery;
+  }
 }
 
 /**