Elasticsearch: Apply ad-hoc filters to annotation queries (#82032)

* Elasticsearch: filter annotations

* fix broken tests

* add a new test case

* allow type assertion

There is a lint rule to not be able to use assertions. In this case, it is needed and correct way to asset AdHocVariableModel[]

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

* simplify the code to add adhoc filters

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

* lint

---------

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

public/app/plugins/datasource/elasticsearch/datasource.test.ts

@@ -1588,6 +1588,9 @@ describe('ElasticDatasource using backend', () => {
 
         const annotations = await ds.annotationQuery({
           annotation: {},
+          dashboard: {
+            getVariables: () => [],
+          },
           range: timeRange,
         });
 
@@ -1619,6 +1622,9 @@ describe('ElasticDatasource using backend', () => {
             tagsField: '@test_tags',
             textField: 'text',
           },
+          dashboard: {
+            getVariables: () => [],
+          },
           range: timeRange,
         });
         expect(annotations).toHaveLength(2);
@@ -1657,6 +1663,9 @@ describe('ElasticDatasource using backend', () => {
             tagsField: '@test_tags',
             textField: 'text',
           },
+          dashboard: {
+            getVariables: () => [],
+          },
           range: {
             from: dateTime(1683291160012),
             to: dateTime(1683291460012),
@@ -1685,6 +1694,9 @@ describe('ElasticDatasource using backend', () => {
 
         await ds.annotationQuery({
           annotation: {},
+          dashboard: {
+            getVariables: () => [],
+          },
           range: {
             from: dateTime(1683291160012),
             to: dateTime(1683291460012),
@@ -1695,6 +1707,63 @@ describe('ElasticDatasource using backend', () => {
           '{"search_type":"query_then_fetch","ignore_unavailable":true,"index":"[test-]YYYY.MM.DD"}\n{"query":{"bool":{"filter":[{"bool":{"should":[{"range":{"@timestamp":{"from":1683291160012,"to":1683291460012,"format":"epoch_millis"}}}],"minimum_should_match":1}}]}},"size":10000}\n'
         );
       });
+
+      it('should process annotation request using dashboard adhoc variables', async () => {
+        const { ds } = getTestContext();
+        const postResourceRequestMock = jest.spyOn(ds, 'postResourceRequest').mockResolvedValue({
+          responses: [
+            {
+              hits: {
+                hits: [
+                  { _source: { '@test_time': 1, '@test_tags': 'foo', text: 'abc' } },
+                  { _source: { '@test_time': 3, '@test_tags': 'bar', text: 'def' } },
+                ],
+              },
+            },
+          ],
+        });
+
+        await ds.annotationQuery({
+          annotation: {
+            timeField: '@test_time',
+            timeEndField: '@time_end_field',
+            name: 'foo',
+            query: 'abc',
+            tagsField: '@test_tags',
+            textField: 'text',
+            datasource: {
+              type: 'elasticsearch',
+              uid: 'gdev-elasticsearch',
+            },
+          },
+          dashboard: {
+            getVariables: () => [
+              {
+                type: 'adhoc',
+                datasource: {
+                  type: 'elasticsearch',
+                  uid: 'gdev-elasticsearch',
+                },
+                filters: [
+                  {
+                    key: 'abc_key',
+                    operator: '=',
+                    value: 'abc_value',
+                  },
+                ],
+              },
+            ],
+          },
+          range: {
+            from: dateTime(1683291160012),
+            to: dateTime(1683291460012),
+          },
+        });
+        expect(postResourceRequestMock).toHaveBeenCalledWith(
+          '_msearch',
+          '{"search_type":"query_then_fetch","ignore_unavailable":true,"index":"[test-]YYYY.MM.DD"}\n{"query":{"bool":{"filter":[{"bool":{"should":[{"range":{"@test_time":{"from":1683291160012,"to":1683291460012,"format":"epoch_millis"}}},{"range":{"@time_end_field":{"from":1683291160012,"to":1683291460012,"format":"epoch_millis"}}}],"minimum_should_match":1}},{"query_string":{"query":"abc AND abc_key:\\"abc_value\\""}}]}},"size":10000}\n'
+        );
+      });
     });
   });
 

public/app/plugins/datasource/elasticsearch/datasource.ts

@@ -37,6 +37,7 @@ import {
   DataSourceGetTagValuesOptions,
   AdHocVariableFilter,
   DataSourceWithQueryModificationSupport,
+  AdHocVariableModel,
 } from '@grafana/data';
 import {
   DataSourceWithBackend,
@@ -46,6 +47,7 @@ import {
   TemplateSrv,
   getTemplateSrv,
 } from '@grafana/runtime';
+import { DashboardModel } from 'app/features/dashboard/state/DashboardModel';
 
 import { IndexPattern, intervalMap } from './IndexPattern';
 import LanguageProvider from './LanguageProvider';
@@ -260,10 +262,20 @@ export class ElasticDatasource
     );
   }
 
-  private prepareAnnotationRequest(options: { annotation: ElasticsearchAnnotationQuery; range: TimeRange }) {
+  private prepareAnnotationRequest(options: {
+    annotation: ElasticsearchAnnotationQuery;
+    dashboard: DashboardModel;
+    range: TimeRange;
+  }) {
     const annotation = options.annotation;
     const timeField = annotation.timeField || '@timestamp';
     const timeEndField = annotation.timeEndField || null;
+    const dashboard = options.dashboard;
+
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    const adhocVariables = dashboard.getVariables().filter((v) => v.type === 'adhoc') as AdHocVariableModel[];
+    const annotationRelatedVariables = adhocVariables.filter((v) => v.datasource?.uid === annotation.datasource.uid);
+    const filters = annotationRelatedVariables.map((v) => v.filters).flat();
 
     // the `target.query` is the "new" location for the query.
     // normally we would write this code as
@@ -296,6 +308,8 @@ export class ElasticDatasource
     }
 
     const queryInterpolated = this.interpolateLuceneQuery(queryString);
+    const finalQuery = this.addAdHocFilters(queryInterpolated, filters);
+
     const query: {
       bool: { filter: Array<Record<string, Record<string, string | number | Array<{ range: RangeMap }>>>> };
     } = {
@@ -311,10 +325,10 @@ export class ElasticDatasource
       },
     };
 
-    if (queryInterpolated) {
+    if (finalQuery) {
       query.bool.filter.push({
         query_string: {
-          query: queryInterpolated,
+          query: finalQuery,
         },
       });
     }

public/app/plugins/datasource/elasticsearch/types.ts

@@ -1,4 +1,5 @@
 import { DataSourceJsonData } from '@grafana/data';
+import { DataSourceRef } from '@grafana/schema';
 
 import {
   BucketAggregationType,
@@ -131,6 +132,7 @@ export interface ElasticsearchAnnotationQuery {
   titleField?: string;
   timeEndField?: string;
   query?: string;
+  datasource: DataSourceRef;
   tagsField?: string;
   textField?: string;
   // @deprecated index is deprecated and will be removed in the future