Elasticsearch: Enable logs samples for metric queries (#70258)

* enable logs samples on elastic ds

* add tests for getSupplementaryQuery

* only display log samples for date_hostogram queries

* changes

* test

* Update public/app/plugins/datasource/elasticsearch/datasource.test.ts

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

* Update public/app/plugins/datasource/elasticsearch/datasource.test.ts

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

* Update public/app/plugins/datasource/elasticsearch/datasource.test.ts

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

* Update public/app/plugins/datasource/elasticsearch/datasource.test.ts

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

* Update public/app/plugins/datasource/elasticsearch/datasource.test.ts

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

* address feedback / tests

---------

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

public/app/plugins/datasource/elasticsearch/components/QueryEditor/index.tsx

@@ -9,7 +9,7 @@ import { ElasticDatasource } from '../../datasource';
 import { useNextId } from '../../hooks/useNextId';
 import { useDispatch } from '../../hooks/useStatelessReducer';
 import { ElasticsearchOptions, ElasticsearchQuery } from '../../types';
-import { isSupportedVersion, unsupportedVersionMessage } from '../../utils';
+import { isSupportedVersion, isTimeSeriesQuery, unsupportedVersionMessage } from '../../utils';
 
 import { BucketAggregationsEditor } from './BucketAggregationsEditor';
 import { ElasticsearchProvider } from './ElasticsearchQueryContext';
@@ -92,8 +92,7 @@ const QueryEditorForm = ({ value }: Props) => {
   const nextId = useNextId();
   const styles = useStyles2(getStyles);
 
-  // To be considered a time series query, the last bucked aggregation must be a Date Histogram
+  const isTimeSeries = isTimeSeriesQuery(value);
-  const isTimeSeriesQuery = value?.bucketAggs?.slice(-1)[0]?.type === 'date_histogram';
 
   const showBucketAggregationsEditor = value.metrics?.every(
     (metric) => metricAggregationConfig[metric.type].impliedQueryType === 'metrics'
@@ -111,7 +110,7 @@ const QueryEditorForm = ({ value }: Props) => {
         <InlineLabel width={17}>Lucene Query</InlineLabel>
         <ElasticSearchQueryField onChange={(query) => dispatch(changeQuery(query))} value={value?.query} />
 
-        {isTimeSeriesQuery && (
+        {isTimeSeries && (
           <InlineField
             label="Alias"
             labelWidth={15}

public/app/plugins/datasource/elasticsearch/datasource.test.ts

@@ -1,5 +1,6 @@
 import { map } from 'lodash';
 import { Observable, of, throwError } from 'rxjs';
+import { getQueryOptions } from 'test/helpers/getQueryOptions';
 
 import {
   CoreApp,
@@ -977,6 +978,102 @@ describe('ElasticDatasource', () => {
         timeField: '',
       });
     });
+
+    it('does not return logs samples for non time series queries', () => {
+      expect(
+        ds.getSupplementaryQuery(
+          { type: SupplementaryQueryType.LogsSample, limit: 100 },
+          {
+            refId: 'A',
+            bucketAggs: [{ type: 'filters', id: '1' }],
+            query: '',
+          }
+        )
+      ).toEqual(undefined);
+    });
+
+    it('returns logs samples for time series queries', () => {
+      expect(
+        ds.getSupplementaryQuery(
+          { type: SupplementaryQueryType.LogsSample, limit: 100 },
+          {
+            refId: 'A',
+            query: '',
+            bucketAggs: [{ type: 'date_histogram', id: '1' }],
+          }
+        )
+      ).toEqual({
+        refId: `log-sample-A`,
+        query: '',
+        metrics: [{ type: 'logs', id: '1', settings: { limit: '100' } }],
+      });
+    });
+  });
+
+  describe('getDataProvider', () => {
+    let ds: ElasticDatasource;
+    beforeEach(() => {
+      ds = getTestContext().ds;
+    });
+
+    it('does not create a logs sample provider for non time series query', () => {
+      const options = getQueryOptions<ElasticsearchQuery>({
+        targets: [
+          {
+            refId: 'A',
+            metrics: [{ type: 'logs', id: '1', settings: { limit: '100' } }],
+          },
+        ],
+      });
+
+      expect(ds.getDataProvider(SupplementaryQueryType.LogsSample, options)).not.toBeDefined();
+    });
+
+    it('does create a logs sample provider for time series query', () => {
+      const options = getQueryOptions<ElasticsearchQuery>({
+        targets: [
+          {
+            refId: 'A',
+            bucketAggs: [{ type: 'date_histogram', id: '1' }],
+          },
+        ],
+      });
+
+      expect(ds.getDataProvider(SupplementaryQueryType.LogsSample, options)).toBeDefined();
+    });
+  });
+
+  describe('getLogsSampleDataProvider', () => {
+    let ds: ElasticDatasource;
+    beforeEach(() => {
+      ds = getTestContext().ds;
+    });
+
+    it("doesn't return a logs sample provider given a non time series query", () => {
+      const request = getQueryOptions<ElasticsearchQuery>({
+        targets: [
+          {
+            refId: 'A',
+            metrics: [{ type: 'logs', id: '1', settings: { limit: '100' } }],
+          },
+        ],
+      });
+
+      expect(ds.getLogsSampleDataProvider(request)).not.toBeDefined();
+    });
+
+    it('returns a logs sample provider given a time series query', () => {
+      const request = getQueryOptions<ElasticsearchQuery>({
+        targets: [
+          {
+            refId: 'A',
+            bucketAggs: [{ type: 'date_histogram', id: '1' }],
+          },
+        ],
+      });
+
+      expect(ds.getLogsSampleDataProvider(request)).toBeDefined();
+    });
   });
 });
 

public/app/plugins/datasource/elasticsearch/datasource.ts

@@ -33,7 +33,7 @@ import {
   AnnotationEvent,
 } from '@grafana/data';
 import { DataSourceWithBackend, getDataSourceSrv, config, BackendSrvRequest } from '@grafana/runtime';
-import { queryLogsVolume } from 'app/core/logsModel';
+import { queryLogsSample, queryLogsVolume } from 'app/core/logsModel';
 import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv';
 import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv';
 
@@ -64,9 +64,11 @@ import {
   ElasticsearchAnnotationQuery,
   RangeMap,
 } from './types';
-import { getScriptValue, isSupportedVersion, unsupportedVersionMessage } from './utils';
+import { getScriptValue, isSupportedVersion, isTimeSeriesQuery, unsupportedVersionMessage } from './utils';
 
 export const REF_ID_STARTER_LOG_VOLUME = 'log-volume-';
+export const REF_ID_STARTER_LOG_SAMPLE = 'log-sample-';
+
 // Those are metadata fields as defined in https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-fields.html#_identity_metadata_fields.
 // custom fields can start with underscores, therefore is not safe to exclude anything that starts with one.
 const ELASTIC_META_FIELDS = [
@@ -520,13 +522,15 @@ export class ElasticDatasource
     switch (type) {
       case SupplementaryQueryType.LogsVolume:
         return this.getLogsVolumeDataProvider(request);
+      case SupplementaryQueryType.LogsSample:
+        return this.getLogsSampleDataProvider(request);
       default:
         return undefined;
     }
   }
 
   getSupportedSupplementaryQueryTypes(): SupplementaryQueryType[] {
-    return [SupplementaryQueryType.LogsVolume];
+    return [SupplementaryQueryType.LogsVolume, SupplementaryQueryType.LogsSample];
   }
 
   getSupplementaryQuery(options: SupplementaryQueryOptions, query: ElasticsearchQuery): ElasticsearchQuery | undefined {
@@ -579,6 +583,27 @@ export class ElasticDatasource
           bucketAggs,
         };
 
+      case SupplementaryQueryType.LogsSample:
+        isQuerySuitable = isTimeSeriesQuery(query);
+
+        if (!isQuerySuitable) {
+          return undefined;
+        }
+
+        if (options.limit) {
+          return {
+            refId: `${REF_ID_STARTER_LOG_SAMPLE}${query.refId}`,
+            query: query.query,
+            metrics: [{ type: 'logs', id: '1', settings: { limit: options.limit.toString() } }],
+          };
+        }
+
+        return {
+          refId: `${REF_ID_STARTER_LOG_SAMPLE}${query.refId}`,
+          query: query.query,
+          metrics: [{ type: 'logs', id: '1' }],
+        };
+
       default:
         return undefined;
     }
@@ -605,6 +630,20 @@ export class ElasticDatasource
     );
   }
 
+  getLogsSampleDataProvider(request: DataQueryRequest<ElasticsearchQuery>): Observable<DataQueryResponse> | undefined {
+    const logsSampleRequest = cloneDeep(request);
+    const targets = logsSampleRequest.targets;
+    const queries = targets.map((query) => {
+      return this.getSupplementaryQuery({ type: SupplementaryQueryType.LogsSample, limit: 100 }, query);
+    });
+    const elasticQueries = queries.filter((query): query is ElasticsearchQuery => !!query);
+
+    if (!elasticQueries.length) {
+      return undefined;
+    }
+    return queryLogsSample(this, { ...logsSampleRequest, targets: elasticQueries });
+  }
+
   query(request: DataQueryRequest<ElasticsearchQuery>): Observable<DataQueryResponse> {
     const { enableElasticsearchBackendQuerying } = config.featureToggles;
     if (enableElasticsearchBackendQuerying) {

public/app/plugins/datasource/elasticsearch/utils.test.ts

@@ -1,4 +1,5 @@
-import { removeEmpty } from './utils';
+import { ElasticsearchQuery } from './types';
+import { isTimeSeriesQuery, removeEmpty } from './utils';
 
 describe('removeEmpty', () => {
   it('Should remove all empty', () => {
@@ -34,3 +35,47 @@ describe('removeEmpty', () => {
     expect(removeEmpty(original)).toStrictEqual(expectedResult);
   });
 });
+
+describe('isTimeSeriesQuery', () => {
+  it('should return false when given a log query', () => {
+    const logsQuery: ElasticsearchQuery = {
+      refId: `A`,
+      metrics: [{ type: 'logs', id: '1' }],
+    };
+
+    expect(isTimeSeriesQuery(logsQuery)).toBe(false);
+  });
+
+  it('should return false when bucket aggs are empty', () => {
+    const query: ElasticsearchQuery = {
+      refId: `A`,
+      bucketAggs: [],
+    };
+
+    expect(isTimeSeriesQuery(query)).toBe(false);
+  });
+
+  it('returns false when empty date_histogram is not last', () => {
+    const query: ElasticsearchQuery = {
+      refId: `A`,
+      bucketAggs: [
+        { id: '1', type: 'date_histogram' },
+        { id: '2', type: 'terms' },
+      ],
+    };
+
+    expect(isTimeSeriesQuery(query)).toBe(false);
+  });
+
+  it('returns true when empty date_histogram is last', () => {
+    const query: ElasticsearchQuery = {
+      refId: `A`,
+      bucketAggs: [
+        { id: '1', type: 'terms' },
+        { id: '2', type: 'date_histogram' },
+      ],
+    };
+
+    expect(isTimeSeriesQuery(query)).toBe(true);
+  });
+});

public/app/plugins/datasource/elasticsearch/utils.ts

@@ -2,7 +2,7 @@ import { gte, SemVer } from 'semver';
 
 import { isMetricAggregationWithField } from './components/QueryEditor/MetricAggregationsEditor/aggregations';
 import { metricAggregationConfig } from './components/QueryEditor/MetricAggregationsEditor/utils';
-import { MetricAggregation, MetricAggregationWithInlineScript } from './types';
+import { ElasticsearchQuery, MetricAggregation, MetricAggregationWithInlineScript } from './types';
 
 export const describeMetric = (metric: MetricAggregation) => {
   if (!isMetricAggregationWithField(metric)) {
@@ -101,3 +101,8 @@ export const isSupportedVersion = (version: SemVer): boolean => {
 
 export const unsupportedVersionMessage =
   'Support for Elasticsearch versions after their end-of-life (currently versions < 7.16) was removed. Using unsupported version of Elasticsearch may lead to unexpected and incorrect results.';
+
+// To be considered a time series query, the last bucked aggregation must be a Date Histogram
+export const isTimeSeriesQuery = (query: ElasticsearchQuery): boolean => {
+  return query?.bucketAggs?.slice(-1)[0]?.type === 'date_histogram';
+};