Elasticsearch: Allow setting a custom limit for log queries (#32422)

public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/SettingsEditor/index.tsx

@@ -72,6 +72,8 @@ export const SettingsEditor: FunctionComponent<Props> = ({ metric, previousMetri
         </InlineField>
       )}
 
+      {metric.type === 'logs' && <SettingField label="Limit" metric={metric} settingName="limit" placeholder="500" />}
+
       {metric.type === 'cardinality' && (
         <SettingField label="Precision Threshold" metric={metric} settingName="precision_threshold" />
       )}

public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/aggregations.ts

@@ -148,6 +148,9 @@ export interface RawData extends BaseMetricAggregation {
 
 export interface Logs extends BaseMetricAggregation {
   type: 'logs';
+  settings?: {
+    limit?: string;
+  };
 }
 
 export interface BasePipelineMetricAggregation extends MetricAggregationWithField {
@@ -289,11 +292,12 @@ export type MetricAggregationWithSettings =
   | Sum
   | Average
   | MovingAverage
-  | MovingFunction;
+  | MovingFunction
+  | Logs;
 
 export type MetricAggregationWithMeta = ExtendedStats;
 
-export type MetricAggregation = Count | Logs | PipelineMetricAggregation | MetricAggregationWithSettings;
+export type MetricAggregation = Count | PipelineMetricAggregation | MetricAggregationWithSettings;
 
 // Guards
 // Given the structure of the aggregations (ie. `settings` field being always optional) we cannot

public/app/plugins/datasource/elasticsearch/components/QueryEditor/MetricAggregationsEditor/utils.ts

@@ -226,10 +226,15 @@ export const metricAggregationConfig: MetricsConfiguration = {
     isPipelineAgg: false,
     supportsMissing: false,
     supportsMultipleBucketPaths: false,
-    hasSettings: false,
+    hasSettings: true,
+    isSingleMetric: true,
     supportsInlineScript: false,
     hasMeta: false,
-    defaults: {},
+    defaults: {
+      settings: {
+        limit: '500',
+      },
+    },
   },
 };
 

public/app/plugins/datasource/elasticsearch/datasource.ts

@@ -31,6 +31,7 @@ import { metricAggregationConfig } from './components/QueryEditor/MetricAggregat
 import {
   isMetricAggregationWithField,
   isPipelineAggregationWithMultipleBucketPaths,
+  Logs,
 } from './components/QueryEditor/MetricAggregationsEditor/aggregations';
 import { bucketAggregationConfig } from './components/QueryEditor/BucketAggregationsEditor/utils';
 import { isBucketAggregationWithField } from './components/QueryEditor/BucketAggregationsEditor/aggregations';
@@ -549,10 +550,19 @@ export class ElasticDatasource extends DataSourceApi<ElasticsearchQuery, Elastic
 
       let queryObj;
       if (hasMetricOfType(target, 'logs')) {
+        // FIXME: All this logic here should be in the query builder.
+        // When moving to the BE-only implementation we should remove this and let the BE
+        // Handle this.
+        // TODO: defaultBucketAgg creates a dete_histogram aggregation without a field, so it fallbacks to
+        // the configured timeField. we should allow people to use a different time field here.
         target.bucketAggs = [defaultBucketAgg()];
+
+        const log = target.metrics?.find((m) => m.type === 'logs') as Logs;
+        const limit = log.settings?.limit ? parseInt(log.settings?.limit, 10) : 500;
+
         target.metrics = [];
         // Setting this for metrics queries that are typed as logs
-        queryObj = this.queryBuilder.getLogsQuery(target, adhocFilters, target.query);
+        queryObj = this.queryBuilder.getLogsQuery(target, limit, adhocFilters, target.query);
       } else {
         if (target.alias) {
           target.alias = this.templateSrv.replace(target.alias, options.scopedVars, 'lucene');

public/app/plugins/datasource/elasticsearch/query_builder.ts

@@ -415,7 +415,7 @@ export class ElasticQueryBuilder {
     return query;
   }
 
-  getLogsQuery(target: ElasticsearchQuery, adhocFilters?: any, querystring?: string) {
+  getLogsQuery(target: ElasticsearchQuery, limit: number, adhocFilters?: any, querystring?: string) {
     let query: any = {
       size: 0,
       query: {
@@ -436,7 +436,7 @@ export class ElasticQueryBuilder {
       });
     }
 
-    query = this.documentQuery(query, 500);
+    query = this.documentQuery(query, limit);
 
     return {
       ...query,

public/app/plugins/datasource/elasticsearch/specs/query_builder.test.ts

@@ -680,7 +680,7 @@ describe('ElasticQueryBuilder', () => {
 
       describe('getLogsQuery', () => {
         it('should return query with defaults', () => {
-          const query = builder.getLogsQuery({ refId: 'A' }, null, '*');
+          const query = builder.getLogsQuery({ refId: 'A' }, 500, null, '*');
 
           expect(query.size).toEqual(500);
 
@@ -714,7 +714,7 @@ describe('ElasticQueryBuilder', () => {
         });
 
         it('with querystring', () => {
-          const query = builder.getLogsQuery({ refId: 'A', query: 'foo' }, null, 'foo');
+          const query = builder.getLogsQuery({ refId: 'A', query: 'foo' }, 500, null, 'foo');
 
           const expectedQuery = {
             bool: {
@@ -737,7 +737,7 @@ describe('ElasticQueryBuilder', () => {
             { key: 'key5', operator: '=~', value: 'value5' },
             { key: 'key6', operator: '!~', value: 'value6' },
           ];
-          const query = builder.getLogsQuery({ refId: 'A' }, adhocFilters, '*');
+          const query = builder.getLogsQuery({ refId: 'A' }, 500, adhocFilters, '*');
 
           expect(query.query.bool.must[0].match_phrase['key1'].query).toBe('value1');
           expect(query.query.bool.must_not[0].match_phrase['key2'].query).toBe('value2');