Elasticsearch: Support nested aggregation (#62301)

* Add nested query support

* Add nested support for alerts

* update nested aggregation

* cleanup types

* Add nested integration test

* Move aggdef to nested

* fixed merge conflict

* fixed lint warning

* mark nested-mode experimental

---------

Co-authored-by: Ethan Gallant <ethan@ziax.com>
Co-authored-by: Ethan J. Gallant <ethan.gallant@acquia.com>

pkg/tsdb/elasticsearch/client/models.go

@@ -236,6 +236,11 @@ type TermsAggregation struct {
 	Missing     *string                `json:"missing,omitempty"`
 }
 
+// NestedAggregation represents a nested aggregation
+type NestedAggregation struct {
+	Path string `json:"path"`
+}
+
 // ExtendedBounds represents extended bounds
 type ExtendedBounds struct {
 	Min int64 `json:"min"`

pkg/tsdb/elasticsearch/client/search_request.go

@@ -270,6 +270,7 @@ type AggBuilder interface {
 	Histogram(key, field string, fn func(a *HistogramAgg, b AggBuilder)) AggBuilder
 	DateHistogram(key, field string, fn func(a *DateHistogramAgg, b AggBuilder)) AggBuilder
 	Terms(key, field string, fn func(a *TermsAggregation, b AggBuilder)) AggBuilder
+	Nested(key, path string, fn func(a *NestedAggregation, b AggBuilder)) AggBuilder
 	Filters(key string, fn func(a *FiltersAggregation, b AggBuilder)) AggBuilder
 	GeoHashGrid(key, field string, fn func(a *GeoHashGridAggregation, b AggBuilder)) AggBuilder
 	Metric(key, metricType, field string, fn func(a *MetricAggregation)) AggBuilder
@@ -382,6 +383,26 @@ func (b *aggBuilderImpl) Terms(key, field string, fn func(a *TermsAggregation, b
 	return b
 }
 
+func (b *aggBuilderImpl) Nested(key, field string, fn func(a *NestedAggregation, b AggBuilder)) AggBuilder {
+	innerAgg := &NestedAggregation{
+		Path: field,
+	}
+	aggDef := newAggDef(key, &aggContainer{
+		Type:        "nested",
+		Aggregation: innerAgg,
+	})
+
+	if fn != nil {
+		builder := newAggBuilder()
+		aggDef.builders = append(aggDef.builders, builder)
+		fn(innerAgg, builder)
+	}
+
+	b.aggDefs = append(b.aggDefs, aggDef)
+
+	return b
+}
+
 func (b *aggBuilderImpl) Filters(key string, fn func(a *FiltersAggregation, b AggBuilder)) AggBuilder {
 	innerAgg := &FiltersAggregation{
 		Filters: make(map[string]interface{}),

pkg/tsdb/elasticsearch/response_parser.go

@@ -22,6 +22,7 @@ const (
 	topMetricsType    = "top_metrics"
 	// Bucket types
 	dateHistType    = "date_histogram"
+	nestedType      = "nested"
 	histogramType   = "histogram"
 	filtersType     = "filters"
 	termsType       = "terms"
@@ -84,6 +85,13 @@ func processBuckets(aggs map[string]interface{}, target *Query,
 		if aggDef == nil {
 			continue
 		}
+		if aggDef.Type == nestedType {
+			err = processBuckets(esAgg.MustMap(), target, queryResult, props, depth+1)
+			if err != nil {
+				return err
+			}
+			continue
+		}
 
 		if depth == maxDepth {
 			if aggDef.Type == dateHistType {

pkg/tsdb/elasticsearch/time_series_query.go

@@ -244,6 +244,14 @@ func addTermsAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg, metrics []*Metr
 	return aggBuilder
 }
 
+func addNestedAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg) es.AggBuilder {
+	aggBuilder.Nested(bucketAgg.ID, bucketAgg.Field, func(a *es.NestedAggregation, b es.AggBuilder) {
+		aggBuilder = b
+	})
+
+	return aggBuilder
+}
+
 func addFiltersAgg(aggBuilder es.AggBuilder, bucketAgg *BucketAgg) es.AggBuilder {
 	filters := make(map[string]interface{})
 	for _, filter := range bucketAgg.Settings.Get("filters").MustArray() {
@@ -361,6 +369,8 @@ func processTimeSeriesQuery(q *Query, b *es.SearchRequestBuilder, from, to int64
 			aggBuilder = addTermsAgg(aggBuilder, bucketAgg, q.Metrics)
 		case geohashGridType:
 			aggBuilder = addGeoHashGridAgg(aggBuilder, bucketAgg)
+		case nestedType:
+			aggBuilder = addNestedAgg(aggBuilder, bucketAgg)
 		}
 	}
 

public/app/plugins/datasource/elasticsearch/ElasticResponse.ts

@@ -288,6 +288,11 @@ export class ElasticResponse {
         continue;
       }
 
+      if (aggDef.type === 'nested') {
+        this.processBuckets(esAgg, target, seriesList, table, props, depth + 1);
+        continue;
+      }
+
       if (depth === maxDepth) {
         if (aggDef.type === 'date_histogram') {
           this.processMetrics(esAgg, target, seriesList, props);

public/app/plugins/datasource/elasticsearch/QueryBuilder.test.ts

@@ -587,6 +587,23 @@ describe('ElasticQueryBuilder', () => {
     expect(firstLevel.histogram.min_doc_count).toBe('2');
   });
 
+  it('with nested', () => {
+    const query = builder.build({
+      refId: 'A',
+      metrics: [{ id: '1', type: 'count' }],
+      bucketAggs: [
+        {
+          type: 'nested',
+          field: 'nested_field',
+          id: '3',
+        },
+      ],
+    });
+
+    const firstLevel = query.aggs['3'];
+    expect(firstLevel.nested.path).toBe('nested_field');
+  });
+
   // This test wasn't migrated, as adhoc variables are going to be interpolated before
   // Or we need to add this to backend query builder (TBD)
   it('with adhoc filters', () => {

public/app/plugins/datasource/elasticsearch/QueryBuilder.ts

@@ -283,6 +283,10 @@ export class ElasticQueryBuilder {
           };
           break;
         }
+        case 'nested': {
+          esAgg['nested'] = { path: aggDef.field };
+          break;
+        }
       }
 
       nestedAggs.aggs = nestedAggs.aggs || {};

public/app/plugins/datasource/elasticsearch/components/QueryEditor/BucketAggregationsEditor/aggregations.ts

@@ -1,6 +1,6 @@
 import { bucketAggregationConfig } from './utils';
 
-export type BucketAggregationType = 'terms' | 'filters' | 'geohash_grid' | 'date_histogram' | 'histogram';
+export type BucketAggregationType = 'terms' | 'filters' | 'geohash_grid' | 'date_histogram' | 'histogram' | 'nested';
 
 interface BaseBucketAggregation {
   id: string;
@@ -62,7 +62,12 @@ interface GeoHashGrid extends BucketAggregationWithField {
   };
 }
 
-export type BucketAggregation = DateHistogram | Histogram | Terms | Filters | GeoHashGrid;
+interface Nested extends BucketAggregationWithField {
+  type: 'nested';
+  settings?: {};
+}
+
+export type BucketAggregation = DateHistogram | Histogram | Terms | Filters | GeoHashGrid | Nested;
 
 export const isBucketAggregationWithField = (
   bucketAgg: BucketAggregation | BucketAggregationWithField
@@ -74,6 +79,7 @@ export const BUCKET_AGGREGATION_TYPES: BucketAggregationType[] = [
   'terms',
   'filters',
   'geohash_grid',
+  'nested',
 ];
 
 export const isBucketAggregationType = (s: BucketAggregationType | string): s is BucketAggregationType =>

public/app/plugins/datasource/elasticsearch/components/QueryEditor/BucketAggregationsEditor/utils.ts

@@ -47,6 +47,11 @@ export const bucketAggregationConfig: BucketsConfiguration = {
       min_doc_count: '0',
     },
   },
+  nested: {
+    label: 'Nested (experimental)',
+    requiresField: true,
+    defaultSettings: {},
+  },
 };
 
 export const orderByOptions: Array<SelectableValue<string>> = [