From dbbbc46351d81b1c97f8dfaed25f6f0609797967 Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Wed, 17 May 2023 14:28:32 +0200 Subject: [PATCH] Elasticsearch: Fix passing of limit and datalinks to logs data frame (#68554) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Elasticsearch: Fix passing of limit and datalinks to logs data frame * Update public/app/core/logsModel.ts Co-authored-by: François Massot --------- Co-authored-by: François Massot --- pkg/tsdb/elasticsearch/response_parser.go | 5 ++- .../elasticsearch/response_parser_test.go | 3 +- .../testdata_response/logs.a.golden.jsonc | 2 + public/app/core/logsModel.test.ts | 42 +++++++++++++++++++ public/app/core/logsModel.ts | 7 ++-- .../elasticsearch/LegacyQueryRunner.ts | 18 +++++++- .../elasticsearch/datasource.test.ts | 3 +- .../datasource/elasticsearch/datasource.ts | 23 +++++----- 8 files changed, 81 insertions(+), 22 deletions(-) diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index 86c3a66b9c4..065de02c40e 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -160,7 +160,7 @@ func processLogsResponse(res *es.SearchResponse, target *Query, configuredFields frames := data.Frames{} frame := data.NewFrame("", fields...) setPreferredVisType(frame, data.VisTypeLogs) - setSearchWords(frame, searchWords) + setLogsCustomMeta(frame, searchWords, stringToIntWithDefaultValue(target.Metrics[0].Settings.Get("limit").MustString(), defaultSize)) frames = append(frames, frame) queryRes.Frames = frames @@ -1137,7 +1137,7 @@ func setPreferredVisType(frame *data.Frame, visType data.VisType) { frame.Meta.PreferredVisualization = visType } -func setSearchWords(frame *data.Frame, searchWords map[string]bool) { +func setLogsCustomMeta(frame *data.Frame, searchWords map[string]bool, limit int) { i := 0 searchWordsList := make([]string, len(searchWords)) for searchWord := range searchWords { @@ -1156,6 +1156,7 @@ func setSearchWords(frame *data.Frame, searchWords map[string]bool) { frame.Meta.Custom = map[string]interface{}{ "searchWords": searchWordsList, + "limit": limit, } } diff --git a/pkg/tsdb/elasticsearch/response_parser_test.go b/pkg/tsdb/elasticsearch/response_parser_test.go index c245be3e776..25b09ca1d76 100644 --- a/pkg/tsdb/elasticsearch/response_parser_test.go +++ b/pkg/tsdb/elasticsearch/response_parser_test.go @@ -105,7 +105,7 @@ func TestProcessLogsResponse(t *testing.T) { logsFrame := frames[0] meta := logsFrame.Meta - require.Equal(t, map[string]interface{}{"searchWords": []string{"hello", "message"}}, meta.Custom) + require.Equal(t, map[string]interface{}{"searchWords": []string{"hello", "message"}, "limit": 500}, meta.Custom) require.Equal(t, data.VisTypeLogs, string(meta.PreferredVisualization)) logsFieldMap := make(map[string]*data.Field) @@ -430,6 +430,7 @@ func TestProcessLogsResponse(t *testing.T) { require.Equal(t, map[string]interface{}{ "searchWords": []string{"hello", "message"}, + "limit": 500, }, customMeta) }) } diff --git a/pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc b/pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc index 1ce38625103..3d349eb866e 100644 --- a/pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc +++ b/pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc @@ -6,6 +6,7 @@ // 0 // ], // "custom": { +// "limit": 500, // "searchWords": [ // "hello", // "message" @@ -40,6 +41,7 @@ 0 ], "custom": { + "limit": 500, "searchWords": [ "hello", "message" diff --git a/public/app/core/logsModel.test.ts b/public/app/core/logsModel.test.ts index d16d1484cb7..77794cf289c 100644 --- a/public/app/core/logsModel.test.ts +++ b/public/app/core/logsModel.test.ts @@ -354,6 +354,48 @@ describe('dataFrameToLogsModel', () => { }); }); + it('given one series with limit as custom meta property should return correct limit', () => { + const series: DataFrame[] = [ + new MutableDataFrame({ + fields: [ + { + name: 'time', + type: FieldType.time, + values: ['2019-04-26T09:28:11.352440161Z', '2019-04-26T14:42:50.991981292Z'], + }, + { + name: 'message', + type: FieldType.string, + values: [ + 't=2019-04-26T11:05:28+0200 lvl=info msg="Initializing DatasourceCacheService" logger=server', + 't=2019-04-26T16:42:50+0200 lvl=eror msg="new token…t unhashed token=56d9fdc5c8b7400bd51b060eea8ca9d7', + ], + labels: { + filename: '/var/log/grafana/grafana.log', + job: 'grafana', + }, + }, + { + name: 'id', + type: FieldType.string, + values: ['foo', 'bar'], + }, + ], + meta: { + custom: { + limit: 1000, + }, + }, + }), + ]; + const logsModel = dataFrameToLogsModel(series, 1); + expect(logsModel.meta![1]).toMatchObject({ + label: LIMIT_LABEL, + value: `1000 (2 returned)`, + kind: LogsMetaKind.String, + }); + }); + it('given one series with labels-field should return expected logs model', () => { const series: DataFrame[] = [ new MutableDataFrame({ diff --git a/public/app/core/logsModel.ts b/public/app/core/logsModel.ts index ed8bc923316..8ad95087138 100644 --- a/public/app/core/logsModel.ts +++ b/public/app/core/logsModel.ts @@ -474,10 +474,11 @@ export function logSeriesToLogsModel(logSeries: DataFrame[], queries: DataQuery[ kind: LogsMetaKind.LabelsMap, }); } - - const limits = logSeries.filter((series) => series.meta && series.meta.limit); + // Data sources that set up searchWords on backend use meta.custom.limit. + // Data sources that set up searchWords through frontend can use meta.limit. + const limits = logSeries.filter((series) => series?.meta?.custom?.limit ?? series?.meta?.limit); const lastLimitPerRef = limits.reduce>((acc, elem) => { - acc[elem.refId ?? ''] = elem.meta?.limit ?? 0; + acc[elem.refId ?? ''] = elem.meta?.custom?.limit ?? elem.meta?.limit ?? 0; return acc; }, {}); diff --git a/public/app/plugins/datasource/elasticsearch/LegacyQueryRunner.ts b/public/app/plugins/datasource/elasticsearch/LegacyQueryRunner.ts index dfc24921b43..fc923f91dd9 100644 --- a/public/app/plugins/datasource/elasticsearch/LegacyQueryRunner.ts +++ b/public/app/plugins/datasource/elasticsearch/LegacyQueryRunner.ts @@ -15,10 +15,10 @@ import { import { BackendSrvRequest, getBackendSrv, TemplateSrv } from '@grafana/runtime'; import { ElasticResponse } from './ElasticResponse'; -import { ElasticDatasource, enhanceDataFrame } from './datasource'; +import { ElasticDatasource, enhanceDataFrameWithDataLinks } from './datasource'; import { defaultBucketAgg, hasMetricOfType } from './queryDef'; import { trackQuery } from './tracking'; -import { ElasticsearchQuery, Logs } from './types'; +import { DataLinkConfig, ElasticsearchQuery, Logs } from './types'; export class LegacyQueryRunner { datasource: ElasticDatasource; @@ -250,3 +250,17 @@ function transformHitsBasedOnDirection(response: any, direction: 'asc' | 'desc') ], }; } + +/** + * Modifies dataFrame and adds dataLinks from the config. + * Exported for tests. + */ +export function enhanceDataFrame(dataFrame: DataFrame, dataLinks: DataLinkConfig[], limit?: number) { + if (limit) { + dataFrame.meta = { + ...dataFrame.meta, + limit, + }; + } + enhanceDataFrameWithDataLinks(dataFrame, dataLinks); +} diff --git a/public/app/plugins/datasource/elasticsearch/datasource.test.ts b/public/app/plugins/datasource/elasticsearch/datasource.test.ts index 8df01e4fa8a..e116d18c360 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.test.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.test.ts @@ -25,7 +25,8 @@ import { TemplateSrv } from 'app/features/templating/template_srv'; import { createFetchResponse } from '../../../../test/helpers/createFetchResponse'; -import { ElasticDatasource, enhanceDataFrame } from './datasource'; +import { enhanceDataFrame } from './LegacyQueryRunner'; +import { ElasticDatasource } from './datasource'; import { createElasticDatasource } from './mocks'; import { Filters, ElasticsearchOptions, ElasticsearchQuery } from './types'; diff --git a/public/app/plugins/datasource/elasticsearch/datasource.ts b/public/app/plugins/datasource/elasticsearch/datasource.ts index e01b5f51c3a..a0f9efa3414 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.ts +++ b/public/app/plugins/datasource/elasticsearch/datasource.ts @@ -626,7 +626,15 @@ export class ElasticDatasource const { enableElasticsearchBackendQuerying } = config.featureToggles; if (enableElasticsearchBackendQuerying) { const start = new Date(); - return super.query(request).pipe(tap((response) => trackQuery(response, request, start))); + return super.query(request).pipe( + tap((response) => trackQuery(response, request, start)), + map((response) => { + response.data.forEach((dataFrame) => { + enhanceDataFrameWithDataLinks(dataFrame, this.dataLinks); + }); + return response; + }) + ); } return this.legacyQueryRunner.query(request); } @@ -1034,18 +1042,7 @@ export class ElasticDatasource }; } -/** - * Modifies dataframe and adds dataLinks from the config. - * Exported for tests. - */ -export function enhanceDataFrame(dataFrame: DataFrame, dataLinks: DataLinkConfig[], limit?: number) { - if (limit) { - dataFrame.meta = { - ...dataFrame.meta, - limit, - }; - } - +export function enhanceDataFrameWithDataLinks(dataFrame: DataFrame, dataLinks: DataLinkConfig[]) { if (!dataLinks.length) { return; }