From 759c088ac50fddaf7c5add182227d646ff3262bb Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Fri, 19 Jan 2024 10:28:08 +0100 Subject: [PATCH] Elasticsearch: Fix showing of logs when `__source` is log message field (#80804) Elasticsearch: Fix showing of logs whe __source is log message field --- pkg/tsdb/elasticsearch/response_parser.go | 9 +- .../elasticsearch/response_parser_test.go | 12 +- .../testdata_response/logs.a.golden.jsonc | 129 ++---------------- 3 files changed, 21 insertions(+), 129 deletions(-) diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index be767a99181..e7d1df81e1f 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -136,8 +136,14 @@ func processLogsResponse(res *es.SearchResponse, target *Query, configuredFields for hitIdx, hit := range res.Hits.Hits { var flattened map[string]interface{} + var sourceString string if hit["_source"] != nil { flattened = flatten(hit["_source"].(map[string]interface{}), 10) + sourceMarshalled, err := json.Marshal(flattened) + if err != nil { + return err + } + sourceString = string(sourceMarshalled) } doc := map[string]interface{}{ @@ -146,7 +152,8 @@ func processLogsResponse(res *es.SearchResponse, target *Query, configuredFields "_index": hit["_index"], "sort": hit["sort"], "highlight": hit["highlight"], - "_source": flattened, + // In case of logs query we want to have the raw source as a string field so it can be visualized in logs panel + "_source": sourceString, } for k, v := range flattened { diff --git a/pkg/tsdb/elasticsearch/response_parser_test.go b/pkg/tsdb/elasticsearch/response_parser_test.go index ee307313ecb..8e0fb11c05f 100644 --- a/pkg/tsdb/elasticsearch/response_parser_test.go +++ b/pkg/tsdb/elasticsearch/response_parser_test.go @@ -129,7 +129,7 @@ func TestProcessLogsResponse(t *testing.T) { require.Equal(t, data.FieldTypeNullableFloat64, logsFieldMap["number"].Type()) require.Contains(t, logsFieldMap, "_source") - require.Equal(t, data.FieldTypeNullableJSON, logsFieldMap["_source"].Type()) + require.Equal(t, data.FieldTypeNullableString, logsFieldMap["_source"].Type()) requireStringAt(t, "fdsfs", logsFieldMap["_id"], 0) requireStringAt(t, "kdospaidopa", logsFieldMap["_id"], 1) @@ -138,10 +138,8 @@ func TestProcessLogsResponse(t *testing.T) { requireStringAt(t, "mock-index", logsFieldMap["_index"], 0) requireStringAt(t, "mock-index", logsFieldMap["_index"], 1) - actualJson1, err := json.Marshal(logsFieldMap["_source"].At(0).(*json.RawMessage)) - require.NoError(t, err) - actualJson2, err := json.Marshal(logsFieldMap["_source"].At(1).(*json.RawMessage)) - require.NoError(t, err) + actualJson1 := logsFieldMap["_source"].At(0).(*string) + actualJson2 := logsFieldMap["_source"].At(1).(*string) expectedJson1 := ` { @@ -165,8 +163,8 @@ func TestProcessLogsResponse(t *testing.T) { "fields.lvl": "info" }` - require.JSONEq(t, expectedJson1, string(actualJson1)) - require.JSONEq(t, expectedJson2, string(actualJson2)) + require.JSONEq(t, expectedJson1, *actualJson1) + require.JSONEq(t, expectedJson2, *actualJson2) }) t.Run("creates correct level field", func(t *testing.T) { diff --git a/pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc b/pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc index 9fe7a4c1337..105279a888e 100644 --- a/pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc +++ b/pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc @@ -19,7 +19,7 @@ // +-----------------------------------+---------------------------+----------------------+-----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+--------------------------------------+---------------+-----------------+-----------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+ // | Name: testtime | Name: line | Name: _id | Name: _index | Name: _source | Name: _type | Name: abc | Name: counter | Name: float | Name: highlight | Name: id | Name: is_true | Name: label | Name: level | Name: location | Name: nested_field.internal.nested | Name: shapes | Name: sort | // | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | -// | Type: []*time.Time | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage | Type: []*string | Type: []*float64 | Type: []*float64 | Type: []*json.RawMessage | Type: []*string | Type: []*bool | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage | +// | Type: []*time.Time | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*string | Type: []*float64 | Type: []*float64 | Type: []*json.RawMessage | Type: []*string | Type: []*bool | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage | // +-----------------------------------+---------------------------+----------------------+-----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------------------------------------------------------------------------------------------------------------+--------------------------------------+---------------+-----------------+-----------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+ // | 2023-02-09 14:40:01.475 +0000 UTC | log text [106619125] | g2aeNoYB7vaC3bq-ezfK | logs-2023.02.09 | {"abc":null,"counter":81,"float":10.911972180833306,"is_true":true,"label":"val3","line":"log text [106619125]","location":"-42.73465234425797, -14.097854057104112","lvl":"info","nested_field.internal.nested":"value1","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}],"testtime":"09/02/2023"} | null | null | 81 | 10.911972180833306 | {"duplicated":["@HIGHLIGHT@hello@/HIGHLIGHT@"],"line":["@HIGHLIGHT@hello@/HIGHLIGHT@, i am a @HIGHLIGHT@message@/HIGHLIGHT@"]} | logs-2023.02.09#g2aeNoYB7vaC3bq-ezfK | true | val3 | info | -42.73465234425797, -14.097854057104112 | value1 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953601475,4] | // | 2023-02-09 14:40:00.513 +0000 UTC | log text with [781660944] | gmaeNoYB7vaC3bq-eDcN | logs-2023.02.09 | {"abc":null,"counter":80,"float":62.94120607636795,"is_true":false,"label":"val3","line":"log text with [781660944]","location":"42.07571917624318, 15.95725088484611","lvl":"error","nested_field.internal.nested":"value2","shapes":[{"type":"triangle"},{"type":"square"}],"testtime":"09/02/2023"} | null | null | 80 | 62.94120607636795 | {"duplicated":["@HIGHLIGHT@hello@/HIGHLIGHT@"],"line":["@HIGHLIGHT@hello@/HIGHLIGHT@, i am a @HIGHLIGHT@message@/HIGHLIGHT@"]} | logs-2023.02.09#gmaeNoYB7vaC3bq-eDcN | false | val3 | error | 42.07571917624318, 15.95725088484611 | value2 | [{"type":"triangle"},{"type":"square"}] | [1675953600513,7] | @@ -96,9 +96,9 @@ }, { "name": "_source", - "type": "other", + "type": "string", "typeInfo": { - "frame": "json.RawMessage", + "frame": "string", "nullable": true }, "config": { @@ -281,124 +281,11 @@ "logs-2023.02.09" ], [ - { - "abc": null, - "counter": 81, - "float": 10.911972180833306, - "is_true": true, - "label": "val3", - "line": "log text [106619125]", - "location": "-42.73465234425797, -14.097854057104112", - "lvl": "info", - "nested_field.internal.nested": "value1", - "shapes": [ - { - "type": "triangle" - }, - { - "type": "triangle" - }, - { - "type": "triangle" - }, - { - "type": "square" - } - ], - "testtime": "09/02/2023" - }, - { - "abc": null, - "counter": 80, - "float": 62.94120607636795, - "is_true": false, - "label": "val3", - "line": "log text with [781660944]", - "location": "42.07571917624318, 15.95725088484611", - "lvl": "error", - "nested_field.internal.nested": "value2", - "shapes": [ - { - "type": "triangle" - }, - { - "type": "square" - } - ], - "testtime": "09/02/2023" - }, - { - "abc": "def", - "counter": 79, - "float": 53.323706427230455, - "is_true": true, - "label": "val1", - "line": "log text [894867430]", - "location": "-38.27341566189766, -23.66739642570781", - "lvl": "info", - "nested_field.internal.nested": "value3", - "shapes": [ - { - "type": "triangle" - }, - { - "type": "square" - } - ], - "testtime": "09/02/2023" - }, - { - "abc": "def", - "counter": 78, - "float": 82.72012623471589, - "is_true": false, - "label": "val1", - "line": "log text [478598889]", - "location": "12.373240290451287, 43.265493464362024", - "lvl": "info", - "nested_field.internal.nested": "value4", - "shapes": [ - { - "type": "triangle" - }, - { - "type": "triangle" - }, - { - "type": "triangle" - }, - { - "type": "square" - } - ], - "testtime": "09/02/2023" - }, - { - "abc": "def", - "counter": 77, - "float": 35.05784443331803, - "is_true": false, - "label": "val3", - "line": "log text [526995818]", - "location": "-31.524344042228194, -32.11254790120572", - "lvl": "info", - "nested_field.internal.nested": "value5", - "shapes": [ - { - "type": "triangle" - }, - { - "type": "triangle" - }, - { - "type": "triangle" - }, - { - "type": "square" - } - ], - "testtime": "09/02/2023" - } + "{\"abc\":null,\"counter\":81,\"float\":10.911972180833306,\"is_true\":true,\"label\":\"val3\",\"line\":\"log text [106619125]\",\"location\":\"-42.73465234425797, -14.097854057104112\",\"lvl\":\"info\",\"nested_field.internal.nested\":\"value1\",\"shapes\":[{\"type\":\"triangle\"},{\"type\":\"triangle\"},{\"type\":\"triangle\"},{\"type\":\"square\"}],\"testtime\":\"09/02/2023\"}", + "{\"abc\":null,\"counter\":80,\"float\":62.94120607636795,\"is_true\":false,\"label\":\"val3\",\"line\":\"log text with [781660944]\",\"location\":\"42.07571917624318, 15.95725088484611\",\"lvl\":\"error\",\"nested_field.internal.nested\":\"value2\",\"shapes\":[{\"type\":\"triangle\"},{\"type\":\"square\"}],\"testtime\":\"09/02/2023\"}", + "{\"abc\":\"def\",\"counter\":79,\"float\":53.323706427230455,\"is_true\":true,\"label\":\"val1\",\"line\":\"log text [894867430]\",\"location\":\"-38.27341566189766, -23.66739642570781\",\"lvl\":\"info\",\"nested_field.internal.nested\":\"value3\",\"shapes\":[{\"type\":\"triangle\"},{\"type\":\"square\"}],\"testtime\":\"09/02/2023\"}", + "{\"abc\":\"def\",\"counter\":78,\"float\":82.72012623471589,\"is_true\":false,\"label\":\"val1\",\"line\":\"log text [478598889]\",\"location\":\"12.373240290451287, 43.265493464362024\",\"lvl\":\"info\",\"nested_field.internal.nested\":\"value4\",\"shapes\":[{\"type\":\"triangle\"},{\"type\":\"triangle\"},{\"type\":\"triangle\"},{\"type\":\"square\"}],\"testtime\":\"09/02/2023\"}", + "{\"abc\":\"def\",\"counter\":77,\"float\":35.05784443331803,\"is_true\":false,\"label\":\"val3\",\"line\":\"log text [526995818]\",\"location\":\"-31.524344042228194, -32.11254790120572\",\"lvl\":\"info\",\"nested_field.internal.nested\":\"value5\",\"shapes\":[{\"type\":\"triangle\"},{\"type\":\"triangle\"},{\"type\":\"triangle\"},{\"type\":\"square\"}],\"testtime\":\"09/02/2023\"}" ], [ null,