mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Elasticsearch: Remove _source field when processing raw data on backend (#64119)
* Elasticsearch: Remove _source field when processing raw data on backend * Update snapshot test
This commit is contained in:
Ivana Huckova
GitHub
@@ -154,7 +154,6 @@ func processRawDataResponse(res *es.SearchResponse, target *Query, configuredFie
|
||||
"_index": hit["_index"],
|
||||
"sort": hit["sort"],
|
||||
"highlight": hit["highlight"],
|
||||
"_source": flattened,
|
||||
}
|
||||
|
||||
for k, v := range flattened {
|
||||
|
||||
@@ -1356,53 +1356,6 @@ func TestTwoBucketScripts(t *testing.T) {
|
||||
requireFloatAt(t, 48.0, fields[4], 1)
|
||||
}
|
||||
|
||||
func TestRawData(t *testing.T) {
|
||||
query := []byte(`
|
||||
[
|
||||
{
|
||||
"refId": "A",
|
||||
"metrics": [{ "type": "raw_data", "id": "1" }],
|
||||
"bucketAggs": []
|
||||
}
|
||||
]
|
||||
`)
|
||||
|
||||
response := []byte(`
|
||||
{
|
||||
"responses": [
|
||||
{
|
||||
"hits": {
|
||||
"total": { "relation": "eq", "value": 1 },
|
||||
"hits": [
|
||||
{
|
||||
"_id": "1",
|
||||
"_type": "_doc",
|
||||
"_index": "index",
|
||||
"_source": { "sourceProp": "asd" }
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
`)
|
||||
|
||||
result, err := queryDataTest(query, response)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, result.response.Responses, 1)
|
||||
// frames := result.response.Responses["A"].Frames
|
||||
// require.True(t, len(frames) > 0) // FIXME
|
||||
|
||||
// for _, field := range frames[0].Fields {
|
||||
// trueValue := true
|
||||
// filterableConfig := data.FieldConfig{Filterable: &trueValue}
|
||||
|
||||
// // we need to test that the only changed setting is `filterable`
|
||||
// require.Equal(t, filterableConfig, *field.Config) // FIXME
|
||||
// }
|
||||
}
|
||||
|
||||
func TestLogsAndCount(t *testing.T) {
|
||||
query := []byte(`
|
||||
[
|
||||
|
||||
@@ -1348,7 +1348,7 @@ func TestResponseParser(t *testing.T) {
|
||||
require.Len(t, dataframes, 1)
|
||||
frame := dataframes[0]
|
||||
|
||||
require.Equal(t, 16, len(frame.Fields))
|
||||
require.Equal(t, 15, len(frame.Fields))
|
||||
// Fields have the correct length
|
||||
require.Equal(t, 2, frame.Fields[0].Len())
|
||||
// First field is timeField
|
||||
@@ -1356,14 +1356,60 @@ func TestResponseParser(t *testing.T) {
|
||||
// Correctly uses string types
|
||||
require.Equal(t, data.FieldTypeNullableString, frame.Fields[1].Type())
|
||||
// Correctly detects float64 types
|
||||
require.Equal(t, data.FieldTypeNullableFloat64, frame.Fields[6].Type())
|
||||
require.Equal(t, data.FieldTypeNullableFloat64, frame.Fields[5].Type())
|
||||
// Correctly detects json types
|
||||
require.Equal(t, data.FieldTypeNullableJSON, frame.Fields[7].Type())
|
||||
require.Equal(t, data.FieldTypeNullableJSON, frame.Fields[6].Type())
|
||||
// Correctly flattens fields
|
||||
require.Equal(t, "nested.field.double_nested", frame.Fields[12].Name)
|
||||
require.Equal(t, data.FieldTypeNullableString, frame.Fields[12].Type())
|
||||
require.Equal(t, "nested.field.double_nested", frame.Fields[11].Name)
|
||||
require.Equal(t, data.FieldTypeNullableString, frame.Fields[11].Type())
|
||||
// Correctly detects type even if first value is null
|
||||
require.Equal(t, data.FieldTypeNullableString, frame.Fields[15].Type())
|
||||
require.Equal(t, data.FieldTypeNullableString, frame.Fields[14].Type())
|
||||
})
|
||||
t.Run("Raw data query filterable fields", func(t *testing.T) {
|
||||
query := []byte(`
|
||||
[
|
||||
{
|
||||
"refId": "A",
|
||||
"metrics": [{ "type": "raw_data", "id": "1" }],
|
||||
"bucketAggs": []
|
||||
}
|
||||
]
|
||||
`)
|
||||
|
||||
response := []byte(`
|
||||
{
|
||||
"responses": [
|
||||
{
|
||||
"hits": {
|
||||
"total": { "relation": "eq", "value": 1 },
|
||||
"hits": [
|
||||
{
|
||||
"_id": "1",
|
||||
"_type": "_doc",
|
||||
"_index": "index",
|
||||
"_source": { "sourceProp": "asd" }
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
`)
|
||||
|
||||
result, err := queryDataTest(query, response)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, result.response.Responses, 1)
|
||||
frames := result.response.Responses["A"].Frames
|
||||
require.True(t, len(frames) > 0) // FIXME
|
||||
|
||||
for _, field := range frames[0].Fields {
|
||||
trueValue := true
|
||||
filterableConfig := data.FieldConfig{Filterable: &trueValue}
|
||||
|
||||
// we need to test that the only changed setting is `filterable`
|
||||
require.Equal(t, filterableConfig, *field.Config) // FIXME
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -2,18 +2,18 @@
|
||||
//
|
||||
// Frame[0]
|
||||
// Name:
|
||||
// Dimensions: 17 Fields by 5 Rows
|
||||
// +--------------------------+----------------------+-----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+---------------------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
|
||||
// | Name: @timestamp | Name: _id | Name: _index | Name: _source | Name: _type | Name: abc | Name: counter | Name: float | Name: highlight | Name: is_true | Name: label | Name: level | Name: line | Name: location | Name: nested_field.internal.nested | Name: shapes | Name: sort |
|
||||
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
|
||||
// | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage | Type: []*string | Type: []*float64 | Type: []*float64 | Type: []*json.RawMessage | Type: []*bool | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage |
|
||||
// +--------------------------+----------------------+-----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+---------------------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
|
||||
// | 2023-02-09T14:40:01.475Z | g2aeNoYB7vaC3bq-ezfK | logs-2023.02.09 | {"@timestamp":"2023-02-09T14:40:01.475Z","abc":null,"counter":81,"float":10.911972180833306,"is_true":true,"label":"val3","level":"info","line":"log text [106619125]","location":"-42.73465234425797, -14.097854057104112","nested_field.internal.nested":"value1","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}]} | null | null | 81 | 10.911972180833306 | null | true | val3 | info | log text [106619125] | -42.73465234425797, -14.097854057104112 | value1 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953601475,4] |
|
||||
// | 2023-02-09T14:40:00.513Z | gmaeNoYB7vaC3bq-eDcN | logs-2023.02.09 | {"@timestamp":"2023-02-09T14:40:00.513Z","abc":null,"counter":80,"float":62.94120607636795,"is_true":false,"label":"val3","level":"error","line":"log text with [781660944]","location":"42.07571917624318, 15.95725088484611","nested_field.internal.nested":"value2","shapes":[{"type":"triangle"},{"type":"square"}]} | null | null | 80 | 62.94120607636795 | null | false | val3 | error | log text with [781660944] | 42.07571917624318, 15.95725088484611 | value2 | [{"type":"triangle"},{"type":"square"}] | [1675953600513,7] |
|
||||
// | 2023-02-09T14:39:59.556Z | gWaeNoYB7vaC3bq-dDdL | logs-2023.02.09 | {"@timestamp":"2023-02-09T14:39:59.556Z","abc":"def","counter":79,"float":53.323706427230455,"is_true":true,"label":"val1","level":"info","line":"log text [894867430]","location":"-38.27341566189766, -23.66739642570781","nested_field.internal.nested":"value3","shapes":[{"type":"triangle"},{"type":"square"}]} | null | def | 79 | 53.323706427230455 | null | true | val1 | info | log text [894867430] | -38.27341566189766, -23.66739642570781 | value3 | [{"type":"triangle"},{"type":"square"}] | [1675953599556,10] |
|
||||
// | 2023-02-09T14:39:58.608Z | gGaeNoYB7vaC3bq-cDeY | logs-2023.02.09 | {"@timestamp":"2023-02-09T14:39:58.608Z","abc":"def","counter":78,"float":82.72012623471589,"is_true":false,"label":"val1","level":"info","line":"log text [478598889]","location":"12.373240290451287, 43.265493464362024","nested_field.internal.nested":"value4","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}]} | null | def | 78 | 82.72012623471589 | null | false | val1 | info | log text [478598889] | 12.373240290451287, 43.265493464362024 | value4 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953598608,15] |
|
||||
// | 2023-02-09T14:39:57.665Z | f2aeNoYB7vaC3bq-bDf7 | logs-2023.02.09 | {"@timestamp":"2023-02-09T14:39:57.665Z","abc":"def","counter":77,"float":35.05784443331803,"is_true":false,"label":"val3","level":"info","line":"log text [526995818]","location":"-31.524344042228194, -32.11254790120572","nested_field.internal.nested":"value5","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}]} | null | def | 77 | 35.05784443331803 | null | false | val3 | info | log text [526995818] | -31.524344042228194, -32.11254790120572 | value5 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953597665,20] |
|
||||
// +--------------------------+----------------------+-----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+---------------------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
|
||||
// Dimensions: 16 Fields by 5 Rows
|
||||
// +--------------------------+----------------------+-----------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+---------------------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
|
||||
// | Name: @timestamp | Name: _id | Name: _index | Name: _type | Name: abc | Name: counter | Name: float | Name: highlight | Name: is_true | Name: label | Name: level | Name: line | Name: location | Name: nested_field.internal.nested | Name: shapes | Name: sort |
|
||||
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
|
||||
// | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*string | Type: []*float64 | Type: []*float64 | Type: []*json.RawMessage | Type: []*bool | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage |
|
||||
// +--------------------------+----------------------+-----------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+---------------------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
|
||||
// | 2023-02-09T14:40:01.475Z | g2aeNoYB7vaC3bq-ezfK | logs-2023.02.09 | null | null | 81 | 10.911972180833306 | null | true | val3 | info | log text [106619125] | -42.73465234425797, -14.097854057104112 | value1 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953601475,4] |
|
||||
// | 2023-02-09T14:40:00.513Z | gmaeNoYB7vaC3bq-eDcN | logs-2023.02.09 | null | null | 80 | 62.94120607636795 | null | false | val3 | error | log text with [781660944] | 42.07571917624318, 15.95725088484611 | value2 | [{"type":"triangle"},{"type":"square"}] | [1675953600513,7] |
|
||||
// | 2023-02-09T14:39:59.556Z | gWaeNoYB7vaC3bq-dDdL | logs-2023.02.09 | null | def | 79 | 53.323706427230455 | null | true | val1 | info | log text [894867430] | -38.27341566189766, -23.66739642570781 | value3 | [{"type":"triangle"},{"type":"square"}] | [1675953599556,10] |
|
||||
// | 2023-02-09T14:39:58.608Z | gGaeNoYB7vaC3bq-cDeY | logs-2023.02.09 | null | def | 78 | 82.72012623471589 | null | false | val1 | info | log text [478598889] | 12.373240290451287, 43.265493464362024 | value4 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953598608,15] |
|
||||
// | 2023-02-09T14:39:57.665Z | f2aeNoYB7vaC3bq-bDf7 | logs-2023.02.09 | null | def | 77 | 35.05784443331803 | null | false | val3 | info | log text [526995818] | -31.524344042228194, -32.11254790120572 | value5 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953597665,20] |
|
||||
// +--------------------------+----------------------+-----------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+---------------------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
|
||||
//
|
||||
//
|
||||
// 🌟 This was machine generated. Do not edit. 🌟
|
||||
@@ -56,17 +56,6 @@
|
||||
"filterable": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "_source",
|
||||
"type": "other",
|
||||
"typeInfo": {
|
||||
"frame": "json.RawMessage",
|
||||
"nullable": true
|
||||
},
|
||||
"config": {
|
||||
"filterable": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "_type",
|
||||
"type": "other",
|
||||
@@ -235,126 +224,6 @@
|
||||
"logs-2023.02.09",
|
||||
"logs-2023.02.09"
|
||||
],
|
||||
[
|
||||
{
|
||||
"@timestamp": "2023-02-09T14:40:01.475Z",
|
||||
"abc": null,
|
||||
"counter": 81,
|
||||
"float": 10.911972180833306,
|
||||
"is_true": true,
|
||||
"label": "val3",
|
||||
"level": "info",
|
||||
"line": "log text [106619125]",
|
||||
"location": "-42.73465234425797, -14.097854057104112",
|
||||
"nested_field.internal.nested": "value1",
|
||||
"shapes": [
|
||||
{
|
||||
"type": "triangle"
|
||||
},
|
||||
{
|
||||
"type": "triangle"
|
||||
},
|
||||
{
|
||||
"type": "triangle"
|
||||
},
|
||||
{
|
||||
"type": "square"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"@timestamp": "2023-02-09T14:40:00.513Z",
|
||||
"abc": null,
|
||||
"counter": 80,
|
||||
"float": 62.94120607636795,
|
||||
"is_true": false,
|
||||
"label": "val3",
|
||||
"level": "error",
|
||||
"line": "log text with [781660944]",
|
||||
"location": "42.07571917624318, 15.95725088484611",
|
||||
"nested_field.internal.nested": "value2",
|
||||
"shapes": [
|
||||
{
|
||||
"type": "triangle"
|
||||
},
|
||||
{
|
||||
"type": "square"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"@timestamp": "2023-02-09T14:39:59.556Z",
|
||||
"abc": "def",
|
||||
"counter": 79,
|
||||
"float": 53.323706427230455,
|
||||
"is_true": true,
|
||||
"label": "val1",
|
||||
"level": "info",
|
||||
"line": "log text [894867430]",
|
||||
"location": "-38.27341566189766, -23.66739642570781",
|
||||
"nested_field.internal.nested": "value3",
|
||||
"shapes": [
|
||||
{
|
||||
"type": "triangle"
|
||||
},
|
||||
{
|
||||
"type": "square"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"@timestamp": "2023-02-09T14:39:58.608Z",
|
||||
"abc": "def",
|
||||
"counter": 78,
|
||||
"float": 82.72012623471589,
|
||||
"is_true": false,
|
||||
"label": "val1",
|
||||
"level": "info",
|
||||
"line": "log text [478598889]",
|
||||
"location": "12.373240290451287, 43.265493464362024",
|
||||
"nested_field.internal.nested": "value4",
|
||||
"shapes": [
|
||||
{
|
||||
"type": "triangle"
|
||||
},
|
||||
{
|
||||
"type": "triangle"
|
||||
},
|
||||
{
|
||||
"type": "triangle"
|
||||
},
|
||||
{
|
||||
"type": "square"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"@timestamp": "2023-02-09T14:39:57.665Z",
|
||||
"abc": "def",
|
||||
"counter": 77,
|
||||
"float": 35.05784443331803,
|
||||
"is_true": false,
|
||||
"label": "val3",
|
||||
"level": "info",
|
||||
"line": "log text [526995818]",
|
||||
"location": "-31.524344042228194, -32.11254790120572",
|
||||
"nested_field.internal.nested": "value5",
|
||||
"shapes": [
|
||||
{
|
||||
"type": "triangle"
|
||||
},
|
||||
{
|
||||
"type": "triangle"
|
||||
},
|
||||
{
|
||||
"type": "triangle"
|
||||
},
|
||||
{
|
||||
"type": "square"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
[
|
||||
null,
|
||||
null,
|
||||
|
||||
Reference in New Issue
Block a user