mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Elasticsearch: Fix processing of raw_data with not-recognized time format (#78262)
* Elasticsearch: Fix non-standard time field in raw data queries * Update snapshot tests * Refactor
This commit is contained in:
parent
cfc67a9e2d
commit
28f4c3ed44
@ -389,6 +389,11 @@ func processDocumentQuery(q *Query, b *es.SearchRequestBuilder, from, to int64,
|
||||
b.Sort(es.SortOrderDesc, defaultTimeField, "boolean")
|
||||
b.Sort(es.SortOrderDesc, "_doc", "")
|
||||
b.AddDocValueField(defaultTimeField)
|
||||
if isRawDataQuery(q) {
|
||||
// For raw_data queries we need to add timeField as field with standardized time format to not receive
|
||||
// invalid formats that elasticsearch can parse, but our frontend can't (e.g. yyyy_MM_dd_HH_mm_ss)
|
||||
b.AddTimeFieldWithStandardizedFormat(defaultTimeField)
|
||||
}
|
||||
b.Size(stringToIntWithDefaultValue(metric.Settings.Get("size").MustString(), defaultSize))
|
||||
}
|
||||
|
||||
|
@ -231,6 +231,15 @@ func processRawDataResponse(res *es.SearchResponse, target *Query, configuredFie
|
||||
doc[k] = v
|
||||
}
|
||||
|
||||
if hit["fields"] != nil {
|
||||
source, ok := hit["fields"].(map[string]interface{})
|
||||
if ok {
|
||||
for k, v := range source {
|
||||
doc[k] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for key := range doc {
|
||||
propNames[key] = true
|
||||
}
|
||||
|
@ -601,6 +601,92 @@ func TestProcessRawDataResponse(t *testing.T) {
|
||||
require.Equal(t, filterableConfig, *field.Config)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("gets correct time field from fields", func(t *testing.T) {
|
||||
query := []byte(`
|
||||
[
|
||||
{
|
||||
"refId": "A",
|
||||
"metrics": [{ "type": "raw_data", "id": "1" }]
|
||||
}
|
||||
]
|
||||
`)
|
||||
|
||||
response := []byte(`
|
||||
{
|
||||
"responses": [
|
||||
{
|
||||
"aggregations": {},
|
||||
"hits": {
|
||||
"hits": [
|
||||
{
|
||||
"_id": "fdsfs",
|
||||
"_type": "_doc",
|
||||
"_index": "mock-index",
|
||||
"_source": {
|
||||
"testtime": "06/24/2019",
|
||||
"host": "djisaodjsoad",
|
||||
"number": 1,
|
||||
"line": "hello, i am a message",
|
||||
"level": "debug",
|
||||
"fields": { "lvl": "debug" }
|
||||
},
|
||||
"highlight": {
|
||||
"message": [
|
||||
"@HIGHLIGHT@hello@/HIGHLIGHT@, i am a @HIGHLIGHT@message@/HIGHLIGHT@"
|
||||
]
|
||||
},
|
||||
"fields": {
|
||||
"testtime": [ "2019-06-24T09:51:19.765Z" ]
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "kdospaidopa",
|
||||
"_type": "_doc",
|
||||
"_index": "mock-index",
|
||||
"_source": {
|
||||
"testtime": "06/24/2019",
|
||||
"host": "dsalkdakdop",
|
||||
"number": 2,
|
||||
"line": "hello, i am also message",
|
||||
"level": "error",
|
||||
"fields": { "lvl": "info" }
|
||||
},
|
||||
"highlight": {
|
||||
"message": [
|
||||
"@HIGHLIGHT@hello@/HIGHLIGHT@, i am a @HIGHLIGHT@message@/HIGHLIGHT@"
|
||||
]
|
||||
},
|
||||
"fields": {
|
||||
"testtime": [ "2019-06-24T09:52:19.765Z" ]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
`)
|
||||
result, err := queryDataTest(query, response)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Len(t, result.response.Responses, 1)
|
||||
frames := result.response.Responses["A"].Frames
|
||||
require.Len(t, frames, 1)
|
||||
|
||||
logsFrame := frames[0]
|
||||
|
||||
logsFieldMap := make(map[string]*data.Field)
|
||||
for _, field := range logsFrame.Fields {
|
||||
logsFieldMap[field.Name] = field
|
||||
}
|
||||
t0 := time.Date(2019, time.June, 24, 9, 51, 19, 765000000, time.UTC)
|
||||
t1 := time.Date(2019, time.June, 24, 9, 52, 19, 765000000, time.UTC)
|
||||
require.Contains(t, logsFieldMap, "testtime")
|
||||
require.Equal(t, data.FieldTypeNullableTime, logsFieldMap["testtime"].Type())
|
||||
require.Equal(t, &t0, logsFieldMap["testtime"].At(0))
|
||||
require.Equal(t, &t1, logsFieldMap["testtime"].At(1))
|
||||
})
|
||||
}
|
||||
|
||||
func TestProcessRawDocumentResponse(t *testing.T) {
|
||||
|
@ -26,5 +26,12 @@
|
||||
"_doc": {
|
||||
"order": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
"fields":
|
||||
[
|
||||
{
|
||||
"field": "testtime",
|
||||
"format": "strict_date_optional_time_nanos"
|
||||
}
|
||||
]
|
||||
}
|
Loading…
Reference in New Issue
Block a user