IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Elasticsearch: Implement processing of logs query results in backend (#63647)

Browse Source 
* Elasticsearch: Add processing of logs query to backend

* Add and fix tests

* Add snapshot tests

* Fix test in ES client

* Small updates, remove redundant logic

* Refactor setPreferredVisType to improve readability
This commit is contained in:
Ivana Huckova
2023-03-01 11:50:56 +01:00
committed by GitHub
parent 8ac92aab19
commit d258c8ef8a
12 changed files with 1061 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
pkg/tsdb/elasticsearch/client/client.go
View File

@@ -25,7 +25,7 @@ type DatasourceInfo struct {
URL string
Database string
ESVersion *semver.Version
TimeField string
ConfiguredFields ConfiguredFields
Interval string
TimeInterval string
MaxConcurrentShardRequests int64
@@ -33,11 +33,17 @@ type DatasourceInfo struct {
XPack bool
}
type ConfiguredFields struct {
TimeField string
LogMessageField string
LogLevelField string
}
const loggerName = "tsdb.elasticsearch.client"
// Client represents a client which can interact with elasticsearch api
type Client interface {
GetTimeField() string
GetConfiguredFields() ConfiguredFields
GetMinInterval(queryInterval string) (time.Duration, error)
ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error)
MultiSearch() *MultiSearchRequestBuilder
@@ -56,29 +62,29 @@ var NewClient = func(ctx context.Context, ds *DatasourceInfo, timeRange backend.
}
logger := log.New(loggerName).FromContext(ctx)
logger.Debug("Creating new client", "version", ds.ESVersion, "timeField", ds.TimeField, "indices", strings.Join(indices, ", "))
logger.Debug("Creating new client", "version", ds.ESVersion, "configuredFields", fmt.Sprintf("%#v", ds.ConfiguredFields), "indices", strings.Join(indices, ", "))
return &baseClientImpl{
logger: logger,
ctx: ctx,
ds: ds,
timeField: ds.TimeField,
indices: indices,
timeRange: timeRange,
logger: logger,
ctx: ctx,
ds: ds,
configuredFields: ds.ConfiguredFields,
indices: indices,
timeRange: timeRange,
}, nil
}
type baseClientImpl struct {
ctx context.Context
ds *DatasourceInfo
timeField string
indices []string
timeRange backend.TimeRange
logger log.Logger
ctx context.Context
ds *DatasourceInfo
configuredFields ConfiguredFields
indices []string
timeRange backend.TimeRange
logger log.Logger
}
func (c *baseClientImpl) GetTimeField() string {
return c.timeField
func (c *baseClientImpl) GetConfiguredFields() ConfiguredFields {
return c.configuredFields
}
func (c *baseClientImpl) GetMinInterval(queryInterval string) (time.Duration, error) {

8
pkg/tsdb/elasticsearch/client/client_test.go
View File

@@ -46,12 +46,18 @@ func TestClient_ExecuteMultisearch(t *testing.T) {
version, err := semver.NewVersion("8.0.0")
require.NoError(t, err)
configuredFields := ConfiguredFields{
TimeField: "testtime",
LogMessageField: "line",
LogLevelField: "lvl",
}
ds := DatasourceInfo{
URL: ts.URL,
HTTPClient: ts.Client(),
Database: "[metrics-]YYYY.MM.DD",
ESVersion: version,
TimeField: "@timestamp",
ConfiguredFields: configuredFields,
Interval: "Daily",
MaxConcurrentShardRequests: 6,
IncludeFrozen: true,

18
pkg/tsdb/elasticsearch/elasticsearch.go
View File

@@ -98,6 +98,16 @@ func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.Inst
return nil, errors.New("elasticsearch time field name is required")
}
logLevelField, ok := jsonData["logLevelField"].(string)
if !ok {
logLevelField = ""
}
logMessageField, ok := jsonData["logMessageField"].(string)
if !ok {
logMessageField = ""
}
interval, ok := jsonData["interval"].(string)
if !ok {
interval = ""
@@ -132,6 +142,12 @@ func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.Inst
xpack = false
}
configuredFields := es.ConfiguredFields{
TimeField: timeField,
LogLevelField: logLevelField,
LogMessageField: logMessageField,
}
model := es.DatasourceInfo{
ID: settings.ID,
URL: settings.URL,
@@ -139,7 +155,7 @@ func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.Inst
Database: settings.Database,
MaxConcurrentShardRequests: int64(maxConcurrentShardRequests),
ESVersion: version,
TimeField: timeField,
ConfiguredFields: configuredFields,
Interval: interval,
TimeInterval: timeInterval,
IncludeFrozen: includeFrozen,

13
pkg/tsdb/elasticsearch/querydata_test.go
View File

@@ -37,15 +37,22 @@ func (rt *queryDataTestRoundTripper) RoundTrip(req *http.Request) (*http.Respons
}
// we setup a fake datasource-info
func newFlowTestDsInfo(body []byte, statusCode int, reuestCallback func(req *http.Request) error) *es.DatasourceInfo {
func newFlowTestDsInfo(body []byte, statusCode int, requestCallback func(req *http.Request) error) *es.DatasourceInfo {
client := http.Client{
Transport: &queryDataTestRoundTripper{body: body, statusCode: statusCode, requestCallback: reuestCallback},
Transport: &queryDataTestRoundTripper{body: body, statusCode: statusCode, requestCallback: requestCallback},
}
configuredFields := es.ConfiguredFields{
TimeField: "testtime",
LogMessageField: "line",
LogLevelField: "lvl",
}
return &es.DatasourceInfo{
ESVersion: semver.MustParse("8.5.0"),
Interval: "Daily",
Database: "[testdb-]YYYY.MM.DD",
TimeField: "testtime",
ConfiguredFields: configuredFields,
TimeInterval: "1s",
URL: "http://localhost:9200",
HTTPClient: &client,

115
pkg/tsdb/elasticsearch/response_parser.go
View File

@@ -36,7 +36,7 @@ const (
logsType = "logs"
)
func parseResponse(responses []*es.SearchResponse, targets []*Query, timeField string) (*backend.QueryDataResponse, error) {
func parseResponse(responses []*es.SearchResponse, targets []*Query, configuredFields es.ConfiguredFields) (*backend.QueryDataResponse, error) {
result := backend.QueryDataResponse{
Responses: backend.Responses{},
}
@@ -58,12 +58,19 @@ func parseResponse(responses []*es.SearchResponse, targets []*Query, timeField s
queryRes := backend.DataResponse{}
if isDocumentQuery(target) {
err := processDocumentResponse(res, target, timeField, &queryRes)
err := processDocumentResponse(res, target, configuredFields, &queryRes)
if err != nil {
return &backend.QueryDataResponse{}, err
}
result.Responses[target.RefID] = queryRes
} else if isLogsQuery(target) {
err := processLogsResponse(res, target, configuredFields, &queryRes)
if err != nil {
return &backend.QueryDataResponse{}, err
}
result.Responses[target.RefID] = queryRes
} else {
// Process as metric query result
props := make(map[string]string)
err := processBuckets(res.Aggregations, target, &queryRes, props, 0)
if err != nil {
@@ -78,9 +85,56 @@ func parseResponse(responses []*es.SearchResponse, targets []*Query, timeField s
return &result, nil
}
func processDocumentResponse(res *es.SearchResponse, target *Query, timeField string, queryRes *backend.DataResponse) error {
docs := make([]map[string]interface{}, len(res.Hits.Hits))
func processLogsResponse(res *es.SearchResponse, target *Query, configuredFields es.ConfiguredFields, queryRes *backend.DataResponse) error {
propNames := make(map[string]bool)
docs := make([]map[string]interface{}, len(res.Hits.Hits))
for hitIdx, hit := range res.Hits.Hits {
var flattened map[string]interface{}
if hit["_source"] != nil {
flattened = flatten(hit["_source"].(map[string]interface{}))
}
doc := map[string]interface{}{
"_id": hit["_id"],
"_type": hit["_type"],
"_index": hit["_index"],
"sort": hit["sort"],
"highlight": hit["highlight"],
"_source": flattened,
}
for k, v := range flattened {
if configuredFields.LogLevelField != "" && k == configuredFields.LogLevelField {
doc["level"] = v
} else {
doc[k] = v
}
}
for key := range doc {
propNames[key] = true
}
// TODO: Implement highlighting
docs[hitIdx] = doc
}
sortedPropNames := sortPropNames(propNames, configuredFields, true)
fields := processDocsToDataFrameFields(docs, sortedPropNames, configuredFields)
frames := data.Frames{}
frame := data.NewFrame("", fields...)
setPreferredVisType(frame, "logs")
frames = append(frames, frame)
queryRes.Frames = frames
return nil
}
func processDocumentResponse(res *es.SearchResponse, target *Query, configuredFields es.ConfiguredFields, queryRes *backend.DataResponse) error {
propNames := make(map[string]bool)
docs := make([]map[string]interface{}, len(res.Hits.Hits))
for hitIdx, hit := range res.Hits.Hits {
var flattened map[string]interface{}
@@ -108,18 +162,28 @@ func processDocumentResponse(res *es.SearchResponse, target *Query, timeField st
docs[hitIdx] = doc
}
sortedPropNames := sortPropNames(propNames, configuredFields, false)
fields := processDocsToDataFrameFields(docs, sortedPropNames, configuredFields)
frames := data.Frames{}
frame := data.NewFrame("", fields...)
frames = append(frames, frame)
queryRes.Frames = frames
return nil
}
func processDocsToDataFrameFields(docs []map[string]interface{}, propNames []string, configuredFields es.ConfiguredFields) []*data.Field {
size := len(docs)
isFilterable := true
allFields := make([]*data.Field, len(propNames))
sortedPropNames := sortPropNames(propNames, timeField)
for propNameIdx, propName := range sortedPropNames {
for propNameIdx, propName := range propNames {
// Special handling for time field
if propName == timeField {
if propName == configuredFields.TimeField {
timeVector := make([]*time.Time, size)
for i, doc := range docs {
timeString, ok := doc[timeField].(string)
timeString, ok := doc[configuredFields.TimeField].(string)
if !ok {
continue
}
@@ -131,7 +195,7 @@ func processDocumentResponse(res *es.SearchResponse, target *Query, timeField st
timeVector[i] = &timeValue
}
}
field := data.NewField(timeField, nil, timeVector)
field := data.NewField(configuredFields.TimeField, nil, timeVector)
field.Config = &data.FieldConfig{Filterable: &isFilterable}
allFields[propNameIdx] = field
continue
@@ -166,12 +230,7 @@ func processDocumentResponse(res *es.SearchResponse, target *Query, timeField st
}
}
frames := data.Frames{}
frame := data.NewFrame("", allFields...)
frames = append(frames, frame)
queryRes.Frames = frames
return nil
return allFields
}
func processBuckets(aggs map[string]interface{}, target *Query,
@@ -894,14 +953,18 @@ func flatten(target map[string]interface{}) map[string]interface{} {
return output
}
// sortPropNames orders propNames so that timeField is first (if it exists) and rest of propNames are ordered alphabetically
func sortPropNames(propNames map[string]bool, timeField string) []string {
// sortPropNames orders propNames so that timeField is first (if it exists), log message field is second
// if shouldSortLogMessageField is true, and rest of propNames are ordered alphabetically
func sortPropNames(propNames map[string]bool, configuredFields es.ConfiguredFields, shouldSortLogMessageField bool) []string {
hasTimeField := false
hasLogMessageField := false
var sortedPropNames []string
for k := range propNames {
if k == timeField {
if configuredFields.TimeField != "" && k == configuredFields.TimeField {
hasTimeField = true
} else if shouldSortLogMessageField && configuredFields.LogMessageField != "" && k == configuredFields.LogMessageField {
hasLogMessageField = true
} else {
sortedPropNames = append(sortedPropNames, k)
}
@@ -909,8 +972,12 @@ func sortPropNames(propNames map[string]bool, timeField string) []string {
sort.Strings(sortedPropNames)
if hasLogMessageField {
sortedPropNames = append([]string{configuredFields.LogMessageField}, sortedPropNames...)
}
if hasTimeField {
sortedPropNames = append([]string{timeField}, sortedPropNames...)
sortedPropNames = append([]string{configuredFields.TimeField}, sortedPropNames...)
}
return sortedPropNames
@@ -939,3 +1006,11 @@ func createFieldOfType[T int | float64 | bool | string](docs []map[string]interf
field.Config = &data.FieldConfig{Filterable: &isFilterable}
return field
}
func setPreferredVisType(frame *data.Frame, visType data.VisType) {
if frame.Meta == nil {
frame.Meta = &data.FrameMeta{}
}
frame.Meta.PreferredVisualization = visType
}

129
pkg/tsdb/elasticsearch/response_parser_test.go
View File

@@ -1043,6 +1043,127 @@ func TestResponseParser(t *testing.T) {
require.Nil(t, frame.Fields[1].Config)
})
t.Run("Log query", func(t *testing.T) {
targets := map[string]string{
"A": `{
"metrics": [{ "type": "logs" }]
}`,
}
response := `{
"responses":[
{
"hits":{
"total":{
"value":109,
"relation":"eq"
},
"max_score":null,
"hits":[
{
"_index":"logs-2023.02.08",
"_id":"GB2UMYYBfCQ-FCMjayJa",
"_score":null,
"_source":{
"@timestamp":"2023-02-08T15:10:55.830Z",
"line":"log text [479231733]",
"counter":"109",
"float":58.253758485091,
"label":"val1",
"lvl":"info",
"location":"17.089705232090438, 41.62861966340297",
"nested": {
"field": {
"double_nested": "value"
}
},
"shapes":[
{
"type":"triangle"
},
{
"type":"square"
}
],
"xyz": null
},
"sort":[
1675869055830,
4
]
},
{
"_index":"logs-2023.02.08",
"_id":"Fx2UMYYBfCQ-FCMjZyJ_",
"_score":null,
"_source":{
"@timestamp":"2023-02-08T15:10:54.835Z",
"line":"log text with ANSI \u001b[31mpart of the text\u001b[0m [493139080]",
"counter":"108",
"float":54.5977098233944,
"label":"val1",
"lvl":"info",
"location":"19.766305918490463, 40.42639175509792",
"nested": {
"field": {
"double_nested": "value"
}
},
"shapes":[
{
"type":"triangle"
},
{
"type":"square"
}
],
"xyz": "def"
},
"sort":[
1675869054835,
7
]
}
]
},
"status":200
}
]
}`
result, err := parseTestResponse(targets, response)
require.NoError(t, err)
require.Len(t, result.Responses, 1)
queryRes := result.Responses["A"]
require.NotNil(t, queryRes)
dataframes := queryRes.Frames
require.Len(t, dataframes, 1)
frame := dataframes[0]
require.Equal(t, 16, len(frame.Fields))
// Fields have the correct length
require.Equal(t, 2, frame.Fields[0].Len())
// First field is timeField
require.Equal(t, data.FieldTypeNullableTime, frame.Fields[0].Type())
// Second is log line
require.Equal(t, data.FieldTypeNullableString, frame.Fields[1].Type())
require.Equal(t, "line", frame.Fields[1].Name)
// Correctly renames lvl field to level
require.Equal(t, "level", frame.Fields[10].Name)
// Correctly uses string types
require.Equal(t, data.FieldTypeNullableString, frame.Fields[1].Type())
// Correctly detects float64 types
require.Equal(t, data.FieldTypeNullableFloat64, frame.Fields[7].Type())
// Correctly detects json types
require.Equal(t, data.FieldTypeNullableJSON, frame.Fields[8].Type())
// Correctly flattens fields
require.Equal(t, "nested.field.double_nested", frame.Fields[12].Name)
require.Equal(t, data.FieldTypeNullableString, frame.Fields[12].Type())
// Correctly detects type even if first value is null
require.Equal(t, data.FieldTypeNullableString, frame.Fields[15].Type())
})
t.Run("Raw data query", func(t *testing.T) {
targets := map[string]string{
"A": `{
@@ -1252,7 +1373,11 @@ func TestResponseParser(t *testing.T) {
func parseTestResponse(tsdbQueries map[string]string, responseBody string) (*backend.QueryDataResponse, error) {
from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC)
to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC)
timeField := "@timestamp"
configuredFields := es.ConfiguredFields{
TimeField: "@timestamp",
LogMessageField: "line",
LogLevelField: "lvl",
}
timeRange := backend.TimeRange{
From: from,
To: to,
@@ -1280,7 +1405,7 @@ func parseTestResponse(tsdbQueries map[string]string, responseBody string) (*bac
return nil, err
}
return parseResponse(response.Responses, queries, timeField)
return parseResponse(response.Responses, queries, configuredFields)
}
func TestLabelOrderInFieldName(t *testing.T) {

1
pkg/tsdb/elasticsearch/snapshot_test.go
View File

@@ -132,6 +132,7 @@ func TestResponseSnapshots(t *testing.T) {
{name: "complex metric test", path: "metric_complex"},
{name: "multi metric test", path: "metric_multi"},
{name: "raw data test", path: "raw_data"},
{name: "logs test", path: "logs"},
}
snapshotCount := findResponseSnapshotCounts(t, "testdata_response")

534
pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc Normal file
View File

@@ -0,0 +1,534 @@
// 🌟 This was machine generated. Do not edit. 🌟
//
// Frame[0] {
// "typeVersion": [
// 0,
// 0
// ],
// "preferredVisualisationType": "logs"
// }
// Name:
// Dimensions: 17 Fields by 5 Rows
// +-----------------------------------+---------------------------+----------------------+-----------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
// | Name: testtime | Name: line | Name: _id | Name: _index | Name: _source | Name: _type | Name: abc | Name: counter | Name: float | Name: highlight | Name: is_true | Name: label | Name: level | Name: location | Name: nested_field.internal.nested | Name: shapes | Name: sort |
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []*time.Time | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage | Type: []*string | Type: []*float64 | Type: []*float64 | Type: []*json.RawMessage | Type: []*bool | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage |
// +-----------------------------------+---------------------------+----------------------+-----------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
// | 2023-02-09 14:40:01.475 +0000 UTC | log text [106619125] | g2aeNoYB7vaC3bq-ezfK | logs-2023.02.09 | {"abc":null,"counter":81,"float":10.911972180833306,"is_true":true,"label":"val3","line":"log text [106619125]","location":"-42.73465234425797, -14.097854057104112","lvl":"info","nested_field.internal.nested":"value1","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:40:01.475Z"} | null | null | 81 | 10.911972180833306 | null | true | val3 | info | -42.73465234425797, -14.097854057104112 | value1 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953601475,4] |
// | 2023-02-09 14:40:00.513 +0000 UTC | log text with [781660944] | gmaeNoYB7vaC3bq-eDcN | logs-2023.02.09 | {"abc":null,"counter":80,"float":62.94120607636795,"is_true":false,"label":"val3","line":"log text with [781660944]","location":"42.07571917624318, 15.95725088484611","lvl":"error","nested_field.internal.nested":"value2","shapes":[{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:40:00.513Z"} | null | null | 80 | 62.94120607636795 | null | false | val3 | error | 42.07571917624318, 15.95725088484611 | value2 | [{"type":"triangle"},{"type":"square"}] | [1675953600513,7] |
// | 2023-02-09 14:39:59.556 +0000 UTC | log text [894867430] | gWaeNoYB7vaC3bq-dDdL | logs-2023.02.09 | {"abc":"def","counter":79,"float":53.323706427230455,"is_true":true,"label":"val1","line":"log text [894867430]","location":"-38.27341566189766, -23.66739642570781","lvl":"info","nested_field.internal.nested":"value3","shapes":[{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:39:59.556Z"} | null | def | 79 | 53.323706427230455 | null | true | val1 | info | -38.27341566189766, -23.66739642570781 | value3 | [{"type":"triangle"},{"type":"square"}] | [1675953599556,10] |
// | 2023-02-09 14:39:58.608 +0000 UTC | log text [478598889] | gGaeNoYB7vaC3bq-cDeY | logs-2023.02.09 | {"abc":"def","counter":78,"float":82.72012623471589,"is_true":false,"label":"val1","line":"log text [478598889]","location":"12.373240290451287, 43.265493464362024","lvl":"info","nested_field.internal.nested":"value4","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:39:58.608Z"} | null | def | 78 | 82.72012623471589 | null | false | val1 | info | 12.373240290451287, 43.265493464362024 | value4 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953598608,15] |
// | 2023-02-09 14:39:57.665 +0000 UTC | log text [526995818] | f2aeNoYB7vaC3bq-bDf7 | logs-2023.02.09 | {"abc":"def","counter":77,"float":35.05784443331803,"is_true":false,"label":"val3","line":"log text [526995818]","location":"-31.524344042228194, -32.11254790120572","lvl":"info","nested_field.internal.nested":"value5","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:39:57.665Z"} | null | def | 77 | 35.05784443331803 | null | false | val3 | info | -31.524344042228194, -32.11254790120572 | value5 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953597665,20] |
// +-----------------------------------+---------------------------+----------------------+-----------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
{
"status": 200,
"frames": [
{
"schema": {
"meta": {
"typeVersion": [
0,
0
],
"preferredVisualisationType": "logs"
},
"fields": [
{
"name": "testtime",
"type": "time",
"typeInfo": {
"frame": "time.Time",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "line",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "_id",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "_index",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "_source",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "_type",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "abc",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "counter",
"type": "number",
"typeInfo": {
"frame": "float64",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "float",
"type": "number",
"typeInfo": {
"frame": "float64",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "highlight",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "is_true",
"type": "boolean",
"typeInfo": {
"frame": "bool",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "label",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "level",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "location",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "nested_field.internal.nested",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "shapes",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "sort",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
},
"config": {
"filterable": true
}
}
]
},
"data": {
"values": [
[
1675953601475,
1675953600513,
1675953599556,
1675953598608,
1675953597665
],
[
"log text [106619125]",
"log text with [781660944]",
"log text [894867430]",
"log text [478598889]",
"log text [526995818]"
],
[
"g2aeNoYB7vaC3bq-ezfK",
"gmaeNoYB7vaC3bq-eDcN",
"gWaeNoYB7vaC3bq-dDdL",
"gGaeNoYB7vaC3bq-cDeY",
"f2aeNoYB7vaC3bq-bDf7"
],
[
"logs-2023.02.09",
"logs-2023.02.09",
"logs-2023.02.09",
"logs-2023.02.09",
"logs-2023.02.09"
],
[
{
"abc": null,
"counter": 81,
"float": 10.911972180833306,
"is_true": true,
"label": "val3",
"line": "log text [106619125]",
"location": "-42.73465234425797, -14.097854057104112",
"lvl": "info",
"nested_field.internal.nested": "value1",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"testtime": "2023-02-09T14:40:01.475Z"
},
{
"abc": null,
"counter": 80,
"float": 62.94120607636795,
"is_true": false,
"label": "val3",
"line": "log text with [781660944]",
"location": "42.07571917624318, 15.95725088484611",
"lvl": "error",
"nested_field.internal.nested": "value2",
"shapes": [
{
"type": "triangle"
},
{
"type": "square"
}
],
"testtime": "2023-02-09T14:40:00.513Z"
},
{
"abc": "def",
"counter": 79,
"float": 53.323706427230455,
"is_true": true,
"label": "val1",
"line": "log text [894867430]",
"location": "-38.27341566189766, -23.66739642570781",
"lvl": "info",
"nested_field.internal.nested": "value3",
"shapes": [
{
"type": "triangle"
},
{
"type": "square"
}
],
"testtime": "2023-02-09T14:39:59.556Z"
},
{
"abc": "def",
"counter": 78,
"float": 82.72012623471589,
"is_true": false,
"label": "val1",
"line": "log text [478598889]",
"location": "12.373240290451287, 43.265493464362024",
"lvl": "info",
"nested_field.internal.nested": "value4",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"testtime": "2023-02-09T14:39:58.608Z"
},
{
"abc": "def",
"counter": 77,
"float": 35.05784443331803,
"is_true": false,
"label": "val3",
"line": "log text [526995818]",
"location": "-31.524344042228194, -32.11254790120572",
"lvl": "info",
"nested_field.internal.nested": "value5",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"testtime": "2023-02-09T14:39:57.665Z"
}
],
[
null,
null,
null,
null,
null
],
[
null,
null,
"def",
"def",
"def"
],
[
81,
80,
79,
78,
77
],
[
10.911972180833306,
62.94120607636795,
53.323706427230455,
82.72012623471589,
35.05784443331803
],
[
null,
null,
null,
null,
null
],
[
true,
false,
true,
false,
false
],
[
"val3",
"val3",
"val1",
"val1",
"val3"
],
[
"info",
"error",
"info",
"info",
"info"
],
[
"-42.73465234425797, -14.097854057104112",
"42.07571917624318, 15.95725088484611",
"-38.27341566189766, -23.66739642570781",
"12.373240290451287, 43.265493464362024",
"-31.524344042228194, -32.11254790120572"
],
[
"value1",
"value2",
"value3",
"value4",
"value5"
],
[
[
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
[
{
"type": "triangle"
},
{
"type": "square"
}
],
[
{
"type": "triangle"
},
{
"type": "square"
}
],
[
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
[
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
]
],
[
[
1675953601475,
4
],
[
1675953600513,
7
],
[
1675953599556,
10
],
[
1675953598608,
15
],
[
1675953597665,
20
]
]
]
}
}
]
}

23
pkg/tsdb/elasticsearch/testdata_response/logs.queries.json Normal file
View File

@@ -0,0 +1,23 @@
[
{
"alias": "",
"datasource": {
"type": "elasticsearch",
"uid": "haha"
},
"datasourceId": 42,
"expression": "",
"hide": true,
"intervalMs": 200,
"maxDataPoints": 1248,
"metrics": [
{
"id": "1",
"type": "logs"
}
],
"query": "",
"refId": "a",
"window": ""
}
]

208
pkg/tsdb/elasticsearch/testdata_response/logs.response.json Normal file
View File

@@ -0,0 +1,208 @@
{
"took": 6,
"responses": [
{
"took": 6,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 81,
"relation": "eq"
},
"max_score": null,
"hits": [
{
"_index": "logs-2023.02.09",
"_id": "g2aeNoYB7vaC3bq-ezfK",
"_score": null,
"_source": {
"abc": null,
"is_true": true,
"testtime": "2023-02-09T14:40:01.475Z",
"line": "log text [106619125]",
"counter": 81,
"float": 10.911972180833306,
"label": "val3",
"lvl": "info",
"location": "-42.73465234425797, -14.097854057104112",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"nested_field": {
"internal": {
"nested": "value1"
}
}
},
"sort": [
1675953601475,
4
]
},
{
"_index": "logs-2023.02.09",
"_id": "gmaeNoYB7vaC3bq-eDcN",
"_score": null,
"_source": {
"abc": null,
"is_true": false,
"testtime": "2023-02-09T14:40:00.513Z",
"line": "log text with [781660944]",
"counter": 80,
"float": 62.94120607636795,
"label": "val3",
"lvl": "error",
"location": "42.07571917624318, 15.95725088484611",
"shapes": [
{
"type": "triangle"
},
{
"type": "square"
}
],
"nested_field": {
"internal": {
"nested": "value2"
}
}
},
"sort": [
1675953600513,
7
]
},
{
"_index": "logs-2023.02.09",
"_id": "gWaeNoYB7vaC3bq-dDdL",
"_score": null,
"_source": {
"abc": "def",
"is_true": true,
"testtime": "2023-02-09T14:39:59.556Z",
"line": "log text [894867430]",
"counter": 79,
"float": 53.323706427230455,
"label": "val1",
"lvl": "info",
"location": "-38.27341566189766, -23.66739642570781",
"shapes": [
{
"type": "triangle"
},
{
"type": "square"
}
],
"nested_field": {
"internal": {
"nested": "value3"
}
}
},
"sort": [
1675953599556,
10
]
},
{
"_index": "logs-2023.02.09",
"_id": "gGaeNoYB7vaC3bq-cDeY",
"_score": null,
"_source": {
"abc": "def",
"is_true": false,
"testtime": "2023-02-09T14:39:58.608Z",
"line": "log text [478598889]",
"counter": 78,
"float": 82.72012623471589,
"label": "val1",
"lvl": "info",
"location": "12.373240290451287, 43.265493464362024",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"nested_field": {
"internal": {
"nested": "value4"
}
}
},
"sort": [
1675953598608,
15
]
},
{
"_index": "logs-2023.02.09",
"_id": "f2aeNoYB7vaC3bq-bDf7",
"_score": null,
"_source": {
"abc": "def",
"is_true": false,
"testtime": "2023-02-09T14:39:57.665Z",
"line": "log text [526995818]",
"counter": 77,
"float": 35.05784443331803,
"label": "val3",
"lvl": "info",
"location": "-31.524344042228194, -32.11254790120572",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"nested_field": {
"internal": {
"nested": "value5"
}
}
},
"sort": [
1675953597665,
20
]
}
]
},
"status": 200
}
]
}

4
pkg/tsdb/elasticsearch/time_series_query.go
View File

@@ -54,7 +54,7 @@ func (e *timeSeriesQuery) execute() (*backend.QueryDataResponse, error) {
return &backend.QueryDataResponse{}, err
}
return parseResponse(res.Responses, queries, e.client.GetTimeField())
return parseResponse(res.Responses, queries, e.client.GetConfiguredFields())
}
func (e *timeSeriesQuery) processQuery(q *Query, ms *es.MultiSearchRequestBuilder, from, to int64) error {
@@ -63,7 +63,7 @@ func (e *timeSeriesQuery) processQuery(q *Query, ms *es.MultiSearchRequestBuilde
return err
}
defaultTimeField := e.client.GetTimeField()
defaultTimeField := e.client.GetConfiguredFields().TimeField
b := ms.Search(q.Interval)
b.Size(0)
filters := b.Query().Bool().Filter()

22
pkg/tsdb/elasticsearch/time_series_query_test.go
View File

@@ -28,7 +28,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) {
require.NoError(t, err)
sr := c.multisearchRequests[0].Requests[0]
rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter)
require.Equal(t, rangeFilter.Key, c.timeField)
require.Equal(t, rangeFilter.Key, c.configuredFields.TimeField)
require.Equal(t, rangeFilter.Lte, toMs)
require.Equal(t, rangeFilter.Gte, fromMs)
require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS)
@@ -418,7 +418,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) {
sr := c.multisearchRequests[0].Requests[0]
rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter)
require.Equal(t, rangeFilter.Key, c.timeField)
require.Equal(t, rangeFilter.Key, c.configuredFields.TimeField)
require.Equal(t, rangeFilter.Lte, toMs)
require.Equal(t, rangeFilter.Gte, fromMs)
require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS)
@@ -439,7 +439,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) {
sr := c.multisearchRequests[0].Requests[0]
rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter)
require.Equal(t, rangeFilter.Key, c.timeField)
require.Equal(t, rangeFilter.Key, c.configuredFields.TimeField)
require.Equal(t, rangeFilter.Lte, toMs)
require.Equal(t, rangeFilter.Gte, fromMs)
require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS)
@@ -1305,7 +1305,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) {
require.Equal(t, sr.Size, defaultSize)
rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter)
require.Equal(t, rangeFilter.Key, c.timeField)
require.Equal(t, rangeFilter.Key, c.configuredFields.TimeField)
require.Equal(t, rangeFilter.Lte, toMs)
require.Equal(t, rangeFilter.Gte, fromMs)
require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS)
@@ -1687,7 +1687,7 @@ func TestSettingsCasting(t *testing.T) {
}
type fakeClient struct {
timeField string
configuredFields es.ConfiguredFields
multiSearchResponse *es.MultiSearchResponse
multiSearchError error
builder *es.MultiSearchRequestBuilder
@@ -1695,15 +1695,21 @@ type fakeClient struct {
}
func newFakeClient() *fakeClient {
configuredFields := es.ConfiguredFields{
TimeField: "@timestamp",
LogMessageField: "line",
LogLevelField: "lvl",
}
return &fakeClient{
timeField: "@timestamp",
configuredFields: configuredFields,
multisearchRequests: make([]*es.MultiSearchRequest, 0),
multiSearchResponse: &es.MultiSearchResponse{},
}
}
func (c *fakeClient) GetTimeField() string {
return c.timeField
func (c *fakeClient) GetConfiguredFields() es.ConfiguredFields {
return c.configuredFields
}
func (c *fakeClient) GetMinInterval(queryInterval string) (time.Duration, error) {