IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

InfluxDB: Fix table parsing with backend mode (#76899)

Browse Source 
* Add resultFormat to query model

* don't add row name if the result format is table

* No need special formatting since we use unified dataframes

* betterer

* specify visualization type in response

* Unit tests

* fix unit tests
This commit is contained in:
ismail simsek 2023-10-23 19:21:30 +02:00 committed by GitHub
parent 59ef1558e8
commit 63abe25b74
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 130 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.betterer.results
View File

@ -6247,8 +6247,8 @@ exports[`better eslint`] = {
[0, 0, 0, "Styles should be written using objects.", "0"]
],
"public/app/plugins/datasource/influxdb/datasource.ts:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"],
[0, 0, 0, "Do not use any type assertions.", "1"],
[0, 0, 0, "Do not use any type assertions.", "0"],
[0, 0, 0, "Unexpected any. Specify a different type.", "1"],
[0, 0, 0, "Unexpected any. Specify a different type.", "2"],
[0, 0, 0, "Unexpected any. Specify a different type.", "3"],
[0, 0, 0, "Unexpected any. Specify a different type.", "4"],
@ -6264,8 +6264,7 @@ exports[`better eslint`] = {
[0, 0, 0, "Unexpected any. Specify a different type.", "14"],
[0, 0, 0, "Unexpected any. Specify a different type.", "15"],
[0, 0, 0, "Unexpected any. Specify a different type.", "16"],
[0, 0, 0, "Unexpected any. Specify a different type.", "17"],
[0, 0, 0, "Do not use any type assertions.", "18"]
[0, 0, 0, "Do not use any type assertions.", "17"]
],
"public/app/plugins/datasource/influxdb/influx_query_model.ts:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"],

34
pkg/tsdb/influxdb/influxql/response_parser.go
View File

@ -19,6 +19,12 @@ var (
legendFormat = regexp.MustCompile(`\[\[([\@\/\w-]+)(\.[\@\/\w-]+)*\]\]*|\$([\@\w-]+?)*`)
)
const (
graphVisType data.VisType = "graph"
tableVisType data.VisType = "table"
logsVisType data.VisType = "logs"
)
func ResponseParse(buf io.ReadCloser, statusCode int, query *models.Query) *backend.DataResponse {
return parse(buf, statusCode, query)
}
@ -155,7 +161,7 @@ func newFrameWithTimeField(row models.Row, column string, colIndex int, query mo
name := string(formatFrameName(row, column, query, frameName[:]))
valueField.SetConfig(&data.FieldConfig{DisplayNameFromDS: name})
return newDataFrame(name, query.RawQuery, timeField, valueField)
return newDataFrame(name, query.RawQuery, timeField, valueField, getVisType(query.ResultFormat))
}
func newFrameWithoutTimeField(row models.Row, retentionPolicyQuery bool, tagValuesQuery bool) *data.Frame {
@ -198,10 +204,11 @@ func newFrameWithoutTimeField(row models.Row, retentionPolicyQuery bool, tagValu
return data.NewFrame(row.Name, field)
}
func newDataFrame(name string, queryString string, timeField *data.Field, valueField *data.Field) *data.Frame {
func newDataFrame(name string, queryString string, timeField *data.Field, valueField *data.Field, visType data.VisType) *data.Frame {
frame := data.NewFrame(name, timeField, valueField)
frame.Meta = &data.FrameMeta{
ExecutedQueryString: queryString,
ExecutedQueryString: queryString,
PreferredVisualization: visType,
}
return frame
@ -209,7 +216,7 @@ func newDataFrame(name string, queryString string, timeField *data.Field, valueF
func formatFrameName(row models.Row, column string, query models.Query, frameName []byte) []byte {
if query.Alias == "" {
return buildFrameNameFromQuery(row, column, frameName)
return buildFrameNameFromQuery(row, column, frameName, query.ResultFormat)
}
nameSegment := strings.Split(row.Name, ".")
@ -247,9 +254,11 @@ func formatFrameName(row models.Row, column string, query models.Query, frameNam
return result
}
func buildFrameNameFromQuery(row models.Row, column string, frameName []byte) []byte {
frameName = append(frameName, row.Name...)
frameName = append(frameName, '.')
func buildFrameNameFromQuery(row models.Row, column string, frameName []byte, resultFormat string) []byte {
if resultFormat != "table" {
frameName = append(frameName, row.Name...)
frameName = append(frameName, '.')
}
frameName = append(frameName, column...)
if len(row.Tags) > 0 {
@ -323,6 +332,17 @@ func parseNumber(value any) *float64 {
return &fvalue
}
func getVisType(resFormat string) data.VisType {
switch resFormat {
case "table":
return tableVisType
case "logs":
return logsVisType
default:
return graphVisType
}
}
func isTagValuesQuery(query models.Query) bool {
return strings.Contains(strings.ToLower(query.RawQuery), strings.ToLower("SHOW TAG VALUES"))
}

83
pkg/tsdb/influxdb/influxql/response_parser_test.go
View File

@ -28,6 +28,10 @@ func generateQuery(query models.Query) *models.Query {
if query.RawQuery == "" {
query.RawQuery = "Test raw query"
}
if query.ResultFormat == "" {
query.ResultFormat = "time_series"
}
return &query
}
@ -82,7 +86,7 @@ func TestInfluxdbResponseParser(t *testing.T) {
}),
floatField,
)
floatFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"}
floatFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"}
string_test := "/usr/path"
stringField := data.NewField("Value", labels, []*string{
@ -98,7 +102,7 @@ func TestInfluxdbResponseParser(t *testing.T) {
}),
stringField,
)
stringFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"}
stringFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"}
bool_true := true
bool_false := false
@ -115,7 +119,7 @@ func TestInfluxdbResponseParser(t *testing.T) {
}),
boolField,
)
boolFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"}
boolFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"}
result := ResponseParse(prepare(response), 200, generateQuery(query))
@ -265,7 +269,7 @@ func TestInfluxdbResponseParser(t *testing.T) {
}),
newField,
)
testFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"}
testFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"}
result := ResponseParse(prepare(response), 200, generateQuery(query))
@ -310,7 +314,7 @@ func TestInfluxdbResponseParser(t *testing.T) {
}),
newField,
)
testFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"}
testFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"}
result := ResponseParse(prepare(response), 200, generateQuery(query))
@ -401,7 +405,7 @@ func TestInfluxdbResponseParser(t *testing.T) {
}),
newField,
)
testFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"}
testFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"}
result := ResponseParse(prepare(response), 200, generateQuery(query))
t.Run("should parse aliases", func(t *testing.T) {
if diff := cmp.Diff(testFrame, result.Frames[0], data.FrameTestCompareOptions()...); diff != "" {
@ -628,7 +632,7 @@ func TestInfluxdbResponseParser(t *testing.T) {
}),
newField,
)
testFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"}
testFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"}
result := ResponseParse(prepare(response), 200, generateQuery(query))
require.EqualError(t, result.Error, "query-timeout limit exceeded")
@ -749,12 +753,14 @@ func TestResponseParser_Parse_RetentionPolicy(t *testing.T) {
func TestResponseParser_Parse(t *testing.T) {
tests := []struct {
name string
input string
f func(t *testing.T, got backend.DataResponse)
name string
resFormat string
input string
f func(t *testing.T, got backend.DataResponse)
}{
{
name: "Influxdb response parser with valid value when null values returned",
name: "Influxdb response parser with valid value when null values returned",
resFormat: "time_series",
input: `{ "results": [ { "series": [ {
"name": "cpu",
"columns": ["time","mean"],
@ -776,12 +782,13 @@ func TestResponseParser_Parse(t *testing.T) {
}),
newField,
)
testFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"}
testFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"}
assert.Equal(t, testFrame, got.Frames[0])
},
},
{
name: "Influxdb response parser with valid value when all values are null",
name: "Influxdb response parser with valid value when all values are null",
resFormat: "time_series",
input: `{ "results": [ { "series": [ {
"name": "cpu",
"columns": ["time","mean"],
@ -803,14 +810,60 @@ func TestResponseParser_Parse(t *testing.T) {
}),
newField,
)
testFrame.Meta = &data.FrameMeta{ExecutedQueryString: "Test raw query"}
testFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"}
assert.Equal(t, testFrame, got.Frames[0])
},
},
{
name: "Influxdb response parser with table result",
resFormat: "table",
input: `{
"results": [
{
"statement_id": 0,
"series": [
{
"name": "Annotation",
"columns": [
"time",
"domain",
"type",
"ASD",
"details"
],
"values": [
[
1697789142916,
"AASD157",
"fghg",
null,
"Something happened AtTime=2023-10-20T08:05:42.902036"
],
[
1697789142918,
"HUY23",
"val23",
null,
"Something else happened AtTime=2023-10-20T08:05:42.902036"
]
]
}
]
}
]
}`,
f: func(t *testing.T, got backend.DataResponse) {
assert.Equal(t, "domain", got.Frames[0].Name)
assert.Equal(t, "domain", got.Frames[0].Fields[1].Config.DisplayNameFromDS)
assert.Equal(t, "ASD", got.Frames[2].Name)
assert.Equal(t, "ASD", got.Frames[2].Fields[1].Config.DisplayNameFromDS)
assert.Equal(t, tableVisType, got.Frames[0].Meta.PreferredVisualization)
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := ResponseParse(prepare(tt.input), 200, generateQuery(models.Query{}))
got := ResponseParse(prepare(tt.input), 200, generateQuery(models.Query{ResultFormat: tt.resFormat}))
require.NotNil(t, got)
if tt.f != nil {
tt.f(t, *got)

29
pkg/tsdb/influxdb/models/model_parser.go
View File

@ -26,8 +26,8 @@ func QueryParse(query backend.DataQuery) (*Query, error) {
limit := model.Get("limit").MustString("")
slimit := model.Get("slimit").MustString("")
orderByTime := model.Get("orderByTime").MustString("")
measurement := model.Get("measurement").MustString("")
resultFormat := model.Get("resultFormat").MustString("")
tags, err := parseTags(model)
if err != nil {
@ -54,19 +54,20 @@ func QueryParse(query backend.DataQuery) (*Query, error) {
}
return &Query{
Measurement: measurement,
Policy: policy,
GroupBy: groupBys,
Tags: tags,
Selects: selects,
RawQuery: rawQuery,
Interval: interval,
Alias: alias,
UseRawQuery: useRawQuery,
Tz: tz,
Limit: limit,
Slimit: slimit,
OrderByTime: orderByTime,
Measurement: measurement,
Policy: policy,
GroupBy: groupBys,
Tags: tags,
Selects: selects,
RawQuery: rawQuery,
Interval: interval,
Alias: alias,
UseRawQuery: useRawQuery,
Tz: tz,
Limit: limit,
Slimit: slimit,
OrderByTime: orderByTime,
ResultFormat: resultFormat,
}, nil
}

29
pkg/tsdb/influxdb/models/models.go
View File

@ -3,20 +3,21 @@ package models
import "time"
type Query struct {
Measurement string
Policy string
Tags []*Tag
GroupBy []*QueryPart
Selects []*Select
RawQuery string
UseRawQuery bool
Alias string
Interval time.Duration
Tz string
Limit string
Slimit string
OrderByTime string
RefID string
Measurement string
Policy string
Tags []*Tag
GroupBy []*QueryPart
Selects []*Select
RawQuery string
UseRawQuery bool
Alias string
Interval time.Duration
Tz string
Limit string
Slimit string
OrderByTime string
RefID string
ResultFormat string
}
type Tag struct {

45
public/app/plugins/datasource/influxdb/datasource.ts
View File

@ -1,4 +1,4 @@
import { cloneDeep, extend, groupBy, has, isString, map as _map, omit, pick, reduce } from 'lodash';
import { cloneDeep, extend, has, isString, map as _map, omit, pick, reduce } from 'lodash';
import { lastValueFrom, merge, Observable, of, throwError } from 'rxjs';
import { catchError, map } from 'rxjs/operators';
@ -138,49 +138,8 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
return merge(...streams);
}
if (this.version === InfluxVersion.Flux || this.version === InfluxVersion.SQL) {
return super.query(filteredRequest);
}
if (this.isMigrationToggleOnAndIsAccessProxy()) {
return super.query(filteredRequest).pipe(
map((res) => {
if (res.error) {
throw {
message: 'InfluxDB Error: ' + res.error.message,
res,
};
}
const groupedFrames = groupBy(res.data, (x) => x.refId);
if (Object.keys(groupedFrames).length === 0) {
return { data: [] };
}
const seriesList: any[] = [];
filteredRequest.targets.forEach((target) => {
const filteredFrames = groupedFrames[target.refId] ?? [];
switch (target.resultFormat) {
case 'logs':
case 'table':
seriesList.push(
this.responseParser.getTable(filteredFrames, target, {
preferredVisualisationType: target.resultFormat,
})
);
break;
default: {
for (let i = 0; i < filteredFrames.length; i++) {
seriesList.push(filteredFrames[i]);
}
break;
}
}
});
return { data: seriesList };
})
);
return super.query(filteredRequest);
}
// Fallback to classic query support