InfluxDB: Parse data for table view to have parity with frontend parser (#78365)

* Use TimeSeriesWide format for table response

* fix group by query result parsing

* handle labels

* provide a test where result has no tags

* parsing results without time column

* clean the code

* remove the comment line

* more cleaning

* lint
This commit is contained in:
ismail simsek 2023-11-22 19:27:42 +01:00 committed by GitHub
parent eef8525a4b
commit c5f3ce1371
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 793 additions and 6 deletions

View File

@ -49,9 +49,13 @@ func parse(buf io.Reader, statusCode int, query *models.Query) *backend.DataResp
result := response.Results[0]
if result.Error != "" {
return &backend.DataResponse{Error: fmt.Errorf(result.Error)}
} else {
return &backend.DataResponse{Frames: transformRows(result.Series, *query)}
}
if query.ResultFormat == "table" {
return &backend.DataResponse{Frames: transformRowsForTable(result.Series, *query)}
}
return &backend.DataResponse{Frames: transformRowsForTimeSeries(result.Series, *query)}
}
func parseJSON(buf io.Reader) (models.Response, error) {
@ -65,7 +69,125 @@ func parseJSON(buf io.Reader) (models.Response, error) {
return response, err
}
func transformRows(rows []models.Row, query models.Query) data.Frames {
func transformRowsForTable(rows []models.Row, query models.Query) data.Frames {
if len(rows) == 0 {
return make([]*data.Frame, 0)
}
frames := make([]*data.Frame, 0, 1)
newFrame := data.NewFrame(rows[0].Name)
newFrame.Meta = &data.FrameMeta{
ExecutedQueryString: query.RawQuery,
PreferredVisualization: getVisType(query.ResultFormat),
}
conLen := len(rows[0].Columns)
if rows[0].Columns[0] == "time" {
newFrame.Fields = append(newFrame.Fields, newTimeField(rows))
} else {
newFrame.Fields = append(newFrame.Fields, newValueFields(rows, nil, 0, 1)...)
}
newFrame.Fields = append(newFrame.Fields, newTagField(rows, nil)...)
newFrame.Fields = append(newFrame.Fields, newValueFields(rows, nil, 1, conLen)...)
frames = append(frames, newFrame)
return frames
}
func newTimeField(rows []models.Row) *data.Field {
var timeArray []time.Time
for _, row := range rows {
for _, valuePair := range row.Values {
timestamp, timestampErr := parseTimestamp(valuePair[0])
// we only add this row if the timestamp is valid
if timestampErr != nil {
continue
}
timeArray = append(timeArray, timestamp)
}
}
timeField := data.NewField("Time", nil, timeArray)
return timeField
}
func newTagField(rows []models.Row, labels data.Labels) []*data.Field {
fields := make([]*data.Field, 0, len(rows[0].Tags))
for key := range rows[0].Tags {
tagField := data.NewField(key, labels, []*string{})
for _, row := range rows {
for range row.Values {
value := row.Tags[key]
tagField.Append(&value)
}
}
tagField.SetConfig(&data.FieldConfig{DisplayNameFromDS: key})
fields = append(fields, tagField)
}
return fields
}
func newValueFields(rows []models.Row, labels data.Labels, colIdxStart, colIdxEnd int) []*data.Field {
fields := make([]*data.Field, 0)
for colIdx := colIdxStart; colIdx < colIdxEnd; colIdx++ {
var valueField *data.Field
var floatArray []*float64
var stringArray []*string
var boolArray []*bool
for _, row := range rows {
valType := typeof(row.Values, colIdx)
for _, valuePair := range row.Values {
switch valType {
case "string":
value, ok := valuePair[colIdx].(string)
if ok {
stringArray = append(stringArray, &value)
} else {
stringArray = append(stringArray, nil)
}
case "json.Number":
value := parseNumber(valuePair[colIdx])
floatArray = append(floatArray, value)
case "bool":
value, ok := valuePair[colIdx].(bool)
if ok {
boolArray = append(boolArray, &value)
} else {
boolArray = append(boolArray, nil)
}
case "null":
floatArray = append(floatArray, nil)
}
}
switch valType {
case "string":
valueField = data.NewField(row.Columns[colIdx], labels, stringArray)
case "json.Number":
valueField = data.NewField(row.Columns[colIdx], labels, floatArray)
case "bool":
valueField = data.NewField(row.Columns[colIdx], labels, boolArray)
case "null":
valueField = data.NewField(row.Columns[colIdx], labels, floatArray)
}
valueField.SetConfig(&data.FieldConfig{DisplayNameFromDS: row.Columns[colIdx]})
}
fields = append(fields, valueField)
}
return fields
}
func transformRowsForTimeSeries(rows []models.Row, query models.Query) data.Frames {
// pre-allocate frames - this can save many allocations
cols := 0
for _, row := range rows {

View File

@ -810,6 +810,97 @@ func TestResponseParser_Parse_RetentionPolicy(t *testing.T) {
})
}
func TestResponseParser_table_format(t *testing.T) {
t.Run("test table result format parsing", func(t *testing.T) {
resp := ResponseParse(prepare(tableResultFormatInfluxResponse1), 200, &models.Query{RefID: "A", RawQuery: `a nice query`, ResultFormat: "table"})
assert.Equal(t, 1, len(resp.Frames))
assert.Equal(t, "a nice query", resp.Frames[0].Meta.ExecutedQueryString)
assert.Equal(t, 3, len(resp.Frames[0].Fields))
for i := range resp.Frames[0].Fields {
assert.Equal(t, resp.Frames[0].Fields[0].Len(), resp.Frames[0].Fields[i].Len())
}
assert.Equal(t, "Time", resp.Frames[0].Fields[0].Name)
assert.Equal(t, "usage_idle", resp.Frames[0].Fields[1].Name)
assert.Equal(t, toPtr(99.09456740445926), resp.Frames[0].Fields[1].At(2))
assert.Equal(t, "usage_iowait", resp.Frames[0].Fields[2].Name)
})
t.Run("test table result format parsing with grouping", func(t *testing.T) {
resp := ResponseParse(prepare(tableResultFormatInfluxResponse2), 200, &models.Query{RefID: "A", RawQuery: `a nice query`, ResultFormat: "table"})
assert.Equal(t, 1, len(resp.Frames))
assert.Equal(t, "a nice query", resp.Frames[0].Meta.ExecutedQueryString)
assert.Equal(t, 7, len(resp.Frames[0].Fields))
for i := range resp.Frames[0].Fields {
assert.Equal(t, resp.Frames[0].Fields[0].Len(), resp.Frames[0].Fields[i].Len())
}
assert.Equal(t, "Time", resp.Frames[0].Fields[0].Name)
assert.Equal(t, "cpu", resp.Frames[0].Fields[1].Name)
assert.Equal(t, toPtr("cpu-total"), resp.Frames[0].Fields[1].At(0))
assert.Equal(t, toPtr("cpu0"), resp.Frames[0].Fields[1].At(1))
assert.Equal(t, toPtr("cpu9"), resp.Frames[0].Fields[1].At(10))
assert.Equal(t, "mean", resp.Frames[0].Fields[2].Name)
assert.Equal(t, "min", resp.Frames[0].Fields[3].Name)
assert.Equal(t, "p90", resp.Frames[0].Fields[4].Name)
assert.Equal(t, "p95", resp.Frames[0].Fields[5].Name)
assert.Equal(t, "max", resp.Frames[0].Fields[6].Name)
})
t.Run("parse result as table group by tag", func(t *testing.T) {
resp := ResponseParse(prepare(tableResultFormatInfluxResponse3), 200, &models.Query{RefID: "A", RawQuery: `a nice query`, ResultFormat: "table"})
assert.Equal(t, 1, len(resp.Frames))
assert.Equal(t, "a nice query", resp.Frames[0].Meta.ExecutedQueryString)
for i := range resp.Frames[0].Fields {
assert.Equal(t, resp.Frames[0].Fields[0].Len(), resp.Frames[0].Fields[i].Len())
}
assert.Equal(t, "Time", resp.Frames[0].Fields[0].Name)
assert.Equal(t, "cpu", resp.Frames[0].Fields[1].Name)
assert.Equal(t, resp.Frames[0].Fields[1].Name, resp.Frames[0].Fields[1].Config.DisplayNameFromDS)
assert.Equal(t, toPtr("cpu-total"), resp.Frames[0].Fields[1].At(0))
assert.Equal(t, toPtr("cpu0"), resp.Frames[0].Fields[1].At(5))
assert.Equal(t, toPtr("cpu2"), resp.Frames[0].Fields[1].At(12))
assert.Equal(t, "mean", resp.Frames[0].Fields[2].Name)
assert.Equal(t, resp.Frames[0].Fields[2].Name, resp.Frames[0].Fields[2].Config.DisplayNameFromDS)
})
t.Run("parse result without tags as table", func(t *testing.T) {
resp := ResponseParse(prepare(tableResultFormatInfluxResponse4), 200, &models.Query{RefID: "A", RawQuery: `a nice query`, ResultFormat: "table"})
assert.Equal(t, 1, len(resp.Frames))
assert.Equal(t, "a nice query", resp.Frames[0].Meta.ExecutedQueryString)
for i := range resp.Frames[0].Fields {
assert.Equal(t, resp.Frames[0].Fields[0].Len(), resp.Frames[0].Fields[i].Len())
}
assert.Equal(t, "Time", resp.Frames[0].Fields[0].Name)
assert.Equal(t, "mean", resp.Frames[0].Fields[1].Name)
assert.Equal(t, resp.Frames[0].Fields[1].Name, resp.Frames[0].Fields[1].Config.DisplayNameFromDS)
})
t.Run("parse show measurements response as table", func(t *testing.T) {
resp := ResponseParse(prepare(showMeasurementsResponse), 200, &models.Query{RefID: "A", RawQuery: `a nice query`, ResultFormat: "table"})
assert.Equal(t, 1, len(resp.Frames))
assert.Equal(t, "a nice query", resp.Frames[0].Meta.ExecutedQueryString)
for i := range resp.Frames[0].Fields {
assert.Equal(t, resp.Frames[0].Fields[0].Len(), resp.Frames[0].Fields[i].Len())
}
assert.Equal(t, 1, len(resp.Frames[0].Fields))
assert.Equal(t, "name", resp.Frames[0].Fields[0].Name)
})
t.Run("parse retention policy response as table", func(t *testing.T) {
resp := ResponseParse(prepare(showRetentionPolicyResponse), 200, &models.Query{RefID: "A", RawQuery: `a nice query`, ResultFormat: "table"})
assert.Equal(t, 1, len(resp.Frames))
assert.Equal(t, "a nice query", resp.Frames[0].Meta.ExecutedQueryString)
for i := range resp.Frames[0].Fields {
assert.Equal(t, resp.Frames[0].Fields[0].Len(), resp.Frames[0].Fields[i].Len())
}
assert.Equal(t, 5, len(resp.Frames[0].Fields))
assert.Equal(t, "name", resp.Frames[0].Fields[0].Name)
assert.Equal(t, "duration", resp.Frames[0].Fields[1].Name)
assert.Equal(t, "shardGroupDuration", resp.Frames[0].Fields[2].Name)
assert.Equal(t, "replicaN", resp.Frames[0].Fields[3].Name)
assert.Equal(t, "default", resp.Frames[0].Fields[4].Name)
})
}
func TestResponseParser_Parse(t *testing.T) {
tests := []struct {
name string
@ -912,10 +1003,9 @@ func TestResponseParser_Parse(t *testing.T) {
]
}`,
f: func(t *testing.T, got backend.DataResponse) {
assert.Equal(t, "domain", got.Frames[0].Name)
assert.Equal(t, "Annotation", got.Frames[0].Name)
assert.Equal(t, "domain", got.Frames[0].Fields[1].Config.DisplayNameFromDS)
assert.Equal(t, "ASD", got.Frames[2].Name)
assert.Equal(t, "ASD", got.Frames[2].Fields[1].Config.DisplayNameFromDS)
assert.Equal(t, "type", got.Frames[0].Fields[2].Config.DisplayNameFromDS)
assert.Equal(t, tableVisType, got.Frames[0].Meta.PreferredVisualization)
},
},
@ -930,3 +1020,578 @@ func TestResponseParser_Parse(t *testing.T) {
})
}
}
func toPtr[T any](v T) *T {
return &v
}
const tableResultFormatInfluxResponse1 = `{
"results": [
{
"statement_id": 0,
"series": [
{
"name": "cpu",
"columns": [
"time",
"usage_idle",
"usage_iowait"
],
"values": [
[
1700090120000,
99.0255173802101,
0.020092425155804713
],
[
1700090120000,
99.29718875523953,
0
],
[
1700090120000,
99.09456740445926,
0
],
[
1700090120000,
99.39455095864957,
0
],
[
1700090120000,
99.09729187566201,
0
]
]
}
]
}
]
}`
const tableResultFormatInfluxResponse2 = `{
"results": [
{
"statement_id": 0,
"series": [
{
"name": "cpu",
"tags": {
"cpu": "cpu-total"
},
"columns": [
"time",
"mean",
"min",
"p90",
"p95",
"max"
],
"values": [
[
1700046000000,
99.06348570053983,
97.3214285712978,
99.2066680055868,
99.24812030075188,
99.31809065366402
]
]
},
{
"name": "cpu",
"tags": {
"cpu": "cpu0"
},
"columns": [
"time",
"mean",
"min",
"p90",
"p95",
"max"
],
"values": [
[
1700046000000,
98.99671817733766,
96.65991902847126,
99.29364278499536,
99.29718875523953,
99.59839357421622
]
]
},
{
"name": "cpu",
"tags": {
"cpu": "cpu1"
},
"columns": [
"time",
"mean",
"min",
"p90",
"p95",
"max"
],
"values": [
[
1700046000000,
99.03148357927465,
96.67673715996412,
99.39698492464545,
99.39759036146867,
99.59798994966731
]
]
},
{
"name": "cpu",
"tags": {
"cpu": "cpu2"
},
"columns": [
"time",
"mean",
"min",
"p90",
"p95",
"max"
],
"values": [
[
1700046000000,
99.03087433486812,
96.03658536600605,
99.29859719431953,
99.39759036146867,
99.59879638908582
]
]
},
{
"name": "cpu",
"tags": {
"cpu": "cpu3"
},
"columns": [
"time",
"mean",
"min",
"p90",
"p95",
"max"
],
"values": [
[
1700046000000,
99.0796957137731,
97.37903225797402,
99.39698492464723,
99.39879759521435,
99.4984954865762
]
]
},
{
"name": "cpu",
"tags": {
"cpu": "cpu4"
},
"columns": [
"time",
"mean",
"min",
"p90",
"p95",
"max"
],
"values": [
[
1700046000000,
99.09573460946685,
97.57330637016123,
99.39759036146867,
99.49698189117252,
99.59839357450608
]
]
},
{
"name": "cpu",
"tags": {
"cpu": "cpu5"
},
"columns": [
"time",
"mean",
"min",
"p90",
"p95",
"max"
],
"values": [
[
1700046000000,
99.0690883079725,
96.65991902847126,
99.39698492464545,
99.39819458377468,
99.59798994995865
]
]
},
{
"name": "cpu",
"tags": {
"cpu": "cpu6"
},
"columns": [
"time",
"mean",
"min",
"p90",
"p95",
"max"
],
"values": [
[
1700046000000,
99.06475215715605,
97.37108190081956,
99.39698492464545,
99.39879759521259,
99.69879518073434
]
]
},
{
"name": "cpu",
"tags": {
"cpu": "cpu7"
},
"columns": [
"time",
"mean",
"min",
"p90",
"p95",
"max"
],
"values": [
[
1700046000000,
99.06204005079694,
97.7596741344093,
99.39637826964127,
99.39759036147042,
99.59879638908698
]
]
},
{
"name": "cpu",
"tags": {
"cpu": "cpu8"
},
"columns": [
"time",
"mean",
"min",
"p90",
"p95",
"max"
],
"values": [
[
1700046000000,
99.0999818796052,
96.56565656568982,
99.39698492464723,
99.39819458377468,
99.59758551299777
]
]
},
{
"name": "cpu",
"tags": {
"cpu": "cpu9"
},
"columns": [
"time",
"mean",
"min",
"p90",
"p95",
"max"
],
"values": [
[
1700046000000,
99.10477313534511,
96.8463886063268,
99.39759036146867,
99.39819458377468,
99.59839357421622
]
]
}
]
}
]
}
`
const tableResultFormatInfluxResponse3 = `{
"results": [
{
"statement_id": 0,
"series": [
{
"name": "cpu",
"tags": {
"cpu": "cpu-total"
},
"columns": [
"time",
"mean"
],
"values": [
[
1700046000000,
99.06919189833442
],
[
1700047200000,
99.13105510262923
],
[
1700048400000,
98.99236330721192
],
[
1700049600000,
98.80510091380069
]
]
},
{
"name": "cpu",
"tags": {
"cpu": "cpu0"
},
"columns": [
"time",
"mean"
],
"values": [
[
1700046000000,
99.01372119142576
],
[
1700047200000,
99.00430308480553
],
[
1700048400000,
98.9737996641964
],
[
1700049600000,
98.79638916754935
]
]
},
{
"name": "cpu",
"tags": {
"cpu": "cpu1"
},
"columns": [
"time",
"mean"
],
"values": [
[
1700046000000,
99.04949983158023
],
[
1700047200000,
99.06989461231551
],
[
1700048400000,
98.97954813782476
],
[
1700049600000,
98.49246231161365
]
]
},
{
"name": "cpu",
"tags": {
"cpu": "cpu2"
},
"columns": [
"time",
"mean"
],
"values": [
[
1700046000000,
99.11296419686643
],
[
1700047200000,
99.01817278917116
],
[
1700048400000,
98.96847021232013
],
[
1700049600000,
98.192771084406
]
]
},
{
"name": "cpu",
"tags": {
"cpu": "cpu3"
},
"columns": [
"time",
"mean"
],
"values": [
[
1700046000000,
99.0742704326151
],
[
1700047200000,
99.17835628293322
],
[
1700048400000,
98.98968994907334
],
[
1700049600000,
98.69215291745849
]
]
}
]
}
]
}
`
const tableResultFormatInfluxResponse4 = `{
"results": [
{
"statement_id": 0,
"series": [
{
"name": "cpu",
"columns": [
"time",
"mean"
],
"values": [
[
1700046000000,
99.0693929754458
],
[
1700047200000,
99.13073313839024
],
[
1700048400000,
98.99278645182834
],
[
1700049600000,
98.77818123433566
]
]
}
]
}
]
}`
const showMeasurementsResponse = `{
"results": [
{
"statement_id": 0,
"series": [
{
"name": "measurements",
"columns": [
"name"
],
"values": [
[
"cpu"
],
[
"disk"
],
[
"diskio"
],
[
"kernel"
]
]
}
]
}
]
}`
const showRetentionPolicyResponse = `{
"results": [
{
"statement_id": 0,
"series": [
{
"columns": [
"name",
"duration",
"shardGroupDuration",
"replicaN",
"default"
],
"values": [
[
"default",
"0s",
"168h0m0s",
1,
true
],
[
"autogen",
"0s",
"168h0m0s",
1,
false
]
]
}
]
}
]
}`