From aea5a9f01a23172bccb1ede3947a9c0a75785436 Mon Sep 17 00:00:00 2001 From: ismail simsek Date: Wed, 29 Nov 2023 17:48:53 +0100 Subject: [PATCH] Chore: Introduce util package for InfluxDB backend testing (#78826) have a util package --- pkg/tsdb/influxdb/influxql/response_parser.go | 161 ++---------------- .../influxdb/influxql/response_parser_test.go | 15 +- pkg/tsdb/influxdb/influxql/util/util.go | 152 +++++++++++++++++ 3 files changed, 170 insertions(+), 158 deletions(-) create mode 100644 pkg/tsdb/influxdb/influxql/util/util.go diff --git a/pkg/tsdb/influxdb/influxql/response_parser.go b/pkg/tsdb/influxdb/influxql/response_parser.go index d3267fde337..79dcfd81c9a 100644 --- a/pkg/tsdb/influxdb/influxql/response_parser.go +++ b/pkg/tsdb/influxdb/influxql/response_parser.go @@ -4,27 +4,16 @@ import ( "encoding/json" "fmt" "io" - "regexp" - "strconv" "strings" "time" "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/data" + "github.com/grafana/grafana/pkg/tsdb/influxdb/influxql/util" "github.com/grafana/grafana/pkg/tsdb/influxdb/models" ) -var ( - legendFormat = regexp.MustCompile(`\[\[([\@\/\w-]+)(\.[\@\/\w-]+)*\]\]*|\$([\@\w-]+?)*`) -) - -const ( - graphVisType data.VisType = "graph" - tableVisType data.VisType = "table" - logsVisType data.VisType = "logs" -) - func ResponseParse(buf io.ReadCloser, statusCode int, query *models.Query) *backend.DataResponse { return parse(buf, statusCode, query) } @@ -79,7 +68,7 @@ func transformRowsForTable(rows []models.Row, query models.Query) data.Frames { newFrame := data.NewFrame(rows[0].Name) newFrame.Meta = &data.FrameMeta{ ExecutedQueryString: query.RawQuery, - PreferredVisualization: getVisType(query.ResultFormat), + PreferredVisualization: util.GetVisType(query.ResultFormat), } conLen := len(rows[0].Columns) @@ -100,7 +89,7 @@ func newTimeField(rows []models.Row) *data.Field { var timeArray []time.Time for _, row := range rows { for _, valuePair := range row.Values { - timestamp, timestampErr := parseTimestamp(valuePair[0]) + timestamp, timestampErr := util.ParseTimestamp(valuePair[0]) // we only add this row if the timestamp is valid if timestampErr != nil { continue @@ -142,7 +131,7 @@ func newValueFields(rows []models.Row, labels data.Labels, colIdxStart, colIdxEn var boolArray []*bool for _, row := range rows { - valType := typeof(row.Values, colIdx) + valType := util.Typeof(row.Values, colIdx) for _, valuePair := range row.Values { switch valType { @@ -154,7 +143,7 @@ func newValueFields(rows []models.Row, labels data.Labels, colIdxStart, colIdxEn stringArray = append(stringArray, nil) } case "json.Number": - value := parseNumber(valuePair[colIdx]) + value := util.ParseNumber(valuePair[colIdx]) floatArray = append(floatArray, value) case "bool": value, ok := valuePair[colIdx].(bool) @@ -226,7 +215,7 @@ func transformRowsForTimeSeries(rows []models.Row, query models.Query) data.Fram if len(frames) == 0 { newFrame.Meta = &data.FrameMeta{ ExecutedQueryString: query.RawQuery, - PreferredVisualization: getVisType(query.ResultFormat), + PreferredVisualization: util.GetVisType(query.ResultFormat), } } frames = append(frames, newFrame) @@ -242,10 +231,10 @@ func newFrameWithTimeField(row models.Row, column string, colIndex int, query mo var floatArray []*float64 var stringArray []*string var boolArray []*bool - valType := typeof(row.Values, colIndex) + valType := util.Typeof(row.Values, colIndex) for _, valuePair := range row.Values { - timestamp, timestampErr := parseTimestamp(valuePair[0]) + timestamp, timestampErr := util.ParseTimestamp(valuePair[0]) // we only add this row if the timestamp is valid if timestampErr != nil { continue @@ -261,7 +250,7 @@ func newFrameWithTimeField(row models.Row, column string, colIndex int, query mo stringArray = append(stringArray, nil) } case "json.Number": - value := parseNumber(valuePair[colIndex]) + value := util.ParseNumber(valuePair[colIndex]) floatArray = append(floatArray, value) case "bool": value, ok := valuePair[colIndex].(bool) @@ -290,7 +279,7 @@ func newFrameWithTimeField(row models.Row, column string, colIndex int, query mo valueField = data.NewField("Value", row.Tags, floatArray) } - name := string(formatFrameName(row, column, query, frameName[:])) + name := string(util.FormatFrameName(row.Name, column, row.Tags, query, frameName[:])) valueField.SetConfig(&data.FieldConfig{DisplayNameFromDS: name}) return data.NewFrame(name, timeField, valueField) } @@ -313,133 +302,3 @@ func newFrameWithoutTimeField(row models.Row, query models.Query) *data.Frame { field := data.NewField("Value", nil, values) return data.NewFrame(row.Name, field) } - -func formatFrameName(row models.Row, column string, query models.Query, frameName []byte) []byte { - if query.Alias == "" { - return buildFrameNameFromQuery(row, column, frameName, query.ResultFormat) - } - nameSegment := strings.Split(row.Name, ".") - - result := legendFormat.ReplaceAllFunc([]byte(query.Alias), func(in []byte) []byte { - aliasFormat := string(in) - aliasFormat = strings.Replace(aliasFormat, "[[", "", 1) - aliasFormat = strings.Replace(aliasFormat, "]]", "", 1) - aliasFormat = strings.Replace(aliasFormat, "$", "", 1) - - if aliasFormat == "m" || aliasFormat == "measurement" { - return []byte(row.Name) - } - if aliasFormat == "col" { - return []byte(column) - } - - pos, err := strconv.Atoi(aliasFormat) - if err == nil && len(nameSegment) > pos { - return []byte(nameSegment[pos]) - } - - if !strings.HasPrefix(aliasFormat, "tag_") { - return in - } - - tagKey := strings.Replace(aliasFormat, "tag_", "", 1) - tagValue, exist := row.Tags[tagKey] - if exist { - return []byte(tagValue) - } - - return in - }) - - return result -} - -func buildFrameNameFromQuery(row models.Row, column string, frameName []byte, resultFormat string) []byte { - if resultFormat != "table" { - frameName = append(frameName, row.Name...) - frameName = append(frameName, '.') - } - frameName = append(frameName, column...) - - if len(row.Tags) > 0 { - frameName = append(frameName, ' ', '{', ' ') - first := true - for k, v := range row.Tags { - if !first { - frameName = append(frameName, ',') - frameName = append(frameName, ' ') - } else { - first = false - } - frameName = append(frameName, k...) - frameName = append(frameName, ':', ' ') - frameName = append(frameName, v...) - } - - frameName = append(frameName, ' ', '}') - } - - return frameName -} - -func parseTimestamp(value any) (time.Time, error) { - timestampNumber, ok := value.(json.Number) - if !ok { - return time.Time{}, fmt.Errorf("timestamp-value has invalid type: %#v", value) - } - timestampInMilliseconds, err := timestampNumber.Int64() - if err != nil { - return time.Time{}, err - } - - // currently in the code the influxdb-timestamps are requested with - // milliseconds-precision, meaning these values are milliseconds - t := time.UnixMilli(timestampInMilliseconds).UTC() - - return t, nil -} - -func typeof(values [][]any, colIndex int) string { - for _, value := range values { - if value != nil && value[colIndex] != nil { - return fmt.Sprintf("%T", value[colIndex]) - } - } - return "null" -} - -func parseNumber(value any) *float64 { - // NOTE: we use pointers-to-float64 because we need - // to represent null-json-values. they come for example - // when we do a group-by with fill(null) - - if value == nil { - // this is what json-nulls become - return nil - } - - number, ok := value.(json.Number) - if !ok { - // in the current implementation, errors become nils - return nil - } - - fvalue, err := number.Float64() - if err != nil { - // in the current implementation, errors become nils - return nil - } - - return &fvalue -} - -func getVisType(resFormat string) data.VisType { - switch resFormat { - case "table": - return tableVisType - case "logs": - return logsVisType - default: - return graphVisType - } -} diff --git a/pkg/tsdb/influxdb/influxql/response_parser_test.go b/pkg/tsdb/influxdb/influxql/response_parser_test.go index 6051065c21f..a3925a34119 100644 --- a/pkg/tsdb/influxdb/influxql/response_parser_test.go +++ b/pkg/tsdb/influxdb/influxql/response_parser_test.go @@ -16,6 +16,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/experimental" "github.com/stretchr/testify/require" + "github.com/grafana/grafana/pkg/tsdb/influxdb/influxql/util" "github.com/grafana/grafana/pkg/tsdb/influxdb/models" ) @@ -118,7 +119,7 @@ func TestInfluxdbResponseParser(t *testing.T) { }), newField, ) - testFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"} + testFrame.Meta = &data.FrameMeta{PreferredVisualization: util.GraphVisType, ExecutedQueryString: "Test raw query"} testFrameWithoutMeta := data.NewFrame("series alias", data.NewField("Time", nil, []time.Time{ @@ -323,17 +324,17 @@ func TestInfluxdbResponseParser(t *testing.T) { }) t.Run("Influxdb response parser parseNumber nil", func(t *testing.T) { - value := parseNumber(nil) + value := util.ParseNumber(nil) require.Nil(t, value) }) t.Run("Influxdb response parser parseNumber valid JSON.number", func(t *testing.T) { - value := parseNumber(json.Number("95.4")) + value := util.ParseNumber(json.Number("95.4")) require.Equal(t, *value, 95.4) }) t.Run("Influxdb response parser parseNumber invalid type", func(t *testing.T) { - value := parseNumber("95.4") + value := util.ParseNumber("95.4") require.Nil(t, value) }) @@ -350,7 +351,7 @@ func TestInfluxdbResponseParser(t *testing.T) { }), newField, ) - testFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"} + testFrame.Meta = &data.FrameMeta{PreferredVisualization: util.GraphVisType, ExecutedQueryString: "Test raw query"} result := ResponseParse(readJsonFile("invalid_timestamp_format"), 200, generateQuery("time_series", "")) @@ -362,13 +363,13 @@ func TestInfluxdbResponseParser(t *testing.T) { t.Run("Influxdb response parser parseTimestamp valid JSON.number", func(t *testing.T) { // currently we use milliseconds-precision with influxdb, so the test works with that. // if we change this to for example nanoseconds-precision, the tests will have to change. - timestamp, err := parseTimestamp(json.Number("1609556645000")) + timestamp, err := util.ParseTimestamp(json.Number("1609556645000")) require.NoError(t, err) require.Equal(t, timestamp.Format(time.RFC3339), "2021-01-02T03:04:05Z") }) t.Run("Influxdb response parser parseNumber invalid type", func(t *testing.T) { - _, err := parseTimestamp("hello") + _, err := util.ParseTimestamp("hello") require.Error(t, err) }) } diff --git a/pkg/tsdb/influxdb/influxql/util/util.go b/pkg/tsdb/influxdb/influxql/util/util.go new file mode 100644 index 00000000000..ede64b21d4b --- /dev/null +++ b/pkg/tsdb/influxdb/influxql/util/util.go @@ -0,0 +1,152 @@ +package util + +import ( + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + "time" + + "github.com/grafana/grafana-plugin-sdk-go/data" + + "github.com/grafana/grafana/pkg/tsdb/influxdb/models" +) + +var ( + legendFormat = regexp.MustCompile(`\[\[([\@\/\w-]+)(\.[\@\/\w-]+)*\]\]*|\$([\@\w-]+?)*`) +) + +const ( + GraphVisType data.VisType = "graph" + TableVisType data.VisType = "table" + LogsVisType data.VisType = "logs" +) + +func FormatFrameName(rowName, column string, tags map[string]string, query models.Query, frameName []byte) []byte { + if query.Alias == "" { + return BuildFrameNameFromQuery(rowName, column, tags, frameName, query.ResultFormat) + } + nameSegment := strings.Split(rowName, ".") + + result := legendFormat.ReplaceAllFunc([]byte(query.Alias), func(in []byte) []byte { + aliasFormat := string(in) + aliasFormat = strings.Replace(aliasFormat, "[[", "", 1) + aliasFormat = strings.Replace(aliasFormat, "]]", "", 1) + aliasFormat = strings.Replace(aliasFormat, "$", "", 1) + + if aliasFormat == "m" || aliasFormat == "measurement" { + return []byte(rowName) + } + if aliasFormat == "col" { + return []byte(column) + } + + pos, err := strconv.Atoi(aliasFormat) + if err == nil && len(nameSegment) > pos { + return []byte(nameSegment[pos]) + } + + if !strings.HasPrefix(aliasFormat, "tag_") { + return in + } + + tagKey := strings.Replace(aliasFormat, "tag_", "", 1) + tagValue, exist := tags[tagKey] + if exist { + return []byte(tagValue) + } + + return in + }) + + return result +} + +func BuildFrameNameFromQuery(rowName, column string, tags map[string]string, frameName []byte, resultFormat string) []byte { + if resultFormat != "table" { + frameName = append(frameName, rowName...) + frameName = append(frameName, '.') + } + frameName = append(frameName, column...) + + if len(tags) == 0 { + return frameName + } + frameName = append(frameName, ' ', '{', ' ') + first := true + for k, v := range tags { + if !first { + frameName = append(frameName, ',') + frameName = append(frameName, ' ') + } else { + first = false + } + frameName = append(frameName, k...) + frameName = append(frameName, ':', ' ') + frameName = append(frameName, v...) + } + return append(frameName, ' ', '}') +} + +func ParseTimestamp(value any) (time.Time, error) { + timestampNumber, ok := value.(json.Number) + if !ok { + return time.Time{}, fmt.Errorf("timestamp-value has invalid type: %#v", value) + } + timestampInMilliseconds, err := timestampNumber.Int64() + if err != nil { + return time.Time{}, err + } + + // currently in the code the influxdb-timestamps are requested with + // milliseconds-precision, meaning these values are milliseconds + t := time.UnixMilli(timestampInMilliseconds).UTC() + + return t, nil +} + +func Typeof(values [][]any, colIndex int) string { + for _, value := range values { + if value != nil && value[colIndex] != nil { + return fmt.Sprintf("%T", value[colIndex]) + } + } + return "null" +} + +func ParseNumber(value any) *float64 { + // NOTE: we use pointers-to-float64 because we need + // to represent null-json-values. they come for example + // when we do a group-by with fill(null) + + if value == nil { + // this is what json-nulls become + return nil + } + + number, ok := value.(json.Number) + if !ok { + // in the current implementation, errors become nils + return nil + } + + fvalue, err := number.Float64() + if err != nil { + // in the current implementation, errors become nils + return nil + } + + return &fvalue +} + +func GetVisType(resFormat string) data.VisType { + switch resFormat { + case "table": + return TableVisType + case "logs": + return LogsVisType + default: + return GraphVisType + } +}