IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2024-11-28 19:54:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

Chore: Introduce util package for InfluxDB backend testing (#78826)

Browse Source 
have a util package
This commit is contained in:
ismail simsek 2023-11-29 17:48:53 +01:00 committed by GitHub
parent 2e2b1cd9e4
commit aea5a9f01a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 170 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
pkg/tsdb/influxdb/influxql/response_parser.go
View File

@ -4,27 +4,16 @@ import (
"encoding/json"
"fmt"
"io"
"regexp"
"strconv"
"strings"
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/tsdb/influxdb/influxql/util"
"github.com/grafana/grafana/pkg/tsdb/influxdb/models"
)
var (
legendFormat = regexp.MustCompile(`\[\[([\@\/\w-]+)(\.[\@\/\w-]+)*\]\]*|\$([\@\w-]+?)*`)
)
const (
graphVisType data.VisType = "graph"
tableVisType data.VisType = "table"
logsVisType data.VisType = "logs"
)
func ResponseParse(buf io.ReadCloser, statusCode int, query *models.Query) *backend.DataResponse {
return parse(buf, statusCode, query)
}
@ -79,7 +68,7 @@ func transformRowsForTable(rows []models.Row, query models.Query) data.Frames {
newFrame := data.NewFrame(rows[0].Name)
newFrame.Meta = &data.FrameMeta{
ExecutedQueryString: query.RawQuery,
PreferredVisualization: getVisType(query.ResultFormat),
PreferredVisualization: util.GetVisType(query.ResultFormat),
}
conLen := len(rows[0].Columns)
@ -100,7 +89,7 @@ func newTimeField(rows []models.Row) *data.Field {
var timeArray []time.Time
for _, row := range rows {
for _, valuePair := range row.Values {
timestamp, timestampErr := parseTimestamp(valuePair[0])
timestamp, timestampErr := util.ParseTimestamp(valuePair[0])
// we only add this row if the timestamp is valid
if timestampErr != nil {
continue
@ -142,7 +131,7 @@ func newValueFields(rows []models.Row, labels data.Labels, colIdxStart, colIdxEn
var boolArray []*bool
for _, row := range rows {
valType := typeof(row.Values, colIdx)
valType := util.Typeof(row.Values, colIdx)
for _, valuePair := range row.Values {
switch valType {
@ -154,7 +143,7 @@ func newValueFields(rows []models.Row, labels data.Labels, colIdxStart, colIdxEn
stringArray = append(stringArray, nil)
}
case "json.Number":
value := parseNumber(valuePair[colIdx])
value := util.ParseNumber(valuePair[colIdx])
floatArray = append(floatArray, value)
case "bool":
value, ok := valuePair[colIdx].(bool)
@ -226,7 +215,7 @@ func transformRowsForTimeSeries(rows []models.Row, query models.Query) data.Fram
if len(frames) == 0 {
newFrame.Meta = &data.FrameMeta{
ExecutedQueryString: query.RawQuery,
PreferredVisualization: getVisType(query.ResultFormat),
PreferredVisualization: util.GetVisType(query.ResultFormat),
}
}
frames = append(frames, newFrame)
@ -242,10 +231,10 @@ func newFrameWithTimeField(row models.Row, column string, colIndex int, query mo
var floatArray []*float64
var stringArray []*string
var boolArray []*bool
valType := typeof(row.Values, colIndex)
valType := util.Typeof(row.Values, colIndex)
for _, valuePair := range row.Values {
timestamp, timestampErr := parseTimestamp(valuePair[0])
timestamp, timestampErr := util.ParseTimestamp(valuePair[0])
// we only add this row if the timestamp is valid
if timestampErr != nil {
continue
@ -261,7 +250,7 @@ func newFrameWithTimeField(row models.Row, column string, colIndex int, query mo
stringArray = append(stringArray, nil)
}
case "json.Number":
value := parseNumber(valuePair[colIndex])
value := util.ParseNumber(valuePair[colIndex])
floatArray = append(floatArray, value)
case "bool":
value, ok := valuePair[colIndex].(bool)
@ -290,7 +279,7 @@ func newFrameWithTimeField(row models.Row, column string, colIndex int, query mo
valueField = data.NewField("Value", row.Tags, floatArray)
}
name := string(formatFrameName(row, column, query, frameName[:]))
name := string(util.FormatFrameName(row.Name, column, row.Tags, query, frameName[:]))
valueField.SetConfig(&data.FieldConfig{DisplayNameFromDS: name})
return data.NewFrame(name, timeField, valueField)
}
@ -313,133 +302,3 @@ func newFrameWithoutTimeField(row models.Row, query models.Query) *data.Frame {
field := data.NewField("Value", nil, values)
return data.NewFrame(row.Name, field)
}
func formatFrameName(row models.Row, column string, query models.Query, frameName []byte) []byte {
if query.Alias == "" {
return buildFrameNameFromQuery(row, column, frameName, query.ResultFormat)
}
nameSegment := strings.Split(row.Name, ".")
result := legendFormat.ReplaceAllFunc([]byte(query.Alias), func(in []byte) []byte {
aliasFormat := string(in)
aliasFormat = strings.Replace(aliasFormat, "[[", "", 1)
aliasFormat = strings.Replace(aliasFormat, "]]", "", 1)
aliasFormat = strings.Replace(aliasFormat, "$", "", 1)
if aliasFormat == "m" || aliasFormat == "measurement" {
return []byte(row.Name)
}
if aliasFormat == "col" {
return []byte(column)
}
pos, err := strconv.Atoi(aliasFormat)
if err == nil && len(nameSegment) > pos {
return []byte(nameSegment[pos])
}
if !strings.HasPrefix(aliasFormat, "tag_") {
return in
}
tagKey := strings.Replace(aliasFormat, "tag_", "", 1)
tagValue, exist := row.Tags[tagKey]
if exist {
return []byte(tagValue)
}
return in
})
return result
}
func buildFrameNameFromQuery(row models.Row, column string, frameName []byte, resultFormat string) []byte {
if resultFormat != "table" {
frameName = append(frameName, row.Name...)
frameName = append(frameName, '.')
}
frameName = append(frameName, column...)
if len(row.Tags) > 0 {
frameName = append(frameName, ' ', '{', ' ')
first := true
for k, v := range row.Tags {
if !first {
frameName = append(frameName, ',')
frameName = append(frameName, ' ')
} else {
first = false
}
frameName = append(frameName, k...)
frameName = append(frameName, ':', ' ')
frameName = append(frameName, v...)
}
frameName = append(frameName, ' ', '}')
}
return frameName
}
func parseTimestamp(value any) (time.Time, error) {
timestampNumber, ok := value.(json.Number)
if !ok {
return time.Time{}, fmt.Errorf("timestamp-value has invalid type: %#v", value)
}
timestampInMilliseconds, err := timestampNumber.Int64()
if err != nil {
return time.Time{}, err
}
// currently in the code the influxdb-timestamps are requested with
// milliseconds-precision, meaning these values are milliseconds
t := time.UnixMilli(timestampInMilliseconds).UTC()
return t, nil
}
func typeof(values [][]any, colIndex int) string {
for _, value := range values {
if value != nil && value[colIndex] != nil {
return fmt.Sprintf("%T", value[colIndex])
}
}
return "null"
}
func parseNumber(value any) *float64 {
// NOTE: we use pointers-to-float64 because we need
// to represent null-json-values. they come for example
// when we do a group-by with fill(null)
if value == nil {
// this is what json-nulls become
return nil
}
number, ok := value.(json.Number)
if !ok {
// in the current implementation, errors become nils
return nil
}
fvalue, err := number.Float64()
if err != nil {
// in the current implementation, errors become nils
return nil
}
return &fvalue
}
func getVisType(resFormat string) data.VisType {
switch resFormat {
case "table":
return tableVisType
case "logs":
return logsVisType
default:
return graphVisType
}
}

15
pkg/tsdb/influxdb/influxql/response_parser_test.go
View File

@ -16,6 +16,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/experimental"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/tsdb/influxdb/influxql/util"
"github.com/grafana/grafana/pkg/tsdb/influxdb/models"
)
@ -118,7 +119,7 @@ func TestInfluxdbResponseParser(t *testing.T) {
}),
newField,
)
testFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"}
testFrame.Meta = &data.FrameMeta{PreferredVisualization: util.GraphVisType, ExecutedQueryString: "Test raw query"}
testFrameWithoutMeta := data.NewFrame("series alias",
data.NewField("Time", nil,
[]time.Time{
@ -323,17 +324,17 @@ func TestInfluxdbResponseParser(t *testing.T) {
})
t.Run("Influxdb response parser parseNumber nil", func(t *testing.T) {
value := parseNumber(nil)
value := util.ParseNumber(nil)
require.Nil(t, value)
})
t.Run("Influxdb response parser parseNumber valid JSON.number", func(t *testing.T) {
value := parseNumber(json.Number("95.4"))
value := util.ParseNumber(json.Number("95.4"))
require.Equal(t, *value, 95.4)
})
t.Run("Influxdb response parser parseNumber invalid type", func(t *testing.T) {
value := parseNumber("95.4")
value := util.ParseNumber("95.4")
require.Nil(t, value)
})
@ -350,7 +351,7 @@ func TestInfluxdbResponseParser(t *testing.T) {
}),
newField,
)
testFrame.Meta = &data.FrameMeta{PreferredVisualization: graphVisType, ExecutedQueryString: "Test raw query"}
testFrame.Meta = &data.FrameMeta{PreferredVisualization: util.GraphVisType, ExecutedQueryString: "Test raw query"}
result := ResponseParse(readJsonFile("invalid_timestamp_format"), 200, generateQuery("time_series", ""))
@ -362,13 +363,13 @@ func TestInfluxdbResponseParser(t *testing.T) {
t.Run("Influxdb response parser parseTimestamp valid JSON.number", func(t *testing.T) {
// currently we use milliseconds-precision with influxdb, so the test works with that.
// if we change this to for example nanoseconds-precision, the tests will have to change.
timestamp, err := parseTimestamp(json.Number("1609556645000"))
timestamp, err := util.ParseTimestamp(json.Number("1609556645000"))
require.NoError(t, err)
require.Equal(t, timestamp.Format(time.RFC3339), "2021-01-02T03:04:05Z")
})
t.Run("Influxdb response parser parseNumber invalid type", func(t *testing.T) {
_, err := parseTimestamp("hello")
_, err := util.ParseTimestamp("hello")
require.Error(t, err)
})
}

152
pkg/tsdb/influxdb/influxql/util/util.go Normal file
View File

@ -0,0 +1,152 @@
package util
import (
"encoding/json"
"fmt"
"regexp"
"strconv"
"strings"
"time"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/tsdb/influxdb/models"
)
var (
legendFormat = regexp.MustCompile(`\[\[([\@\/\w-]+)(\.[\@\/\w-]+)*\]\]*|\$([\@\w-]+?)*`)
)
const (
GraphVisType data.VisType = "graph"
TableVisType data.VisType = "table"
LogsVisType data.VisType = "logs"
)
func FormatFrameName(rowName, column string, tags map[string]string, query models.Query, frameName []byte) []byte {
if query.Alias == "" {
return BuildFrameNameFromQuery(rowName, column, tags, frameName, query.ResultFormat)
}
nameSegment := strings.Split(rowName, ".")
result := legendFormat.ReplaceAllFunc([]byte(query.Alias), func(in []byte) []byte {
aliasFormat := string(in)
aliasFormat = strings.Replace(aliasFormat, "[[", "", 1)
aliasFormat = strings.Replace(aliasFormat, "]]", "", 1)
aliasFormat = strings.Replace(aliasFormat, "$", "", 1)
if aliasFormat == "m" || aliasFormat == "measurement" {
return []byte(rowName)
}
if aliasFormat == "col" {
return []byte(column)
}
pos, err := strconv.Atoi(aliasFormat)
if err == nil && len(nameSegment) > pos {
return []byte(nameSegment[pos])
}
if !strings.HasPrefix(aliasFormat, "tag_") {
return in
}
tagKey := strings.Replace(aliasFormat, "tag_", "", 1)
tagValue, exist := tags[tagKey]
if exist {
return []byte(tagValue)
}
return in
})
return result
}
func BuildFrameNameFromQuery(rowName, column string, tags map[string]string, frameName []byte, resultFormat string) []byte {
if resultFormat != "table" {
frameName = append(frameName, rowName...)
frameName = append(frameName, '.')
}
frameName = append(frameName, column...)
if len(tags) == 0 {
return frameName
}
frameName = append(frameName, ' ', '{', ' ')
first := true
for k, v := range tags {
if !first {
frameName = append(frameName, ',')
frameName = append(frameName, ' ')
} else {
first = false
}
frameName = append(frameName, k...)
frameName = append(frameName, ':', ' ')
frameName = append(frameName, v...)
}
return append(frameName, ' ', '}')
}
func ParseTimestamp(value any) (time.Time, error) {
timestampNumber, ok := value.(json.Number)
if !ok {
return time.Time{}, fmt.Errorf("timestamp-value has invalid type: %#v", value)
}
timestampInMilliseconds, err := timestampNumber.Int64()
if err != nil {
return time.Time{}, err
}
// currently in the code the influxdb-timestamps are requested with
// milliseconds-precision, meaning these values are milliseconds
t := time.UnixMilli(timestampInMilliseconds).UTC()
return t, nil
}
func Typeof(values [][]any, colIndex int) string {
for _, value := range values {
if value != nil && value[colIndex] != nil {
return fmt.Sprintf("%T", value[colIndex])
}
}
return "null"
}
func ParseNumber(value any) *float64 {
// NOTE: we use pointers-to-float64 because we need
// to represent null-json-values. they come for example
// when we do a group-by with fill(null)
if value == nil {
// this is what json-nulls become
return nil
}
number, ok := value.(json.Number)
if !ok {
// in the current implementation, errors become nils
return nil
}
fvalue, err := number.Float64()
if err != nil {
// in the current implementation, errors become nils
return nil
}
return &fvalue
}
func GetVisType(resFormat string) data.VisType {
switch resFormat {
case "table":
return TableVisType
case "logs":
return LogsVisType
default:
return GraphVisType
}
}