From 4afd40018babbbb9f50d921ccfb2fe167175b4d3 Mon Sep 17 00:00:00 2001 From: Kyle Brandt Date: Tue, 26 Nov 2019 13:28:16 -0500 Subject: [PATCH] transform: update to use sdk with frame.labels moved to frame.[]field.labels (#20670) sdk v0.4.0 --- go.mod | 2 +- go.sum | 4 + pkg/tsdb/frame_util.go | 6 +- .../grafana-plugin-sdk-go/dataframe/arrow.go | 81 +++++++++++-------- .../dataframe/dataframe.go | 28 +++---- .../grafana-plugin-sdk-go/dataframe/vector.go | 36 +++++++++ .../dataframe/vector_bool.go | 4 + .../dataframe/vector_bool_nullable.go | 4 + .../dataframe/vector_float.go | 4 + .../dataframe/vector_float_nullable.go | 4 + .../dataframe/vector_int.go | 4 + .../dataframe/vector_int_nullable.go | 4 + .../dataframe/vector_string.go | 4 + .../dataframe/vector_string_nullable.go | 4 + .../dataframe/vector_time.go | 5 +- .../dataframe/vector_time_nullable.go | 4 + .../dataframe/vector_uint.go | 4 + .../dataframe/vector_uint_nullable.go | 4 + .../transform/transform.go | 2 +- vendor/modules.txt | 2 +- 20 files changed, 155 insertions(+), 55 deletions(-) diff --git a/go.mod b/go.mod index 1f5d4ad79d9..1cc76c3779d 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/gorilla/websocket v1.4.0 github.com/gosimple/slug v1.4.2 github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4 - github.com/grafana/grafana-plugin-sdk-go v0.2.0 + github.com/grafana/grafana-plugin-sdk-go v0.4.0 github.com/hashicorp/go-hclog v0.8.0 github.com/hashicorp/go-plugin v1.0.1 github.com/hashicorp/go-version v1.1.0 diff --git a/go.sum b/go.sum index 396030c5b22..7de5a2307d6 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,10 @@ github.com/grafana/grafana-plugin-sdk-go v0.1.1-0.20191115194829-9bfc5937b406 h1 github.com/grafana/grafana-plugin-sdk-go v0.1.1-0.20191115194829-9bfc5937b406/go.mod h1:yA268OaX+C71ubT39tyACEfFwyhEzS1kbEVHUCgkKS8= github.com/grafana/grafana-plugin-sdk-go v0.2.0 h1:MgcTjCuzIkZcjb/2vCPK1RvLEHfRnQtFK7AF0W3SQm0= github.com/grafana/grafana-plugin-sdk-go v0.2.0/go.mod h1:yA268OaX+C71ubT39tyACEfFwyhEzS1kbEVHUCgkKS8= +github.com/grafana/grafana-plugin-sdk-go v0.3.1-0.20191125180836-d77f6ffe8e05 h1:COdehD2bs2CJ3zrGAOueGrqCOaCG/M9aYiO4y+J4MUk= +github.com/grafana/grafana-plugin-sdk-go v0.3.1-0.20191125180836-d77f6ffe8e05/go.mod h1:yA268OaX+C71ubT39tyACEfFwyhEzS1kbEVHUCgkKS8= +github.com/grafana/grafana-plugin-sdk-go v0.4.0 h1:bypT7gwGL9i584JEUQ1twcLxoUPO/60XW3VM8VYndYI= +github.com/grafana/grafana-plugin-sdk-go v0.4.0/go.mod h1:yA268OaX+C71ubT39tyACEfFwyhEzS1kbEVHUCgkKS8= github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= github.com/hashicorp/go-hclog v0.8.0 h1:z3ollgGRg8RjfJH6UVBaG54R70GFd++QOkvnJH3VSBY= github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= diff --git a/pkg/tsdb/frame_util.go b/pkg/tsdb/frame_util.go index 735a7229420..cc2d9427d00 100644 --- a/pkg/tsdb/frame_util.go +++ b/pkg/tsdb/frame_util.go @@ -13,9 +13,9 @@ func SeriesToFrame(series *TimeSeries) (*dataframe.Frame, error) { for idx, point := range series.Points { timeVec[idx], floatVec[idx] = convertTSDBTimePoint(point) } - frame := dataframe.New(series.Name, dataframe.Labels(series.Tags), - dataframe.NewField("time", timeVec), - dataframe.NewField("value", floatVec), + frame := dataframe.New(series.Name, + dataframe.NewField("time", nil, timeVec), + dataframe.NewField("value", dataframe.Labels(series.Tags), floatVec), ) return frame, nil diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/arrow.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/arrow.go index fdc2a028f67..dcb0fd8179a 100644 --- a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/arrow.go +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/arrow.go @@ -80,7 +80,8 @@ func buildArrowFields(f *Frame) ([]arrow.Field, error) { } fieldMeta := map[string]string{ - "name": field.Name, + "name": field.Name, + "labels": field.Labels.String(), } arrowFields[i] = arrow.Field{ @@ -146,10 +147,6 @@ func buildArrowSchema(f *Frame, fs []arrow.Field) (*arrow.Schema, error) { "refId": f.RefID, } - if f.Labels != nil { - tableMetaMap["labels"] = f.Labels.String() - } - tableMeta := arrow.MetadataFrom(tableMetaMap) return arrow.NewSchema(fs, &tableMeta), nil @@ -195,39 +192,27 @@ func fieldToArrow(f *Field) (arrow.DataType, bool, error) { } } -// UnMarshalArrow converts a byte representation of an arrow table to a Frame -// TODO: Break up this function. -func UnMarshalArrow(b []byte) (*Frame, error) { - fB := filebuffer.New(b) - fR, err := ipc.NewFileReader(fB) - if err != nil { - return nil, err +func getMDKey(key string, metaData arrow.Metadata) (string, bool) { + idx := metaData.FindKey(key) + if idx < 0 { + return "", false } - defer fR.Close() + return metaData.Values()[idx], true +} - schema := fR.Schema() - metaData := schema.Metadata() - frame := &Frame{} - getMDKey := func(key string) (string, bool) { - idx := metaData.FindKey(key) - if idx < 0 { - return "", false - } - return metaData.Values()[idx], true - } - frame.Name, _ = getMDKey("name") // No need to check ok, zero value ("") is returned - frame.RefID, _ = getMDKey("refId") - if labelsAsString, ok := getMDKey("labels"); ok { - frame.Labels, err = LabelsFromString(labelsAsString) - if err != nil { - return nil, err - } - } +func initializeFrameFields(schema *arrow.Schema, frame *Frame) ([]bool, error) { nullable := make([]bool, len(schema.Fields())) for idx, field := range schema.Fields() { sdkField := &Field{ Name: field.Name, } + if labelsAsString, ok := getMDKey("labels", field.Metadata); ok { + var err error + sdkField.Labels, err = LabelsFromString(labelsAsString) + if err != nil { + return nil, err + } + } nullable[idx] = field.Nullable switch field.Type.ID() { case arrow.STRING: @@ -267,18 +252,21 @@ func UnMarshalArrow(b []byte) (*Frame, error) { } sdkField.Vector = newTimeVector(0) default: - return nil, fmt.Errorf("unsupported conversion from arrow to sdk type for arrow type %v", field.Type.ID().String()) + return nullable, fmt.Errorf("unsupported conversion from arrow to sdk type for arrow type %v", field.Type.ID().String()) } frame.Fields = append(frame.Fields, sdkField) } + return nullable, nil +} +func populateFrameFields(fR *ipc.FileReader, nullable []bool, frame *Frame) error { for { record, err := fR.Read() if err == io.EOF { break } if err != nil { - return nil, err + return err } for i := 0; i < len(frame.Fields); i++ { col := record.Column(i) @@ -374,10 +362,35 @@ func UnMarshalArrow(b []byte) (*Frame, error) { frame.Fields[i].Vector.Append(t) } default: - return nil, fmt.Errorf("unsupported arrow type %s for conversion", col.DataType().ID()) + return fmt.Errorf("unsupported arrow type %s for conversion", col.DataType().ID()) } } } + return nil +} +// UnmarshalArrow converts a byte representation of an arrow table to a Frame +func UnmarshalArrow(b []byte) (*Frame, error) { + fB := filebuffer.New(b) + fR, err := ipc.NewFileReader(fB) + if err != nil { + return nil, err + } + defer fR.Close() + + schema := fR.Schema() + metaData := schema.Metadata() + frame := &Frame{} + frame.Name, _ = getMDKey("name", metaData) // No need to check ok, zero value ("") is returned + frame.RefID, _ = getMDKey("refId", metaData) + nullable, err := initializeFrameFields(schema, frame) + if err != nil { + return nil, err + } + + err = populateFrameFields(fR, nullable, frame) + if err != nil { + return nil, err + } return frame, nil } diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/dataframe.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/dataframe.go index b11abee1097..2f174c0f7b9 100644 --- a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/dataframe.go +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/dataframe.go @@ -7,17 +7,26 @@ import ( "time" ) +// Frame represents a columnar storage with optional labels. +type Frame struct { + Name string + Fields []*Field + + RefID string +} + // Field represents a column of data with a specific type. type Field struct { Name string Vector Vector + Labels Labels } // Fields is a slice of Field pointers. type Fields []*Field // NewField returns a new instance of Field. -func NewField(name string, values interface{}) *Field { +func NewField(name string, labels Labels, values interface{}) *Field { var vec Vector switch v := values.(type) { case []int64: @@ -87,6 +96,7 @@ func NewField(name string, values interface{}) *Field { return &Field{ Name: name, Vector: vec, + Labels: labels, } } @@ -152,10 +162,10 @@ func (l Labels) String() string { // LabelsFromString parses the output of Labels.String() into // a Labels object. It probably has some flaws. func LabelsFromString(s string) (Labels, error) { - labels := make(map[string]string) if s == "" { - return labels, nil + return nil, nil } + labels := make(map[string]string) for _, rawKV := range strings.Split(s, ", ") { kV := strings.SplitN(rawKV, "=", 2) @@ -168,20 +178,10 @@ func LabelsFromString(s string) (Labels, error) { return labels, nil } -// Frame represents a columnar storage with optional labels. -type Frame struct { - Name string - Labels Labels - Fields []*Field - - RefID string -} - // New returns a new instance of a Frame. -func New(name string, labels Labels, fields ...*Field) *Frame { +func New(name string, fields ...*Field) *Frame { return &Frame{ Name: name, - Labels: labels, Fields: fields, } } diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector.go index 6877d1fb4d6..d6939872516 100644 --- a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector.go +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector.go @@ -11,6 +11,7 @@ type Vector interface { Append(i interface{}) At(i int) interface{} Len() int + PrimitiveType() VectorPType } func newVector(t interface{}, n int) (v Vector) { @@ -44,3 +45,38 @@ func newVector(t interface{}, n int) (v Vector) { } return } + +// VectorPType indicates the go type underlying the Vector. +type VectorPType int + +const ( + // VectorPTypeInt64 indicates the underlying primitive is a []int64. + VectorPTypeInt64 VectorPType = iota + // VectorPTypeNullableInt64 indicates the underlying primitive is a []*int64. + VectorPTypeNullableInt64 + + // VectorPTypeUint64 indicates the underlying primitive is a []uint64. + VectorPTypeUint64 + // VectorPTypeNullableUInt64 indicates the underlying primitive is a []*uint64. + VectorPTypeNullableUInt64 + + // VectorPTypeFloat64 indicates the underlying primitive is a []float64. + VectorPTypeFloat64 + // VectorPTypeNullableFloat64 indicates the underlying primitive is a []*float64. + VectorPTypeNullableFloat64 + + // VectorPTypeString indicates the underlying primitive is a []string. + VectorPTypeString + // VectorPTypeNullableString indicates the underlying primitive is a []*string. + VectorPTypeNullableString + + // VectorPTypeBool indicates the underlying primitive is a []bool. + VectorPTypeBool + // VectorPTypeNullableBool indicates the underlying primitive is a []*bool. + VectorPTypeNullableBool + + // VectorPTypeTime indicates the underlying primitive is a []time.Time. + VectorPTypeTime + // VectorPTypeNullableTime indicates the underlying primitive is a []*time.Time. + VectorPTypeNullableTime +) diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_bool.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_bool.go index 2204afed744..44da59864fb 100644 --- a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_bool.go +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_bool.go @@ -22,3 +22,7 @@ func (v *boolVector) At(i int) interface{} { func (v *boolVector) Len() int { return len(*v) } + +func (v *boolVector) PrimitiveType() VectorPType { + return VectorPTypeBool +} \ No newline at end of file diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_bool_nullable.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_bool_nullable.go index 2b78705b3a8..0b805cad2ca 100644 --- a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_bool_nullable.go +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_bool_nullable.go @@ -22,3 +22,7 @@ func (v *nullableBoolVector) At(i int) interface{} { func (v *nullableBoolVector) Len() int { return len(*v) } + +func (v *nullableBoolVector) PrimitiveType() VectorPType { + return VectorPTypeNullableBool +} diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_float.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_float.go index 924bd4ede09..356580f55a2 100644 --- a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_float.go +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_float.go @@ -22,3 +22,7 @@ func (v *floatVector) At(i int) interface{} { func (v *floatVector) Len() int { return len(*v) } + +func (v *floatVector) PrimitiveType() VectorPType { + return VectorPTypeFloat64 +} diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_float_nullable.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_float_nullable.go index c6377c2626d..645466c6265 100644 --- a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_float_nullable.go +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_float_nullable.go @@ -18,3 +18,7 @@ func (v *nullableFloatVector) Append(val interface{}) { func (v *nullableFloatVector) At(i int) interface{} { return (*v)[i] } func (v *nullableFloatVector) Len() int { return len(*v) } + +func (v *nullableFloatVector) PrimitiveType() VectorPType { + return VectorPTypeNullableFloat64 +} diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_int.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_int.go index de80989a571..cb1ff5ad011 100644 --- a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_int.go +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_int.go @@ -22,3 +22,7 @@ func (v *intVector) At(i int) interface{} { func (v *intVector) Len() int { return len(*v) } + +func (v *intVector) PrimitiveType() VectorPType { + return VectorPTypeInt64 +} diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_int_nullable.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_int_nullable.go index 749a0210f90..8db9957f11a 100644 --- a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_int_nullable.go +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_int_nullable.go @@ -18,3 +18,7 @@ func (v *nullableIntVector) Append(val interface{}) { func (v *nullableIntVector) At(i int) interface{} { return (*v)[i] } func (v *nullableIntVector) Len() int { return len(*v) } + +func (v *nullableIntVector) PrimitiveType() VectorPType { + return VectorPTypeNullableInt64 +} diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_string.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_string.go index b33761a0943..76c544444c4 100644 --- a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_string.go +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_string.go @@ -21,4 +21,8 @@ func (v *stringVector) At(i int) interface{} { func (v *stringVector) Len() int { return len(*v) +} + +func (v *stringVector) PrimitiveType() VectorPType { + return VectorPTypeString } \ No newline at end of file diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_string_nullable.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_string_nullable.go index 35f32ca8bbe..383541dec3b 100644 --- a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_string_nullable.go +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_string_nullable.go @@ -22,3 +22,7 @@ func (v *nullableStringVector) At(i int) interface{} { func (v *nullableStringVector) Len() int { return len(*v) } + +func (v *nullableStringVector) PrimitiveType() VectorPType { + return VectorPTypeNullableString +} \ No newline at end of file diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_time.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_time.go index 7db593437f6..7f89ad42f91 100644 --- a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_time.go +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_time.go @@ -26,4 +26,7 @@ func (v *timeVector) At(i int) interface{} { func (v *timeVector) Len() int { return len(*v) } - \ No newline at end of file + +func (v *timeVector) PrimitiveType() VectorPType { + return VectorPTypeTime +} \ No newline at end of file diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_time_nullable.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_time_nullable.go index 602eabc31e3..2d4e3354e62 100644 --- a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_time_nullable.go +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_time_nullable.go @@ -26,3 +26,7 @@ func (v *nullableTimeVector) At(i int) interface{} { func (v *nullableTimeVector) Len() int { return len(*v) } + +func (v *nullableTimeVector) PrimitiveType() VectorPType { + return VectorPTypeNullableTime +} diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_uint.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_uint.go index efcb037f63b..0af664640df 100644 --- a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_uint.go +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_uint.go @@ -22,3 +22,7 @@ func (v *uintVector) At(i int) interface{} { func (v *uintVector) Len() int { return len(*v) } + +func (v *uintVector) PrimitiveType() VectorPType { + return VectorPTypeUint64 +} diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_uint_nullable.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_uint_nullable.go index 9f91864c4cc..fa88268f5c5 100644 --- a/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_uint_nullable.go +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/dataframe/vector_uint_nullable.go @@ -18,3 +18,7 @@ func (v *nullableUintVector) Append(val interface{}) { func (v *nullableUintVector) At(i int) interface{} { return (*v)[i] } func (v *nullableUintVector) Len() int { return len(*v) } + +func (v *nullableUintVector) PrimitiveType() VectorPType { + return VectorPTypeNullableUInt64 +} diff --git a/vendor/github.com/grafana/grafana-plugin-sdk-go/transform/transform.go b/vendor/github.com/grafana/grafana-plugin-sdk-go/transform/transform.go index eb71cec0e5a..4d599507675 100644 --- a/vendor/github.com/grafana/grafana-plugin-sdk-go/transform/transform.go +++ b/vendor/github.com/grafana/grafana-plugin-sdk-go/transform/transform.go @@ -152,7 +152,7 @@ func (w *grafanaAPIWrapper) QueryDatasource(ctx context.Context, orgID int64, da // TODO Error property etc dfs := make([]*dataframe.Frame, len(rawRes.Dataframes)) for dfIdx, b := range rawRes.Dataframes { - dfs[dfIdx], err = dataframe.UnMarshalArrow(b) + dfs[dfIdx], err = dataframe.UnmarshalArrow(b) if err != nil { return nil, err } diff --git a/vendor/modules.txt b/vendor/modules.txt index 4733ce2a32a..364cab040d5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -131,7 +131,7 @@ github.com/gosimple/slug # github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4 github.com/grafana/grafana-plugin-model/go/datasource github.com/grafana/grafana-plugin-model/go/renderer -# github.com/grafana/grafana-plugin-sdk-go v0.2.0 +# github.com/grafana/grafana-plugin-sdk-go v0.4.0 github.com/grafana/grafana-plugin-sdk-go/common github.com/grafana/grafana-plugin-sdk-go/dataframe github.com/grafana/grafana-plugin-sdk-go/datasource