From 1e7f2e93e6669cf98630e1ce7c0f0390472adfe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Farkas?= Date: Tue, 23 Feb 2021 16:44:33 +0100 Subject: [PATCH] InfluxDB: handle columns named "table" (#30985) * influxdb: flux: manually calculate table-group-key * influxdb: flux: added explanation about the group-key --- pkg/tsdb/influxdb/flux/builder.go | 78 +++++++++++++++---- pkg/tsdb/influxdb/flux/executor_test.go | 5 ++ pkg/tsdb/influxdb/flux/testdata/table.csv | 6 ++ .../influxdb/flux/testdata/table.golden.txt | 16 ++++ 4 files changed, 89 insertions(+), 16 deletions(-) create mode 100644 pkg/tsdb/influxdb/flux/testdata/table.csv create mode 100644 pkg/tsdb/influxdb/flux/testdata/table.golden.txt diff --git a/pkg/tsdb/influxdb/flux/builder.go b/pkg/tsdb/influxdb/flux/builder.go index f9358972ffc..2b8b9dd7fff 100644 --- a/pkg/tsdb/influxdb/flux/builder.go +++ b/pkg/tsdb/influxdb/flux/builder.go @@ -30,18 +30,19 @@ type columnInfo struct { // frameBuilder is an interface to help testing. type frameBuilder struct { - tableID int64 - active *data.Frame - frames []*data.Frame - value *data.FieldConverter - columns []columnInfo - labels []string - maxPoints int // max points in a series - maxSeries int // max number of series - totalSeries int - isTimeSeries bool - timeColumn string // sometimes it is not `_time` - timeDisplay string + currentGroupKey []interface{} + groupKeyColumnNames []string + active *data.Frame + frames []*data.Frame + value *data.FieldConverter + columns []columnInfo + labels []string + maxPoints int // max points in a series + maxSeries int // max number of series + totalSeries int + isTimeSeries bool + timeColumn string // sometimes it is not `_time` + timeDisplay string } func isTag(schk string) bool { @@ -95,11 +96,18 @@ func getConverter(t string) (*data.FieldConverter, error) { func (fb *frameBuilder) Init(metadata *query.FluxTableMetadata) error { columns := metadata.Columns() fb.frames = make([]*data.Frame, 0) - fb.tableID = -1 + fb.currentGroupKey = nil fb.value = nil fb.columns = make([]columnInfo, 0) fb.isTimeSeries = false fb.timeColumn = "" + fb.groupKeyColumnNames = make([]string, 0) + + for _, col := range columns { + if col.IsGroup() { + fb.groupKeyColumnNames = append(fb.groupKeyColumnNames, col.Name()) + } + } for _, col := range columns { switch { @@ -175,14 +183,52 @@ func getTimeSeriesTimeColumn(columns []*query.FluxColumn) *query.FluxColumn { return nil } +func getTableID(record *query.FluxRecord, groupColumns []string) []interface{} { + result := make([]interface{}, len(groupColumns)) + + // Flux does not allow duplicate column-names, + // so we can be sure there is no confusion in the record. + // + // ( it does allow for a column named "table" to exist, + // and shadow the table-id "table" column, but the potentially + // shadowed table-id column is not a part of the group-key, + // so we should be safe ) + + for i, colName := range groupColumns { + result[i] = record.ValueByKey(colName) + } + + return result +} + +func isTableIDEqual(id1 []interface{}, id2 []interface{}) bool { + if (id1 == nil) || (id2 == nil) { + return false + } + + if len(id1) != len(id2) { + return false + } + + for i, id1Val := range id1 { + id2Val := id2[i] + + if id1Val != id2Val { + return false + } + } + + return true +} + // Append appends a single entry from an influxdb2 record to a data frame // Values are appended to _value // Tags are appended as labels // _measurement holds the dataframe name // _field holds the field name. func (fb *frameBuilder) Append(record *query.FluxRecord) error { - table, ok := record.ValueByKey("table").(int64) - if ok && table != fb.tableID { + table := getTableID(record, fb.groupKeyColumnNames) + if (fb.currentGroupKey == nil) || !isTableIDEqual(table, fb.currentGroupKey) { fb.totalSeries++ if fb.totalSeries > fb.maxSeries { return fmt.Errorf("results are truncated, max series reached (%d)", fb.maxSeries) @@ -226,7 +272,7 @@ func (fb *frameBuilder) Append(record *query.FluxRecord) error { } fb.frames = append(fb.frames, fb.active) - fb.tableID = table + fb.currentGroupKey = table } if fb.isTimeSeries { diff --git a/pkg/tsdb/influxdb/flux/executor_test.go b/pkg/tsdb/influxdb/flux/executor_test.go index 46f92bc3e3b..f623a164814 100644 --- a/pkg/tsdb/influxdb/flux/executor_test.go +++ b/pkg/tsdb/influxdb/flux/executor_test.go @@ -103,6 +103,11 @@ func TestExecuteMultiple(t *testing.T) { fmt.Println("----------------------") } +func TestExecuteColumnNamedTable(t *testing.T) { + dr := verifyGoldenResponse(t, "table") + require.Len(t, dr.Frames, 1) +} + func TestExecuteGrouping(t *testing.T) { dr := verifyGoldenResponse(t, "grouping") require.Len(t, dr.Frames, 3) diff --git a/pkg/tsdb/influxdb/flux/testdata/table.csv b/pkg/tsdb/influxdb/flux/testdata/table.csv new file mode 100644 index 00000000000..927b9166598 --- /dev/null +++ b/pkg/tsdb/influxdb/flux/testdata/table.csv @@ -0,0 +1,6 @@ +#group,false,false,true,true,false,false,true +#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,long,string +#default,last,,,,,, +,result,table,_start,_stop,_time,_value,table +,,0,2021-02-05T14:02:04.7009051Z,2021-02-05T14:03:04.7009051Z,2021-02-05T14:03:04.7009051Z,42,something + diff --git a/pkg/tsdb/influxdb/flux/testdata/table.golden.txt b/pkg/tsdb/influxdb/flux/testdata/table.golden.txt new file mode 100644 index 00000000000..82bf6e4a7db --- /dev/null +++ b/pkg/tsdb/influxdb/flux/testdata/table.golden.txt @@ -0,0 +1,16 @@ +🌟 This was machine generated. Do not edit. 🌟 + +Frame[0] {} +Name: +Dimensions: 2 Fields by 1 Rows ++---------------------------------------+----------------+ +| Name: Time | Name: | +| Labels: | Labels: | +| Type: []time.Time | Type: []*int64 | ++---------------------------------------+----------------+ +| 2021-02-05 14:03:04.7009051 +0000 UTC | 42 | ++---------------------------------------+----------------+ + + +====== TEST DATA RESPONSE (arrow base64) ====== +FRAME=QVJST1cxAAD/////yAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAHQAAAADAAAATAAAACgAAAAEAAAAzP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADs/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAz///8IAAAADAAAAAIAAAB7fQAABAAAAG1ldGEAAAAAAgAAALAAAAAYAAAAAAASABgAFAATABIADAAAAAgABAASAAAAFAAAAFwAAABkAAAAAAACAWgAAAACAAAAKAAAAAQAAABw////CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAJD///8IAAAADAAAAAIAAAB7fQAABgAAAGxhYmVscwAAAAAAAAgADAAIAAcACAAAAAAAAAFAAAAAAAAAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAATAAAAAAAAApMAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAAAAAAAP////+4AAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAEAAAAAAAAAAUAAAAAAAAAwMACgAYAAwACAAEAAoAAAAUAAAAWAAAAAEAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAAAAAAAIAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAIzmaQHv3mAWKgAAAAAAAAAQAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAOAAAAAAAAwABAAAA2AEAAAAAAADAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAoADAAAAAgABAAKAAAACAAAAHQAAAADAAAATAAAACgAAAAEAAAAzP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADs/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAz///8IAAAADAAAAAIAAAB7fQAABAAAAG1ldGEAAAAAAgAAALAAAAAYAAAAAAASABgAFAATABIADAAAAAgABAASAAAAFAAAAFwAAABkAAAAAAACAWgAAAACAAAAKAAAAAQAAABw////CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAJD///8IAAAADAAAAAIAAAB7fQAABgAAAGxhYmVscwAAAAAAAAgADAAIAAcACAAAAAAAAAFAAAAAAAAAAAAAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAATAAAAAAAAApMAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAA8AEAAEFSUk9XMQ==