From 113bfb3d3eafeaad1453d371e3cab889b9d13f70 Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 9 Apr 2018 13:28:32 +0200 Subject: [PATCH 1/6] don't convert to uint64 --- pkg/tsdb/postgres/macros.go | 12 ++++++------ pkg/tsdb/postgres/macros_test.go | 7 +++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pkg/tsdb/postgres/macros.go b/pkg/tsdb/postgres/macros.go index 23daeebec5a..f96b3896041 100644 --- a/pkg/tsdb/postgres/macros.go +++ b/pkg/tsdb/postgres/macros.go @@ -83,11 +83,11 @@ func (m *PostgresMacroEngine) evaluateMacro(name string, args []string) (string, if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("extract(epoch from %s) BETWEEN %d AND %d", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("extract(epoch from %s) BETWEEN %d AND %d", args[0], int64(m.TimeRange.GetFromAsMsEpoch()/1000), int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil case "__timeFrom": - return fmt.Sprintf("to_timestamp(%d)", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil + return fmt.Sprintf("to_timestamp(%d)", int64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil case "__timeTo": - return fmt.Sprintf("to_timestamp(%d)", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("to_timestamp(%d)", int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil case "__timeGroup": if len(args) < 2 { return "", fmt.Errorf("macro %v needs time column and interval and optional fill value", name) @@ -114,11 +114,11 @@ func (m *PostgresMacroEngine) evaluateMacro(name string, args []string) (string, if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], int64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil case "__unixEpochFrom": - return fmt.Sprintf("%d", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil + return fmt.Sprintf("%d", int64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil case "__unixEpochTo": - return fmt.Sprintf("%d", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%d", int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil default: return "", fmt.Errorf("Unknown macro %v", name) } diff --git a/pkg/tsdb/postgres/macros_test.go b/pkg/tsdb/postgres/macros_test.go index b18acced963..1485f8a261f 100644 --- a/pkg/tsdb/postgres/macros_test.go +++ b/pkg/tsdb/postgres/macros_test.go @@ -85,5 +85,12 @@ func TestMacroEngine(t *testing.T) { So(sql, ShouldEqual, "select 18446744066914187038") }) + timeRange := &tsdb.TimeRange{From: "-315622800000", To: "315529200000"} // 1960-1980 + Convey("interpolate __timeFilter function before epoch", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "WHERE extract(epoch from time_column) BETWEEN -315622800 AND 315529200") + }) }) } From 7d6c8aa61299d48f2308b56d057c45a3feb35f38 Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 9 Apr 2018 13:34:35 +0200 Subject: [PATCH 2/6] add mssql and mysql --- pkg/tsdb/mssql/macros.go | 12 ++++++------ pkg/tsdb/mysql/macros.go | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/tsdb/mssql/macros.go b/pkg/tsdb/mssql/macros.go index 9d41cd03255..6fd1550b048 100644 --- a/pkg/tsdb/mssql/macros.go +++ b/pkg/tsdb/mssql/macros.go @@ -82,11 +82,11 @@ func (m *MsSqlMacroEngine) evaluateMacro(name string, args []string) (string, er if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= DATEADD(s, %d, '1970-01-01') AND %s <= DATEADD(s, %d, '1970-01-01')", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%s >= DATEADD(s, %d, '1970-01-01') AND %s <= DATEADD(s, %d, '1970-01-01')", args[0], int64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil case "__timeFrom": - return fmt.Sprintf("DATEADD(second, %d, '1970-01-01')", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil + return fmt.Sprintf("DATEADD(second, %d, '1970-01-01')", int64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil case "__timeTo": - return fmt.Sprintf("DATEADD(second, %d, '1970-01-01')", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("DATEADD(second, %d, '1970-01-01')", int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil case "__timeGroup": if len(args) < 2 { return "", fmt.Errorf("macro %v needs time column and interval", name) @@ -113,11 +113,11 @@ func (m *MsSqlMacroEngine) evaluateMacro(name string, args []string) (string, er if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], int64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil case "__unixEpochFrom": - return fmt.Sprintf("%d", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil + return fmt.Sprintf("%d", int64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil case "__unixEpochTo": - return fmt.Sprintf("%d", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%d", int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil default: return "", fmt.Errorf("Unknown macro %v", name) } diff --git a/pkg/tsdb/mysql/macros.go b/pkg/tsdb/mysql/macros.go index a292f209429..92f6968f12a 100644 --- a/pkg/tsdb/mysql/macros.go +++ b/pkg/tsdb/mysql/macros.go @@ -77,11 +77,11 @@ func (m *MySqlMacroEngine) evaluateMacro(name string, args []string) (string, er if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= FROM_UNIXTIME(%d) AND %s <= FROM_UNIXTIME(%d)", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%s >= FROM_UNIXTIME(%d) AND %s <= FROM_UNIXTIME(%d)", args[0], int64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil case "__timeFrom": - return fmt.Sprintf("FROM_UNIXTIME(%d)", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil + return fmt.Sprintf("FROM_UNIXTIME(%d)", int64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil case "__timeTo": - return fmt.Sprintf("FROM_UNIXTIME(%d)", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("FROM_UNIXTIME(%d)", int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil case "__timeGroup": if len(args) < 2 { return "", fmt.Errorf("macro %v needs time column and interval", name) @@ -108,11 +108,11 @@ func (m *MySqlMacroEngine) evaluateMacro(name string, args []string) (string, er if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], int64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil case "__unixEpochFrom": - return fmt.Sprintf("%d", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil + return fmt.Sprintf("%d", int64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil case "__unixEpochTo": - return fmt.Sprintf("%d", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%d", int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil default: return "", fmt.Errorf("Unknown macro %v", name) } From 920a0c4fec5149fc1bc3b42646a909bc036178dd Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 9 Apr 2018 13:49:13 +0200 Subject: [PATCH 3/6] skip mssql fix --- pkg/tsdb/mssql/macros.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/tsdb/mssql/macros.go b/pkg/tsdb/mssql/macros.go index 6fd1550b048..9d41cd03255 100644 --- a/pkg/tsdb/mssql/macros.go +++ b/pkg/tsdb/mssql/macros.go @@ -82,11 +82,11 @@ func (m *MsSqlMacroEngine) evaluateMacro(name string, args []string) (string, er if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= DATEADD(s, %d, '1970-01-01') AND %s <= DATEADD(s, %d, '1970-01-01')", args[0], int64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%s >= DATEADD(s, %d, '1970-01-01') AND %s <= DATEADD(s, %d, '1970-01-01')", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil case "__timeFrom": - return fmt.Sprintf("DATEADD(second, %d, '1970-01-01')", int64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil + return fmt.Sprintf("DATEADD(second, %d, '1970-01-01')", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil case "__timeTo": - return fmt.Sprintf("DATEADD(second, %d, '1970-01-01')", int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("DATEADD(second, %d, '1970-01-01')", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil case "__timeGroup": if len(args) < 2 { return "", fmt.Errorf("macro %v needs time column and interval", name) @@ -113,11 +113,11 @@ func (m *MsSqlMacroEngine) evaluateMacro(name string, args []string) (string, er if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], int64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil case "__unixEpochFrom": - return fmt.Sprintf("%d", int64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil + return fmt.Sprintf("%d", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil case "__unixEpochTo": - return fmt.Sprintf("%d", int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%d", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil default: return "", fmt.Errorf("Unknown macro %v", name) } From 1c9ebd5bd8c46cb5b32e7cbe0f94d4f5bded1447 Mon Sep 17 00:00:00 2001 From: ryan Date: Mon, 9 Apr 2018 14:01:09 +0200 Subject: [PATCH 4/6] fix test --- pkg/tsdb/postgres/macros_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tsdb/postgres/macros_test.go b/pkg/tsdb/postgres/macros_test.go index 1485f8a261f..838a73ce240 100644 --- a/pkg/tsdb/postgres/macros_test.go +++ b/pkg/tsdb/postgres/macros_test.go @@ -85,7 +85,7 @@ func TestMacroEngine(t *testing.T) { So(sql, ShouldEqual, "select 18446744066914187038") }) - timeRange := &tsdb.TimeRange{From: "-315622800000", To: "315529200000"} // 1960-1980 + timeRange = &tsdb.TimeRange{From: "-315622800000", To: "315529200000"} // 1960-1980 Convey("interpolate __timeFilter function before epoch", func() { sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") So(err, ShouldBeNil) From f5586b1270ebd92e5fc2ebb8a22f1c1bbc186f43 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Thu, 12 Apr 2018 18:53:12 +0200 Subject: [PATCH 5/6] tsdb: sql data sources should handle time ranges before epoch start correctly --- pkg/tsdb/mssql/macros.go | 12 +- pkg/tsdb/mssql/macros_test.go | 271 +++++++++++++++++++++---------- pkg/tsdb/mysql/macros.go | 12 +- pkg/tsdb/mysql/macros_test.go | 233 ++++++++++++++++++-------- pkg/tsdb/postgres/macros.go | 12 +- pkg/tsdb/postgres/macros_test.go | 218 ++++++++++++++++++------- pkg/tsdb/time_range.go | 16 ++ 7 files changed, 546 insertions(+), 228 deletions(-) diff --git a/pkg/tsdb/mssql/macros.go b/pkg/tsdb/mssql/macros.go index 9d41cd03255..f53e06131be 100644 --- a/pkg/tsdb/mssql/macros.go +++ b/pkg/tsdb/mssql/macros.go @@ -82,11 +82,11 @@ func (m *MsSqlMacroEngine) evaluateMacro(name string, args []string) (string, er if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= DATEADD(s, %d, '1970-01-01') AND %s <= DATEADD(s, %d, '1970-01-01')", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%s >= DATEADD(s, %d, '1970-01-01') AND %s <= DATEADD(s, %d, '1970-01-01')", args[0], int64(m.TimeRange.GetFromAsSecondsEpoch()), args[0], int64(m.TimeRange.GetToAsSecondsEpoch())), nil case "__timeFrom": - return fmt.Sprintf("DATEADD(second, %d, '1970-01-01')", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil + return fmt.Sprintf("DATEADD(second, %d, '1970-01-01')", int64(m.TimeRange.GetFromAsSecondsEpoch())), nil case "__timeTo": - return fmt.Sprintf("DATEADD(second, %d, '1970-01-01')", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("DATEADD(second, %d, '1970-01-01')", int64(m.TimeRange.GetToAsSecondsEpoch())), nil case "__timeGroup": if len(args) < 2 { return "", fmt.Errorf("macro %v needs time column and interval", name) @@ -113,11 +113,11 @@ func (m *MsSqlMacroEngine) evaluateMacro(name string, args []string) (string, er if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], uint64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], int64(m.TimeRange.GetFromAsSecondsEpoch()), args[0], int64(m.TimeRange.GetToAsSecondsEpoch())), nil case "__unixEpochFrom": - return fmt.Sprintf("%d", uint64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil + return fmt.Sprintf("%d", int64(m.TimeRange.GetFromAsSecondsEpoch())), nil case "__unixEpochTo": - return fmt.Sprintf("%d", uint64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%d", int64(m.TimeRange.GetToAsSecondsEpoch())), nil default: return "", fmt.Errorf("Unknown macro %v", name) } diff --git a/pkg/tsdb/mssql/macros_test.go b/pkg/tsdb/mssql/macros_test.go index 12a9b0d82be..ae0d4f67d2b 100644 --- a/pkg/tsdb/mssql/macros_test.go +++ b/pkg/tsdb/mssql/macros_test.go @@ -1,6 +1,8 @@ package mssql import ( + "fmt" + "strconv" "testing" "time" @@ -13,112 +15,213 @@ import ( func TestMacroEngine(t *testing.T) { Convey("MacroEngine", t, func() { engine := &MsSqlMacroEngine{} - timeRange := &tsdb.TimeRange{From: "5m", To: "now"} query := &tsdb.Query{ Model: simplejson.New(), } - Convey("interpolate __time function", func() { - sql, err := engine.Interpolate(query, nil, "select $__time(time_column)") - So(err, ShouldBeNil) + Convey("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func() { + from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC) + to := from.Add(5 * time.Minute) + timeRange := tsdb.NewFakeTimeRange("5m", "now", to) - So(sql, ShouldEqual, "select time_column AS time") + Convey("interpolate __time function", func() { + sql, err := engine.Interpolate(query, nil, "select $__time(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select time_column AS time") + }) + + Convey("interpolate __timeEpoch function", func() { + sql, err := engine.Interpolate(query, nil, "select $__timeEpoch(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select DATEDIFF(second, '1970-01-01', time_column) AS time") + }) + + Convey("interpolate __timeEpoch function wrapped in aggregation", func() { + sql, err := engine.Interpolate(query, nil, "select min($__timeEpoch(time_column))") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select min(DATEDIFF(second, '1970-01-01', time_column) AS time)") + }) + + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("WHERE time_column >= DATEADD(s, %d, '1970-01-01') AND time_column <= DATEADD(s, %d, '1970-01-01')", from.Unix(), to.Unix())) + }) + + Convey("interpolate __timeGroup function", func() { + sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "GROUP BY CAST(ROUND(DATEDIFF(second, '1970-01-01', time_column)/300.0, 0) as bigint)*300") + }) + + Convey("interpolate __timeGroup function with spaces around arguments", func() { + sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column , '5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "GROUP BY CAST(ROUND(DATEDIFF(second, '1970-01-01', time_column)/300.0, 0) as bigint)*300") + }) + + Convey("interpolate __timeGroup function with fill (value = NULL)", func() { + _, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m', NULL)") + + fill := query.Model.Get("fill").MustBool() + fillNull := query.Model.Get("fillNull").MustBool() + fillInterval := query.Model.Get("fillInterval").MustInt() + + So(err, ShouldBeNil) + So(fill, ShouldBeTrue) + So(fillNull, ShouldBeTrue) + So(fillInterval, ShouldEqual, 5*time.Minute.Seconds()) + }) + + Convey("interpolate __timeGroup function with fill (value = float)", func() { + _, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m', 1.5)") + + fill := query.Model.Get("fill").MustBool() + fillValue := query.Model.Get("fillValue").MustFloat64() + fillInterval := query.Model.Get("fillInterval").MustInt() + + So(err, ShouldBeNil) + So(fill, ShouldBeTrue) + So(fillValue, ShouldEqual, 1.5) + So(fillInterval, ShouldEqual, 5*time.Minute.Seconds()) + }) + + Convey("interpolate __timeFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeFrom(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select DATEADD(second, %d, '1970-01-01')", from.Unix())) + }) + + Convey("interpolate __timeTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select DATEADD(second, %d, '1970-01-01')", to.Unix())) + }) + + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select time_column >= %d AND time_column <= %d", from.Unix(), to.Unix())) + }) + + Convey("interpolate __unixEpochFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFrom()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select %d", from.Unix())) + }) + + Convey("interpolate __unixEpochTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochTo()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select %d", to.Unix())) + }) }) - Convey("interpolate __timeEpoch function", func() { - sql, err := engine.Interpolate(query, nil, "select $__timeEpoch(time_column)") - So(err, ShouldBeNil) + Convey("Given a time range between 1960-02-01 07:00 and 1965-02-03 08:00", func() { + from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC) + to := time.Date(1965, 2, 3, 8, 0, 0, 0, time.UTC) + timeRange := tsdb.NewTimeRange(strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10), strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10)) - So(sql, ShouldEqual, "select DATEDIFF(second, '1970-01-01', time_column) AS time") + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("WHERE time_column >= DATEADD(s, %d, '1970-01-01') AND time_column <= DATEADD(s, %d, '1970-01-01')", from.Unix(), to.Unix())) + }) + + Convey("interpolate __timeFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeFrom(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select DATEADD(second, %d, '1970-01-01')", from.Unix())) + }) + + Convey("interpolate __timeTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select DATEADD(second, %d, '1970-01-01')", to.Unix())) + }) + + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select time_column >= %d AND time_column <= %d", from.Unix(), to.Unix())) + }) + + Convey("interpolate __unixEpochFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFrom()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select %d", from.Unix())) + }) + + Convey("interpolate __unixEpochTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochTo()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select %d", to.Unix())) + }) }) - Convey("interpolate __timeEpoch function wrapped in aggregation", func() { - sql, err := engine.Interpolate(query, nil, "select min($__timeEpoch(time_column))") - So(err, ShouldBeNil) + Convey("Given a time range between 1960-02-01 07:00 and 1980-02-03 08:00", func() { + from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC) + to := time.Date(1980, 2, 3, 8, 0, 0, 0, time.UTC) + timeRange := tsdb.NewTimeRange(strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10), strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10)) - So(sql, ShouldEqual, "select min(DATEDIFF(second, '1970-01-01', time_column) AS time)") - }) + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) - Convey("interpolate __timeFilter function", func() { - sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") - So(err, ShouldBeNil) + So(sql, ShouldEqual, fmt.Sprintf("WHERE time_column >= DATEADD(s, %d, '1970-01-01') AND time_column <= DATEADD(s, %d, '1970-01-01')", from.Unix(), to.Unix())) + }) - So(sql, ShouldEqual, "WHERE time_column >= DATEADD(s, 18446744066914186738, '1970-01-01') AND time_column <= DATEADD(s, 18446744066914187038, '1970-01-01')") - }) + Convey("interpolate __timeFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeFrom(time_column)") + So(err, ShouldBeNil) - Convey("interpolate __timeGroup function", func() { - sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m')") - So(err, ShouldBeNil) + So(sql, ShouldEqual, fmt.Sprintf("select DATEADD(second, %d, '1970-01-01')", from.Unix())) + }) - So(sql, ShouldEqual, "GROUP BY CAST(ROUND(DATEDIFF(second, '1970-01-01', time_column)/300.0, 0) as bigint)*300") - }) + Convey("interpolate __timeTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)") + So(err, ShouldBeNil) - Convey("interpolate __timeGroup function with spaces around arguments", func() { - sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column , '5m')") - So(err, ShouldBeNil) + So(sql, ShouldEqual, fmt.Sprintf("select DATEADD(second, %d, '1970-01-01')", to.Unix())) + }) - So(sql, ShouldEqual, "GROUP BY CAST(ROUND(DATEDIFF(second, '1970-01-01', time_column)/300.0, 0) as bigint)*300") - }) + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time_column)") + So(err, ShouldBeNil) - Convey("interpolate __timeGroup function with fill (value = NULL)", func() { - _, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m', NULL)") + So(sql, ShouldEqual, fmt.Sprintf("select time_column >= %d AND time_column <= %d", from.Unix(), to.Unix())) + }) - fill := query.Model.Get("fill").MustBool() - fillNull := query.Model.Get("fillNull").MustBool() - fillInterval := query.Model.Get("fillInterval").MustInt() + Convey("interpolate __unixEpochFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFrom()") + So(err, ShouldBeNil) - So(err, ShouldBeNil) - So(fill, ShouldBeTrue) - So(fillNull, ShouldBeTrue) - So(fillInterval, ShouldEqual, 5*time.Minute.Seconds()) - }) + So(sql, ShouldEqual, fmt.Sprintf("select %d", from.Unix())) + }) - Convey("interpolate __timeGroup function with fill (value = float)", func() { - _, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m', 1.5)") + Convey("interpolate __unixEpochTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochTo()") + So(err, ShouldBeNil) - fill := query.Model.Get("fill").MustBool() - fillValue := query.Model.Get("fillValue").MustFloat64() - fillInterval := query.Model.Get("fillInterval").MustInt() - - So(err, ShouldBeNil) - So(fill, ShouldBeTrue) - So(fillValue, ShouldEqual, 1.5) - So(fillInterval, ShouldEqual, 5*time.Minute.Seconds()) - }) - - Convey("interpolate __timeFrom function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__timeFrom(time_column)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select DATEADD(second, 18446744066914186738, '1970-01-01')") - }) - - Convey("interpolate __timeTo function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select DATEADD(second, 18446744066914187038, '1970-01-01')") - }) - - Convey("interpolate __unixEpochFilter function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time_column)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select time_column >= 18446744066914186738 AND time_column <= 18446744066914187038") - }) - - Convey("interpolate __unixEpochFrom function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFrom()") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select 18446744066914186738") - }) - - Convey("interpolate __unixEpochTo function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochTo()") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select 18446744066914187038") + So(sql, ShouldEqual, fmt.Sprintf("select %d", to.Unix())) + }) }) }) } diff --git a/pkg/tsdb/mysql/macros.go b/pkg/tsdb/mysql/macros.go index 92f6968f12a..d4a08bd6241 100644 --- a/pkg/tsdb/mysql/macros.go +++ b/pkg/tsdb/mysql/macros.go @@ -77,11 +77,11 @@ func (m *MySqlMacroEngine) evaluateMacro(name string, args []string) (string, er if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= FROM_UNIXTIME(%d) AND %s <= FROM_UNIXTIME(%d)", args[0], int64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%s >= FROM_UNIXTIME(%d) AND %s <= FROM_UNIXTIME(%d)", args[0], int64(m.TimeRange.GetFromAsSecondsEpoch()), args[0], int64(m.TimeRange.GetToAsSecondsEpoch())), nil case "__timeFrom": - return fmt.Sprintf("FROM_UNIXTIME(%d)", int64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil + return fmt.Sprintf("FROM_UNIXTIME(%d)", int64(m.TimeRange.GetFromAsSecondsEpoch())), nil case "__timeTo": - return fmt.Sprintf("FROM_UNIXTIME(%d)", int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("FROM_UNIXTIME(%d)", int64(m.TimeRange.GetToAsSecondsEpoch())), nil case "__timeGroup": if len(args) < 2 { return "", fmt.Errorf("macro %v needs time column and interval", name) @@ -108,11 +108,11 @@ func (m *MySqlMacroEngine) evaluateMacro(name string, args []string) (string, er if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], int64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], int64(m.TimeRange.GetFromAsSecondsEpoch()), args[0], int64(m.TimeRange.GetToAsSecondsEpoch())), nil case "__unixEpochFrom": - return fmt.Sprintf("%d", int64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil + return fmt.Sprintf("%d", int64(m.TimeRange.GetFromAsSecondsEpoch())), nil case "__unixEpochTo": - return fmt.Sprintf("%d", int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%d", int64(m.TimeRange.GetToAsSecondsEpoch())), nil default: return "", fmt.Errorf("Unknown macro %v", name) } diff --git a/pkg/tsdb/mysql/macros_test.go b/pkg/tsdb/mysql/macros_test.go index a89ba16ab78..66ec143eac8 100644 --- a/pkg/tsdb/mysql/macros_test.go +++ b/pkg/tsdb/mysql/macros_test.go @@ -1,7 +1,10 @@ package mysql import ( + "fmt" + "strconv" "testing" + "time" "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" @@ -11,79 +14,179 @@ func TestMacroEngine(t *testing.T) { Convey("MacroEngine", t, func() { engine := &MySqlMacroEngine{} query := &tsdb.Query{} - timeRange := &tsdb.TimeRange{From: "5m", To: "now"} - Convey("interpolate __time function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__time(time_column)") - So(err, ShouldBeNil) + Convey("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func() { + from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC) + to := from.Add(5 * time.Minute) + timeRange := tsdb.NewFakeTimeRange("5m", "now", to) - So(sql, ShouldEqual, "select UNIX_TIMESTAMP(time_column) as time_sec") + Convey("interpolate __time function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__time(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select UNIX_TIMESTAMP(time_column) as time_sec") + }) + + Convey("interpolate __time function wrapped in aggregation", func() { + sql, err := engine.Interpolate(query, timeRange, "select min($__time(time_column))") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select min(UNIX_TIMESTAMP(time_column) as time_sec)") + }) + + Convey("interpolate __timeGroup function", func() { + + sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "GROUP BY cast(cast(UNIX_TIMESTAMP(time_column)/(300) as signed)*300 as signed)") + }) + + Convey("interpolate __timeGroup function with spaces around arguments", func() { + + sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column , '5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "GROUP BY cast(cast(UNIX_TIMESTAMP(time_column)/(300) as signed)*300 as signed)") + }) + + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("WHERE time_column >= FROM_UNIXTIME(%d) AND time_column <= FROM_UNIXTIME(%d)", from.Unix(), to.Unix())) + }) + + Convey("interpolate __timeFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeFrom(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select FROM_UNIXTIME(%d)", from.Unix())) + }) + + Convey("interpolate __timeTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select FROM_UNIXTIME(%d)", to.Unix())) + }) + + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select time >= %d AND time <= %d", from.Unix(), to.Unix())) + }) + + Convey("interpolate __unixEpochFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFrom()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select %d", from.Unix())) + }) + + Convey("interpolate __unixEpochTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochTo()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select %d", to.Unix())) + }) }) - Convey("interpolate __time function wrapped in aggregation", func() { - sql, err := engine.Interpolate(query, timeRange, "select min($__time(time_column))") - So(err, ShouldBeNil) + Convey("Given a time range between 1960-02-01 07:00 and 1965-02-03 08:00", func() { + from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC) + to := time.Date(1965, 2, 3, 8, 0, 0, 0, time.UTC) + timeRange := tsdb.NewTimeRange(strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10), strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10)) - So(sql, ShouldEqual, "select min(UNIX_TIMESTAMP(time_column) as time_sec)") + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("WHERE time_column >= FROM_UNIXTIME(%d) AND time_column <= FROM_UNIXTIME(%d)", from.Unix(), to.Unix())) + }) + + Convey("interpolate __timeFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeFrom(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select FROM_UNIXTIME(%d)", from.Unix())) + }) + + Convey("interpolate __timeTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select FROM_UNIXTIME(%d)", to.Unix())) + }) + + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select time >= %d AND time <= %d", from.Unix(), to.Unix())) + }) + + Convey("interpolate __unixEpochFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFrom()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select %d", from.Unix())) + }) + + Convey("interpolate __unixEpochTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochTo()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select %d", to.Unix())) + }) }) - Convey("interpolate __timeFilter function", func() { - sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") - So(err, ShouldBeNil) + Convey("Given a time range between 1960-02-01 07:00 and 1980-02-03 08:00", func() { + from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC) + to := time.Date(1980, 2, 3, 8, 0, 0, 0, time.UTC) + timeRange := tsdb.NewTimeRange(strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10), strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10)) - So(sql, ShouldEqual, "WHERE time_column >= FROM_UNIXTIME(18446744066914186738) AND time_column <= FROM_UNIXTIME(18446744066914187038)") + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("WHERE time_column >= FROM_UNIXTIME(%d) AND time_column <= FROM_UNIXTIME(%d)", from.Unix(), to.Unix())) + }) + + Convey("interpolate __timeFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeFrom(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select FROM_UNIXTIME(%d)", from.Unix())) + }) + + Convey("interpolate __timeTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select FROM_UNIXTIME(%d)", to.Unix())) + }) + + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select time >= %d AND time <= %d", from.Unix(), to.Unix())) + }) + + Convey("interpolate __unixEpochFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFrom()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select %d", from.Unix())) + }) + + Convey("interpolate __unixEpochTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochTo()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select %d", to.Unix())) + }) }) - - Convey("interpolate __timeFrom function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__timeFrom(time_column)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select FROM_UNIXTIME(18446744066914186738)") - }) - - Convey("interpolate __timeGroup function", func() { - - sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m')") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "GROUP BY cast(cast(UNIX_TIMESTAMP(time_column)/(300) as signed)*300 as signed)") - }) - - Convey("interpolate __timeGroup function with spaces around arguments", func() { - - sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column , '5m')") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "GROUP BY cast(cast(UNIX_TIMESTAMP(time_column)/(300) as signed)*300 as signed)") - }) - - Convey("interpolate __timeTo function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select FROM_UNIXTIME(18446744066914187038)") - }) - - Convey("interpolate __unixEpochFilter function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(18446744066914186738)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select 18446744066914186738 >= 18446744066914186738 AND 18446744066914186738 <= 18446744066914187038") - }) - - Convey("interpolate __unixEpochFrom function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFrom()") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select 18446744066914186738") - }) - - Convey("interpolate __unixEpochTo function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochTo()") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select 18446744066914187038") - }) - }) } diff --git a/pkg/tsdb/postgres/macros.go b/pkg/tsdb/postgres/macros.go index f96b3896041..b28b7bd36da 100644 --- a/pkg/tsdb/postgres/macros.go +++ b/pkg/tsdb/postgres/macros.go @@ -83,11 +83,11 @@ func (m *PostgresMacroEngine) evaluateMacro(name string, args []string) (string, if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("extract(epoch from %s) BETWEEN %d AND %d", args[0], int64(m.TimeRange.GetFromAsMsEpoch()/1000), int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("extract(epoch from %s) BETWEEN %d AND %d", args[0], int64(m.TimeRange.GetFromAsSecondsEpoch()), int64(m.TimeRange.GetToAsSecondsEpoch())), nil case "__timeFrom": - return fmt.Sprintf("to_timestamp(%d)", int64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil + return fmt.Sprintf("to_timestamp(%d)", int64(m.TimeRange.GetFromAsSecondsEpoch())), nil case "__timeTo": - return fmt.Sprintf("to_timestamp(%d)", int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("to_timestamp(%d)", int64(m.TimeRange.GetToAsSecondsEpoch())), nil case "__timeGroup": if len(args) < 2 { return "", fmt.Errorf("macro %v needs time column and interval and optional fill value", name) @@ -114,11 +114,11 @@ func (m *PostgresMacroEngine) evaluateMacro(name string, args []string) (string, if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], int64(m.TimeRange.GetFromAsMsEpoch()/1000), args[0], int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], int64(m.TimeRange.GetFromAsSecondsEpoch()), args[0], int64(m.TimeRange.GetToAsSecondsEpoch())), nil case "__unixEpochFrom": - return fmt.Sprintf("%d", int64(m.TimeRange.GetFromAsMsEpoch()/1000)), nil + return fmt.Sprintf("%d", int64(m.TimeRange.GetFromAsSecondsEpoch())), nil case "__unixEpochTo": - return fmt.Sprintf("%d", int64(m.TimeRange.GetToAsMsEpoch()/1000)), nil + return fmt.Sprintf("%d", int64(m.TimeRange.GetToAsSecondsEpoch())), nil default: return "", fmt.Errorf("Unknown macro %v", name) } diff --git a/pkg/tsdb/postgres/macros_test.go b/pkg/tsdb/postgres/macros_test.go index 838a73ce240..f441690a429 100644 --- a/pkg/tsdb/postgres/macros_test.go +++ b/pkg/tsdb/postgres/macros_test.go @@ -1,7 +1,10 @@ package postgres import ( + "fmt" + "strconv" "testing" + "time" "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" @@ -11,86 +14,179 @@ func TestMacroEngine(t *testing.T) { Convey("MacroEngine", t, func() { engine := &PostgresMacroEngine{} query := &tsdb.Query{} - timeRange := &tsdb.TimeRange{From: "5m", To: "now"} - Convey("interpolate __time function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__time(time_column)") - So(err, ShouldBeNil) + Convey("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func() { + from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC) + to := from.Add(5 * time.Minute) + timeRange := tsdb.NewFakeTimeRange("5m", "now", to) - So(sql, ShouldEqual, "select time_column AS \"time\"") + Convey("interpolate __time function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__time(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select time_column AS \"time\"") + }) + + Convey("interpolate __time function wrapped in aggregation", func() { + sql, err := engine.Interpolate(query, timeRange, "select min($__time(time_column))") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "select min(time_column AS \"time\")") + }) + + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("WHERE extract(epoch from time_column) BETWEEN %d AND %d", from.Unix(), to.Unix())) + }) + + Convey("interpolate __timeFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeFrom(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select to_timestamp(%d)", from.Unix())) + }) + + Convey("interpolate __timeGroup function", func() { + + sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "GROUP BY (extract(epoch from time_column)/300)::bigint*300 AS time") + }) + + Convey("interpolate __timeGroup function with spaces between args", func() { + + sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column , '5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "GROUP BY (extract(epoch from time_column)/300)::bigint*300 AS time") + }) + + Convey("interpolate __timeTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select to_timestamp(%d)", to.Unix())) + }) + + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select time >= %d AND time <= %d", from.Unix(), to.Unix())) + }) + + Convey("interpolate __unixEpochFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFrom()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select %d", from.Unix())) + }) + + Convey("interpolate __unixEpochTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochTo()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select %d", to.Unix())) + }) }) - Convey("interpolate __time function wrapped in aggregation", func() { - sql, err := engine.Interpolate(query, timeRange, "select min($__time(time_column))") - So(err, ShouldBeNil) + Convey("Given a time range between 1960-02-01 07:00 and 1965-02-03 08:00", func() { + from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC) + to := time.Date(1965, 2, 3, 8, 0, 0, 0, time.UTC) + timeRange := tsdb.NewTimeRange(strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10), strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10)) - So(sql, ShouldEqual, "select min(time_column AS \"time\")") + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("WHERE extract(epoch from time_column) BETWEEN %d AND %d", from.Unix(), to.Unix())) + }) + + Convey("interpolate __timeFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeFrom(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select to_timestamp(%d)", from.Unix())) + }) + + Convey("interpolate __timeTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select to_timestamp(%d)", to.Unix())) + }) + + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time)") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select time >= %d AND time <= %d", from.Unix(), to.Unix())) + }) + + Convey("interpolate __unixEpochFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFrom()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select %d", from.Unix())) + }) + + Convey("interpolate __unixEpochTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochTo()") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, fmt.Sprintf("select %d", to.Unix())) + }) }) - Convey("interpolate __timeFilter function", func() { - sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") - So(err, ShouldBeNil) + Convey("Given a time range between 1960-02-01 07:00 and 1980-02-03 08:00", func() { + from := time.Date(1960, 2, 1, 7, 0, 0, 0, time.UTC) + to := time.Date(1980, 2, 3, 8, 0, 0, 0, time.UTC) + timeRange := tsdb.NewTimeRange(strconv.FormatInt(from.UnixNano()/int64(time.Millisecond), 10), strconv.FormatInt(to.UnixNano()/int64(time.Millisecond), 10)) - So(sql, ShouldEqual, "WHERE extract(epoch from time_column) BETWEEN 18446744066914186738 AND 18446744066914187038") - }) + Convey("interpolate __timeFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") + So(err, ShouldBeNil) - Convey("interpolate __timeFrom function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__timeFrom(time_column)") - So(err, ShouldBeNil) + So(sql, ShouldEqual, fmt.Sprintf("WHERE extract(epoch from time_column) BETWEEN %d AND %d", from.Unix(), to.Unix())) + }) - So(sql, ShouldEqual, "select to_timestamp(18446744066914186738)") - }) + Convey("interpolate __timeFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeFrom(time_column)") + So(err, ShouldBeNil) - Convey("interpolate __timeGroup function", func() { + So(sql, ShouldEqual, fmt.Sprintf("select to_timestamp(%d)", from.Unix())) + }) - sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column,'5m')") - So(err, ShouldBeNil) + Convey("interpolate __timeTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)") + So(err, ShouldBeNil) - So(sql, ShouldEqual, "GROUP BY (extract(epoch from time_column)/300)::bigint*300 AS time") - }) + So(sql, ShouldEqual, fmt.Sprintf("select to_timestamp(%d)", to.Unix())) + }) - Convey("interpolate __timeGroup function with spaces between args", func() { + Convey("interpolate __unixEpochFilter function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(time)") + So(err, ShouldBeNil) - sql, err := engine.Interpolate(query, timeRange, "GROUP BY $__timeGroup(time_column , '5m')") - So(err, ShouldBeNil) + So(sql, ShouldEqual, fmt.Sprintf("select time >= %d AND time <= %d", from.Unix(), to.Unix())) + }) - So(sql, ShouldEqual, "GROUP BY (extract(epoch from time_column)/300)::bigint*300 AS time") - }) + Convey("interpolate __unixEpochFrom function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFrom()") + So(err, ShouldBeNil) - Convey("interpolate __timeTo function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)") - So(err, ShouldBeNil) + So(sql, ShouldEqual, fmt.Sprintf("select %d", from.Unix())) + }) - So(sql, ShouldEqual, "select to_timestamp(18446744066914187038)") - }) + Convey("interpolate __unixEpochTo function", func() { + sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochTo()") + So(err, ShouldBeNil) - Convey("interpolate __unixEpochFilter function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFilter(18446744066914186738)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select 18446744066914186738 >= 18446744066914186738 AND 18446744066914186738 <= 18446744066914187038") - }) - - Convey("interpolate __unixEpochFrom function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochFrom()") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select 18446744066914186738") - }) - - Convey("interpolate __unixEpochTo function", func() { - sql, err := engine.Interpolate(query, timeRange, "select $__unixEpochTo()") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "select 18446744066914187038") - }) - - timeRange = &tsdb.TimeRange{From: "-315622800000", To: "315529200000"} // 1960-1980 - Convey("interpolate __timeFilter function before epoch", func() { - sql, err := engine.Interpolate(query, timeRange, "WHERE $__timeFilter(time_column)") - So(err, ShouldBeNil) - - So(sql, ShouldEqual, "WHERE extract(epoch from time_column) BETWEEN -315622800 AND 315529200") + So(sql, ShouldEqual, fmt.Sprintf("select %d", to.Unix())) + }) }) }) } diff --git a/pkg/tsdb/time_range.go b/pkg/tsdb/time_range.go index fd0cb3f8e82..5ebad362bf7 100644 --- a/pkg/tsdb/time_range.go +++ b/pkg/tsdb/time_range.go @@ -15,6 +15,14 @@ func NewTimeRange(from, to string) *TimeRange { } } +func NewFakeTimeRange(from, to string, now time.Time) *TimeRange { + return &TimeRange{ + From: from, + To: to, + now: now, + } +} + type TimeRange struct { From string To string @@ -25,10 +33,18 @@ func (tr *TimeRange) GetFromAsMsEpoch() int64 { return tr.MustGetFrom().UnixNano() / int64(time.Millisecond) } +func (tr *TimeRange) GetFromAsSecondsEpoch() int64 { + return tr.GetFromAsMsEpoch() / 1000 +} + func (tr *TimeRange) GetToAsMsEpoch() int64 { return tr.MustGetTo().UnixNano() / int64(time.Millisecond) } +func (tr *TimeRange) GetToAsSecondsEpoch() int64 { + return tr.GetToAsMsEpoch() / 1000 +} + func (tr *TimeRange) MustGetFrom() time.Time { if res, err := tr.ParseFrom(); err != nil { return time.Unix(0, 0) From a86ee304ff9d7f30f044eebdf673c13fcbc0a7b9 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Thu, 12 Apr 2018 19:08:35 +0200 Subject: [PATCH 6/6] tsdb: remove unnecessary type casts in sql data sources macro engines --- pkg/tsdb/mssql/macros.go | 12 ++++++------ pkg/tsdb/mysql/macros.go | 12 ++++++------ pkg/tsdb/postgres/macros.go | 12 ++++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/pkg/tsdb/mssql/macros.go b/pkg/tsdb/mssql/macros.go index f53e06131be..bb9489cd654 100644 --- a/pkg/tsdb/mssql/macros.go +++ b/pkg/tsdb/mssql/macros.go @@ -82,11 +82,11 @@ func (m *MsSqlMacroEngine) evaluateMacro(name string, args []string) (string, er if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= DATEADD(s, %d, '1970-01-01') AND %s <= DATEADD(s, %d, '1970-01-01')", args[0], int64(m.TimeRange.GetFromAsSecondsEpoch()), args[0], int64(m.TimeRange.GetToAsSecondsEpoch())), nil + return fmt.Sprintf("%s >= DATEADD(s, %d, '1970-01-01') AND %s <= DATEADD(s, %d, '1970-01-01')", args[0], m.TimeRange.GetFromAsSecondsEpoch(), args[0], m.TimeRange.GetToAsSecondsEpoch()), nil case "__timeFrom": - return fmt.Sprintf("DATEADD(second, %d, '1970-01-01')", int64(m.TimeRange.GetFromAsSecondsEpoch())), nil + return fmt.Sprintf("DATEADD(second, %d, '1970-01-01')", m.TimeRange.GetFromAsSecondsEpoch()), nil case "__timeTo": - return fmt.Sprintf("DATEADD(second, %d, '1970-01-01')", int64(m.TimeRange.GetToAsSecondsEpoch())), nil + return fmt.Sprintf("DATEADD(second, %d, '1970-01-01')", m.TimeRange.GetToAsSecondsEpoch()), nil case "__timeGroup": if len(args) < 2 { return "", fmt.Errorf("macro %v needs time column and interval", name) @@ -113,11 +113,11 @@ func (m *MsSqlMacroEngine) evaluateMacro(name string, args []string) (string, er if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], int64(m.TimeRange.GetFromAsSecondsEpoch()), args[0], int64(m.TimeRange.GetToAsSecondsEpoch())), nil + return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], m.TimeRange.GetFromAsSecondsEpoch(), args[0], m.TimeRange.GetToAsSecondsEpoch()), nil case "__unixEpochFrom": - return fmt.Sprintf("%d", int64(m.TimeRange.GetFromAsSecondsEpoch())), nil + return fmt.Sprintf("%d", m.TimeRange.GetFromAsSecondsEpoch()), nil case "__unixEpochTo": - return fmt.Sprintf("%d", int64(m.TimeRange.GetToAsSecondsEpoch())), nil + return fmt.Sprintf("%d", m.TimeRange.GetToAsSecondsEpoch()), nil default: return "", fmt.Errorf("Unknown macro %v", name) } diff --git a/pkg/tsdb/mysql/macros.go b/pkg/tsdb/mysql/macros.go index d4a08bd6241..fadcbe4edbc 100644 --- a/pkg/tsdb/mysql/macros.go +++ b/pkg/tsdb/mysql/macros.go @@ -77,11 +77,11 @@ func (m *MySqlMacroEngine) evaluateMacro(name string, args []string) (string, er if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= FROM_UNIXTIME(%d) AND %s <= FROM_UNIXTIME(%d)", args[0], int64(m.TimeRange.GetFromAsSecondsEpoch()), args[0], int64(m.TimeRange.GetToAsSecondsEpoch())), nil + return fmt.Sprintf("%s >= FROM_UNIXTIME(%d) AND %s <= FROM_UNIXTIME(%d)", args[0], m.TimeRange.GetFromAsSecondsEpoch(), args[0], m.TimeRange.GetToAsSecondsEpoch()), nil case "__timeFrom": - return fmt.Sprintf("FROM_UNIXTIME(%d)", int64(m.TimeRange.GetFromAsSecondsEpoch())), nil + return fmt.Sprintf("FROM_UNIXTIME(%d)", m.TimeRange.GetFromAsSecondsEpoch()), nil case "__timeTo": - return fmt.Sprintf("FROM_UNIXTIME(%d)", int64(m.TimeRange.GetToAsSecondsEpoch())), nil + return fmt.Sprintf("FROM_UNIXTIME(%d)", m.TimeRange.GetToAsSecondsEpoch()), nil case "__timeGroup": if len(args) < 2 { return "", fmt.Errorf("macro %v needs time column and interval", name) @@ -108,11 +108,11 @@ func (m *MySqlMacroEngine) evaluateMacro(name string, args []string) (string, er if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], int64(m.TimeRange.GetFromAsSecondsEpoch()), args[0], int64(m.TimeRange.GetToAsSecondsEpoch())), nil + return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], m.TimeRange.GetFromAsSecondsEpoch(), args[0], m.TimeRange.GetToAsSecondsEpoch()), nil case "__unixEpochFrom": - return fmt.Sprintf("%d", int64(m.TimeRange.GetFromAsSecondsEpoch())), nil + return fmt.Sprintf("%d", m.TimeRange.GetFromAsSecondsEpoch()), nil case "__unixEpochTo": - return fmt.Sprintf("%d", int64(m.TimeRange.GetToAsSecondsEpoch())), nil + return fmt.Sprintf("%d", m.TimeRange.GetToAsSecondsEpoch()), nil default: return "", fmt.Errorf("Unknown macro %v", name) } diff --git a/pkg/tsdb/postgres/macros.go b/pkg/tsdb/postgres/macros.go index b28b7bd36da..bd0ac0cc620 100644 --- a/pkg/tsdb/postgres/macros.go +++ b/pkg/tsdb/postgres/macros.go @@ -83,11 +83,11 @@ func (m *PostgresMacroEngine) evaluateMacro(name string, args []string) (string, if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("extract(epoch from %s) BETWEEN %d AND %d", args[0], int64(m.TimeRange.GetFromAsSecondsEpoch()), int64(m.TimeRange.GetToAsSecondsEpoch())), nil + return fmt.Sprintf("extract(epoch from %s) BETWEEN %d AND %d", args[0], m.TimeRange.GetFromAsSecondsEpoch(), m.TimeRange.GetToAsSecondsEpoch()), nil case "__timeFrom": - return fmt.Sprintf("to_timestamp(%d)", int64(m.TimeRange.GetFromAsSecondsEpoch())), nil + return fmt.Sprintf("to_timestamp(%d)", m.TimeRange.GetFromAsSecondsEpoch()), nil case "__timeTo": - return fmt.Sprintf("to_timestamp(%d)", int64(m.TimeRange.GetToAsSecondsEpoch())), nil + return fmt.Sprintf("to_timestamp(%d)", m.TimeRange.GetToAsSecondsEpoch()), nil case "__timeGroup": if len(args) < 2 { return "", fmt.Errorf("macro %v needs time column and interval and optional fill value", name) @@ -114,11 +114,11 @@ func (m *PostgresMacroEngine) evaluateMacro(name string, args []string) (string, if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) } - return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], int64(m.TimeRange.GetFromAsSecondsEpoch()), args[0], int64(m.TimeRange.GetToAsSecondsEpoch())), nil + return fmt.Sprintf("%s >= %d AND %s <= %d", args[0], m.TimeRange.GetFromAsSecondsEpoch(), args[0], m.TimeRange.GetToAsSecondsEpoch()), nil case "__unixEpochFrom": - return fmt.Sprintf("%d", int64(m.TimeRange.GetFromAsSecondsEpoch())), nil + return fmt.Sprintf("%d", m.TimeRange.GetFromAsSecondsEpoch()), nil case "__unixEpochTo": - return fmt.Sprintf("%d", int64(m.TimeRange.GetToAsSecondsEpoch())), nil + return fmt.Sprintf("%d", m.TimeRange.GetToAsSecondsEpoch()), nil default: return "", fmt.Errorf("Unknown macro %v", name) }