From b987aee7cffe873a01380b4ed6dbf60838f97736 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 21 Jul 2018 20:00:26 +0200 Subject: [PATCH] add timescaledb option to postgres datasource This adds an option to the postgres datasource config for timescaledb support. When set to auto it will check for timescaledb when testing the datasource. When this option is enabled the $__timeGroup macro will use the time_bucket function from timescaledb to group times by an interval. This also passes the datasource edit control to testDatasource to allow for setting additional settings, this might be useful for other datasources aswell which have optional or version dependant features which can be queried. --- pkg/tsdb/postgres/macros.go | 8 +++- pkg/tsdb/postgres/macros_test.go | 22 +++++++++- pkg/tsdb/postgres/postgres_test.go | 23 ++++++++++- public/app/features/plugins/ds_edit_ctrl.ts | 2 +- .../plugins/datasource/postgres/datasource.ts | 40 +++++++++---------- .../datasource/postgres/partials/config.html | 14 +++++++ 6 files changed, 85 insertions(+), 24 deletions(-) diff --git a/pkg/tsdb/postgres/macros.go b/pkg/tsdb/postgres/macros.go index aebdc55d1d7..4f1d3f72558 100644 --- a/pkg/tsdb/postgres/macros.go +++ b/pkg/tsdb/postgres/macros.go @@ -130,13 +130,19 @@ func (m *postgresMacroEngine) evaluateMacro(name string, args []string) (string, m.query.Model.Set("fillValue", floatVal) } } - return fmt.Sprintf("floor(extract(epoch from %s)/%v)*%v", args[0], interval.Seconds(), interval.Seconds()), nil + + if m.query.DataSource.JsonData.Get("timescaledb").MustString("auto") == "enabled" { + return fmt.Sprintf("time_bucket('%vs',%s) AS time", interval.Seconds(), args[0]), nil + } else { + return fmt.Sprintf("floor(extract(epoch from %s)/%v)*%v AS time", args[0], interval.Seconds(), interval.Seconds()), nil + } case "__timeGroupAlias": tg, err := m.evaluateMacro("__timeGroup", args) if err == nil { return tg + " AS \"time\"", err } return "", err + case "__unixEpochFilter": if len(args) == 0 { return "", fmt.Errorf("missing time column argument for macro %v", name) diff --git a/pkg/tsdb/postgres/macros_test.go b/pkg/tsdb/postgres/macros_test.go index beeea93893b..6c4ba8305b1 100644 --- a/pkg/tsdb/postgres/macros_test.go +++ b/pkg/tsdb/postgres/macros_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tsdb" . "github.com/smartystreets/goconvey/convey" ) @@ -13,7 +15,9 @@ import ( func TestMacroEngine(t *testing.T) { Convey("MacroEngine", t, func() { engine := newPostgresMacroEngine() - query := &tsdb.Query{} + query := &tsdb.Query{DataSource: &models.DataSource{JsonData: simplejson.New()}} + queryTS := &tsdb.Query{DataSource: &models.DataSource{JsonData: simplejson.New()}} + queryTS.DataSource.JsonData.Set("timescaledb", "enabled") Convey("Given a time range between 2018-04-12 00:00 and 2018-04-12 00:05", func() { from := time.Date(2018, 4, 12, 18, 0, 0, 0, time.UTC) @@ -83,6 +87,22 @@ func TestMacroEngine(t *testing.T) { So(sql2, ShouldEqual, sql+" AS \"time\"") }) + Convey("interpolate __timeGroup function with TimescaleDB enabled", func() { + + sql, err := engine.Interpolate(queryTS, timeRange, "GROUP BY $__timeGroup(time_column,'5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "GROUP BY time_bucket('300s',time_column) AS time") + }) + + Convey("interpolate __timeGroup function with spaces between args and TimescaleDB enabled", func() { + + sql, err := engine.Interpolate(queryTS, timeRange, "GROUP BY $__timeGroup(time_column , '5m')") + So(err, ShouldBeNil) + + So(sql, ShouldEqual, "GROUP BY time_bucket('300s',time_column) AS time") + }) + Convey("interpolate __timeTo function", func() { sql, err := engine.Interpolate(query, timeRange, "select $__timeTo(time_column)") So(err, ShouldBeNil) diff --git a/pkg/tsdb/postgres/postgres_test.go b/pkg/tsdb/postgres/postgres_test.go index 9e363529df1..27888b318a9 100644 --- a/pkg/tsdb/postgres/postgres_test.go +++ b/pkg/tsdb/postgres/postgres_test.go @@ -27,7 +27,7 @@ import ( // use to verify that the generated data are vizualized as expected, see // devenv/README.md for setup instructions. func TestPostgres(t *testing.T) { - // change to true to run the MySQL tests + // change to true to run the PostgreSQL tests runPostgresTests := false // runPostgresTests := true @@ -102,6 +102,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": "SELECT * FROM postgres_types", "format": "table", @@ -182,6 +183,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": "SELECT $__timeGroup(time, '5m') AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", "format": "time_series", @@ -226,6 +228,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": "SELECT $__timeGroup(time, '5m', NULL) AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", "format": "time_series", @@ -280,6 +283,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": "SELECT $__timeGroup(time, '5m', 1.5) AS time, avg(value) as value FROM metric GROUP BY 1 ORDER BY 1", "format": "time_series", @@ -401,6 +405,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeInt64" as time, "timeInt64" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -423,6 +428,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeInt64Nullable" as time, "timeInt64Nullable" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -445,6 +451,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeFloat64" as time, "timeFloat64" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -467,6 +474,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeFloat64Nullable" as time, "timeFloat64Nullable" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -511,6 +519,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeInt32Nullable" as time, "timeInt32Nullable" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -533,6 +542,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeFloat32" as time, "timeFloat32" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -555,6 +565,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "timeFloat32Nullable" as time, "timeFloat32Nullable" FROM metric_values ORDER BY time LIMIT 1`, "format": "time_series", @@ -577,6 +588,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT $__timeEpoch(time), measurement || ' - value one' as metric, "valueOne" FROM metric_values ORDER BY 1`, "format": "time_series", @@ -625,6 +637,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT $__timeEpoch(time), "valueOne", "valueTwo" FROM metric_values ORDER BY 1`, "format": "time_series", @@ -682,6 +695,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "time_sec" as time, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='deploy' ORDER BY 1 ASC`, "format": "table", @@ -705,6 +719,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT "time_sec" as time, description as text, tags FROM event WHERE $__unixEpochFilter(time_sec) AND tags='ticket' ORDER BY 1 ASC`, "format": "table", @@ -731,6 +746,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": fmt.Sprintf(`SELECT CAST('%s' AS TIMESTAMP) as time, @@ -761,6 +777,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": fmt.Sprintf(`SELECT %d as time, @@ -791,6 +808,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": fmt.Sprintf(`SELECT cast(%d as bigint) as time, @@ -821,6 +839,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": fmt.Sprintf(`SELECT %d as time, @@ -849,6 +868,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT cast(null as bigint) as time, @@ -877,6 +897,7 @@ func TestPostgres(t *testing.T) { query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { + DataSource: &models.DataSource{JsonData: simplejson.New()}, Model: simplejson.NewFromAny(map[string]interface{}{ "rawSql": `SELECT cast(null as timestamp) as time, diff --git a/public/app/features/plugins/ds_edit_ctrl.ts b/public/app/features/plugins/ds_edit_ctrl.ts index 542e9cc3648..6e05ddc36be 100644 --- a/public/app/features/plugins/ds_edit_ctrl.ts +++ b/public/app/features/plugins/ds_edit_ctrl.ts @@ -132,7 +132,7 @@ export class DataSourceEditCtrl { this.backendSrv .withNoBackendCache(() => { return datasource - .testDatasource() + .testDatasource(this) .then(result => { this.testing.message = result.message; this.testing.status = result.status; diff --git a/public/app/plugins/datasource/postgres/datasource.ts b/public/app/plugins/datasource/postgres/datasource.ts index 644c9e48b9b..88c928e425a 100644 --- a/public/app/plugins/datasource/postgres/datasource.ts +++ b/public/app/plugins/datasource/postgres/datasource.ts @@ -123,27 +123,27 @@ export class PostgresDatasource { .then(data => this.responseParser.parseMetricFindQueryResult(refId, data)); } - testDatasource() { - return this.backendSrv - .datasourceRequest({ - url: '/api/tsdb/query', - method: 'POST', - data: { - from: '5m', - to: 'now', - queries: [ - { - refId: 'A', - intervalMs: 1, - maxDataPoints: 1, - datasourceId: this.id, - rawSql: 'SELECT 1', - format: 'table', - }, - ], - }, - }) + testDatasource(control) { + return this.metricFindQuery('SELECT 1', {}) .then(res => { + if (control.current.jsonData.timescaledb === 'auto') { + return this.metricFindQuery("SELECT 1 FROM pg_extension WHERE extname='timescaledb'", {}) + .then(res => { + if (res.length === 1) { + control.current.jsonData.timescaledb = 'enabled'; + return this.backendSrv.put('/api/datasources/' + this.id, control.current).then(settings => { + control.current = settings.datasource; + control.updateFrontendSettings(); + return { status: 'success', message: 'Database Connection OK, TimescaleDB found' }; + }); + } + throw new Error('timescaledb not found'); + }) + .catch(err => { + // query errored out or empty so timescaledb is not available + return { status: 'success', message: 'Database Connection OK' }; + }); + } return { status: 'success', message: 'Database Connection OK' }; }) .catch(err => { diff --git a/public/app/plugins/datasource/postgres/partials/config.html b/public/app/plugins/datasource/postgres/partials/config.html index 77f0dcfa4a5..07568fdc459 100644 --- a/public/app/plugins/datasource/postgres/partials/config.html +++ b/public/app/plugins/datasource/postgres/partials/config.html @@ -38,6 +38,20 @@ +

PostgreSQL details

+ +
+
+ +
+ + + This option determines whether TimescaleDB features will be used. + +
+
+
+
User Permission