From 33ac22bfdb53eeb7655966a1aed469a8a6f62a7b Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 21 Jan 2018 22:08:18 +0100 Subject: [PATCH 001/310] start query builder ui --- .../postgres/partials/query.editor.html | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 163970a9ad5..635c3e6f222 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -1,10 +1,23 @@ - -
-
- - -
-
+ + +
+
+
+ + +
+
+
+ +
+
+
+ + + +
+
+
From 17be31e2167ef92af57884a2bb9975a33f7968fb Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 28 Jan 2018 22:16:51 +0100 Subject: [PATCH 002/310] call render in query --- .../app/plugins/datasource/postgres/datasource.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/public/app/plugins/datasource/postgres/datasource.ts b/public/app/plugins/datasource/postgres/datasource.ts index 8eee389d1a5..3a4bd27bb4c 100644 --- a/public/app/plugins/datasource/postgres/datasource.ts +++ b/public/app/plugins/datasource/postgres/datasource.ts @@ -1,5 +1,6 @@ import _ from 'lodash'; import ResponseParser from './response_parser'; +import PostgresQuery from 'app/plugins/datasource/postgres/postgres_query'; export class PostgresDatasource { id: any; @@ -33,16 +34,18 @@ export class PostgresDatasource { } query(options) { - var queries = _.filter(options.targets, item => { - return item.hide !== true; - }).map(item => { + var queries = _.filter(options.targets, target => { + return target.hide !== true; + }).map(target => { + var queryModel = new PostgresQuery(target, this.templateSrv, options.scopedVars); + return { - refId: item.refId, + refId: target.refId, intervalMs: options.intervalMs, maxDataPoints: options.maxDataPoints, datasourceId: this.id, - rawSql: this.templateSrv.replace(item.rawSql, options.scopedVars, this.interpolateVariable), - format: item.format, + rawSql: queryModel.render(this.interpolateVariable), + format: target.format, }; }); From a59e052a0f9ac9aab7467d49dd5586de0d08e124 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 30 Jan 2018 14:08:10 +0100 Subject: [PATCH 003/310] more query builder components --- .../postgres/partials/query.editor.html | 94 +++++++++++++------ 1 file changed, 64 insertions(+), 30 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 635c3e6f222..400855cf82f 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -1,12 +1,12 @@
-
-
- - -
-
+
+
+ + +
+
@@ -15,42 +15,76 @@ +
+ +
+
+ +
+ +
+ + +
+ +
+ +
+ +
+
+
+
+ +
+
+ + + +
+
+
- -
- -
-
-
+ +
+ +
+
+
-
-
+
+
-
-
-
-
-
+ +
+
+
+ -
- 
{{ctrl.lastQueryMeta.sql}}
-
+
+ 
{{ctrl.lastQueryMeta.sql}}
+
-
- 
Time series:
+  

+    
Time series:
 - return column named time (UTC in seconds or timestamp)
 - return column(s) with numeric datatype as values
 - (Optional: return column named metric to represent the series name. If no column named metric is found the column name of the value column is used as series name)
@@ -78,13 +112,13 @@ Or build your own conditionals using these macros which just return the values:
 - $__timeTo() ->  to_timestamp(1492750877)
 - $__unixEpochFrom() ->  1492750877
 - $__unixEpochTo() ->  1492750877
-		

-	

+
+
- + -
- 
{{ctrl.lastQueryError}}
-
+
+ 
{{ctrl.lastQueryError}}
+
From 438b10bcd68abd115dbe86acad1a70a53a791c79 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 30 Jan 2018 14:10:38 +0100 Subject: [PATCH 004/310] query builder changes --- .../plugins/datasource/postgres/query_ctrl.ts | 179 ++++++++++++++++-- 1 file changed, 168 insertions(+), 11 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 7afd0cf7253..4e39105370b 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -1,12 +1,7 @@ import _ from 'lodash'; import { QueryCtrl } from 'app/plugins/sdk'; - -export interface PostgresQuery { - refId: string; - format: string; - alias: string; - rawSql: string; -} +import queryPart from './query_part'; +import PostgresQuery from './postgres_query'; export interface QueryMeta { sql: string; @@ -26,17 +21,21 @@ export class PostgresQueryCtrl extends QueryCtrl { showLastQuerySQL: boolean; formats: any[]; - target: PostgresQuery; + queryModel: PostgresQuery; lastQueryMeta: QueryMeta; lastQueryError: string; showHelp: boolean; + schemaSegment: any; + tableSegment: any; + timeColumnSegment: any; + selectMenu: any; /** @ngInject **/ - constructor($scope, $injector) { + constructor($scope, $injector, private templateSrv, private $q, private uiSegmentSrv) { super($scope, $injector); + this.target = this.target; + this.queryModel = new PostgresQuery(this.target, templateSrv, this.panel.scopedVars); - this.target.format = this.target.format || 'time_series'; - this.target.alias = ''; this.formats = [{ text: 'Time series', value: 'time_series' }, { text: 'Table', value: 'table' }]; if (!this.target.rawSql) { @@ -49,10 +48,104 @@ export class PostgresQueryCtrl extends QueryCtrl { } } + this.schemaSegment= uiSegmentSrv.newSegment(this.target.schema); + + if (!this.target.table) { + this.tableSegment = uiSegmentSrv.newSegment({value: 'select table',fake: true}); + } else { + this.tableSegment= uiSegmentSrv.newSegment(this.target.table); + } + + this.timeColumnSegment = uiSegmentSrv.newSegment(this.target.timeColumn); + + this.buildSelectMenu(); this.panelCtrl.events.on('data-received', this.onDataReceived.bind(this), $scope); this.panelCtrl.events.on('data-error', this.onDataError.bind(this), $scope); } + buildSelectMenu() { + var categories = queryPart.getCategories(); + this.selectMenu = _.reduce( + categories, + function(memo, cat, key) { + var menu = { + text: key, + submenu: cat.map(item => { + return { text: item.type, value: item.type }; + }), + }; + memo.push(menu); + return memo; + }, + [] + ); + } + + toggleEditorMode() { + try { +// this.target.query = this.queryModel.render(false); + } catch (err) { + console.log('query render error'); + } + this.target.rawQuery = !this.target.rawQuery; + } + + getSchemaSegments() { + var schemaQuery = "SELECT schema_name FROM information_schema.schemata WHERE"; + schemaQuery += " schema_name NOT LIKE 'pg_%' AND schema_name <> 'information_schema';"; + return this.datasource + .metricFindQuery(schemaQuery) + .then(this.transformToSegments(true)) + .catch(this.handleQueryError.bind(this)); + } + + getTableSegments() { + var tableQuery = "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + this.target.schema + "';"; + return this.datasource + .metricFindQuery(tableQuery) + .then(this.transformToSegments(true)) + .catch(this.handleQueryError.bind(this)); + } + + getTimeColumnSegments() { + var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; + columnQuery += " table_schema = '" + this.target.schema + "'"; + columnQuery += " AND table_name = '" + this.target.table + "'"; + columnQuery += " AND data_type IN ('timestamp without time zone','timestamp with time zone','bigint','integer','double precision','real');"; + + return this.datasource + .metricFindQuery(columnQuery) + .then(this.transformToSegments(true)) + .catch(this.handleQueryError.bind(this)); + } + + getColumnSegments() { + var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; + columnQuery += " table_schema = '" + this.target.schema + "'"; + columnQuery += " AND table_name = '" + this.target.table + "'"; + columnQuery += " AND data_type IN ('bigint','integer','double precision','real');"; + + return this.datasource + .metricFindQuery(columnQuery) + .then(this.transformToSegments(true)) + .catch(this.handleQueryError.bind(this)); + } + + tableChanged() { + this.target.table = this.tableSegment.value; + this.panelCtrl.refresh(); + } + + schemaChanged() { + this.target.schema = this.schemaSegment.value; + this.panelCtrl.refresh(); + } + + timeColumnChanged() { + this.target.time = this.timeColumnSegment.value; + this.panelCtrl.refresh(); + } + onDataReceived(dataList) { this.lastQueryMeta = null; this.lastQueryError = null; @@ -72,4 +165,68 @@ export class PostgresQueryCtrl extends QueryCtrl { } } } + + transformToSegments(addTemplateVars) { + return results => { + var segments = _.map(results, segment => { + return this.uiSegmentSrv.newSegment({ + value: segment.text, + expandable: segment.expandable, + }); + }); + + if (addTemplateVars) { + for (let variable of this.templateSrv.variables) { + segments.unshift( + this.uiSegmentSrv.newSegment({ + type: 'template', + value: '/^$' + variable.name + '$/', + expandable: true, + }) + ); + } + } + + return segments; + }; + } + + addSelectPart(selectParts, cat, subitem) { + this.queryModel.addSelectPart(selectParts, subitem.value); + this.panelCtrl.refresh(); + } + + handleSelectPartEvent(selectParts, part, evt) { + switch (evt.name) { + case 'get-param-options': { + var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; + columnQuery += " table_schema = '" + this.target.schema + "'"; + columnQuery += " AND table_name = '" + this.target.table + "'"; + columnQuery += " AND data_type IN ('bigint','integer','double precision','real');"; + + return this.datasource + .metricFindQuery(columnQuery) + .then(this.transformToSegments(true)) + .catch(this.handleQueryError.bind(this)); + } + case 'part-param-changed': { + this.panelCtrl.refresh(); + break; + } + case 'action': { + this.queryModel.removeSelectPart(selectParts, part); + this.panelCtrl.refresh(); + break; + } + case 'get-part-actions': { + return this.$q.when([{ text: 'Remove', value: 'remove-part' }]); + } + } + } + + handleQueryError(err) { + this.error = err.message || 'Failed to issue metric query'; + return []; + } + } From 443504517a1bcd58c533feef202530248a6d7050 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 30 Jan 2018 22:13:55 +0100 Subject: [PATCH 005/310] add postgres_query.ts --- .../datasource/postgres/postgres_query.ts | 239 ++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 public/app/plugins/datasource/postgres/postgres_query.ts diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts new file mode 100644 index 00000000000..3424ae24c78 --- /dev/null +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -0,0 +1,239 @@ +import _ from 'lodash'; +import queryPart from './query_part'; +import kbn from 'app/core/utils/kbn'; + +export default class PostgresQuery { + target: any; + selectModels: any[]; + queryBuilder: any; + groupByParts: any; + templateSrv: any; + scopedVars: any; + + /** @ngInject */ + constructor(target, templateSrv?, scopedVars?) { + this.target = target; + this.templateSrv = templateSrv; + this.scopedVars = scopedVars; + + target.schema = target.schema || 'public'; + target.format = target.format || 'time_series'; + target.timeColumn = target.timeColumn || 'time'; + target.alias = ''; + + target.orderByTime = target.orderByTime || 'ASC'; +// target.groupBy = target.groupBy || [{ type: 'time', params: ['$__interval'] }, { type: 'fill', params: ['null'] }]; + target.select = target.select || [[{ type: 'field', params: ['value'] }]]; + + this.updateProjection(); + } + + updateProjection() { + this.selectModels = _.map(this.target.select, function(parts: any) { + return _.map(parts, queryPart.create); + }); + this.groupByParts = _.map(this.target.groupBy, queryPart.create); + } + + updatePersistedParts() { + this.target.select = _.map(this.selectModels, function(selectParts) { + return _.map(selectParts, function(part: any) { + return { type: part.def.type, params: part.params }; + }); + }); + } + + hasGroupByTime() { + return _.find(this.target.groupBy, (g: any) => g.type === 'time'); + } + + hasFill() { + return _.find(this.target.groupBy, (g: any) => g.type === 'fill'); + } + + addGroupBy(value) { + var stringParts = value.match(/^(\w+)\((.*)\)$/); + var typePart = stringParts[1]; + var arg = stringParts[2]; + var partModel = queryPart.create({ type: typePart, params: [arg] }); + var partCount = this.target.groupBy.length; + + if (partCount === 0) { + this.target.groupBy.push(partModel.part); + } else if (typePart === 'time') { + this.target.groupBy.splice(0, 0, partModel.part); + } else if (typePart === 'tag') { + if (this.target.groupBy[partCount - 1].type === 'fill') { + this.target.groupBy.splice(partCount - 1, 0, partModel.part); + } else { + this.target.groupBy.push(partModel.part); + } + } else { + this.target.groupBy.push(partModel.part); + } + + this.updateProjection(); + } + + removeGroupByPart(part, index) { + var categories = queryPart.getCategories(); + + if (part.def.type === 'time') { + // remove fill + this.target.groupBy = _.filter(this.target.groupBy, (g: any) => g.type !== 'fill'); + // remove aggregations + this.target.select = _.map(this.target.select, (s: any) => { + return _.filter(s, (part: any) => { + var partModel = queryPart.create(part); + if (partModel.def.category === categories.Aggregations) { + return false; + } + if (partModel.def.category === categories.Selectors) { + return false; + } + return true; + }); + }); + } + + this.target.groupBy.splice(index, 1); + this.updateProjection(); + } + + removeSelect(index: number) { + this.target.select.splice(index, 1); + this.updateProjection(); + } + + removeSelectPart(selectParts, part) { + // if we remove the field remove the whole statement + if (part.def.type === 'field') { + if (this.selectModels.length > 1) { + var modelsIndex = _.indexOf(this.selectModels, selectParts); + this.selectModels.splice(modelsIndex, 1); + } + } else { + var partIndex = _.indexOf(selectParts, part); + selectParts.splice(partIndex, 1); + } + + this.updatePersistedParts(); + } + + addSelectPart(selectParts, type) { + var partModel = queryPart.create({ type: type }); + partModel.def.addStrategy(selectParts, partModel, this); + this.updatePersistedParts(); + } + + private renderTagCondition(tag, index, interpolate) { + var str = ''; + var operator = tag.operator; + var value = tag.value; + if (index > 0) { + str = (tag.condition || 'AND') + ' '; + } + + if (!operator) { + if (/^\/.*\/$/.test(value)) { + operator = '=~'; + } else { + operator = '='; + } + } + + // quote value unless regex + if (operator !== '=~' && operator !== '!~') { + if (interpolate) { + value = this.templateSrv.replace(value, this.scopedVars); + } + if (operator !== '>' && operator !== '<') { + value = "'" + value.replace(/\\/g, '\\\\') + "'"; + } + } else if (interpolate) { + value = this.templateSrv.replace(value, this.scopedVars, 'regex'); + } + + return str + '"' + tag.key + '" ' + operator + ' ' + value; + } + + interpolateQueryStr(value, variable, defaultFormatFn) { + // if no multi or include all do not regexEscape + if (!variable.multi && !variable.includeAll) { + return value; + } + + if (typeof value === 'string') { + return kbn.regexEscape(value); + } + + var escapedValues = _.map(value, kbn.regexEscape); + return '(' + escapedValues.join('|') + ')'; + } + + render(interpolate?) { + var target = this.target; + + if (target.rawQuery) { + if (interpolate) { + return this.templateSrv.replace(target.rawSql, this.scopedVars, this.interpolateQueryStr); + } else { + return target.rawSql; + } + } + + var query = 'SELECT '; + query += target.timeColumn + ' AS time,'; + + var i, y; + for (i = 0; i < this.selectModels.length; i++) { + let parts = this.selectModels[i]; + var selectText = ''; + for (y = 0; y < parts.length; y++) { + let part = parts[y]; + selectText = part.render(selectText); + } + + if (i > 0) { + query += ', '; + } + query += selectText; + } + + query += ' FROM ' + target.schema + '.' + target.table + ' WHERE '; + var conditions = _.map(target.tags, (tag, index) => { + return this.renderTagCondition(tag, index, interpolate); + }); + + if (conditions.length > 0) { + query += '(' + conditions.join(' ') + ') AND '; + } + + query += '$__timeFilter(time)'; + + var groupBySection = ''; + for (i = 0; i < this.groupByParts.length; i++) { + var part = this.groupByParts[i]; + if (i > 0) { + // for some reason fill has no seperator + groupBySection += part.def.type === 'fill' ? ' ' : ', '; + } + groupBySection += part.render(''); + } + + if (groupBySection.length) { + query += ' GROUP BY ' + groupBySection; + } + + query += ' ORDER BY time'; + + return query; + } + + renderAdhocFilters(filters) { + var conditions = _.map(filters, (tag, index) => { + return this.renderTagCondition(tag, index, false); + }); + return conditions.join(' '); + } +} From 571ecdc740b87aad130ce05bd827135fd2e570d4 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 30 Jan 2018 22:57:38 +0100 Subject: [PATCH 006/310] enhance render function --- .../datasource/postgres/postgres_query.ts | 8 +- .../plugins/datasource/postgres/query_part.ts | 380 ++++++++++++++++++ 2 files changed, 386 insertions(+), 2 deletions(-) create mode 100644 public/app/plugins/datasource/postgres/query_part.ts diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 3424ae24c78..e6d84306a9b 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -28,6 +28,10 @@ export default class PostgresQuery { this.updateProjection(); } + quoteIdentifier(field) { + return '"' + field + '"'; + } + updateProjection() { this.selectModels = _.map(this.target.select, function(parts: any) { return _.map(parts, queryPart.create); @@ -183,7 +187,7 @@ export default class PostgresQuery { } var query = 'SELECT '; - query += target.timeColumn + ' AS time,'; + query += this.quoteIdentifier(target.timeColumn) + ' AS time,'; var i, y; for (i = 0; i < this.selectModels.length; i++) { @@ -209,7 +213,7 @@ export default class PostgresQuery { query += '(' + conditions.join(' ') + ') AND '; } - query += '$__timeFilter(time)'; + query += '$__timeFilter(' + this.quoteIdentifier(target.timeColumn) + ')'; var groupBySection = ''; for (i = 0; i < this.groupByParts.length; i++) { diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts new file mode 100644 index 00000000000..dffea15dbd1 --- /dev/null +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -0,0 +1,380 @@ +import _ from 'lodash'; +import { QueryPartDef, QueryPart, functionRenderer, suffixRenderer } from 'app/core/components/query_part/query_part'; + +var index = []; +var categories = { + Aggregations: [], + Selectors: [], + Transformations: [], + Predictors: [], + Math: [], + Aliasing: [], + Fields: [], +}; + +function createPart(part): any { + var def = index[part.type]; + if (!def) { + throw { message: 'Could not find query part ' + part.type }; + } + + return new QueryPart(part, def); +} + +function register(options: any) { + index[options.type] = new QueryPartDef(options); + options.category.push(index[options.type]); +} + +var groupByTimeFunctions = []; + +function aliasRenderer(part, innerExpr) { + return innerExpr + ' AS ' + '"' + part.params[0] + '"'; +} + +function fieldRenderer(part, innerExpr) { + return '"' + part.params[0] + '"'; +} + +function replaceAggregationAddStrategy(selectParts, partModel) { + // look for existing aggregation + for (var i = 0; i < selectParts.length; i++) { + var part = selectParts[i]; + if (part.def.category === categories.Aggregations) { + selectParts[i] = partModel; + return; + } + if (part.def.category === categories.Selectors) { + selectParts[i] = partModel; + return; + } + } + + selectParts.splice(1, 0, partModel); +} + +function addTransformationStrategy(selectParts, partModel) { + var i; + // look for index to add transformation + for (i = 0; i < selectParts.length; i++) { + var part = selectParts[i]; + if (part.def.category === categories.Math || part.def.category === categories.Aliasing) { + break; + } + } + + selectParts.splice(i, 0, partModel); +} + +function addMathStrategy(selectParts, partModel) { + var partCount = selectParts.length; + if (partCount > 0) { + // if last is math, replace it + if (selectParts[partCount - 1].def.type === 'math') { + selectParts[partCount - 1] = partModel; + return; + } + // if next to last is math, replace it + if (partCount > 1 && selectParts[partCount - 2].def.type === 'math') { + selectParts[partCount - 2] = partModel; + return; + } else if (selectParts[partCount - 1].def.type === 'alias') { + // if last is alias add it before + selectParts.splice(partCount - 1, 0, partModel); + return; + } + } + selectParts.push(partModel); +} + +function addAliasStrategy(selectParts, partModel) { + var partCount = selectParts.length; + if (partCount > 0) { + // if last is alias, replace it + if (selectParts[partCount - 1].def.type === 'alias') { + selectParts[partCount - 1] = partModel; + return; + } + } + selectParts.push(partModel); +} + +function addFieldStrategy(selectParts, partModel, query) { + // copy all parts + var parts = _.map(selectParts, function(part: any) { + return createPart({ type: part.def.type, params: _.clone(part.params) }); + }); + + query.selectModels.push(parts); +} + +register({ + type: 'field', + addStrategy: addFieldStrategy, + category: categories.Fields, + params: [{ type: 'field', dynamicLookup: true }], + defaultParams: ['value'], + renderer: fieldRenderer, +}); + +// Aggregations +register({ + type: 'avg', + addStrategy: replaceAggregationAddStrategy, + category: categories.Aggregations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'count', + addStrategy: replaceAggregationAddStrategy, + category: categories.Aggregations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'sum', + addStrategy: replaceAggregationAddStrategy, + category: categories.Aggregations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +// transformations + +register({ + type: 'derivative', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [ + { + name: 'duration', + type: 'interval', + options: ['1s', '10s', '1m', '5m', '10m', '15m', '1h'], + }, + ], + defaultParams: ['10s'], + renderer: functionRenderer, +}); + +register({ + type: 'spread', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'non_negative_derivative', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [ + { + name: 'duration', + type: 'interval', + options: ['1s', '10s', '1m', '5m', '10m', '15m', '1h'], + }, + ], + defaultParams: ['10s'], + renderer: functionRenderer, +}); + +register({ + type: 'difference', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'non_negative_difference', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'moving_average', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [{ name: 'window', type: 'int', options: [5, 10, 20, 30, 40] }], + defaultParams: [10], + renderer: functionRenderer, +}); + +register({ + type: 'cumulative_sum', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'stddev', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'time', + category: groupByTimeFunctions, + params: [ + { + name: 'interval', + type: 'time', + options: ['$__interval', '1s', '10s', '1m', '5m', '10m', '15m', '1h'], + }, + ], + defaultParams: ['$__interval'], + renderer: functionRenderer, +}); + +register({ + type: 'fill', + category: groupByTimeFunctions, + params: [ + { + name: 'fill', + type: 'string', + options: ['none', 'null', '0', 'previous', 'linear'], + }, + ], + defaultParams: ['null'], + renderer: functionRenderer, +}); + +register({ + type: 'elapsed', + addStrategy: addTransformationStrategy, + category: categories.Transformations, + params: [ + { + name: 'duration', + type: 'interval', + options: ['1s', '10s', '1m', '5m', '10m', '15m', '1h'], + }, + ], + defaultParams: ['10s'], + renderer: functionRenderer, +}); + +// predictions +register({ + type: 'holt_winters', + addStrategy: addTransformationStrategy, + category: categories.Predictors, + params: [ + { name: 'number', type: 'int', options: [5, 10, 20, 30, 40] }, + { name: 'season', type: 'int', options: [0, 1, 2, 5, 10] }, + ], + defaultParams: [10, 2], + renderer: functionRenderer, +}); + +register({ + type: 'holt_winters_with_fit', + addStrategy: addTransformationStrategy, + category: categories.Predictors, + params: [ + { name: 'number', type: 'int', options: [5, 10, 20, 30, 40] }, + { name: 'season', type: 'int', options: [0, 1, 2, 5, 10] }, + ], + defaultParams: [10, 2], + renderer: functionRenderer, +}); + +// Selectors +register({ + type: 'bottom', + addStrategy: replaceAggregationAddStrategy, + category: categories.Selectors, + params: [{ name: 'count', type: 'int' }], + defaultParams: [3], + renderer: functionRenderer, +}); + +register({ + type: 'max', + addStrategy: replaceAggregationAddStrategy, + category: categories.Selectors, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'min', + addStrategy: replaceAggregationAddStrategy, + category: categories.Selectors, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + +register({ + type: 'percentile', + addStrategy: replaceAggregationAddStrategy, + category: categories.Selectors, + params: [{ name: 'nth', type: 'int' }], + defaultParams: [95], + renderer: functionRenderer, +}); + +register({ + type: 'top', + addStrategy: replaceAggregationAddStrategy, + category: categories.Selectors, + params: [{ name: 'count', type: 'int' }], + defaultParams: [3], + renderer: functionRenderer, +}); + +register({ + type: 'tag', + category: groupByTimeFunctions, + params: [{ name: 'tag', type: 'string', dynamicLookup: true }], + defaultParams: ['tag'], + renderer: fieldRenderer, +}); + +register({ + type: 'math', + addStrategy: addMathStrategy, + category: categories.Math, + params: [{ name: 'expr', type: 'string' }], + defaultParams: [' / 100'], + renderer: suffixRenderer, +}); + +register({ + type: 'alias', + addStrategy: addAliasStrategy, + category: categories.Aliasing, + params: [{ name: 'name', type: 'string', quote: 'double' }], + defaultParams: ['alias'], + renderMode: 'suffix', + renderer: aliasRenderer, +}); + +export default { + create: createPart, + getCategories: function() { + return categories; + }, +}; From 4dbd83fac18a3cc5e59208d396a32538f2ab1a5e Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 31 Jan 2018 15:32:32 +0100 Subject: [PATCH 007/310] add groupby to querybuilder remove unused aggregations --- .../postgres/partials/query.editor.html | 21 +++ .../datasource/postgres/postgres_query.ts | 5 +- .../plugins/datasource/postgres/query_ctrl.ts | 84 ++++++++++ .../plugins/datasource/postgres/query_part.ts | 150 +----------------- 4 files changed, 110 insertions(+), 150 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 400855cf82f..3e921e1c631 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -51,6 +51,27 @@ +
+
+ + + + +
+ +
+ +
+ +
+
+
+
+
diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index e6d84306a9b..389ce85aae9 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -22,7 +22,7 @@ export default class PostgresQuery { target.alias = ''; target.orderByTime = target.orderByTime || 'ASC'; -// target.groupBy = target.groupBy || [{ type: 'time', params: ['$__interval'] }, { type: 'fill', params: ['null'] }]; + target.groupBy = target.groupBy || [{ type: 'time', params: ['$__interval'] }, { type: 'fill', params: ['null'] }]; target.select = target.select || [[{ type: 'field', params: ['value'] }]]; this.updateProjection(); @@ -92,9 +92,6 @@ export default class PostgresQuery { if (partModel.def.category === categories.Aggregations) { return false; } - if (partModel.def.category === categories.Selectors) { - return false; - } return true; }); }); diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 4e39105370b..020d8007789 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -29,6 +29,7 @@ export class PostgresQueryCtrl extends QueryCtrl { tableSegment: any; timeColumnSegment: any; selectMenu: any; + groupBySegment: any; /** @ngInject **/ constructor($scope, $injector, private templateSrv, private $q, private uiSegmentSrv) { @@ -59,6 +60,8 @@ export class PostgresQueryCtrl extends QueryCtrl { this.timeColumnSegment = uiSegmentSrv.newSegment(this.target.timeColumn); this.buildSelectMenu(); + this.groupBySegment = this.uiSegmentSrv.newPlusButton(); + this.panelCtrl.events.on('data-received', this.onDataReceived.bind(this), $scope); this.panelCtrl.events.on('data-error', this.onDataError.bind(this), $scope); } @@ -224,6 +227,87 @@ export class PostgresQueryCtrl extends QueryCtrl { } } + handleGroupByPartEvent(part, index, evt) { + switch (evt.name) { + case 'get-param-options': { + var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; + columnQuery += " table_schema = '" + this.target.schema + "'"; + columnQuery += " AND table_name = '" + this.target.table + "'"; + + return this.datasource + .metricFindQuery(columnQuery) + .then(this.transformToSegments(true)) + .catch(this.handleQueryError.bind(this)); + } + case 'part-param-changed': { + this.panelCtrl.refresh(); + break; + } + case 'action': { + this.queryModel.removeGroupByPart(part, index); + this.panelCtrl.refresh(); + break; + } + case 'get-part-actions': { + return this.$q.when([{ text: 'Remove', value: 'remove-part' }]); + } + } + } + + getGroupByOptions() { + var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; + columnQuery += " table_schema = '" + this.target.schema + "'"; + columnQuery += " AND table_name = '" + this.target.table + "'"; + + + return this.datasource + .metricFindQuery(columnQuery) + .then(tags => { + var options = []; + if (!this.queryModel.hasFill()) { + options.push(this.uiSegmentSrv.newSegment({ value: 'fill(null)' })); + } + if (!this.target.limit) { + options.push(this.uiSegmentSrv.newSegment({ value: 'LIMIT' })); + } + if (!this.target.slimit) { + options.push(this.uiSegmentSrv.newSegment({ value: 'SLIMIT' })); + } + if (this.target.orderByTime === 'ASC') { + options.push(this.uiSegmentSrv.newSegment({ value: 'ORDER BY time DESC' })); + } + if (!this.queryModel.hasGroupByTime()) { + options.push(this.uiSegmentSrv.newSegment({ value: 'time($interval)' })); + } + for (let tag of tags) { + options.push(this.uiSegmentSrv.newSegment({ value: 'tag(' + tag.text + ')' })); + } + return options; + }) + .catch(this.handleQueryError.bind(this)); + } + + groupByAction() { + switch (this.groupBySegment.value) { + case 'LIMIT': { + this.target.limit = 10; + break; + } + case 'ORDER BY time DESC': { + this.target.orderByTime = 'DESC'; + break; + } + default: { + this.queryModel.addGroupBy(this.groupBySegment.value); + } + } + + var plusButton = this.uiSegmentSrv.newPlusButton(); + this.groupBySegment.value = plusButton.value; + this.groupBySegment.html = plusButton.html; + this.panelCtrl.refresh(); + } + handleQueryError(err) { this.error = err.message || 'Failed to issue metric query'; return []; diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts index dffea15dbd1..5828515ec06 100644 --- a/public/app/plugins/datasource/postgres/query_part.ts +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -4,9 +4,6 @@ import { QueryPartDef, QueryPart, functionRenderer, suffixRenderer } from 'app/c var index = []; var categories = { Aggregations: [], - Selectors: [], - Transformations: [], - Predictors: [], Math: [], Aliasing: [], Fields: [], @@ -44,10 +41,6 @@ function replaceAggregationAddStrategy(selectParts, partModel) { selectParts[i] = partModel; return; } - if (part.def.category === categories.Selectors) { - selectParts[i] = partModel; - return; - } } selectParts.splice(1, 0, partModel); @@ -147,34 +140,10 @@ register({ // transformations -register({ - type: 'derivative', - addStrategy: addTransformationStrategy, - category: categories.Transformations, - params: [ - { - name: 'duration', - type: 'interval', - options: ['1s', '10s', '1m', '5m', '10m', '15m', '1h'], - }, - ], - defaultParams: ['10s'], - renderer: functionRenderer, -}); - -register({ - type: 'spread', - addStrategy: addTransformationStrategy, - category: categories.Transformations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - register({ type: 'non_negative_derivative', addStrategy: addTransformationStrategy, - category: categories.Transformations, + category: categories.Aggregations, params: [ { name: 'duration', @@ -186,46 +155,10 @@ register({ renderer: functionRenderer, }); -register({ - type: 'difference', - addStrategy: addTransformationStrategy, - category: categories.Transformations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - -register({ - type: 'non_negative_difference', - addStrategy: addTransformationStrategy, - category: categories.Transformations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - -register({ - type: 'moving_average', - addStrategy: addTransformationStrategy, - category: categories.Transformations, - params: [{ name: 'window', type: 'int', options: [5, 10, 20, 30, 40] }], - defaultParams: [10], - renderer: functionRenderer, -}); - -register({ - type: 'cumulative_sum', - addStrategy: addTransformationStrategy, - category: categories.Transformations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - register({ type: 'stddev', addStrategy: addTransformationStrategy, - category: categories.Transformations, + category: categories.Aggregations, params: [], defaultParams: [], renderer: functionRenderer, @@ -259,60 +192,11 @@ register({ renderer: functionRenderer, }); -register({ - type: 'elapsed', - addStrategy: addTransformationStrategy, - category: categories.Transformations, - params: [ - { - name: 'duration', - type: 'interval', - options: ['1s', '10s', '1m', '5m', '10m', '15m', '1h'], - }, - ], - defaultParams: ['10s'], - renderer: functionRenderer, -}); - -// predictions -register({ - type: 'holt_winters', - addStrategy: addTransformationStrategy, - category: categories.Predictors, - params: [ - { name: 'number', type: 'int', options: [5, 10, 20, 30, 40] }, - { name: 'season', type: 'int', options: [0, 1, 2, 5, 10] }, - ], - defaultParams: [10, 2], - renderer: functionRenderer, -}); - -register({ - type: 'holt_winters_with_fit', - addStrategy: addTransformationStrategy, - category: categories.Predictors, - params: [ - { name: 'number', type: 'int', options: [5, 10, 20, 30, 40] }, - { name: 'season', type: 'int', options: [0, 1, 2, 5, 10] }, - ], - defaultParams: [10, 2], - renderer: functionRenderer, -}); - // Selectors -register({ - type: 'bottom', - addStrategy: replaceAggregationAddStrategy, - category: categories.Selectors, - params: [{ name: 'count', type: 'int' }], - defaultParams: [3], - renderer: functionRenderer, -}); - register({ type: 'max', addStrategy: replaceAggregationAddStrategy, - category: categories.Selectors, + category: categories.Aggregations, params: [], defaultParams: [], renderer: functionRenderer, @@ -321,38 +205,12 @@ register({ register({ type: 'min', addStrategy: replaceAggregationAddStrategy, - category: categories.Selectors, + category: categories.Aggregations, params: [], defaultParams: [], renderer: functionRenderer, }); -register({ - type: 'percentile', - addStrategy: replaceAggregationAddStrategy, - category: categories.Selectors, - params: [{ name: 'nth', type: 'int' }], - defaultParams: [95], - renderer: functionRenderer, -}); - -register({ - type: 'top', - addStrategy: replaceAggregationAddStrategy, - category: categories.Selectors, - params: [{ name: 'count', type: 'int' }], - defaultParams: [3], - renderer: functionRenderer, -}); - -register({ - type: 'tag', - category: groupByTimeFunctions, - params: [{ name: 'tag', type: 'string', dynamicLookup: true }], - defaultParams: ['tag'], - renderer: fieldRenderer, -}); - register({ type: 'math', addStrategy: addMathStrategy, From c65a964cdda6e4cabcc570f0199dcd5378331781 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 3 Feb 2018 18:03:00 +0100 Subject: [PATCH 008/310] add metric column selector --- .../postgres/partials/query.editor.html | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 3e921e1c631..e266050dff1 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -17,6 +17,15 @@
+ +
+ +
+ +
+
+
+
@@ -46,9 +55,15 @@
- + +
+ +
+
+
+
From 382a5254772d29b67658dc1ed8e38a34b7df0a11 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 3 Feb 2018 18:26:57 +0100 Subject: [PATCH 009/310] make metricColumn functional --- .../app/plugins/datasource/postgres/query_ctrl.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 020d8007789..0b2bce7fb05 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -27,7 +27,9 @@ export class PostgresQueryCtrl extends QueryCtrl { showHelp: boolean; schemaSegment: any; tableSegment: any; + whereSegment: any; timeColumnSegment: any; + metricColumnSegment: any; selectMenu: any; groupBySegment: any; @@ -58,6 +60,7 @@ export class PostgresQueryCtrl extends QueryCtrl { } this.timeColumnSegment = uiSegmentSrv.newSegment(this.target.timeColumn); + this.metricColumnSegment = uiSegmentSrv.newSegment(this.target.metricColumn); this.buildSelectMenu(); this.groupBySegment = this.uiSegmentSrv.newPlusButton(); @@ -122,11 +125,11 @@ export class PostgresQueryCtrl extends QueryCtrl { .catch(this.handleQueryError.bind(this)); } - getColumnSegments() { + getMetricColumnSegments() { var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; columnQuery += " table_schema = '" + this.target.schema + "'"; columnQuery += " AND table_name = '" + this.target.table + "'"; - columnQuery += " AND data_type IN ('bigint','integer','double precision','real');"; + columnQuery += " AND data_type IN ('text','char','varchar');"; return this.datasource .metricFindQuery(columnQuery) @@ -145,7 +148,12 @@ export class PostgresQueryCtrl extends QueryCtrl { } timeColumnChanged() { - this.target.time = this.timeColumnSegment.value; + this.target.timeColumn = this.timeColumnSegment.value; + this.panelCtrl.refresh(); + } + + metricColumnChanged() { + this.target.metricColumn = this.metricColumnSegment.value; this.panelCtrl.refresh(); } From 3bce45d8a66abf984644f0cad508e73e60b595bc Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 8 Feb 2018 10:19:43 +0100 Subject: [PATCH 010/310] add query_builder --- .../datasource/postgres/query_builder.ts | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 public/app/plugins/datasource/postgres/query_builder.ts diff --git a/public/app/plugins/datasource/postgres/query_builder.ts b/public/app/plugins/datasource/postgres/query_builder.ts new file mode 100644 index 00000000000..7a227f33b93 --- /dev/null +++ b/public/app/plugins/datasource/postgres/query_builder.ts @@ -0,0 +1,50 @@ + +export class PostgresQueryBuilder { + constructor(private target, private queryModel) {} + + buildSchemaQuery() { + var query = "SELECT schema_name FROM information_schema.schemata WHERE"; + query += " schema_name NOT LIKE 'pg_%' AND schema_name NOT LIKE '\\_%' AND schema_name <> 'information_schema';"; + + return query; + } + + buildTableQuery() { + var query = "SELECT table_name FROM information_schema.tables WHERE "; + query += "table_schema = " + this.queryModel.quoteLiteral(this.target.schema); + return query; + } + + buildColumnQuery(type?: string) { + var query = "SELECT column_name FROM information_schema.columns WHERE "; + query += "table_schema = " + this.queryModel.quoteLiteral(this.target.schema); + query += " AND table_name = " + this.queryModel.quoteLiteral(this.target.table); + + switch (type) { + case "time": { + query += " AND data_type IN ('timestamp without time zone','timestamp with time zone','bigint','integer','double precision','real')"; + break; + } + case "metric": { + query += " AND data_type IN ('text','char','varchar')"; + break; + } + case "value": { + query += " AND data_type IN ('bigint','integer','double precision','real')"; + break; + } + } + + return query; + } + + buildValueQuery(column: string) { + var query = "SELECT DISTINCT " + this.queryModel.quoteIdentifier(column) + "::text"; + query += " FROM " + this.queryModel.quoteIdentifier(this.target.schema); + query += "." + this.queryModel.quoteIdentifier(this.target.table); + query += " ORDER BY " + this.queryModel.quoteIdentifier(column); + query += " LIMIT 100"; + return query; + } + +} From ef18eb7fcb0449c7bc9709b8837ff8fa202ccc93 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 8 Feb 2018 10:25:32 +0100 Subject: [PATCH 011/310] add where constraint handling --- .../postgres/partials/query.editor.html | 2 +- .../datasource/postgres/postgres_query.ts | 15 +- .../plugins/datasource/postgres/query_ctrl.ts | 192 ++++++++++++++---- 3 files changed, 163 insertions(+), 46 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index e266050dff1..e25a41b11c2 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -19,7 +19,7 @@
- +
diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 389ce85aae9..9a5a14c2b6a 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -19,17 +19,22 @@ export default class PostgresQuery { target.schema = target.schema || 'public'; target.format = target.format || 'time_series'; target.timeColumn = target.timeColumn || 'time'; - target.alias = ''; + target.metricColumn = target.metricColumn || 'None'; target.orderByTime = target.orderByTime || 'ASC'; - target.groupBy = target.groupBy || [{ type: 'time', params: ['$__interval'] }, { type: 'fill', params: ['null'] }]; + target.groupBy = target.groupBy || []; + target.where = target.where || []; target.select = target.select || [[{ type: 'field', params: ['value'] }]]; this.updateProjection(); } - quoteIdentifier(field) { - return '"' + field + '"'; + quoteIdentifier(value) { + return '"' + value + '"'; + } + + quoteLiteral(value) { + return "'" + value + "'"; } updateProjection() { @@ -202,7 +207,7 @@ export default class PostgresQuery { } query += ' FROM ' + target.schema + '.' + target.table + ' WHERE '; - var conditions = _.map(target.tags, (tag, index) => { + var conditions = _.map(target.where, (tag, index) => { return this.renderTagCondition(tag, index, interpolate); }); diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 0b2bce7fb05..f6f6641eff0 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -1,4 +1,6 @@ +import angular from 'angular'; import _ from 'lodash'; +import { PostgresQueryBuilder } from './query_builder'; import { QueryCtrl } from 'app/plugins/sdk'; import queryPart from './query_part'; import PostgresQuery from './postgres_query'; @@ -22,22 +24,25 @@ export class PostgresQueryCtrl extends QueryCtrl { showLastQuerySQL: boolean; formats: any[]; queryModel: PostgresQuery; + queryBuilder: PostgresQueryBuilder; lastQueryMeta: QueryMeta; lastQueryError: string; showHelp: boolean; schemaSegment: any; tableSegment: any; - whereSegment: any; + whereSegments: any; timeColumnSegment: any; metricColumnSegment: any; selectMenu: any; groupBySegment: any; + removeWhereFilterSegment: any; /** @ngInject **/ constructor($scope, $injector, private templateSrv, private $q, private uiSegmentSrv) { super($scope, $injector); this.target = this.target; this.queryModel = new PostgresQuery(this.target, templateSrv, this.panel.scopedVars); + this.queryBuilder = new PostgresQueryBuilder(this.target, this.queryModel); this.formats = [{ text: 'Time series', value: 'time_series' }, { text: 'Table', value: 'table' }]; @@ -63,8 +68,32 @@ export class PostgresQueryCtrl extends QueryCtrl { this.metricColumnSegment = uiSegmentSrv.newSegment(this.target.metricColumn); this.buildSelectMenu(); + this.whereSegments = []; + for (let tag of this.target.where) { + if (!tag.operator) { + if (/^\/.*\/$/.test(tag.value)) { + tag.operator = '=~'; + } else { + tag.operator = '='; + } + } + + if (tag.condition) { + this.whereSegments.push(uiSegmentSrv.newCondition(tag.condition)); + } + + this.whereSegments.push(uiSegmentSrv.newKey(tag.key)); + this.whereSegments.push(uiSegmentSrv.newOperator(tag.operator)); + this.whereSegments.push(uiSegmentSrv.newKeyValue(tag.value)); + } + + this.fixWhereSegments(); this.groupBySegment = this.uiSegmentSrv.newPlusButton(); + this.removeWhereFilterSegment = uiSegmentSrv.newSegment({ + fake: true, + value: '-- remove tag filter --', + }); this.panelCtrl.events.on('data-received', this.onDataReceived.bind(this), $scope); this.panelCtrl.events.on('data-error', this.onDataError.bind(this), $scope); } @@ -97,42 +126,29 @@ export class PostgresQueryCtrl extends QueryCtrl { } getSchemaSegments() { - var schemaQuery = "SELECT schema_name FROM information_schema.schemata WHERE"; - schemaQuery += " schema_name NOT LIKE 'pg_%' AND schema_name <> 'information_schema';"; return this.datasource - .metricFindQuery(schemaQuery) + .metricFindQuery(this.queryBuilder.buildSchemaQuery()) .then(this.transformToSegments(true)) .catch(this.handleQueryError.bind(this)); } getTableSegments() { - var tableQuery = "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + this.target.schema + "';"; return this.datasource - .metricFindQuery(tableQuery) + .metricFindQuery(this.queryBuilder.buildTableQuery()) .then(this.transformToSegments(true)) .catch(this.handleQueryError.bind(this)); } getTimeColumnSegments() { - var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; - columnQuery += " table_schema = '" + this.target.schema + "'"; - columnQuery += " AND table_name = '" + this.target.table + "'"; - columnQuery += " AND data_type IN ('timestamp without time zone','timestamp with time zone','bigint','integer','double precision','real');"; - return this.datasource - .metricFindQuery(columnQuery) + .metricFindQuery(this.queryBuilder.buildColumnQuery("time")) .then(this.transformToSegments(true)) .catch(this.handleQueryError.bind(this)); } getMetricColumnSegments() { - var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; - columnQuery += " table_schema = '" + this.target.schema + "'"; - columnQuery += " AND table_name = '" + this.target.table + "'"; - columnQuery += " AND data_type IN ('text','char','varchar');"; - return this.datasource - .metricFindQuery(columnQuery) + .metricFindQuery(this.queryBuilder.buildColumnQuery("metric")) .then(this.transformToSegments(true)) .catch(this.handleQueryError.bind(this)); } @@ -210,13 +226,8 @@ export class PostgresQueryCtrl extends QueryCtrl { handleSelectPartEvent(selectParts, part, evt) { switch (evt.name) { case 'get-param-options': { - var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; - columnQuery += " table_schema = '" + this.target.schema + "'"; - columnQuery += " AND table_name = '" + this.target.table + "'"; - columnQuery += " AND data_type IN ('bigint','integer','double precision','real');"; - return this.datasource - .metricFindQuery(columnQuery) + .metricFindQuery(this.queryBuilder.buildColumnQuery("value")) .then(this.transformToSegments(true)) .catch(this.handleQueryError.bind(this)); } @@ -238,12 +249,8 @@ export class PostgresQueryCtrl extends QueryCtrl { handleGroupByPartEvent(part, index, evt) { switch (evt.name) { case 'get-param-options': { - var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; - columnQuery += " table_schema = '" + this.target.schema + "'"; - columnQuery += " AND table_name = '" + this.target.table + "'"; - return this.datasource - .metricFindQuery(columnQuery) + .metricFindQuery(this.queryBuilder.buildColumnQuery()) .then(this.transformToSegments(true)) .catch(this.handleQueryError.bind(this)); } @@ -262,14 +269,125 @@ export class PostgresQueryCtrl extends QueryCtrl { } } - getGroupByOptions() { - var columnQuery = "SELECT column_name FROM information_schema.columns WHERE "; - columnQuery += " table_schema = '" + this.target.schema + "'"; - columnQuery += " AND table_name = '" + this.target.table + "'"; + fixWhereSegments() { + var count = this.whereSegments.length; + var lastSegment = this.whereSegments[Math.max(count - 1, 0)]; + if (!lastSegment || lastSegment.type !== 'plus-button') { + this.whereSegments.push(this.uiSegmentSrv.newPlusButton()); + } + } + + getTagsOrValues(segment, index) { + if (segment.type === 'condition') { + return this.$q.when([this.uiSegmentSrv.newSegment('AND'), this.uiSegmentSrv.newSegment('OR')]); + } + if (segment.type === 'operator') { + var nextValue = this.whereSegments[index + 1].value; + if (/^\/.*\/$/.test(nextValue)) { + return this.$q.when(this.uiSegmentSrv.newOperators(['=~', '!~'])); + } else { + return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '<>', '<', '>'])); + } + } + + var query, addTemplateVars; + if (segment.type === 'key' || segment.type === 'plus-button') { + query = this.queryBuilder.buildColumnQuery(); + + addTemplateVars = false; + } else if (segment.type === 'value') { + query = this.queryBuilder.buildValueQuery(this.whereSegments[index -2].value); + addTemplateVars = true; + } return this.datasource - .metricFindQuery(columnQuery) + .metricFindQuery(query) + .then(this.transformToSegments(addTemplateVars)) + .then(results => { + if (segment.type === 'key') { + results.splice(0, 0, angular.copy(this.removeWhereFilterSegment)); + } + return results; + }) + .catch(this.handleQueryError.bind(this)); + } + + getTagValueOperator(tagValue, tagOperator): string { + if (tagOperator !== '=~' && tagOperator !== '!~' && /^\/.*\/$/.test(tagValue)) { + return '=~'; + } else if ((tagOperator === '=~' || tagOperator === '!~') && /^(?!\/.*\/$)/.test(tagValue)) { + return '='; + } + return null; + } + + whereSegmentUpdated(segment, index) { + this.whereSegments[index] = segment; + + // handle remove where condition + if (segment.value === this.removeWhereFilterSegment.value) { + this.whereSegments.splice(index, 3); + if (this.whereSegments.length === 0) { + this.whereSegments.push(this.uiSegmentSrv.newPlusButton()); + } else if (this.whereSegments.length > 2) { + this.whereSegments.splice(Math.max(index - 1, 0), 1); + if (this.whereSegments[this.whereSegments.length - 1].type !== 'plus-button') { + this.whereSegments.push(this.uiSegmentSrv.newPlusButton()); + } + } + } else { + if (segment.type === 'plus-button') { + if (index > 2) { + this.whereSegments.splice(index, 0, this.uiSegmentSrv.newCondition('AND')); + } + this.whereSegments.push(this.uiSegmentSrv.newOperator('=')); + this.whereSegments.push(this.uiSegmentSrv.newFake('select value', 'value', 'query-segment-value')); + segment.type = 'key'; + segment.cssClass = 'query-segment-key'; + } + + if (index + 1 === this.whereSegments.length) { + this.whereSegments.push(this.uiSegmentSrv.newPlusButton()); + } + } + + this.rebuildTargetWhereConditions(); + } + + rebuildTargetWhereConditions() { + var where = []; + var tagIndex = 0; + var tagOperator = ''; + + _.each(this.whereSegments, (segment2, index) => { + if (segment2.type === 'key') { + if (where.length === 0) { + where.push({}); + } + where[tagIndex].key = segment2.value; + } else if (segment2.type === 'value') { + tagOperator = this.getTagValueOperator(segment2.value, where[tagIndex].operator); + if (tagOperator) { + this.whereSegments[index - 1] = this.uiSegmentSrv.newOperator(tagOperator); + where[tagIndex].operator = tagOperator; + } + where[tagIndex].value = segment2.value; + } else if (segment2.type === 'condition') { + where.push({ condition: segment2.value }); + tagIndex += 1; + } else if (segment2.type === 'operator') { + where[tagIndex].operator = segment2.value; + } + }); + + this.target.where = where; + this.panelCtrl.refresh(); + } + + getGroupByOptions() { + return this.datasource + .metricFindQuery(this.queryBuilder.buildColumnQuery()) .then(tags => { var options = []; if (!this.queryModel.hasFill()) { @@ -278,12 +396,6 @@ export class PostgresQueryCtrl extends QueryCtrl { if (!this.target.limit) { options.push(this.uiSegmentSrv.newSegment({ value: 'LIMIT' })); } - if (!this.target.slimit) { - options.push(this.uiSegmentSrv.newSegment({ value: 'SLIMIT' })); - } - if (this.target.orderByTime === 'ASC') { - options.push(this.uiSegmentSrv.newSegment({ value: 'ORDER BY time DESC' })); - } if (!this.queryModel.hasGroupByTime()) { options.push(this.uiSegmentSrv.newSegment({ value: 'time($interval)' })); } From fd518846b1865385eb9775332ffd04fc5388dcc9 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 3 Mar 2018 20:57:00 +0100 Subject: [PATCH 012/310] rename field to column --- .../datasource/postgres/postgres_query.ts | 22 ++++++++----- .../plugins/datasource/postgres/query_ctrl.ts | 7 ++--- .../plugins/datasource/postgres/query_part.ts | 31 +++++++------------ 3 files changed, 27 insertions(+), 33 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 9a5a14c2b6a..78e9c9c1a3f 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -24,7 +24,7 @@ export default class PostgresQuery { target.orderByTime = target.orderByTime || 'ASC'; target.groupBy = target.groupBy || []; target.where = target.where || []; - target.select = target.select || [[{ type: 'field', params: ['value'] }]]; + target.select = target.select || [[{ type: 'column', params: ['value'] }]]; this.updateProjection(); } @@ -88,8 +88,6 @@ export default class PostgresQuery { var categories = queryPart.getCategories(); if (part.def.type === 'time') { - // remove fill - this.target.groupBy = _.filter(this.target.groupBy, (g: any) => g.type !== 'fill'); // remove aggregations this.target.select = _.map(this.target.select, (s: any) => { return _.filter(s, (part: any) => { @@ -113,7 +111,7 @@ export default class PostgresQuery { removeSelectPart(selectParts, part) { // if we remove the field remove the whole statement - if (part.def.type === 'field') { + if (part.def.type === 'column') { if (this.selectModels.length > 1) { var modelsIndex = _.indexOf(this.selectModels, selectParts); this.selectModels.splice(modelsIndex, 1); @@ -189,7 +187,12 @@ export default class PostgresQuery { } var query = 'SELECT '; - query += this.quoteIdentifier(target.timeColumn) + ' AS time,'; + + if (this.hasGroupByTime()) { + query += '$__timeGroup(' + this.quoteIdentifier(target.timeColumn) + ',1m),'; + } else { + query += this.quoteIdentifier(target.timeColumn) + ' AS time,'; + } var i, y; for (i = 0; i < this.selectModels.length; i++) { @@ -221,10 +224,13 @@ export default class PostgresQuery { for (i = 0; i < this.groupByParts.length; i++) { var part = this.groupByParts[i]; if (i > 0) { - // for some reason fill has no seperator - groupBySection += part.def.type === 'fill' ? ' ' : ', '; + groupBySection += ', '; + } + if (part.def.type === 'time') { + groupBySection += 'time'; + } else { + groupBySection += part.render(''); } - groupBySection += part.render(''); } if (groupBySection.length) { diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index f6f6641eff0..0c30e2c51ff 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -92,7 +92,7 @@ export class PostgresQueryCtrl extends QueryCtrl { this.removeWhereFilterSegment = uiSegmentSrv.newSegment({ fake: true, - value: '-- remove tag filter --', + value: '-- remove filter --', }); this.panelCtrl.events.on('data-received', this.onDataReceived.bind(this), $scope); this.panelCtrl.events.on('data-error', this.onDataError.bind(this), $scope); @@ -390,9 +390,6 @@ export class PostgresQueryCtrl extends QueryCtrl { .metricFindQuery(this.queryBuilder.buildColumnQuery()) .then(tags => { var options = []; - if (!this.queryModel.hasFill()) { - options.push(this.uiSegmentSrv.newSegment({ value: 'fill(null)' })); - } if (!this.target.limit) { options.push(this.uiSegmentSrv.newSegment({ value: 'LIMIT' })); } @@ -400,7 +397,7 @@ export class PostgresQueryCtrl extends QueryCtrl { options.push(this.uiSegmentSrv.newSegment({ value: 'time($interval)' })); } for (let tag of tags) { - options.push(this.uiSegmentSrv.newSegment({ value: 'tag(' + tag.text + ')' })); + options.push(this.uiSegmentSrv.newSegment({ value: tag.text })); } return options; }) diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts index 5828515ec06..0086b45b848 100644 --- a/public/app/plugins/datasource/postgres/query_part.ts +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -6,7 +6,7 @@ var categories = { Aggregations: [], Math: [], Aliasing: [], - Fields: [], + Columns: [], }; function createPart(part): any { @@ -29,7 +29,7 @@ function aliasRenderer(part, innerExpr) { return innerExpr + ' AS ' + '"' + part.params[0] + '"'; } -function fieldRenderer(part, innerExpr) { +function columnRenderer(part, innerExpr) { return '"' + part.params[0] + '"'; } @@ -92,7 +92,7 @@ function addAliasStrategy(selectParts, partModel) { selectParts.push(partModel); } -function addFieldStrategy(selectParts, partModel, query) { +function addColumnStrategy(selectParts, partModel, query) { // copy all parts var parts = _.map(selectParts, function(part: any) { return createPart({ type: part.def.type, params: _.clone(part.params) }); @@ -102,12 +102,12 @@ function addFieldStrategy(selectParts, partModel, query) { } register({ - type: 'field', - addStrategy: addFieldStrategy, - category: categories.Fields, - params: [{ type: 'field', dynamicLookup: true }], + type: 'column', + addStrategy: addColumnStrategy, + category: categories.Columns, + params: [{ type: 'column', dynamicLookup: true }], defaultParams: ['value'], - renderer: fieldRenderer, + renderer: columnRenderer, }); // Aggregations @@ -170,25 +170,16 @@ register({ params: [ { name: 'interval', - type: 'time', + type: 'interval', options: ['$__interval', '1s', '10s', '1m', '5m', '10m', '15m', '1h'], }, - ], - defaultParams: ['$__interval'], - renderer: functionRenderer, -}); - -register({ - type: 'fill', - category: groupByTimeFunctions, - params: [ { name: 'fill', type: 'string', - options: ['none', 'null', '0', 'previous', 'linear'], + options: ['none', 'null', '0'], }, ], - defaultParams: ['null'], + defaultParams: ['$__interval','none'], renderer: functionRenderer, }); From 7104e6f9f8dccd5bbba53519daaa40a7cce6a329 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 3 Mar 2018 22:11:51 +0100 Subject: [PATCH 013/310] fix variable interpolation --- public/app/plugins/datasource/postgres/postgres_query.ts | 3 +++ public/app/plugins/datasource/postgres/query_ctrl.ts | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 78e9c9c1a3f..2f8b3911ebc 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -239,6 +239,9 @@ export default class PostgresQuery { query += ' ORDER BY time'; + if (interpolate) { + query = this.templateSrv.replace(query, this.scopedVars, this.interpolateQueryStr); + } return query; } diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 0c30e2c51ff..bde4010e226 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -207,7 +207,7 @@ export class PostgresQueryCtrl extends QueryCtrl { segments.unshift( this.uiSegmentSrv.newSegment({ type: 'template', - value: '/^$' + variable.name + '$/', + value: '$' + variable.name, expandable: true, }) ); From e8c6341fed3811b3d6339dc5e2fc2a8815caa285 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 4 Mar 2018 10:18:11 +0100 Subject: [PATCH 014/310] clean up aggregation functions --- .../plugins/datasource/postgres/query_part.ts | 87 ++++++------------- 1 file changed, 28 insertions(+), 59 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts index 0086b45b848..600723be00d 100644 --- a/public/app/plugins/datasource/postgres/query_part.ts +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -46,19 +46,6 @@ function replaceAggregationAddStrategy(selectParts, partModel) { selectParts.splice(1, 0, partModel); } -function addTransformationStrategy(selectParts, partModel) { - var i; - // look for index to add transformation - for (i = 0; i < selectParts.length; i++) { - var part = selectParts[i]; - if (part.def.category === categories.Math || part.def.category === categories.Aliasing) { - break; - } - } - - selectParts.splice(i, 0, partModel); -} - function addMathStrategy(selectParts, partModel) { var partCount = selectParts.length; if (partCount > 0) { @@ -138,54 +125,8 @@ register({ renderer: functionRenderer, }); -// transformations - -register({ - type: 'non_negative_derivative', - addStrategy: addTransformationStrategy, - category: categories.Aggregations, - params: [ - { - name: 'duration', - type: 'interval', - options: ['1s', '10s', '1m', '5m', '10m', '15m', '1h'], - }, - ], - defaultParams: ['10s'], - renderer: functionRenderer, -}); - register({ type: 'stddev', - addStrategy: addTransformationStrategy, - category: categories.Aggregations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - -register({ - type: 'time', - category: groupByTimeFunctions, - params: [ - { - name: 'interval', - type: 'interval', - options: ['$__interval', '1s', '10s', '1m', '5m', '10m', '15m', '1h'], - }, - { - name: 'fill', - type: 'string', - options: ['none', 'null', '0'], - }, - ], - defaultParams: ['$__interval','none'], - renderer: functionRenderer, -}); - -// Selectors -register({ - type: 'max', addStrategy: replaceAggregationAddStrategy, category: categories.Aggregations, params: [], @@ -202,6 +143,15 @@ register({ renderer: functionRenderer, }); +register({ + type: 'max', + addStrategy: replaceAggregationAddStrategy, + category: categories.Aggregations, + params: [], + defaultParams: [], + renderer: functionRenderer, +}); + register({ type: 'math', addStrategy: addMathStrategy, @@ -221,6 +171,25 @@ register({ renderer: aliasRenderer, }); +register({ + type: 'time', + category: groupByTimeFunctions, + params: [ + { + name: 'interval', + type: 'interval', + options: ['$__interval', '1s', '10s', '1m', '5m', '10m', '15m', '1h'], + }, + { + name: 'fill', + type: 'string', + options: ['none', 'NULL', '0'], + }, + ], + defaultParams: ['$__interval','none'], + renderer: functionRenderer, +}); + export default { create: createPart, getCategories: function() { From 26e09b598c809e93a120dfbc30214af8ff9ac690 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 4 Mar 2018 19:35:43 +0100 Subject: [PATCH 015/310] fix group by column --- .../datasource/postgres/postgres_query.ts | 24 ++++++++++++------- .../plugins/datasource/postgres/query_ctrl.ts | 4 ++-- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 2f8b3911ebc..a236fa91b18 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -61,17 +61,17 @@ export default class PostgresQuery { } addGroupBy(value) { - var stringParts = value.match(/^(\w+)\((.*)\)$/); + var stringParts = value.match(/^(\w+)(\((.*)\))?$/); var typePart = stringParts[1]; - var arg = stringParts[2]; - var partModel = queryPart.create({ type: typePart, params: [arg] }); + var args = stringParts[3].split(","); + var partModel = queryPart.create({ type: typePart, params: args }); var partCount = this.target.groupBy.length; if (partCount === 0) { this.target.groupBy.push(partModel.part); } else if (typePart === 'time') { this.target.groupBy.splice(0, 0, partModel.part); - } else if (typePart === 'tag') { + } else if (typePart === 'column') { if (this.target.groupBy[partCount - 1].type === 'fill') { this.target.groupBy.splice(partCount - 1, 0, partModel.part); } else { @@ -188,8 +188,16 @@ export default class PostgresQuery { var query = 'SELECT '; - if (this.hasGroupByTime()) { - query += '$__timeGroup(' + this.quoteIdentifier(target.timeColumn) + ',1m),'; + var timeGroup = this.hasGroupByTime(); + + if (timeGroup) { + var args; + if (timeGroup.params.length > 1 && timeGroup.params[1] !== "none") { + args = timeGroup.params.join(","); + } else { + args = timeGroup.params[0]; + } + query += '$__timeGroup(' + this.quoteIdentifier(target.timeColumn) + ',' + args + '),'; } else { query += this.quoteIdentifier(target.timeColumn) + ' AS time,'; } @@ -227,7 +235,7 @@ export default class PostgresQuery { groupBySection += ', '; } if (part.def.type === 'time') { - groupBySection += 'time'; + groupBySection += '1'; } else { groupBySection += part.render(''); } @@ -237,7 +245,7 @@ export default class PostgresQuery { query += ' GROUP BY ' + groupBySection; } - query += ' ORDER BY time'; + query += ' ORDER BY 1'; if (interpolate) { query = this.templateSrv.replace(query, this.scopedVars, this.interpolateQueryStr); diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index bde4010e226..b0d5d7ab9ca 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -394,10 +394,10 @@ export class PostgresQueryCtrl extends QueryCtrl { options.push(this.uiSegmentSrv.newSegment({ value: 'LIMIT' })); } if (!this.queryModel.hasGroupByTime()) { - options.push(this.uiSegmentSrv.newSegment({ value: 'time($interval)' })); + options.push(this.uiSegmentSrv.newSegment({ type: 'time', value: 'time(1m,none)' })); } for (let tag of tags) { - options.push(this.uiSegmentSrv.newSegment({ value: tag.text })); + options.push(this.uiSegmentSrv.newSegment({ type: 'column', value: 'column(' + tag.text + ')' })); } return options; }) From bf4a30d30f93aa7ac1cf01c319666242022272fe Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 4 Mar 2018 19:46:11 +0100 Subject: [PATCH 016/310] set rawSQL when rendering query builder query --- public/app/plugins/datasource/postgres/postgres_query.ts | 1 + public/app/plugins/datasource/postgres/query_ctrl.ts | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index a236fa91b18..a708f384972 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -247,6 +247,7 @@ export default class PostgresQuery { query += ' ORDER BY 1'; + this.target.rawSql = query; if (interpolate) { query = this.templateSrv.replace(query, this.scopedVars, this.interpolateQueryStr); } diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index b0d5d7ab9ca..4e36e8699ae 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -117,11 +117,6 @@ export class PostgresQueryCtrl extends QueryCtrl { } toggleEditorMode() { - try { -// this.target.query = this.queryModel.render(false); - } catch (err) { - console.log('query render error'); - } this.target.rawQuery = !this.target.rawQuery; } From 83200c289a454e978ac4c0ade432ac7ccb31dbc4 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 4 Mar 2018 23:32:23 +0100 Subject: [PATCH 017/310] use metricColumn in query builder --- public/app/plugins/datasource/postgres/postgres_query.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index a708f384972..4541ebae7de 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -217,6 +217,10 @@ export default class PostgresQuery { query += selectText; } + if (this.target.metricColumn !== 'None') { + query += "," + this.quoteIdentifier(this.target.metricColumn) + " AS metric"; + } + query += ' FROM ' + target.schema + '.' + target.table + ' WHERE '; var conditions = _.map(target.where, (tag, index) => { return this.renderTagCondition(tag, index, interpolate); From 340f679d0f7c531eaa332c036a3141eb40119bff Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 9 Mar 2018 18:09:59 +0100 Subject: [PATCH 018/310] quote schema and table --- public/app/plugins/datasource/postgres/postgres_query.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 4541ebae7de..bd66ab5eca9 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -221,7 +221,7 @@ export default class PostgresQuery { query += "," + this.quoteIdentifier(this.target.metricColumn) + " AS metric"; } - query += ' FROM ' + target.schema + '.' + target.table + ' WHERE '; + query += ' FROM ' + this.quoteIdentifier(target.schema) + '.' + this.quoteIdentifier(target.table) + ' WHERE '; var conditions = _.map(target.where, (tag, index) => { return this.renderTagCondition(tag, index, interpolate); }); From 1d8540ac69d37b84689eaf0016b8d155d8e899a3 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 9 Mar 2018 18:18:12 +0100 Subject: [PATCH 019/310] properly quote where constraint parts --- .../datasource/postgres/postgres_query.ts | 28 ++++--------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index bd66ab5eca9..9e682d2f186 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -130,7 +130,7 @@ export default class PostgresQuery { this.updatePersistedParts(); } - private renderTagCondition(tag, index, interpolate) { + private renderWhereConstraint(tag, index, interpolate) { var str = ''; var operator = tag.operator; var value = tag.value; @@ -138,27 +138,11 @@ export default class PostgresQuery { str = (tag.condition || 'AND') + ' '; } - if (!operator) { - if (/^\/.*\/$/.test(value)) { - operator = '=~'; - } else { - operator = '='; - } + if (interpolate) { + value = this.templateSrv.replace(value, this.scopedVars); } - // quote value unless regex - if (operator !== '=~' && operator !== '!~') { - if (interpolate) { - value = this.templateSrv.replace(value, this.scopedVars); - } - if (operator !== '>' && operator !== '<') { - value = "'" + value.replace(/\\/g, '\\\\') + "'"; - } - } else if (interpolate) { - value = this.templateSrv.replace(value, this.scopedVars, 'regex'); - } - - return str + '"' + tag.key + '" ' + operator + ' ' + value; + return str + this.quoteIdentifier(tag.key) + ' ' + operator + ' ' + this.quoteLiteral(value); } interpolateQueryStr(value, variable, defaultFormatFn) { @@ -223,7 +207,7 @@ export default class PostgresQuery { query += ' FROM ' + this.quoteIdentifier(target.schema) + '.' + this.quoteIdentifier(target.table) + ' WHERE '; var conditions = _.map(target.where, (tag, index) => { - return this.renderTagCondition(tag, index, interpolate); + return this.renderWhereConstraint(tag, index, interpolate); }); if (conditions.length > 0) { @@ -260,7 +244,7 @@ export default class PostgresQuery { renderAdhocFilters(filters) { var conditions = _.map(filters, (tag, index) => { - return this.renderTagCondition(tag, index, false); + return this.renderWhereConstraint(tag, index, false); }); return conditions.join(' '); } From cb5278d413e85ec7fbd21490ca4edc467f691518 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 10 Mar 2018 20:18:03 +0100 Subject: [PATCH 020/310] handle variables in where constraints --- .../datasource/postgres/postgres_query.ts | 16 ++++++---------- .../plugins/datasource/postgres/query_ctrl.ts | 8 ++------ 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 9e682d2f186..9c48a94862f 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -56,10 +56,6 @@ export default class PostgresQuery { return _.find(this.target.groupBy, (g: any) => g.type === 'time'); } - hasFill() { - return _.find(this.target.groupBy, (g: any) => g.type === 'fill'); - } - addGroupBy(value) { var stringParts = value.match(/^(\w+)(\((.*)\))?$/); var typePart = stringParts[1]; @@ -130,19 +126,19 @@ export default class PostgresQuery { this.updatePersistedParts(); } - private renderWhereConstraint(tag, index, interpolate) { + private renderWhereConstraint(constraint, index, interpolate) { var str = ''; - var operator = tag.operator; - var value = tag.value; + var operator = constraint.operator; + var value = constraint.value; if (index > 0) { - str = (tag.condition || 'AND') + ' '; + str = (constraint.condition || 'AND') + ' '; } if (interpolate) { value = this.templateSrv.replace(value, this.scopedVars); } - return str + this.quoteIdentifier(tag.key) + ' ' + operator + ' ' + this.quoteLiteral(value); + return str + this.quoteIdentifier(constraint.key) + ' ' + operator + ' ' + this.quoteLiteral(value); } interpolateQueryStr(value, variable, defaultFormatFn) { @@ -207,7 +203,7 @@ export default class PostgresQuery { query += ' FROM ' + this.quoteIdentifier(target.schema) + '.' + this.quoteIdentifier(target.table) + ' WHERE '; var conditions = _.map(target.where, (tag, index) => { - return this.renderWhereConstraint(tag, index, interpolate); + return this.renderWhereConstraint(tag, index, false); }); if (conditions.length > 0) { diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 4e36e8699ae..da8c28eab9c 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -353,7 +353,6 @@ export class PostgresQueryCtrl extends QueryCtrl { rebuildTargetWhereConditions() { var where = []; var tagIndex = 0; - var tagOperator = ''; _.each(this.whereSegments, (segment2, index) => { if (segment2.type === 'key') { @@ -362,11 +361,8 @@ export class PostgresQueryCtrl extends QueryCtrl { } where[tagIndex].key = segment2.value; } else if (segment2.type === 'value') { - tagOperator = this.getTagValueOperator(segment2.value, where[tagIndex].operator); - if (tagOperator) { - this.whereSegments[index - 1] = this.uiSegmentSrv.newOperator(tagOperator); - where[tagIndex].operator = tagOperator; - } + where[tagIndex].value = segment2.value; + } else if (segment2.type === 'template') { where[tagIndex].value = segment2.value; } else if (segment2.type === 'condition') { where.push({ condition: segment2.value }); From 0b358ff5f30930401a3cfa9ca35699d2903786dc Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 10 Mar 2018 22:39:42 +0100 Subject: [PATCH 021/310] remove limit --- .../postgres/partials/query.editor.html | 2 +- .../plugins/datasource/postgres/query_ctrl.ts | 31 ++----------------- 2 files changed, 4 insertions(+), 29 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index e25a41b11c2..0706cf3a6cc 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -19,7 +19,7 @@
- +
diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index da8c28eab9c..101d52ee096 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -273,17 +273,12 @@ export class PostgresQueryCtrl extends QueryCtrl { } } - getTagsOrValues(segment, index) { + getWhereSegments(segment, index) { if (segment.type === 'condition') { return this.$q.when([this.uiSegmentSrv.newSegment('AND'), this.uiSegmentSrv.newSegment('OR')]); } if (segment.type === 'operator') { - var nextValue = this.whereSegments[index + 1].value; - if (/^\/.*\/$/.test(nextValue)) { - return this.$q.when(this.uiSegmentSrv.newOperators(['=~', '!~'])); - } else { - return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '<>', '<', '>'])); - } + return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '<>', '<', '>'])); } var query, addTemplateVars; @@ -308,15 +303,6 @@ export class PostgresQueryCtrl extends QueryCtrl { .catch(this.handleQueryError.bind(this)); } - getTagValueOperator(tagValue, tagOperator): string { - if (tagOperator !== '=~' && tagOperator !== '!~' && /^\/.*\/$/.test(tagValue)) { - return '=~'; - } else if ((tagOperator === '=~' || tagOperator === '!~') && /^(?!\/.*\/$)/.test(tagValue)) { - return '='; - } - return null; - } - whereSegmentUpdated(segment, index) { this.whereSegments[index] = segment; @@ -381,14 +367,11 @@ export class PostgresQueryCtrl extends QueryCtrl { .metricFindQuery(this.queryBuilder.buildColumnQuery()) .then(tags => { var options = []; - if (!this.target.limit) { - options.push(this.uiSegmentSrv.newSegment({ value: 'LIMIT' })); - } if (!this.queryModel.hasGroupByTime()) { options.push(this.uiSegmentSrv.newSegment({ type: 'time', value: 'time(1m,none)' })); } for (let tag of tags) { - options.push(this.uiSegmentSrv.newSegment({ type: 'column', value: 'column(' + tag.text + ')' })); + options.push(this.uiSegmentSrv.newSegment({ type: 'column', value: tag.text })); } return options; }) @@ -397,14 +380,6 @@ export class PostgresQueryCtrl extends QueryCtrl { groupByAction() { switch (this.groupBySegment.value) { - case 'LIMIT': { - this.target.limit = 10; - break; - } - case 'ORDER BY time DESC': { - this.target.orderByTime = 'DESC'; - break; - } default: { this.queryModel.addGroupBy(this.groupBySegment.value); } From e780b1bce5140a6c74fcc4782090e15035c5268a Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 11 Mar 2018 12:06:54 +0100 Subject: [PATCH 022/310] cleanup where segment handling --- .../plugins/datasource/postgres/query_ctrl.ts | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 101d52ee096..afde342474b 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -68,26 +68,7 @@ export class PostgresQueryCtrl extends QueryCtrl { this.metricColumnSegment = uiSegmentSrv.newSegment(this.target.metricColumn); this.buildSelectMenu(); - this.whereSegments = []; - for (let tag of this.target.where) { - if (!tag.operator) { - if (/^\/.*\/$/.test(tag.value)) { - tag.operator = '=~'; - } else { - tag.operator = '='; - } - } - - if (tag.condition) { - this.whereSegments.push(uiSegmentSrv.newCondition(tag.condition)); - } - - this.whereSegments.push(uiSegmentSrv.newKey(tag.key)); - this.whereSegments.push(uiSegmentSrv.newOperator(tag.operator)); - this.whereSegments.push(uiSegmentSrv.newKeyValue(tag.value)); - } - - this.fixWhereSegments(); + this.buildWhereSegments(); this.groupBySegment = this.uiSegmentSrv.newPlusButton(); this.removeWhereFilterSegment = uiSegmentSrv.newSegment({ @@ -264,7 +245,18 @@ export class PostgresQueryCtrl extends QueryCtrl { } } - fixWhereSegments() { + buildWhereSegments() { + this.whereSegments = []; + for (let constraint of this.target.where) { + + if (constraint.condition) { + this.whereSegments.push(this.uiSegmentSrv.newCondition(constraint.condition)); + } + this.whereSegments.push(this.uiSegmentSrv.newKey(constraint.key)); + this.whereSegments.push(this.uiSegmentSrv.newOperator(constraint.operator)); + this.whereSegments.push(this.uiSegmentSrv.newKeyValue(constraint.value)); + } + var count = this.whereSegments.length; var lastSegment = this.whereSegments[Math.max(count - 1, 0)]; From cdb4e2ba0b44741ed9efc09fbf060a7dd5d0c6ac Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 13 Mar 2018 21:31:07 +0100 Subject: [PATCH 023/310] remove unused setting --- public/app/plugins/datasource/postgres/postgres_query.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 9c48a94862f..f9b87e89541 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -21,7 +21,6 @@ export default class PostgresQuery { target.timeColumn = target.timeColumn || 'time'; target.metricColumn = target.metricColumn || 'None'; - target.orderByTime = target.orderByTime || 'ASC'; target.groupBy = target.groupBy || []; target.where = target.where || []; target.select = target.select || [[{ type: 'column', params: ['value'] }]]; From b7c7030a4681e3c5c8d3e565588a02e9e8a2e9db Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 13 Mar 2018 23:06:39 +0100 Subject: [PATCH 024/310] add regex operators --- .../datasource/postgres/query_builder.ts | 8 ++++++++ .../plugins/datasource/postgres/query_ctrl.ts | 18 ++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_builder.ts b/public/app/plugins/datasource/postgres/query_builder.ts index 7a227f33b93..23691830b17 100644 --- a/public/app/plugins/datasource/postgres/query_builder.ts +++ b/public/app/plugins/datasource/postgres/query_builder.ts @@ -47,4 +47,12 @@ export class PostgresQueryBuilder { return query; } + buildDatatypeQuery(column: string) { + var query = "SELECT data_type FROM information_schema.columns WHERE "; + query += " table_schema = " + this.queryModel.quoteLiteral(this.target.schema); + query += " AND table_name = " + this.queryModel.quoteLiteral(this.target.table); + query += " AND column_name = " + this.queryModel.quoteLiteral(column); + return query; + } + } diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index afde342474b..b84b0ce0750 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -266,14 +266,28 @@ export class PostgresQueryCtrl extends QueryCtrl { } getWhereSegments(segment, index) { + var query, addTemplateVars; + if (segment.type === 'condition') { return this.$q.when([this.uiSegmentSrv.newSegment('AND'), this.uiSegmentSrv.newSegment('OR')]); } if (segment.type === 'operator') { - return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '<>', '<', '>'])); + var columnName = this.whereSegments[index - 1].value; + query = this.queryBuilder.buildDatatypeQuery(columnName); + return this.datasource.metricFindQuery(query) + .then(results => { + var datatype = results[0].text; + switch (datatype) { + case "text": + case "character varying": + return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '~', '~*','!~','!~*','IN'])); + default: + return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '<', '<=', '>', '>='])); + } + }) + .catch(this.handleQueryError.bind(this)); } - var query, addTemplateVars; if (segment.type === 'key' || segment.type === 'plus-button') { query = this.queryBuilder.buildColumnQuery(); From 958646d976ad64a3b32707c2a7b686a906cd9fc1 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 13 Mar 2018 23:15:25 +0100 Subject: [PATCH 025/310] dont quote where constraints --- public/app/plugins/datasource/postgres/postgres_query.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index f9b87e89541..48f7efae760 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -137,7 +137,7 @@ export default class PostgresQuery { value = this.templateSrv.replace(value, this.scopedVars); } - return str + this.quoteIdentifier(constraint.key) + ' ' + operator + ' ' + this.quoteLiteral(value); + return str + constraint.key + ' ' + operator + ' ' + value; } interpolateQueryStr(value, variable, defaultFormatFn) { From 5e9a66de5f36c8e29ebeb5370f503529ac2dd7c7 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 13 Mar 2018 23:19:56 +0100 Subject: [PATCH 026/310] put values for IN in parens --- public/app/plugins/datasource/postgres/postgres_query.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 48f7efae760..e06cb3f2126 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -137,7 +137,11 @@ export default class PostgresQuery { value = this.templateSrv.replace(value, this.scopedVars); } - return str + constraint.key + ' ' + operator + ' ' + value; + if (operator === "IN") { + return str + constraint.key + ' ' + operator + ' (' + value + ')'; + } else { + return str + constraint.key + ' ' + operator + ' ' + value; + } } interpolateQueryStr(value, variable, defaultFormatFn) { From e6501f0f0ecec19ccafd9191bcfcdef4a1e19c6f Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 13 Mar 2018 23:24:26 +0100 Subject: [PATCH 027/310] revert special handling for IN --- public/app/plugins/datasource/postgres/postgres_query.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index e06cb3f2126..48f7efae760 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -137,11 +137,7 @@ export default class PostgresQuery { value = this.templateSrv.replace(value, this.scopedVars); } - if (operator === "IN") { - return str + constraint.key + ' ' + operator + ' (' + value + ')'; - } else { - return str + constraint.key + ' ' + operator + ' ' + value; - } + return str + constraint.key + ' ' + operator + ' ' + value; } interpolateQueryStr(value, variable, defaultFormatFn) { From 6793fa5e549dfe31c04dd3dacfa476a9091d3f3a Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 13 Mar 2018 23:33:40 +0100 Subject: [PATCH 028/310] join multivalue variables with , --- public/app/plugins/datasource/postgres/postgres_query.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 48f7efae760..3a11c344f3e 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -151,7 +151,7 @@ export default class PostgresQuery { } var escapedValues = _.map(value, kbn.regexEscape); - return '(' + escapedValues.join('|') + ')'; + return '(' + escapedValues.join(',') + ')'; } render(interpolate?) { From 64fa1ce8a0a3bfe6b7bb4e3c7cbc4227d0c42af4 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 14 Mar 2018 18:09:47 +0100 Subject: [PATCH 029/310] properly handle IN queries --- .../app/plugins/datasource/postgres/postgres_query.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 3a11c344f3e..8f9f1261340 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -1,6 +1,5 @@ import _ from 'lodash'; import queryPart from './query_part'; -import kbn from 'app/core/utils/kbn'; export default class PostgresQuery { target: any; @@ -26,14 +25,16 @@ export default class PostgresQuery { target.select = target.select || [[{ type: 'column', params: ['value'] }]]; this.updateProjection(); + // give interpolateQueryStr access to this + this.interpolateQueryStr = this.interpolateQueryStr.bind(this); } quoteIdentifier(value) { - return '"' + value + '"'; + return '"' + value.replace('"','""') + '"'; } quoteLiteral(value) { - return "'" + value + "'"; + return "'" + value.replace("'","''") + "'"; } updateProjection() { @@ -147,10 +148,10 @@ export default class PostgresQuery { } if (typeof value === 'string') { - return kbn.regexEscape(value); + return this.quoteLiteral(value); } - var escapedValues = _.map(value, kbn.regexEscape); + var escapedValues = _.map(value, this.quoteLiteral); return '(' + escapedValues.join(',') + ')'; } From 0c3afd0e9c098c83f04f44f048f72468496aec5c Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 14 Mar 2018 22:59:48 +0100 Subject: [PATCH 030/310] add buildAggregateQuery --- public/app/plugins/datasource/postgres/query_builder.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/public/app/plugins/datasource/postgres/query_builder.ts b/public/app/plugins/datasource/postgres/query_builder.ts index 23691830b17..275d15492fe 100644 --- a/public/app/plugins/datasource/postgres/query_builder.ts +++ b/public/app/plugins/datasource/postgres/query_builder.ts @@ -55,4 +55,12 @@ export class PostgresQueryBuilder { return query; } + buildAggregateQuery() { + var query = "SELECT DISTINCT proname FROM pg_aggregate "; + query += "INNER JOIN pg_proc ON pg_aggregate.aggfnoid = pg_proc.oid "; + query += "INNER JOIN pg_type ON pg_type.oid=pg_proc.prorettype "; + query += "WHERE pronargs=1 AND typname IN ('int8','float8') AND aggkind='n' ORDER BY 1"; + return query; + } + } From 46c229188e985f74ed35d7505c21d9ec723d7b99 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 14 Mar 2018 23:03:32 +0100 Subject: [PATCH 031/310] read aggregate functions from database --- .../datasource/postgres/postgres_query.ts | 3 ++- .../plugins/datasource/postgres/query_ctrl.ts | 11 +++++++++++ .../plugins/datasource/postgres/query_part.ts | 17 +++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 8f9f1261340..4516bf4a4be 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -24,9 +24,10 @@ export default class PostgresQuery { target.where = target.where || []; target.select = target.select || [[{ type: 'column', params: ['value'] }]]; - this.updateProjection(); // give interpolateQueryStr access to this this.interpolateQueryStr = this.interpolateQueryStr.bind(this); + + this.updateProjection(); } quoteIdentifier(value) { diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index b84b0ce0750..4a2a0ce6c1c 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -77,9 +77,19 @@ export class PostgresQueryCtrl extends QueryCtrl { }); this.panelCtrl.events.on('data-received', this.onDataReceived.bind(this), $scope); this.panelCtrl.events.on('data-error', this.onDataError.bind(this), $scope); + } buildSelectMenu() { + + if (!queryPart.hasAggregates()) { + this.datasource.metricFindQuery(this.queryBuilder.buildAggregateQuery()) + .then(results => { + queryPart.clearAggregates(); + _.map(results, segment => { queryPart.registerAggregate(segment.text); }); + }) + .catch(this.handleQueryError.bind(this)); + } var categories = queryPart.getCategories(); this.selectMenu = _.reduce( categories, @@ -279,6 +289,7 @@ export class PostgresQueryCtrl extends QueryCtrl { var datatype = results[0].text; switch (datatype) { case "text": + case "character": case "character varying": return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '~', '~*','!~','!~*','IN'])); default: diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts index 600723be00d..b63ebfae0c1 100644 --- a/public/app/plugins/datasource/postgres/query_part.ts +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -23,6 +23,17 @@ function register(options: any) { options.category.push(index[options.type]); } +function registerAggregate(name: string) { + register({ + type: name, + addStrategy: replaceAggregationAddStrategy, + category: categories.Aggregations, + params: [], + defaultParams: [], + renderer: functionRenderer, + }); +} + var groupByTimeFunctions = []; function aliasRenderer(part, innerExpr) { @@ -192,6 +203,12 @@ register({ export default { create: createPart, + registerAggregate: registerAggregate, + clearAggregates: function() { categories.Aggregations = []; }, + hasAggregates: function() { + // FIXME + return categories.Aggregations.length > 6; + }, getCategories: function() { return categories; }, From 12600a0e959866036058092d35f6b7414e98dd65 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 26 Mar 2018 13:19:14 +0200 Subject: [PATCH 032/310] support non-nested menu entries --- public/app/plugins/datasource/postgres/query_ctrl.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 4a2a0ce6c1c..8a5cb273ef7 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -205,7 +205,11 @@ export class PostgresQueryCtrl extends QueryCtrl { } addSelectPart(selectParts, cat, subitem) { - this.queryModel.addSelectPart(selectParts, subitem.value); + if ("submenu" in cat) { + this.queryModel.addSelectPart(selectParts, subitem.value); + } else { + this.queryModel.addSelectPart(selectParts, cat.value); + } this.panelCtrl.refresh(); } From 6c2ef7dca6b34f189ef44c416e98c386117d6010 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 26 Mar 2018 18:34:40 +0200 Subject: [PATCH 033/310] handle aggregate functions more generic --- .../plugins/datasource/postgres/query_ctrl.ts | 47 +++++------- .../plugins/datasource/postgres/query_part.ts | 75 ++----------------- 2 files changed, 27 insertions(+), 95 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 8a5cb273ef7..d0ec5dc59be 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -81,30 +81,12 @@ export class PostgresQueryCtrl extends QueryCtrl { } buildSelectMenu() { - - if (!queryPart.hasAggregates()) { - this.datasource.metricFindQuery(this.queryBuilder.buildAggregateQuery()) - .then(results => { - queryPart.clearAggregates(); - _.map(results, segment => { queryPart.registerAggregate(segment.text); }); - }) - .catch(this.handleQueryError.bind(this)); - } - var categories = queryPart.getCategories(); - this.selectMenu = _.reduce( - categories, - function(memo, cat, key) { - var menu = { - text: key, - submenu: cat.map(item => { - return { text: item.type, value: item.type }; - }), - }; - memo.push(menu); - return memo; - }, - [] - ); + this.selectMenu = [ + {text: "aggregate", value: "aggregate"}, + {text: "math", value: "math"}, + {text: "alias", value: "alias"}, + {text: "column", value: "column"}, + ]; } toggleEditorMode() { @@ -216,10 +198,19 @@ export class PostgresQueryCtrl extends QueryCtrl { handleSelectPartEvent(selectParts, part, evt) { switch (evt.name) { case 'get-param-options': { - return this.datasource - .metricFindQuery(this.queryBuilder.buildColumnQuery("value")) - .then(this.transformToSegments(true)) - .catch(this.handleQueryError.bind(this)); + switch (part.def.type) { + case "aggregate": + return this.datasource + .metricFindQuery(this.queryBuilder.buildAggregateQuery()) + .then(this.transformToSegments(false)) + .catch(this.handleQueryError.bind(this)); + case "column": + return this.datasource + .metricFindQuery(this.queryBuilder.buildColumnQuery("value")) + .then(this.transformToSegments(true)) + .catch(this.handleQueryError.bind(this)); + } + } case 'part-param-changed': { this.panelCtrl.refresh(); diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts index b63ebfae0c1..044f51a2457 100644 --- a/public/app/plugins/datasource/postgres/query_part.ts +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -23,23 +23,16 @@ function register(options: any) { options.category.push(index[options.type]); } -function registerAggregate(name: string) { - register({ - type: name, - addStrategy: replaceAggregationAddStrategy, - category: categories.Aggregations, - params: [], - defaultParams: [], - renderer: functionRenderer, - }); -} - var groupByTimeFunctions = []; function aliasRenderer(part, innerExpr) { return innerExpr + ' AS ' + '"' + part.params[0] + '"'; } +function aggregateRenderer(part, innerExpr) { + return part.params[0] + '(' + innerExpr + ')'; +} + function columnRenderer(part, innerExpr) { return '"' + part.params[0] + '"'; } @@ -108,59 +101,13 @@ register({ renderer: columnRenderer, }); -// Aggregations register({ - type: 'avg', + type: 'aggregate', addStrategy: replaceAggregationAddStrategy, category: categories.Aggregations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - -register({ - type: 'count', - addStrategy: replaceAggregationAddStrategy, - category: categories.Aggregations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - -register({ - type: 'sum', - addStrategy: replaceAggregationAddStrategy, - category: categories.Aggregations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - -register({ - type: 'stddev', - addStrategy: replaceAggregationAddStrategy, - category: categories.Aggregations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - -register({ - type: 'min', - addStrategy: replaceAggregationAddStrategy, - category: categories.Aggregations, - params: [], - defaultParams: [], - renderer: functionRenderer, -}); - -register({ - type: 'max', - addStrategy: replaceAggregationAddStrategy, - category: categories.Aggregations, - params: [], - defaultParams: [], - renderer: functionRenderer, + params: [{name: 'name', type: 'string', dynamicLookup: true}], + defaultParams: ['avg'], + renderer: aggregateRenderer, }); register({ @@ -203,12 +150,6 @@ register({ export default { create: createPart, - registerAggregate: registerAggregate, - clearAggregates: function() { categories.Aggregations = []; }, - hasAggregates: function() { - // FIXME - return categories.Aggregations.length > 6; - }, getCategories: function() { return categories; }, From d6ac7aee899db14d2306ab9828cb39ea9854d853 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 26 Mar 2018 18:50:03 +0200 Subject: [PATCH 034/310] remove unused import --- public/app/plugins/datasource/postgres/query_ctrl.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index d0ec5dc59be..dd1da1c75cf 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -2,7 +2,6 @@ import angular from 'angular'; import _ from 'lodash'; import { PostgresQueryBuilder } from './query_builder'; import { QueryCtrl } from 'app/plugins/sdk'; -import queryPart from './query_part'; import PostgresQuery from './postgres_query'; export interface QueryMeta { From 8b3c3081689236be24a21645ca852a928d33d9c7 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 26 Mar 2018 20:15:16 +0200 Subject: [PATCH 035/310] remove categories from queryPart --- .../datasource/postgres/postgres_query.ts | 5 +---- .../plugins/datasource/postgres/query_part.ts | 19 +------------------ 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 4516bf4a4be..a804cb70294 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -82,14 +82,11 @@ export default class PostgresQuery { } removeGroupByPart(part, index) { - var categories = queryPart.getCategories(); - if (part.def.type === 'time') { // remove aggregations this.target.select = _.map(this.target.select, (s: any) => { return _.filter(s, (part: any) => { - var partModel = queryPart.create(part); - if (partModel.def.category === categories.Aggregations) { + if (part.type === "aggregate") { return false; } return true; diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts index 044f51a2457..0d2b365fe66 100644 --- a/public/app/plugins/datasource/postgres/query_part.ts +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -2,12 +2,6 @@ import _ from 'lodash'; import { QueryPartDef, QueryPart, functionRenderer, suffixRenderer } from 'app/core/components/query_part/query_part'; var index = []; -var categories = { - Aggregations: [], - Math: [], - Aliasing: [], - Columns: [], -}; function createPart(part): any { var def = index[part.type]; @@ -20,11 +14,8 @@ function createPart(part): any { function register(options: any) { index[options.type] = new QueryPartDef(options); - options.category.push(index[options.type]); } -var groupByTimeFunctions = []; - function aliasRenderer(part, innerExpr) { return innerExpr + ' AS ' + '"' + part.params[0] + '"'; } @@ -41,7 +32,7 @@ function replaceAggregationAddStrategy(selectParts, partModel) { // look for existing aggregation for (var i = 0; i < selectParts.length; i++) { var part = selectParts[i]; - if (part.def.category === categories.Aggregations) { + if (part.def.type === "aggregate") { selectParts[i] = partModel; return; } @@ -95,7 +86,6 @@ function addColumnStrategy(selectParts, partModel, query) { register({ type: 'column', addStrategy: addColumnStrategy, - category: categories.Columns, params: [{ type: 'column', dynamicLookup: true }], defaultParams: ['value'], renderer: columnRenderer, @@ -104,7 +94,6 @@ register({ register({ type: 'aggregate', addStrategy: replaceAggregationAddStrategy, - category: categories.Aggregations, params: [{name: 'name', type: 'string', dynamicLookup: true}], defaultParams: ['avg'], renderer: aggregateRenderer, @@ -113,7 +102,6 @@ register({ register({ type: 'math', addStrategy: addMathStrategy, - category: categories.Math, params: [{ name: 'expr', type: 'string' }], defaultParams: [' / 100'], renderer: suffixRenderer, @@ -122,7 +110,6 @@ register({ register({ type: 'alias', addStrategy: addAliasStrategy, - category: categories.Aliasing, params: [{ name: 'name', type: 'string', quote: 'double' }], defaultParams: ['alias'], renderMode: 'suffix', @@ -131,7 +118,6 @@ register({ register({ type: 'time', - category: groupByTimeFunctions, params: [ { name: 'interval', @@ -150,7 +136,4 @@ register({ export default { create: createPart, - getCategories: function() { - return categories; - }, }; From 731c7520b393ae7e82603032860390771b35dc7d Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 19 May 2018 15:34:48 +0200 Subject: [PATCH 036/310] return values quotes for suggestions in where expression --- public/app/plugins/datasource/postgres/query_builder.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_builder.ts b/public/app/plugins/datasource/postgres/query_builder.ts index 275d15492fe..3dfbf99fefd 100644 --- a/public/app/plugins/datasource/postgres/query_builder.ts +++ b/public/app/plugins/datasource/postgres/query_builder.ts @@ -39,11 +39,10 @@ export class PostgresQueryBuilder { } buildValueQuery(column: string) { - var query = "SELECT DISTINCT " + this.queryModel.quoteIdentifier(column) + "::text"; + var query = "SELECT DISTINCT quote_literal(" + column + ")"; query += " FROM " + this.queryModel.quoteIdentifier(this.target.schema); query += "." + this.queryModel.quoteIdentifier(this.target.table); - query += " ORDER BY " + this.queryModel.quoteIdentifier(column); - query += " LIMIT 100"; + query += " ORDER BY 1 LIMIT 100"; return query; } From b12d049f6f1b24ff733b0e6f71f910977751e5a2 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 19 May 2018 17:21:53 +0200 Subject: [PATCH 037/310] quote column name in buildValueQuery --- public/app/plugins/datasource/postgres/query_builder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/query_builder.ts b/public/app/plugins/datasource/postgres/query_builder.ts index 3dfbf99fefd..dd6f95b550c 100644 --- a/public/app/plugins/datasource/postgres/query_builder.ts +++ b/public/app/plugins/datasource/postgres/query_builder.ts @@ -39,7 +39,7 @@ export class PostgresQueryBuilder { } buildValueQuery(column: string) { - var query = "SELECT DISTINCT quote_literal(" + column + ")"; + var query = "SELECT DISTINCT quote_literal(" + this.queryModel.quoteIdentifier(column) + ")"; query += " FROM " + this.queryModel.quoteIdentifier(this.target.schema); query += "." + this.queryModel.quoteIdentifier(this.target.table); query += " ORDER BY 1 LIMIT 100"; From 181dfdba04d6ff31ae468eea11a5fe334290f617 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 21 May 2018 09:37:04 +0200 Subject: [PATCH 038/310] add sql_part component --- .../app/core/components/sql_part/sql_part.ts | 118 +++++++++++ .../components/sql_part/sql_part_editor.ts | 185 ++++++++++++++++++ public/app/core/core.ts | 2 + 3 files changed, 305 insertions(+) create mode 100644 public/app/core/components/sql_part/sql_part.ts create mode 100644 public/app/core/components/sql_part/sql_part_editor.ts diff --git a/public/app/core/components/sql_part/sql_part.ts b/public/app/core/components/sql_part/sql_part.ts new file mode 100644 index 00000000000..77366e6d09b --- /dev/null +++ b/public/app/core/components/sql_part/sql_part.ts @@ -0,0 +1,118 @@ +import _ from 'lodash'; + +export class SqlPartDef { + type: string; + params: any[]; + defaultParams: any[]; + renderer: any; + category: any; + addStrategy: any; + + constructor(options: any) { + this.type = options.type; + this.params = options.params; + this.defaultParams = options.defaultParams; + this.renderer = options.renderer; + this.category = options.category; + this.addStrategy = options.addStrategy; + } +} + +export class SqlPart { + part: any; + def: SqlPartDef; + params: any[]; + text: string; + + constructor(part: any, def: any) { + this.part = part; + this.def = def; + if (!this.def) { + throw { message: 'Could not find query part ' + part.type }; + } + + part.params = part.params || _.clone(this.def.defaultParams); + this.params = part.params; + this.updateText(); + } + + render(innerExpr: string) { + return this.def.renderer(this, innerExpr); + } + + hasMultipleParamsInString(strValue, index) { + if (strValue.indexOf(',') === -1) { + return false; + } + + return this.def.params[index + 1] && this.def.params[index + 1].optional; + } + + updateParam(strValue, index) { + // handle optional parameters + // if string contains ',' and next param is optional, split and update both + if (this.hasMultipleParamsInString(strValue, index)) { + _.each(strValue.split(','), (partVal, idx) => { + this.updateParam(partVal.trim(), idx); + }); + return; + } + + if (strValue === '' && this.def.params[index].optional) { + this.params.splice(index, 1); + } else { + this.params[index] = strValue; + } + + this.part.params = this.params; + this.updateText(); + } + + updateText() { + if (this.params.length === 0) { + this.text = this.def.type + '()'; + return; + } + + var text = this.def.type + '('; + text += this.params.join(', '); + text += ')'; + this.text = text; + } +} + +export function functionRenderer(part, innerExpr) { + var str = part.def.type + '('; + var parameters = _.map(part.params, (value, index) => { + var paramType = part.def.params[index]; + if (paramType.type === 'time') { + if (value === 'auto') { + value = '$__interval'; + } + } + if (paramType.quote === 'single') { + return "'" + value + "'"; + } else if (paramType.quote === 'double') { + return '"' + value + '"'; + } + + return value; + }); + + if (innerExpr) { + parameters.unshift(innerExpr); + } + return str + parameters.join(', ') + ')'; +} + +export function suffixRenderer(part, innerExpr) { + return innerExpr + ' ' + part.params[0]; +} + +export function identityRenderer(part, innerExpr) { + return part.params[0]; +} + +export function quotedIdentityRenderer(part, innerExpr) { + return '"' + part.params[0] + '"'; +} diff --git a/public/app/core/components/sql_part/sql_part_editor.ts b/public/app/core/components/sql_part/sql_part_editor.ts new file mode 100644 index 00000000000..837f280e66d --- /dev/null +++ b/public/app/core/components/sql_part/sql_part_editor.ts @@ -0,0 +1,185 @@ +import _ from 'lodash'; +import $ from 'jquery'; +import coreModule from 'app/core/core_module'; + +var template = ` +
+{{part.def.type}} +() + +`; + +/** @ngInject */ +export function sqlPartEditorDirective($compile, templateSrv) { + var paramTemplate = ''; + + return { + restrict: 'E', + template: template, + scope: { + part: '=', + handleEvent: '&', + debounce: '@', + }, + link: function postLink($scope, elem) { + var part = $scope.part; + var partDef = part.def; + var $paramsContainer = elem.find('.query-part-parameters'); + var debounceLookup = $scope.debounce; + + $scope.partActions = []; + + function clickFuncParam(paramIndex) { + /*jshint validthis:true */ + var $link = $(this); + var $input = $link.next(); + + $input.val(part.params[paramIndex]); + $input.css('width', $link.width() + 16 + 'px'); + + $link.hide(); + $input.show(); + $input.focus(); + $input.select(); + + var typeahead = $input.data('typeahead'); + if (typeahead) { + $input.val(''); + typeahead.lookup(); + } + } + + function inputBlur(paramIndex) { + /*jshint validthis:true */ + var $input = $(this); + var $link = $input.prev(); + var newValue = $input.val(); + + if (newValue !== '' || part.def.params[paramIndex].optional) { + $link.html(templateSrv.highlightVariablesAsHtml(newValue)); + + part.updateParam($input.val(), paramIndex); + $scope.$apply(() => { + $scope.handleEvent({ $event: { name: 'part-param-changed' } }); + }); + } + + $input.hide(); + $link.show(); + } + + function inputKeyPress(paramIndex, e) { + /*jshint validthis:true */ + if (e.which === 13) { + inputBlur.call(this, paramIndex); + } + } + + function inputKeyDown() { + /*jshint validthis:true */ + this.style.width = (3 + this.value.length) * 8 + 'px'; + } + + function addTypeahead($input, param, paramIndex) { + if (!param.options && !param.dynamicLookup) { + return; + } + + var typeaheadSource = function(query, callback) { + if (param.options) { + var options = param.options; + if (param.type === 'int') { + options = _.map(options, function(val) { + return val.toString(); + }); + } + return options; + } + + $scope.$apply(function() { + $scope.handleEvent({ $event: { name: 'get-param-options' } }).then(function(result) { + var dynamicOptions = _.map(result, function(op) { + return op.value; + }); + callback(dynamicOptions); + }); + }); + }; + + $input.attr('data-provide', 'typeahead'); + + $input.typeahead({ + source: typeaheadSource, + minLength: 0, + items: 1000, + updater: function(value) { + setTimeout(function() { + inputBlur.call($input[0], paramIndex); + }, 0); + return value; + }, + }); + + var typeahead = $input.data('typeahead'); + typeahead.lookup = function() { + this.query = this.$element.val() || ''; + var items = this.source(this.query, $.proxy(this.process, this)); + return items ? this.process(items) : items; + }; + + if (debounceLookup) { + typeahead.lookup = _.debounce(typeahead.lookup, 500, { leading: true }); + } + } + + $scope.showActionsMenu = function() { + $scope.handleEvent({ $event: { name: 'get-part-actions' } }).then(res => { + $scope.partActions = res; + }); + }; + + $scope.triggerPartAction = function(action) { + $scope.handleEvent({ $event: { name: 'action', action: action } }); + }; + + function addElementsAndCompile() { + _.each(partDef.params, function(param, index) { + if (param.optional && part.params.length <= index) { + return; + } + + if (index > 0) { + $(', ').appendTo($paramsContainer); + } + + var paramValue = templateSrv.highlightVariablesAsHtml(part.params[index]); + var $paramLink = $('' + paramValue + ''); + var $input = $(paramTemplate); + + $paramLink.appendTo($paramsContainer); + $input.appendTo($paramsContainer); + + $input.blur(_.partial(inputBlur, index)); + $input.keyup(inputKeyDown); + $input.keypress(_.partial(inputKeyPress, index)); + $paramLink.click(_.partial(clickFuncParam, index)); + + addTypeahead($input, param, index); + }); + } + + function relink() { + $paramsContainer.empty(); + addElementsAndCompile(); + } + + relink(); + }, + }; +} + +coreModule.directive('sqlPartEditor', sqlPartEditorDirective); diff --git a/public/app/core/core.ts b/public/app/core/core.ts index fb7021fe883..2bfe84a74df 100644 --- a/public/app/core/core.ts +++ b/public/app/core/core.ts @@ -31,6 +31,7 @@ import { layoutSelector } from './components/layout_selector/layout_selector'; import { switchDirective } from './components/switch'; import { dashboardSelector } from './components/dashboard_selector'; import { queryPartEditorDirective } from './components/query_part/query_part_editor'; +import { sqlPartEditorDirective } from './components/sql_part/sql_part_editor'; import { formDropdownDirective } from './components/form_dropdown/form_dropdown'; import 'app/core/controllers/all'; import 'app/core/services/all'; @@ -74,6 +75,7 @@ export { appEvents, dashboardSelector, queryPartEditorDirective, + sqlPartEditorDirective, colors, formDropdownDirective, assignModelProperties, From de9a117dc67aee70aff6cc108fea4981a8228559 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 21 May 2018 11:04:52 +0200 Subject: [PATCH 039/310] use sql part component --- .../datasource/postgres/partials/query.editor.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 1833b845d41..b0614a3ee9e 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -36,8 +36,8 @@
- - + +
@@ -72,10 +72,10 @@ GROUP BY - - +
From e93276b1f822edb434b280662fb914a1e0e4a626 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 21 May 2018 11:10:00 +0200 Subject: [PATCH 040/310] use sql part component --- public/app/plugins/datasource/postgres/query_part.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts index 0d2b365fe66..d54a66783eb 100644 --- a/public/app/plugins/datasource/postgres/query_part.ts +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -1,5 +1,5 @@ import _ from 'lodash'; -import { QueryPartDef, QueryPart, functionRenderer, suffixRenderer } from 'app/core/components/query_part/query_part'; +import { SqlPartDef, SqlPart, functionRenderer, suffixRenderer } from 'app/core/components/sql_part/sql_part'; var index = []; @@ -9,11 +9,11 @@ function createPart(part): any { throw { message: 'Could not find query part ' + part.type }; } - return new QueryPart(part, def); + return new SqlPart(part, def); } function register(options: any) { - index[options.type] = new QueryPartDef(options); + index[options.type] = new SqlPartDef(options); } function aliasRenderer(part, innerExpr) { From 3af4e4e0d696ba62731bcd9b527472b5c2acef68 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 21 May 2018 11:44:37 +0200 Subject: [PATCH 041/310] separate label in template from type --- public/app/core/components/sql_part/sql_part.ts | 2 ++ public/app/core/components/sql_part/sql_part_editor.ts | 2 +- public/app/plugins/datasource/postgres/query_part.ts | 5 +++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/public/app/core/components/sql_part/sql_part.ts b/public/app/core/components/sql_part/sql_part.ts index 77366e6d09b..94dd0ff6ae8 100644 --- a/public/app/core/components/sql_part/sql_part.ts +++ b/public/app/core/components/sql_part/sql_part.ts @@ -2,6 +2,7 @@ import _ from 'lodash'; export class SqlPartDef { type: string; + label: string; params: any[]; defaultParams: any[]; renderer: any; @@ -10,6 +11,7 @@ export class SqlPartDef { constructor(options: any) { this.type = options.type; + this.label = options.label; this.params = options.params; this.defaultParams = options.defaultParams; this.renderer = options.renderer; diff --git a/public/app/core/components/sql_part/sql_part_editor.ts b/public/app/core/components/sql_part/sql_part_editor.ts index 837f280e66d..2ebeca5753a 100644 --- a/public/app/core/components/sql_part/sql_part_editor.ts +++ b/public/app/core/components/sql_part/sql_part_editor.ts @@ -4,7 +4,7 @@ import coreModule from 'app/core/core_module'; var template = `
-{{part.def.type}} +{{part.def.label}} ()
  • diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts index d54a66783eb..029820da776 100644 --- a/public/app/plugins/datasource/postgres/query_part.ts +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -85,6 +85,7 @@ function addColumnStrategy(selectParts, partModel, query) { register({ type: 'column', + label: 'Column:', addStrategy: addColumnStrategy, params: [{ type: 'column', dynamicLookup: true }], defaultParams: ['value'], @@ -93,6 +94,7 @@ register({ register({ type: 'aggregate', + label: 'Aggregate:', addStrategy: replaceAggregationAddStrategy, params: [{name: 'name', type: 'string', dynamicLookup: true}], defaultParams: ['avg'], @@ -101,6 +103,7 @@ register({ register({ type: 'math', + label: 'Math:', addStrategy: addMathStrategy, params: [{ name: 'expr', type: 'string' }], defaultParams: [' / 100'], @@ -109,6 +112,7 @@ register({ register({ type: 'alias', + label: 'Alias:', addStrategy: addAliasStrategy, params: [{ name: 'name', type: 'string', quote: 'double' }], defaultParams: ['alias'], @@ -118,6 +122,7 @@ register({ register({ type: 'time', + label: 'time', params: [ { name: 'interval', From a5bcd8eac02f52900c0839dd2642a3dbd02ced23 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 21 May 2018 12:10:18 +0200 Subject: [PATCH 042/310] fix metric column when using group by --- .../postgres/partials/query.editor.html | 28 +++++++++---------- .../datasource/postgres/postgres_query.ts | 20 ++++++------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index b0614a3ee9e..220f11fdfef 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -28,6 +28,20 @@
+
+
+ + + + +
+ +
+
+
+ +
+
-
-
- - - - -
- -
-
-
- -
-
From 3cb0e27e1c474e5d203eb32428b2f39ee5fb3216 Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Sat, 19 May 2018 16:21:00 -0400 Subject: [PATCH 050/310] Revert changes post code review and move them to notification page --- pkg/api/dtos/alerting.go | 25 +++---- pkg/models/alert.go | 9 --- pkg/models/alert_notifications.go | 71 ++++++++++++++----- pkg/services/alerting/eval_context.go | 15 ++++ pkg/services/alerting/extractor.go | 2 - pkg/services/alerting/interfaces.go | 2 + pkg/services/alerting/notifier.go | 12 +++- .../alerting/notifiers/alertmanager.go | 2 +- pkg/services/alerting/notifiers/base.go | 25 +++++-- pkg/services/alerting/notifiers/dingding.go | 2 +- pkg/services/alerting/notifiers/discord.go | 2 +- pkg/services/alerting/notifiers/email.go | 2 +- pkg/services/alerting/notifiers/hipchat.go | 2 +- pkg/services/alerting/notifiers/kafka.go | 2 +- pkg/services/alerting/notifiers/line.go | 2 +- pkg/services/alerting/notifiers/opsgenie.go | 2 +- pkg/services/alerting/notifiers/pagerduty.go | 2 +- pkg/services/alerting/notifiers/pushover.go | 2 +- pkg/services/alerting/notifiers/sensu.go | 2 +- pkg/services/alerting/notifiers/slack.go | 2 +- pkg/services/alerting/notifiers/teams.go | 2 +- pkg/services/alerting/notifiers/telegram.go | 2 +- pkg/services/alerting/notifiers/threema.go | 2 +- pkg/services/alerting/notifiers/victorops.go | 2 +- pkg/services/alerting/notifiers/webhook.go | 2 +- pkg/services/alerting/result_handler.go | 1 - pkg/services/alerting/rule.go | 6 -- pkg/services/sqlstore/alert.go | 22 +----- pkg/services/sqlstore/alert_notification.go | 58 +++++++++++++-- pkg/services/sqlstore/migrations/alert_mig.go | 27 ++++++- .../app/features/alerting/alert_tab_ctrl.ts | 3 - .../alerting/notification_edit_ctrl.ts | 2 + .../features/alerting/partials/alert_tab.html | 3 - .../alerting/partials/notification_edit.html | 5 ++ 34 files changed, 215 insertions(+), 107 deletions(-) diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go index 64dd619a4eb..5e0196c20d1 100644 --- a/pkg/api/dtos/alerting.go +++ b/pkg/api/dtos/alerting.go @@ -21,18 +21,17 @@ type AlertRule struct { ExecutionError string `json:"executionError"` Url string `json:"url"` CanEdit bool `json:"canEdit"` - NotifyOnce bool `json:"notifyOnce"` - NotifyEval uint64 `json:"notifyEval"` - NotifyFreq uint64 `json:"notifyFrequency"` } type AlertNotification struct { - Id int64 `json:"id"` - Name string `json:"name"` - Type string `json:"type"` - IsDefault bool `json:"isDefault"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` + Id int64 `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + IsDefault bool `json:"isDefault"` + NotifyOnce bool `json:"notifyOnce"` + Frequency bool `json:"frequency"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` } type AlertTestCommand struct { @@ -62,9 +61,11 @@ type EvalMatch struct { } type NotificationTestCommand struct { - Name string `json:"name"` - Type string `json:"type"` - Settings *simplejson.Json `json:"settings"` + Name string `json:"name"` + Type string `json:"type"` + NotifyOnce bool `json:"notifyOnce"` + Frequency time.Duration `json:"frequency"` + Settings *simplejson.Json `json:"settings"` } type PauseAlertCommand struct { diff --git a/pkg/models/alert.go b/pkg/models/alert.go index 56ceeb2cbf7..fba2aa63df9 100644 --- a/pkg/models/alert.go +++ b/pkg/models/alert.go @@ -72,9 +72,6 @@ type Alert struct { Silenced bool ExecutionError string Frequency int64 - NotifyOnce bool - NotifyFreq uint64 - NotifyEval uint64 EvalData *simplejson.Json NewStateDate time.Time @@ -98,8 +95,6 @@ func (this *Alert) ContainsUpdates(other *Alert) bool { result := false result = result || this.Name != other.Name result = result || this.Message != other.Message - result = result || this.NotifyOnce != other.NotifyOnce - result = result || (!other.NotifyOnce && this.NotifyFreq != other.NotifyFreq) if this.Settings != nil && other.Settings != nil { json1, err1 := this.Settings.Encode() @@ -164,10 +159,6 @@ type SetAlertStateCommand struct { Timestamp time.Time } -type IncAlertEvalCommand struct { - AlertId int64 -} - //Queries type GetAlertsQuery struct { OrgId int64 diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index 87b515f370c..cba62a51527 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -7,32 +7,38 @@ import ( ) type AlertNotification struct { - Id int64 `json:"id"` - OrgId int64 `json:"-"` - Name string `json:"name"` - Type string `json:"type"` - IsDefault bool `json:"isDefault"` - Settings *simplejson.Json `json:"settings"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` + Id int64 `json:"id"` + OrgId int64 `json:"-"` + Name string `json:"name"` + Type string `json:"type"` + NotifyOnce bool `json:"notifyOnce"` + Frequency time.Duration `json:"frequency"` + IsDefault bool `json:"isDefault"` + Settings *simplejson.Json `json:"settings"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` } type CreateAlertNotificationCommand struct { - Name string `json:"name" binding:"Required"` - Type string `json:"type" binding:"Required"` - IsDefault bool `json:"isDefault"` - Settings *simplejson.Json `json:"settings"` + Name string `json:"name" binding:"Required"` + Type string `json:"type" binding:"Required"` + NotifyOnce bool `json:"notifyOnce" binding:"Required"` + Frequency time.Duration `json:"frequency"` + IsDefault bool `json:"isDefault"` + Settings *simplejson.Json `json:"settings"` OrgId int64 `json:"-"` Result *AlertNotification } type UpdateAlertNotificationCommand struct { - Id int64 `json:"id" binding:"Required"` - Name string `json:"name" binding:"Required"` - Type string `json:"type" binding:"Required"` - IsDefault bool `json:"isDefault"` - Settings *simplejson.Json `json:"settings" binding:"Required"` + Id int64 `json:"id" binding:"Required"` + Name string `json:"name" binding:"Required"` + Type string `json:"type" binding:"Required"` + NotifyOnce string `json:"notifyOnce" binding:"Required"` + Frequency string `json:"frequency"` + IsDefault bool `json:"isDefault"` + Settings *simplejson.Json `json:"settings" binding:"Required"` OrgId int64 `json:"-"` Result *AlertNotification @@ -63,3 +69,34 @@ type GetAllAlertNotificationsQuery struct { Result []*AlertNotification } + +type NotificationJournal struct { + Id int64 + OrgId int64 + AlertId int64 + NotifierId int64 + SentAt time.Time + Success bool +} + +type RecordNotificationJournalCommand struct { + OrgId int64 + AlertId int64 + NotifierId int64 + SentAt time.Time + Success bool +} + +type GetLatestNotificationQuery struct { + OrgId int64 + AlertId int64 + NotifierId int64 + + Result *NotificationJournal +} + +type CleanNotificationJournalCommand struct { + OrgId int64 + AlertId int64 + NotifierId int64 +} diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go index d0441d379b7..b451d188a64 100644 --- a/pkg/services/alerting/eval_context.go +++ b/pkg/services/alerting/eval_context.go @@ -143,3 +143,18 @@ func (c *EvalContext) GetNewState() m.AlertStateType { return m.AlertStateOK } + +func (c *EvalContext) LastNotify(notifierId int64) *time.Time { + cmd := &m.GetLatestNotificationQuery{ + OrgId: c.Rule.OrgId, + AlertId: c.Rule.Id, + NotifierId: notifierId, + } + if err := bus.Dispatch(cmd); err != nil { + c.log.Warn("Could not determine last time alert", + c.Rule.Name, "notified") + return nil + } + + return &cmd.Result.SentAt +} diff --git a/pkg/services/alerting/extractor.go b/pkg/services/alerting/extractor.go index f820e546a93..e1c1bfacb2e 100644 --- a/pkg/services/alerting/extractor.go +++ b/pkg/services/alerting/extractor.go @@ -122,8 +122,6 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json, Handler: jsonAlert.Get("handler").MustInt64(), Message: jsonAlert.Get("message").MustString(), Frequency: frequency, - NotifyOnce: jsonAlert.Get("notifyOnce").MustBool(), - NotifyFreq: jsonAlert.Get("notifyFrequency").MustUint64(), } for _, condition := range jsonAlert.Get("conditions").MustArray() { diff --git a/pkg/services/alerting/interfaces.go b/pkg/services/alerting/interfaces.go index 18f969ba1b9..8842b35fba2 100644 --- a/pkg/services/alerting/interfaces.go +++ b/pkg/services/alerting/interfaces.go @@ -19,6 +19,8 @@ type Notifier interface { GetNotifierId() int64 GetIsDefault() bool + GetNotifyOnce() bool + GetFrequency() time.Duration } type NotifierSlice []Notifier diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 2ea68cf5085..53923a420fe 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -66,7 +66,17 @@ func (n *notificationService) sendNotifications(context *EvalContext, notifiers not := notifier //avoid updating scope variable in go routine n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault()) metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc() - g.Go(func() error { return not.Notify(context) }) + g.Go(func() error { + success := not.Notify(context) == nil + cmd := &m.RecordNotificationJournalCommand{ + OrgId: context.Rule.OrgId, + AlertId: context.Rule.Id, + NotifierId: not.GetNotifierId(), + SentAt: time.Now(), + Success: success, + } + return bus.Dispatch(cmd) + }) } return g.Wait() diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go index d449167de13..3eeb25986e0 100644 --- a/pkg/services/alerting/notifiers/alertmanager.go +++ b/pkg/services/alerting/notifiers/alertmanager.go @@ -33,7 +33,7 @@ func NewAlertmanagerNotifier(model *m.AlertNotification) (alerting.Notifier, err } return &AlertmanagerNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Url: url, log: log.New("alerting.notifier.prometheus-alertmanager"), }, nil diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 498bae3a6e6..e9e32020c6d 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -1,6 +1,8 @@ package notifiers import ( + "time" + "github.com/grafana/grafana/pkg/components/simplejson" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting" @@ -12,9 +14,11 @@ type NotifierBase struct { Id int64 IsDeault bool UploadImage bool + NotifyOnce bool + Frequency time.Duration } -func NewNotifierBase(id int64, isDefault bool, name, notifierType string, model *simplejson.Json) NotifierBase { +func NewNotifierBase(id int64, isDefault bool, name, notifierType string, notifyOnce bool, frequency time.Duration, model *simplejson.Json) NotifierBase { uploadImage := true value, exist := model.CheckGet("uploadImage") if exist { @@ -27,15 +31,17 @@ func NewNotifierBase(id int64, isDefault bool, name, notifierType string, model IsDeault: isDefault, Type: notifierType, UploadImage: uploadImage, + NotifyOnce: notifyOnce, + Frequency: frequency, } } -func defaultShouldNotify(context *alerting.EvalContext) bool { +func defaultShouldNotify(context *alerting.EvalContext, notifyOnce bool, frequency time.Duration, lastNotify *time.Time) bool { // Only notify on state change. - if context.PrevAlertState == context.Rule.State && context.Rule.NotifyOnce { + if context.PrevAlertState == context.Rule.State && notifyOnce { return false } - if !context.Rule.NotifyOnce && context.Rule.NotifyEval != 0 { + if !notifyOnce && lastNotify != nil && lastNotify.Add(frequency).After(time.Now()) { return false } // Do not notify when we become OK for the first time. @@ -46,7 +52,8 @@ func defaultShouldNotify(context *alerting.EvalContext) bool { } func (n *NotifierBase) ShouldNotify(context *alerting.EvalContext) bool { - return defaultShouldNotify(context) + lastNotify := context.LastNotify(n.Id) + return defaultShouldNotify(context, n.NotifyOnce, n.Frequency, lastNotify) } func (n *NotifierBase) GetType() string { @@ -64,3 +71,11 @@ func (n *NotifierBase) GetNotifierId() int64 { func (n *NotifierBase) GetIsDefault() bool { return n.IsDeault } + +func (n *NotifierBase) GetNotifyOnce() bool { + return n.NotifyOnce +} + +func (n *NotifierBase) GetFrequency() time.Duration { + return n.Frequency +} diff --git a/pkg/services/alerting/notifiers/dingding.go b/pkg/services/alerting/notifiers/dingding.go index 14eacef5831..78446c56f88 100644 --- a/pkg/services/alerting/notifiers/dingding.go +++ b/pkg/services/alerting/notifiers/dingding.go @@ -32,7 +32,7 @@ func NewDingDingNotifier(model *m.AlertNotification) (alerting.Notifier, error) } return &DingDingNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Url: url, log: log.New("alerting.notifier.dingding"), }, nil diff --git a/pkg/services/alerting/notifiers/discord.go b/pkg/services/alerting/notifiers/discord.go index 3ffa7484870..693ed31e206 100644 --- a/pkg/services/alerting/notifiers/discord.go +++ b/pkg/services/alerting/notifiers/discord.go @@ -39,7 +39,7 @@ func NewDiscordNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &DiscordNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), WebhookURL: url, log: log.New("alerting.notifier.discord"), }, nil diff --git a/pkg/services/alerting/notifiers/email.go b/pkg/services/alerting/notifiers/email.go index 562ffbe1269..234a4f8e756 100644 --- a/pkg/services/alerting/notifiers/email.go +++ b/pkg/services/alerting/notifiers/email.go @@ -52,7 +52,7 @@ func NewEmailNotifier(model *m.AlertNotification) (alerting.Notifier, error) { }) return &EmailNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Addresses: addresses, log: log.New("alerting.notifier.email"), }, nil diff --git a/pkg/services/alerting/notifiers/hipchat.go b/pkg/services/alerting/notifiers/hipchat.go index 58e1b7bd71e..4eb5b78811e 100644 --- a/pkg/services/alerting/notifiers/hipchat.go +++ b/pkg/services/alerting/notifiers/hipchat.go @@ -59,7 +59,7 @@ func NewHipChatNotifier(model *models.AlertNotification) (alerting.Notifier, err roomId := model.Settings.Get("roomid").MustString() return &HipChatNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Url: url, ApiKey: apikey, RoomId: roomId, diff --git a/pkg/services/alerting/notifiers/kafka.go b/pkg/services/alerting/notifiers/kafka.go index 92f6489106b..0dab556d5e1 100644 --- a/pkg/services/alerting/notifiers/kafka.go +++ b/pkg/services/alerting/notifiers/kafka.go @@ -43,7 +43,7 @@ func NewKafkaNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &KafkaNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Endpoint: endpoint, Topic: topic, log: log.New("alerting.notifier.kafka"), diff --git a/pkg/services/alerting/notifiers/line.go b/pkg/services/alerting/notifiers/line.go index 4814662f3a9..0ee252e6447 100644 --- a/pkg/services/alerting/notifiers/line.go +++ b/pkg/services/alerting/notifiers/line.go @@ -39,7 +39,7 @@ func NewLINENotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &LineNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Token: token, log: log.New("alerting.notifier.line"), }, nil diff --git a/pkg/services/alerting/notifiers/opsgenie.go b/pkg/services/alerting/notifiers/opsgenie.go index f0f5142cf05..991afd5ce9b 100644 --- a/pkg/services/alerting/notifiers/opsgenie.go +++ b/pkg/services/alerting/notifiers/opsgenie.go @@ -56,7 +56,7 @@ func NewOpsGenieNotifier(model *m.AlertNotification) (alerting.Notifier, error) } return &OpsGenieNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), ApiKey: apiKey, ApiUrl: apiUrl, AutoClose: autoClose, diff --git a/pkg/services/alerting/notifiers/pagerduty.go b/pkg/services/alerting/notifiers/pagerduty.go index 02219b2203d..afa0ba63eca 100644 --- a/pkg/services/alerting/notifiers/pagerduty.go +++ b/pkg/services/alerting/notifiers/pagerduty.go @@ -51,7 +51,7 @@ func NewPagerdutyNotifier(model *m.AlertNotification) (alerting.Notifier, error) } return &PagerdutyNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Key: key, AutoResolve: autoResolve, log: log.New("alerting.notifier.pagerduty"), diff --git a/pkg/services/alerting/notifiers/pushover.go b/pkg/services/alerting/notifiers/pushover.go index cbe9e16801a..09dfd6f0f9b 100644 --- a/pkg/services/alerting/notifiers/pushover.go +++ b/pkg/services/alerting/notifiers/pushover.go @@ -99,7 +99,7 @@ func NewPushoverNotifier(model *m.AlertNotification) (alerting.Notifier, error) return nil, alerting.ValidationError{Reason: "API token not given"} } return &PushoverNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), UserKey: userKey, ApiToken: apiToken, Priority: priority, diff --git a/pkg/services/alerting/notifiers/sensu.go b/pkg/services/alerting/notifiers/sensu.go index 9f77801d458..e6b94d3223e 100644 --- a/pkg/services/alerting/notifiers/sensu.go +++ b/pkg/services/alerting/notifiers/sensu.go @@ -51,7 +51,7 @@ func NewSensuNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &SensuNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Url: url, User: model.Settings.Get("username").MustString(), Source: model.Settings.Get("source").MustString(), diff --git a/pkg/services/alerting/notifiers/slack.go b/pkg/services/alerting/notifiers/slack.go index a8139b62726..fbbe4b3e59d 100644 --- a/pkg/services/alerting/notifiers/slack.go +++ b/pkg/services/alerting/notifiers/slack.go @@ -78,7 +78,7 @@ func NewSlackNotifier(model *m.AlertNotification) (alerting.Notifier, error) { uploadImage := model.Settings.Get("uploadImage").MustBool(true) return &SlackNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Url: url, Recipient: recipient, Mention: mention, diff --git a/pkg/services/alerting/notifiers/teams.go b/pkg/services/alerting/notifiers/teams.go index 7f62340d0e1..362a367e1f2 100644 --- a/pkg/services/alerting/notifiers/teams.go +++ b/pkg/services/alerting/notifiers/teams.go @@ -33,7 +33,7 @@ func NewTeamsNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &TeamsNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Url: url, log: log.New("alerting.notifier.teams"), }, nil diff --git a/pkg/services/alerting/notifiers/telegram.go b/pkg/services/alerting/notifiers/telegram.go index ca24c996914..97696b2290c 100644 --- a/pkg/services/alerting/notifiers/telegram.go +++ b/pkg/services/alerting/notifiers/telegram.go @@ -78,7 +78,7 @@ func NewTelegramNotifier(model *m.AlertNotification) (alerting.Notifier, error) } return &TelegramNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), BotToken: botToken, ChatID: chatId, UploadImage: uploadImage, diff --git a/pkg/services/alerting/notifiers/threema.go b/pkg/services/alerting/notifiers/threema.go index e4ffffc9108..e7fb39f27db 100644 --- a/pkg/services/alerting/notifiers/threema.go +++ b/pkg/services/alerting/notifiers/threema.go @@ -106,7 +106,7 @@ func NewThreemaNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &ThreemaNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), GatewayID: gatewayID, RecipientID: recipientID, APISecret: apiSecret, diff --git a/pkg/services/alerting/notifiers/victorops.go b/pkg/services/alerting/notifiers/victorops.go index a753ca3cbf6..c6c1cf76047 100644 --- a/pkg/services/alerting/notifiers/victorops.go +++ b/pkg/services/alerting/notifiers/victorops.go @@ -51,7 +51,7 @@ func NewVictoropsNotifier(model *models.AlertNotification) (alerting.Notifier, e } return &VictoropsNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), URL: url, AutoResolve: autoResolve, log: log.New("alerting.notifier.victorops"), diff --git a/pkg/services/alerting/notifiers/webhook.go b/pkg/services/alerting/notifiers/webhook.go index 4c97ed2b75e..26989873e9e 100644 --- a/pkg/services/alerting/notifiers/webhook.go +++ b/pkg/services/alerting/notifiers/webhook.go @@ -47,7 +47,7 @@ func NewWebHookNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &WebhookNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.Settings), + NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), Url: url, User: model.Settings.Get("username").MustString(), Password: model.Settings.Get("password").MustString(), diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go index 56d299001f0..c57b28c7c3e 100644 --- a/pkg/services/alerting/result_handler.go +++ b/pkg/services/alerting/result_handler.go @@ -88,7 +88,6 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { } } - bus.Dispatch(&m.IncAlertEvalCommand{AlertId: evalContext.Rule.Id}) handler.notifier.SendIfNeeded(evalContext) return nil diff --git a/pkg/services/alerting/rule.go b/pkg/services/alerting/rule.go index 0003fe791e3..018d138dbe4 100644 --- a/pkg/services/alerting/rule.go +++ b/pkg/services/alerting/rule.go @@ -23,9 +23,6 @@ type Rule struct { State m.AlertStateType Conditions []Condition Notifications []int64 - NotifyOnce bool - NotifyFreq uint64 - NotifyEval uint64 } type ValidationError struct { @@ -100,9 +97,6 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) { model.Name = ruleDef.Name model.Message = ruleDef.Message model.Frequency = ruleDef.Frequency - model.NotifyOnce = ruleDef.NotifyOnce - model.NotifyFreq = ruleDef.NotifyFreq - model.NotifyEval = ruleDef.NotifyEval model.State = ruleDef.State model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data")) model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting")) diff --git a/pkg/services/sqlstore/alert.go b/pkg/services/sqlstore/alert.go index 9ab28be84ee..58ec7e2857a 100644 --- a/pkg/services/sqlstore/alert.go +++ b/pkg/services/sqlstore/alert.go @@ -22,7 +22,6 @@ func init() { bus.AddHandler("sql", GetAlertStatesForDashboard) bus.AddHandler("sql", PauseAlert) bus.AddHandler("sql", PauseAllAlerts) - bus.AddHandler("sql", IncAlertEval) } func GetAlertById(query *m.GetAlertByIdQuery) error { @@ -189,7 +188,7 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS if alertToUpdate.ContainsUpdates(alert) { alert.Updated = timeNow() alert.State = alertToUpdate.State - sess.MustCols("message", "notify_freq", "notify_once") + sess.MustCols("message") _, err := sess.Id(alert.Id).Update(alert) if err != nil { return err @@ -344,22 +343,3 @@ func GetAlertStatesForDashboard(query *m.GetAlertStatesForDashboardQuery) error return err } - -func IncAlertEval(cmd *m.IncAlertEvalCommand) error { - return inTransaction(func(sess *DBSession) error { - alert := m.Alert{} - - if _, err := sess.Id(cmd.AlertId).Get(&alert); err != nil { - return err - } - - alert.NotifyEval = (alert.NotifyEval + 1) % alert.NotifyFreq - - sess.MustCols("notify_eval") - if _, err := sess.Id(cmd.AlertId).Update(alert); err != nil { - return err - } - - return nil - }) -} diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 651241f7714..8bb17143042 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -17,6 +17,9 @@ func init() { bus.AddHandler("sql", DeleteAlertNotification) bus.AddHandler("sql", GetAlertNotificationsToSend) bus.AddHandler("sql", GetAllAlertNotifications) + bus.AddHandler("sql", RecordNotificationJournal) + bus.AddHandler("sql", GetLatestNotification) + bus.AddHandler("sql", CleanNotificationJournal) } func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error { @@ -138,13 +141,15 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error } alertNotification := &m.AlertNotification{ - OrgId: cmd.OrgId, - Name: cmd.Name, - Type: cmd.Type, - Settings: cmd.Settings, - Created: time.Now(), - Updated: time.Now(), - IsDefault: cmd.IsDefault, + OrgId: cmd.OrgId, + Name: cmd.Name, + Type: cmd.Type, + Settings: cmd.Settings, + NotifyOnce: cmd.NotifyOnce, + Frequency: cmd.Frequency, + Created: time.Now(), + Updated: time.Now(), + IsDefault: cmd.IsDefault, } if _, err = sess.Insert(alertNotification); err != nil { @@ -192,3 +197,42 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { return nil }) } + +func RecordNotificationJournal(cmd *m.RecordNotificationJournalCommand) error { + return inTransaction(func(sess *DBSession) error { + journalEntry := &m.NotificationJournal{ + OrgId: cmd.OrgId, + AlertId: cmd.AlertId, + NotifierId: cmd.NotifierId, + SentAt: cmd.SentAt, + Success: cmd.Success, + } + + if _, err := sess.Insert(journalEntry); err != nil { + return err + } + + return nil + }) +} + +func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error { + return inTransaction(func(sess *DBSession) error { + notificationJournal := &m.NotificationJournal{} + _, err := sess.OrderBy("notification_journal.sent_at").Desc().Where("notification_journal.org_id = ? AND notification_journal.alert_id = ? AND notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal) + if err != nil { + return err + } + + cmd.Result = notificationJournal + return nil + }) +} + +func CleanNotificationJournal(cmd *m.CleanNotificationJournalCommand) error { + return inTransaction(func(sess *DBSession) error { + sql := "DELETE FROM notification_journal WHERE notification_journal.org_id = ? AND notification_journal.alert_id = ? AND notification_journal.notifier_id = ?" + _, err := sess.Exec(sql, cmd.OrgId, cmd.AlertId, cmd.NotifierId) + return err + }) +} diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index 3452e5710cc..d045f611fb2 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -29,9 +29,6 @@ func addAlertMigrations(mg *Migrator) { {Name: "state_changes", Type: DB_Int, Nullable: false}, {Name: "created", Type: DB_DateTime, Nullable: false}, {Name: "updated", Type: DB_DateTime, Nullable: false}, - {Name: "notify_once", Type: DB_Bool, Nullable: false}, - {Name: "notify_freq", Type: DB_Int, Nullable: false}, - {Name: "notify_eval", Type: DB_Int, Nullable: false}, }, Indices: []*Index{ {Cols: []string{"org_id", "id"}, Type: IndexType}, @@ -68,8 +65,32 @@ func addAlertMigrations(mg *Migrator) { mg.AddMigration("Add column is_default", NewAddColumnMigration(alert_notification, &Column{ Name: "is_default", Type: DB_Bool, Nullable: false, Default: "0", })) + mg.AddMigration("Add column frequency", NewAddColumnMigration(alert_notification, &Column{ + Name: "frequency", Type: DB_BigInt, Nullable: true, + })) + mg.AddMigration("Add column notify_once", NewAddColumnMigration(alert_notification, &Column{ + Name: "notify_once", Type: DB_Bool, Nullable: false, Default: "1", + })) mg.AddMigration("add index alert_notification org_id & name", NewAddIndexMigration(alert_notification, alert_notification.Indices[0])) + notification_journal := Table{ + Name: "notification_journal", + Columns: []*Column{ + {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, + {Name: "org_id", Type: DB_BigInt, Nullable: false}, + {Name: "alert_id", Type: DB_BigInt, Nullable: false}, + {Name: "notifier_id", Type: DB_BigInt, Nullable: false}, + {Name: "sent_at", Type: DB_DateTime, Nullable: false}, + {Name: "success", Type: DB_Bool, Nullable: false}, + }, + Indices: []*Index{ + {Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: IndexType}, + }, + } + + mg.AddMigration("create notification_journal table v1", NewAddTableMigration(notification_journal)) + mg.AddMigration("add index notification_journal org_id & alert_id & notifier_id", NewAddIndexMigration(notification_journal, notification_journal.Indices[0])) + mg.AddMigration("Update alert table charset", NewTableCharsetMigration("alert", []*Column{ {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, {Name: "message", Type: DB_Text, Nullable: false}, diff --git a/public/app/features/alerting/alert_tab_ctrl.ts b/public/app/features/alerting/alert_tab_ctrl.ts index f0d965ae81e..79baa1e3f5a 100644 --- a/public/app/features/alerting/alert_tab_ctrl.ts +++ b/public/app/features/alerting/alert_tab_ctrl.ts @@ -167,9 +167,6 @@ export class AlertTabCtrl { alert.noDataState = alert.noDataState || 'no_data'; alert.executionErrorState = alert.executionErrorState || 'alerting'; alert.frequency = alert.frequency || '60s'; - alert.notifyFrequency = alert.notifyFrequency || 10; - alert.notifyOnce = alert.notifyOnce == null ? true : alert.notifyOnce; - alert.frequency = alert.frequency || '60s'; alert.handler = alert.handler || 1; alert.notifications = alert.notifications || []; diff --git a/public/app/features/alerting/notification_edit_ctrl.ts b/public/app/features/alerting/notification_edit_ctrl.ts index 18b1c4d1d55..2fd185bee29 100644 --- a/public/app/features/alerting/notification_edit_ctrl.ts +++ b/public/app/features/alerting/notification_edit_ctrl.ts @@ -11,6 +11,7 @@ export class AlertNotificationEditCtrl { model: any; defaults: any = { type: 'email', + notifyOnce: true, settings: { httpMethod: 'POST', autoResolve: true, @@ -102,6 +103,7 @@ export class AlertNotificationEditCtrl { var payload = { name: this.model.name, type: this.model.type, + frequency: this.model.frequency, settings: this.model.settings, }; diff --git a/public/app/features/alerting/partials/alert_tab.html b/public/app/features/alerting/partials/alert_tab.html index 084aeb2036a..cb101672aa4 100644 --- a/public/app/features/alerting/partials/alert_tab.html +++ b/public/app/features/alerting/partials/alert_tab.html @@ -31,9 +31,6 @@ Evaluate every - {{ ctrl.alert.notifyOnce ? 'Notify on state change' : 'Notify every' }} - - evaluations
diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html index d20b9031a8f..ccdb9ef1073 100644 --- a/public/app/features/alerting/partials/notification_edit.html +++ b/public/app/features/alerting/partials/notification_edit.html @@ -18,6 +18,11 @@
+
+ {{ ctrl.model.notifyOnce ? 'Notify on state change' : 'Notify at most every' }} + + seconds +
Date: Sun, 20 May 2018 12:12:10 -0400 Subject: [PATCH 051/310] Fix multiple bugs --- pkg/api/alerting.go | 57 ++++++++++++++++--- pkg/api/dtos/alerting.go | 19 ++++--- pkg/models/alert_notifications.go | 6 +- pkg/services/alerting/eval_context.go | 4 +- pkg/services/alerting/notifiers/base.go | 1 + pkg/services/alerting/notifiers/base_test.go | 13 +++-- pkg/services/sqlstore/alert_notification.go | 28 +++++++-- .../alerting/partials/notification_edit.html | 3 +- 8 files changed, 95 insertions(+), 36 deletions(-) diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go index 961fc11b2dc..c5b47270f4d 100644 --- a/pkg/api/alerting.go +++ b/pkg/api/alerting.go @@ -193,12 +193,15 @@ func GetAlertNotifications(c *m.ReqContext) Response { for _, notification := range query.Result { result = append(result, &dtos.AlertNotification{ - Id: notification.Id, - Name: notification.Name, - Type: notification.Type, - IsDefault: notification.IsDefault, - Created: notification.Created, - Updated: notification.Updated, + Id: notification.Id, + Name: notification.Name, + Type: notification.Type, + IsDefault: notification.IsDefault, + Created: notification.Created, + Updated: notification.Updated, + Frequency: notification.Frequency.String(), + NotifyOnce: notification.NotifyOnce, + Settings: notification.Settings, }) } @@ -215,7 +218,19 @@ func GetAlertNotificationByID(c *m.ReqContext) Response { return Error(500, "Failed to get alert notifications", err) } - return JSON(200, query.Result) + result := &dtos.AlertNotification{ + Id: query.Result.Id, + Name: query.Result.Name, + Type: query.Result.Type, + IsDefault: query.Result.IsDefault, + Created: query.Result.Created, + Updated: query.Result.Updated, + Frequency: query.Result.Frequency.String(), + NotifyOnce: query.Result.NotifyOnce, + Settings: query.Result.Settings, + } + + return JSON(200, result) } func CreateAlertNotification(c *m.ReqContext, cmd m.CreateAlertNotificationCommand) Response { @@ -225,7 +240,19 @@ func CreateAlertNotification(c *m.ReqContext, cmd m.CreateAlertNotificationComma return Error(500, "Failed to create alert notification", err) } - return JSON(200, cmd.Result) + result := &dtos.AlertNotification{ + Id: cmd.Result.Id, + Name: cmd.Result.Name, + Type: cmd.Result.Type, + IsDefault: cmd.Result.IsDefault, + Created: cmd.Result.Created, + Updated: cmd.Result.Updated, + Frequency: cmd.Result.Frequency.String(), + NotifyOnce: cmd.Result.NotifyOnce, + Settings: cmd.Result.Settings, + } + + return JSON(200, result) } func UpdateAlertNotification(c *m.ReqContext, cmd m.UpdateAlertNotificationCommand) Response { @@ -235,7 +262,19 @@ func UpdateAlertNotification(c *m.ReqContext, cmd m.UpdateAlertNotificationComma return Error(500, "Failed to update alert notification", err) } - return JSON(200, cmd.Result) + result := &dtos.AlertNotification{ + Id: cmd.Result.Id, + Name: cmd.Result.Name, + Type: cmd.Result.Type, + IsDefault: cmd.Result.IsDefault, + Created: cmd.Result.Created, + Updated: cmd.Result.Updated, + Frequency: cmd.Result.Frequency.String(), + NotifyOnce: cmd.Result.NotifyOnce, + Settings: cmd.Result.Settings, + } + + return JSON(200, result) } func DeleteAlertNotification(c *m.ReqContext) Response { diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go index 5e0196c20d1..7d4201fba87 100644 --- a/pkg/api/dtos/alerting.go +++ b/pkg/api/dtos/alerting.go @@ -24,14 +24,15 @@ type AlertRule struct { } type AlertNotification struct { - Id int64 `json:"id"` - Name string `json:"name"` - Type string `json:"type"` - IsDefault bool `json:"isDefault"` - NotifyOnce bool `json:"notifyOnce"` - Frequency bool `json:"frequency"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` + Id int64 `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + IsDefault bool `json:"isDefault"` + NotifyOnce bool `json:"notifyOnce"` + Frequency string `json:"frequency"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` + Settings *simplejson.Json `json:"settings"` } type AlertTestCommand struct { @@ -64,7 +65,7 @@ type NotificationTestCommand struct { Name string `json:"name"` Type string `json:"type"` NotifyOnce bool `json:"notifyOnce"` - Frequency time.Duration `json:"frequency"` + Frequency string `json:"frequency"` Settings *simplejson.Json `json:"settings"` } diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index cba62a51527..6715eb21395 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -22,8 +22,8 @@ type AlertNotification struct { type CreateAlertNotificationCommand struct { Name string `json:"name" binding:"Required"` Type string `json:"type" binding:"Required"` - NotifyOnce bool `json:"notifyOnce" binding:"Required"` - Frequency time.Duration `json:"frequency"` + NotifyOnce bool `json:"notifyOnce"` + Frequency string `json:"frequency"` IsDefault bool `json:"isDefault"` Settings *simplejson.Json `json:"settings"` @@ -35,7 +35,7 @@ type UpdateAlertNotificationCommand struct { Id int64 `json:"id" binding:"Required"` Name string `json:"name" binding:"Required"` Type string `json:"type" binding:"Required"` - NotifyOnce string `json:"notifyOnce" binding:"Required"` + NotifyOnce bool `json:"notifyOnce"` Frequency string `json:"frequency"` IsDefault bool `json:"isDefault"` Settings *simplejson.Json `json:"settings" binding:"Required"` diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go index b451d188a64..3817f4b4a3c 100644 --- a/pkg/services/alerting/eval_context.go +++ b/pkg/services/alerting/eval_context.go @@ -151,8 +151,8 @@ func (c *EvalContext) LastNotify(notifierId int64) *time.Time { NotifierId: notifierId, } if err := bus.Dispatch(cmd); err != nil { - c.log.Warn("Could not determine last time alert", - c.Rule.Name, "notified") + c.log.Warn("Could not determine last time alert notifier fired", + "Alert name", c.Rule.Name, "Error", err) return nil } diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index e9e32020c6d..734b5e56b28 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -41,6 +41,7 @@ func defaultShouldNotify(context *alerting.EvalContext, notifyOnce bool, frequen if context.PrevAlertState == context.Rule.State && notifyOnce { return false } + // Do not notify if interval has not elapsed if !notifyOnce && lastNotify != nil && lastNotify.Add(frequency).After(time.Now()) { return false } diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go index b7142d144cc..5f2d4989063 100644 --- a/pkg/services/alerting/notifiers/base_test.go +++ b/pkg/services/alerting/notifiers/base_test.go @@ -3,6 +3,7 @@ package notifiers import ( "context" "testing" + "time" "github.com/grafana/grafana/pkg/components/simplejson" m "github.com/grafana/grafana/pkg/models" @@ -18,19 +19,19 @@ func TestBaseNotifier(t *testing.T) { Convey("can parse false value", func() { bJson.Set("uploadImage", false) - base := NewNotifierBase(1, false, "name", "email", bJson) + base := NewNotifierBase(1, false, "name", "email", true, 0, bJson) So(base.UploadImage, ShouldBeFalse) }) Convey("can parse true value", func() { bJson.Set("uploadImage", true) - base := NewNotifierBase(1, false, "name", "email", bJson) + base := NewNotifierBase(1, false, "name", "email", true, 0, bJson) So(base.UploadImage, ShouldBeTrue) }) Convey("default value should be true for backwards compatibility", func() { - base := NewNotifierBase(1, false, "name", "email", bJson) + base := NewNotifierBase(1, false, "name", "email", true, 0, bJson) So(base.UploadImage, ShouldBeTrue) }) }) @@ -41,7 +42,8 @@ func TestBaseNotifier(t *testing.T) { State: m.AlertStatePending, }) context.Rule.State = m.AlertStateOK - So(defaultShouldNotify(context), ShouldBeFalse) + timeNow := time.Now() + So(defaultShouldNotify(context, true, 0, &timeNow), ShouldBeFalse) }) Convey("ok -> alerting", func() { @@ -49,7 +51,8 @@ func TestBaseNotifier(t *testing.T) { State: m.AlertStateOK, }) context.Rule.State = m.AlertStateAlerting - So(defaultShouldNotify(context), ShouldBeTrue) + timeNow := time.Now() + So(defaultShouldNotify(context, true, 0, &timeNow), ShouldBeTrue) }) }) }) diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 8bb17143042..a2cfa37ce5c 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -56,7 +56,9 @@ func GetAlertNotificationsToSend(query *m.GetAlertNotificationsToSendQuery) erro alert_notification.created, alert_notification.updated, alert_notification.settings, - alert_notification.is_default + alert_notification.is_default, + alert_notification.notify_once, + alert_notification.frequency FROM alert_notification `) @@ -94,7 +96,9 @@ func getAlertNotificationInternal(query *m.GetAlertNotificationsQuery, sess *DBS alert_notification.created, alert_notification.updated, alert_notification.settings, - alert_notification.is_default + alert_notification.is_default, + alert_notification.notify_once, + alert_notification.frequency FROM alert_notification `) @@ -140,19 +144,24 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error return fmt.Errorf("Alert notification name %s already exists", cmd.Name) } + frequency, err_convert := time.ParseDuration(cmd.Frequency) + if err_convert != nil { + return err + } + alertNotification := &m.AlertNotification{ OrgId: cmd.OrgId, Name: cmd.Name, Type: cmd.Type, Settings: cmd.Settings, NotifyOnce: cmd.NotifyOnce, - Frequency: cmd.Frequency, + Frequency: frequency, Created: time.Now(), Updated: time.Now(), IsDefault: cmd.IsDefault, } - if _, err = sess.Insert(alertNotification); err != nil { + if _, err = sess.MustCols("notify_once").Insert(alertNotification); err != nil { return err } @@ -184,8 +193,15 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { current.Name = cmd.Name current.Type = cmd.Type current.IsDefault = cmd.IsDefault + current.NotifyOnce = cmd.NotifyOnce - sess.UseBool("is_default") + frequency, err_convert := time.ParseDuration(cmd.Frequency) + if err_convert != nil { + return err + } + current.Frequency = frequency + + sess.UseBool("is_default", "notify_once") if affected, err := sess.ID(cmd.Id).Update(current); err != nil { return err @@ -219,7 +235,7 @@ func RecordNotificationJournal(cmd *m.RecordNotificationJournalCommand) error { func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error { return inTransaction(func(sess *DBSession) error { notificationJournal := &m.NotificationJournal{} - _, err := sess.OrderBy("notification_journal.sent_at").Desc().Where("notification_journal.org_id = ? AND notification_journal.alert_id = ? AND notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal) + _, err := sess.Desc("notification_journal.sent_at").Limit(1).Where("notification_journal.org_id = ? AND notification_journal.alert_id = ? AND notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal) if err != nil { return err } diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html index ccdb9ef1073..dd56564cb95 100644 --- a/public/app/features/alerting/partials/notification_edit.html +++ b/public/app/features/alerting/partials/notification_edit.html @@ -20,8 +20,7 @@
{{ ctrl.model.notifyOnce ? 'Notify on state change' : 'Notify at most every' }} - - seconds +
Date: Sun, 20 May 2018 16:08:42 -0400 Subject: [PATCH 052/310] Fix tests --- pkg/services/sqlstore/alert_notification.go | 8 +++++ .../sqlstore/alert_notification_test.go | 32 +++++++++++-------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index a2cfa37ce5c..0ecd6a18818 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -144,6 +144,10 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error return fmt.Errorf("Alert notification name %s already exists", cmd.Name) } + if cmd.Frequency == "" { + return fmt.Errorf("Alert notification frequency required") + } + frequency, err_convert := time.ParseDuration(cmd.Frequency) if err_convert != nil { return err @@ -195,6 +199,10 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { current.IsDefault = cmd.IsDefault current.NotifyOnce = cmd.NotifyOnce + if cmd.Frequency == "" { + return fmt.Errorf("Alert notification frequency required") + } + frequency, err_convert := time.ParseDuration(cmd.Frequency) if err_convert != nil { return err diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index 2dbf9de5ca8..01c6c3aebd6 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -26,10 +26,12 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Can save Alert Notification", func() { cmd := &m.CreateAlertNotificationCommand{ - Name: "ops", - Type: "email", - OrgId: 1, - Settings: simplejson.New(), + Name: "ops", + Type: "email", + OrgId: 1, + NotifyOnce: true, + Frequency: "10s", + Settings: simplejson.New(), } err = CreateAlertNotificationCommand(cmd) @@ -45,11 +47,13 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Can update alert notification", func() { newCmd := &m.UpdateAlertNotificationCommand{ - Name: "NewName", - Type: "webhook", - OrgId: cmd.Result.OrgId, - Settings: simplejson.New(), - Id: cmd.Result.Id, + Name: "NewName", + Type: "webhook", + OrgId: cmd.Result.OrgId, + NotifyOnce: true, + Frequency: "10s", + Settings: simplejson.New(), + Id: cmd.Result.Id, } err := UpdateAlertNotification(newCmd) So(err, ShouldBeNil) @@ -58,12 +62,12 @@ func TestAlertNotificationSQLAccess(t *testing.T) { }) Convey("Can search using an array of ids", func() { - cmd1 := m.CreateAlertNotificationCommand{Name: "nagios", Type: "webhook", OrgId: 1, Settings: simplejson.New()} - cmd2 := m.CreateAlertNotificationCommand{Name: "slack", Type: "webhook", OrgId: 1, Settings: simplejson.New()} - cmd3 := m.CreateAlertNotificationCommand{Name: "ops2", Type: "email", OrgId: 1, Settings: simplejson.New()} - cmd4 := m.CreateAlertNotificationCommand{IsDefault: true, Name: "default", Type: "email", OrgId: 1, Settings: simplejson.New()} + cmd1 := m.CreateAlertNotificationCommand{Name: "nagios", Type: "webhook", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()} + cmd2 := m.CreateAlertNotificationCommand{Name: "slack", Type: "webhook", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()} + cmd3 := m.CreateAlertNotificationCommand{Name: "ops2", Type: "email", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()} + cmd4 := m.CreateAlertNotificationCommand{IsDefault: true, Name: "default", Type: "email", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()} - otherOrg := m.CreateAlertNotificationCommand{Name: "default", Type: "email", OrgId: 2, Settings: simplejson.New()} + otherOrg := m.CreateAlertNotificationCommand{Name: "default", Type: "email", OrgId: 2, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()} So(CreateAlertNotificationCommand(&cmd1), ShouldBeNil) So(CreateAlertNotificationCommand(&cmd2), ShouldBeNil) From 5c5951bc4274f3b4ff1ea3b41507e394faaeb22f Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Sun, 20 May 2018 19:01:10 -0400 Subject: [PATCH 053/310] Bug fix for repeated alerting even on OK state and add notification_journal cleanup when alert resolves --- pkg/services/alerting/engine.go | 14 ++++++++++++++ pkg/services/alerting/notifiers/base.go | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 0f8e24bcef5..43f6db66771 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -10,7 +10,9 @@ import ( tlog "github.com/opentracing/opentracing-go/log" "github.com/benbjohnson/clock" + "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/log" + m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/services/rendering" "github.com/grafana/grafana/pkg/setting" @@ -205,6 +207,18 @@ func (e *AlertingService) processJob(attemptID int, attemptChan chan int, cancel } evalContext.Rule.State = evalContext.GetNewState() + if evalContext.Rule.State == m.AlertStateOK && evalContext.PrevAlertState != m.AlertStateOK { + for _, notifierId := range evalContext.Rule.Notifications { + cmd := &m.CleanNotificationJournalCommand{ + AlertId: evalContext.Rule.Id, + NotifierId: notifierId, + OrgId: evalContext.Rule.OrgId, + } + if err := bus.Dispatch(cmd); err != nil { + e.log.Error("Failed to clean up old notification records", "notifier", notifierId, "alert", evalContext.Rule.Id, "Error", err) + } + } + } e.resultHandler.Handle(evalContext) span.Finish() e.log.Debug("Job Execution completed", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.Id, "name", evalContext.Rule.Name, "firing", evalContext.Firing, "attemptID", attemptID) diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 734b5e56b28..7672d397491 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -45,6 +45,10 @@ func defaultShouldNotify(context *alerting.EvalContext, notifyOnce bool, frequen if !notifyOnce && lastNotify != nil && lastNotify.Add(frequency).After(time.Now()) { return false } + // Do not notify if alert state if OK or pending even on repeated notify + if !notifyOnce && (context.Rule.State == m.AlertStateOK || context.Rule.State == m.AlertStatePending) { + return false + } // Do not notify when we become OK for the first time. if (context.PrevAlertState == m.AlertStatePending) && (context.Rule.State == m.AlertStateOK) { return false From bdf433594add113b05b2bbd4f0381c1090fd2d6b Mon Sep 17 00:00:00 2001 From: John Baublitz Date: Fri, 25 May 2018 14:14:33 -0400 Subject: [PATCH 054/310] Implement code review changes --- pkg/models/alert_notifications.go | 5 +++++ pkg/services/alerting/engine.go | 14 -------------- pkg/services/alerting/result_handler.go | 12 ++++++++++++ pkg/services/sqlstore/alert_notification.go | 7 ++++--- .../features/alerting/notification_edit_ctrl.ts | 1 + .../alerting/partials/notification_edit.html | 15 +++++++++++---- 6 files changed, 33 insertions(+), 21 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index 6715eb21395..ed6b8f372d1 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -1,11 +1,16 @@ package models import ( + "errors" "time" "github.com/grafana/grafana/pkg/components/simplejson" ) +var ( + ErrNotificationFrequencyNotFound = errors.New("Notification frequency not specified") +) + type AlertNotification struct { Id int64 `json:"id"` OrgId int64 `json:"-"` diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 43f6db66771..0f8e24bcef5 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -10,9 +10,7 @@ import ( tlog "github.com/opentracing/opentracing-go/log" "github.com/benbjohnson/clock" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/log" - m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/services/rendering" "github.com/grafana/grafana/pkg/setting" @@ -207,18 +205,6 @@ func (e *AlertingService) processJob(attemptID int, attemptChan chan int, cancel } evalContext.Rule.State = evalContext.GetNewState() - if evalContext.Rule.State == m.AlertStateOK && evalContext.PrevAlertState != m.AlertStateOK { - for _, notifierId := range evalContext.Rule.Notifications { - cmd := &m.CleanNotificationJournalCommand{ - AlertId: evalContext.Rule.Id, - NotifierId: notifierId, - OrgId: evalContext.Rule.OrgId, - } - if err := bus.Dispatch(cmd); err != nil { - e.log.Error("Failed to clean up old notification records", "notifier", notifierId, "alert", evalContext.Rule.Id, "Error", err) - } - } - } e.resultHandler.Handle(evalContext) span.Finish() e.log.Debug("Job Execution completed", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.Id, "name", evalContext.Rule.Name, "firing", evalContext.Firing, "attemptID", attemptID) diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go index c57b28c7c3e..c4c20bd8beb 100644 --- a/pkg/services/alerting/result_handler.go +++ b/pkg/services/alerting/result_handler.go @@ -88,6 +88,18 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { } } + if evalContext.Rule.State == m.AlertStateOK && evalContext.PrevAlertState != m.AlertStateOK { + for _, notifierId := range evalContext.Rule.Notifications { + cmd := &m.CleanNotificationJournalCommand{ + AlertId: evalContext.Rule.Id, + NotifierId: notifierId, + OrgId: evalContext.Rule.OrgId, + } + if err := bus.Dispatch(cmd); err != nil { + handler.log.Error("Failed to clean up old notification records", "notifier", notifierId, "alert", evalContext.Rule.Id, "Error", err) + } + } + } handler.notifier.SendIfNeeded(evalContext) return nil diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 0ecd6a18818..6913009a163 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -148,8 +148,9 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error return fmt.Errorf("Alert notification frequency required") } - frequency, err_convert := time.ParseDuration(cmd.Frequency) - if err_convert != nil { + var frequency time.Duration + frequency, err = time.ParseDuration(cmd.Frequency) + if err != nil { return err } @@ -200,7 +201,7 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { current.NotifyOnce = cmd.NotifyOnce if cmd.Frequency == "" { - return fmt.Errorf("Alert notification frequency required") + return m.ErrNotificationFrequencyNotFound } frequency, err_convert := time.ParseDuration(cmd.Frequency) diff --git a/public/app/features/alerting/notification_edit_ctrl.ts b/public/app/features/alerting/notification_edit_ctrl.ts index 2fd185bee29..9d20e871c7c 100644 --- a/public/app/features/alerting/notification_edit_ctrl.ts +++ b/public/app/features/alerting/notification_edit_ctrl.ts @@ -12,6 +12,7 @@ export class AlertNotificationEditCtrl { defaults: any = { type: 'email', notifyOnce: true, + frequency: '15m', settings: { httpMethod: 'POST', autoResolve: true, diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html index dd56564cb95..48d44b74581 100644 --- a/public/app/features/alerting/partials/notification_edit.html +++ b/public/app/features/alerting/partials/notification_edit.html @@ -18,10 +18,6 @@ -
- {{ ctrl.model.notifyOnce ? 'Notify on state change' : 'Notify at most every' }} - -
+ + +
+ Notify every + +
From 86e65f84f9ba86b202ff38d7f51f1b7d2e75f02f Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 4 Jun 2018 17:30:57 +0200 Subject: [PATCH 055/310] alerting: fixes invalid error handling --- pkg/services/sqlstore/alert_notification.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 6913009a163..4f79035063e 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -204,8 +204,8 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { return m.ErrNotificationFrequencyNotFound } - frequency, err_convert := time.ParseDuration(cmd.Frequency) - if err_convert != nil { + frequency, err := time.ParseDuration(cmd.Frequency) + if err != nil { return err } current.Frequency = frequency From 93124f38fae77627cdb0a7b4a5f71171c5088eca Mon Sep 17 00:00:00 2001 From: bergquist Date: Mon, 4 Jun 2018 22:19:27 +0200 Subject: [PATCH 056/310] alerting: only check frequency when not send once --- pkg/api/alerting.go | 54 ++---------------- pkg/api/alerting_test.go | 19 +++++++ pkg/api/dtos/alerting.go | 53 +++++++++++++----- pkg/services/sqlstore/alert_notification.go | 35 +++++++----- .../sqlstore/alert_notification_test.go | 55 ++++++++++++++++++- 5 files changed, 135 insertions(+), 81 deletions(-) diff --git a/pkg/api/alerting.go b/pkg/api/alerting.go index 4e9b89fefd6..a936d696207 100644 --- a/pkg/api/alerting.go +++ b/pkg/api/alerting.go @@ -192,17 +192,7 @@ func GetAlertNotifications(c *m.ReqContext) Response { result := make([]*dtos.AlertNotification, 0) for _, notification := range query.Result { - result = append(result, &dtos.AlertNotification{ - Id: notification.Id, - Name: notification.Name, - Type: notification.Type, - IsDefault: notification.IsDefault, - Created: notification.Created, - Updated: notification.Updated, - Frequency: notification.Frequency.String(), - NotifyOnce: notification.NotifyOnce, - Settings: notification.Settings, - }) + result = append(result, dtos.NewAlertNotification(notification)) } return JSON(200, result) @@ -218,19 +208,7 @@ func GetAlertNotificationByID(c *m.ReqContext) Response { return Error(500, "Failed to get alert notifications", err) } - result := &dtos.AlertNotification{ - Id: query.Result.Id, - Name: query.Result.Name, - Type: query.Result.Type, - IsDefault: query.Result.IsDefault, - Created: query.Result.Created, - Updated: query.Result.Updated, - Frequency: query.Result.Frequency.String(), - NotifyOnce: query.Result.NotifyOnce, - Settings: query.Result.Settings, - } - - return JSON(200, result) + return JSON(200, dtos.NewAlertNotification(query.Result)) } func CreateAlertNotification(c *m.ReqContext, cmd m.CreateAlertNotificationCommand) Response { @@ -240,19 +218,7 @@ func CreateAlertNotification(c *m.ReqContext, cmd m.CreateAlertNotificationComma return Error(500, "Failed to create alert notification", err) } - result := &dtos.AlertNotification{ - Id: cmd.Result.Id, - Name: cmd.Result.Name, - Type: cmd.Result.Type, - IsDefault: cmd.Result.IsDefault, - Created: cmd.Result.Created, - Updated: cmd.Result.Updated, - Frequency: cmd.Result.Frequency.String(), - NotifyOnce: cmd.Result.NotifyOnce, - Settings: cmd.Result.Settings, - } - - return JSON(200, result) + return JSON(200, dtos.NewAlertNotification(cmd.Result)) } func UpdateAlertNotification(c *m.ReqContext, cmd m.UpdateAlertNotificationCommand) Response { @@ -262,19 +228,7 @@ func UpdateAlertNotification(c *m.ReqContext, cmd m.UpdateAlertNotificationComma return Error(500, "Failed to update alert notification", err) } - result := &dtos.AlertNotification{ - Id: cmd.Result.Id, - Name: cmd.Result.Name, - Type: cmd.Result.Type, - IsDefault: cmd.Result.IsDefault, - Created: cmd.Result.Created, - Updated: cmd.Result.Updated, - Frequency: cmd.Result.Frequency.String(), - NotifyOnce: cmd.Result.NotifyOnce, - Settings: cmd.Result.Settings, - } - - return JSON(200, result) + return JSON(200, dtos.NewAlertNotification(cmd.Result)) } func DeleteAlertNotification(c *m.ReqContext) Response { diff --git a/pkg/api/alerting_test.go b/pkg/api/alerting_test.go index abfdfb66322..3e50487190c 100644 --- a/pkg/api/alerting_test.go +++ b/pkg/api/alerting_test.go @@ -2,6 +2,7 @@ package api import ( "testing" + "time" "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/bus" @@ -11,6 +12,24 @@ import ( . "github.com/smartystreets/goconvey/convey" ) +func TestRemoveZeroUnitsFromInterval(t *testing.T) { + tcs := []struct { + interval time.Duration + expected string + }{ + {interval: time.Duration(time.Hour), expected: "1h"}, + {interval: time.Duration(time.Hour + time.Minute), expected: "1h1m"}, + {interval: time.Duration((time.Hour * 10) + time.Minute), expected: "10h1m"}, + } + + for _, tc := range tcs { + got := removeZeroesFromDuration(tc.interval) + if got != tc.expected { + t.Errorf("expected %s got %s internval: %v", tc.expected, got, tc.interval) + } + } +} + func TestAlertingApiEndpoint(t *testing.T) { Convey("Given an alert in a dashboard with an acl", t, func() { diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go index 7d4201fba87..786fccc10b5 100644 --- a/pkg/api/dtos/alerting.go +++ b/pkg/api/dtos/alerting.go @@ -1,26 +1,51 @@ package dtos import ( + "strings" "time" "github.com/grafana/grafana/pkg/components/null" "github.com/grafana/grafana/pkg/components/simplejson" - m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/models" ) type AlertRule struct { - Id int64 `json:"id"` - DashboardId int64 `json:"dashboardId"` - PanelId int64 `json:"panelId"` - Name string `json:"name"` - Message string `json:"message"` - State m.AlertStateType `json:"state"` - NewStateDate time.Time `json:"newStateDate"` - EvalDate time.Time `json:"evalDate"` - EvalData *simplejson.Json `json:"evalData"` - ExecutionError string `json:"executionError"` - Url string `json:"url"` - CanEdit bool `json:"canEdit"` + Id int64 `json:"id"` + DashboardId int64 `json:"dashboardId"` + PanelId int64 `json:"panelId"` + Name string `json:"name"` + Message string `json:"message"` + State models.AlertStateType `json:"state"` + NewStateDate time.Time `json:"newStateDate"` + EvalDate time.Time `json:"evalDate"` + EvalData *simplejson.Json `json:"evalData"` + ExecutionError string `json:"executionError"` + Url string `json:"url"` + CanEdit bool `json:"canEdit"` +} + +func removeZeroesFromDuration(interval time.Duration) string { + frequency := interval.String() + + frequency = strings.Replace(frequency, "0h", "", 1) + frequency = strings.Replace(frequency, "0m", "", 1) + frequency = strings.Replace(frequency, "0s", "", 1) + + return frequency +} + +func NewAlertNotification(notification *models.AlertNotification) *AlertNotification { + return &AlertNotification{ + Id: notification.Id, + Name: notification.Name, + Type: notification.Type, + IsDefault: notification.IsDefault, + Created: notification.Created, + Updated: notification.Updated, + Frequency: removeZeroesFromDuration(notification.Frequency), + NotifyOnce: notification.NotifyOnce, + Settings: notification.Settings, + } } type AlertNotification struct { @@ -42,7 +67,7 @@ type AlertTestCommand struct { type AlertTestResult struct { Firing bool `json:"firing"` - State m.AlertStateType `json:"state"` + State models.AlertStateType `json:"state"` ConditionEvals string `json:"conditionEvals"` TimeMs string `json:"timeMs"` Error string `json:"error,omitempty"` diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 4f79035063e..ff36c38b1a5 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -144,14 +144,16 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error return fmt.Errorf("Alert notification name %s already exists", cmd.Name) } - if cmd.Frequency == "" { - return fmt.Errorf("Alert notification frequency required") - } - var frequency time.Duration - frequency, err = time.ParseDuration(cmd.Frequency) - if err != nil { - return err + if !cmd.NotifyOnce { + if cmd.Frequency == "" { + return m.ErrNotificationFrequencyNotFound + } + + frequency, err = time.ParseDuration(cmd.Frequency) + if err != nil { + return err + } } alertNotification := &m.AlertNotification{ @@ -200,22 +202,25 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { current.IsDefault = cmd.IsDefault current.NotifyOnce = cmd.NotifyOnce - if cmd.Frequency == "" { - return m.ErrNotificationFrequencyNotFound - } + if !current.NotifyOnce { + if cmd.Frequency == "" { + return m.ErrNotificationFrequencyNotFound + } - frequency, err := time.ParseDuration(cmd.Frequency) - if err != nil { - return err + frequency, err := time.ParseDuration(cmd.Frequency) + if err != nil { + return err + } + + current.Frequency = frequency } - current.Frequency = frequency sess.UseBool("is_default", "notify_once") if affected, err := sess.ID(cmd.Id).Update(current); err != nil { return err } else if affected == 0 { - return fmt.Errorf("Could not find alert notification") + return fmt.Errorf("Could not update alert notification") } cmd.Result = ¤t diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index 01c6c3aebd6..578a53f34ad 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -11,7 +11,6 @@ import ( func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Testing Alert notification sql access", t, func() { InitTestDB(t) - var err error Convey("Alert notifications should be empty", func() { cmd := &m.GetAlertNotificationsQuery{ @@ -24,6 +23,58 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(cmd.Result, ShouldBeNil) }) + Convey("Cannot save alert notifier with notitfyonce = false", func() { + cmd := &m.CreateAlertNotificationCommand{ + Name: "ops", + Type: "email", + OrgId: 1, + NotifyOnce: false, + Settings: simplejson.New(), + } + + Convey("and missing frequency", func() { + err := CreateAlertNotificationCommand(cmd) + So(err, ShouldEqual, m.ErrNotificationFrequencyNotFound) + }) + + Convey("invalid frequency", func() { + cmd.Frequency = "invalid duration" + + err := CreateAlertNotificationCommand(cmd) + So(err.Error(), ShouldEqual, "time: invalid duration invalid duration") + }) + }) + + Convey("Cannot update alert notifier with notitfyonce = false", func() { + cmd := &m.CreateAlertNotificationCommand{ + Name: "ops update", + Type: "email", + OrgId: 1, + NotifyOnce: true, + Settings: simplejson.New(), + } + + err := CreateAlertNotificationCommand(cmd) + So(err, ShouldBeNil) + + updateCmd := &m.UpdateAlertNotificationCommand{ + Id: cmd.Result.Id, + NotifyOnce: false, + } + + Convey("and missing frequency", func() { + err := UpdateAlertNotification(updateCmd) + So(err, ShouldEqual, m.ErrNotificationFrequencyNotFound) + }) + + Convey("invalid frequency", func() { + updateCmd.Frequency = "invalid duration" + + err := UpdateAlertNotification(updateCmd) + So(err.Error(), ShouldEqual, "time: invalid duration invalid duration") + }) + }) + Convey("Can save Alert Notification", func() { cmd := &m.CreateAlertNotificationCommand{ Name: "ops", @@ -34,7 +85,7 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Settings: simplejson.New(), } - err = CreateAlertNotificationCommand(cmd) + err := CreateAlertNotificationCommand(cmd) So(err, ShouldBeNil) So(cmd.Result.Id, ShouldNotEqual, 0) So(cmd.Result.OrgId, ShouldNotEqual, 0) From 0c6d8398a14e3a4c15b10af5ad0b4eda2965d988 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 5 Jun 2018 08:42:39 +0200 Subject: [PATCH 057/310] alerting: remove zero units from duration --- pkg/api/alerting_test.go | 19 ------------------- pkg/api/dtos/alerting.go | 29 +++++++++++++++++++++-------- pkg/api/dtos/alerting_test.go | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 27 deletions(-) create mode 100644 pkg/api/dtos/alerting_test.go diff --git a/pkg/api/alerting_test.go b/pkg/api/alerting_test.go index 3e50487190c..abfdfb66322 100644 --- a/pkg/api/alerting_test.go +++ b/pkg/api/alerting_test.go @@ -2,7 +2,6 @@ package api import ( "testing" - "time" "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/bus" @@ -12,24 +11,6 @@ import ( . "github.com/smartystreets/goconvey/convey" ) -func TestRemoveZeroUnitsFromInterval(t *testing.T) { - tcs := []struct { - interval time.Duration - expected string - }{ - {interval: time.Duration(time.Hour), expected: "1h"}, - {interval: time.Duration(time.Hour + time.Minute), expected: "1h1m"}, - {interval: time.Duration((time.Hour * 10) + time.Minute), expected: "10h1m"}, - } - - for _, tc := range tcs { - got := removeZeroesFromDuration(tc.interval) - if got != tc.expected { - t.Errorf("expected %s got %s internval: %v", tc.expected, got, tc.interval) - } - } -} - func TestAlertingApiEndpoint(t *testing.T) { Convey("Given an alert in a dashboard with an acl", t, func() { diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go index 786fccc10b5..f8671978148 100644 --- a/pkg/api/dtos/alerting.go +++ b/pkg/api/dtos/alerting.go @@ -1,7 +1,7 @@ package dtos import ( - "strings" + "fmt" "time" "github.com/grafana/grafana/pkg/components/null" @@ -24,14 +24,27 @@ type AlertRule struct { CanEdit bool `json:"canEdit"` } -func removeZeroesFromDuration(interval time.Duration) string { - frequency := interval.String() +func formatShort(interval time.Duration) string { + var result string - frequency = strings.Replace(frequency, "0h", "", 1) - frequency = strings.Replace(frequency, "0m", "", 1) - frequency = strings.Replace(frequency, "0s", "", 1) + hours := interval / time.Hour + if hours > 0 { + result += fmt.Sprintf("%dh", hours) + } - return frequency + remaining := interval - (hours * time.Hour) + mins := remaining / time.Minute + if mins > 0 { + result += fmt.Sprintf("%dm", mins) + } + + remaining = remaining - (mins * time.Minute) + seconds := remaining / time.Second + if seconds > 0 { + result += fmt.Sprintf("%ds", seconds) + } + + return result } func NewAlertNotification(notification *models.AlertNotification) *AlertNotification { @@ -42,7 +55,7 @@ func NewAlertNotification(notification *models.AlertNotification) *AlertNotifica IsDefault: notification.IsDefault, Created: notification.Created, Updated: notification.Updated, - Frequency: removeZeroesFromDuration(notification.Frequency), + Frequency: formatShort(notification.Frequency), NotifyOnce: notification.NotifyOnce, Settings: notification.Settings, } diff --git a/pkg/api/dtos/alerting_test.go b/pkg/api/dtos/alerting_test.go new file mode 100644 index 00000000000..ea4c97fb4cc --- /dev/null +++ b/pkg/api/dtos/alerting_test.go @@ -0,0 +1,34 @@ +package dtos + +import ( + "testing" + "time" +) + +func TestFormatShort(t *testing.T) { + tcs := []struct { + interval time.Duration + expected string + }{ + {interval: time.Duration(time.Hour), expected: "1h"}, + {interval: time.Duration(time.Hour + time.Minute), expected: "1h1m"}, + {interval: time.Duration((time.Hour * 10) + time.Minute), expected: "10h1m"}, + {interval: time.Duration((time.Hour * 10) + (time.Minute * 10) + time.Second), expected: "10h10m1s"}, + } + + for _, tc := range tcs { + got := formatShort(tc.interval) + if got != tc.expected { + t.Errorf("expected %s got %s interval: %v", tc.expected, got, tc.interval) + } + + parsed, err := time.ParseDuration(tc.expected) + if err != nil { + t.Fatalf("could not parse expected duration") + } + + if parsed != tc.interval { + t.Errorf("expectes the parsed duration to equal the interval. Got %v expected: %v", parsed, tc.interval) + } + } +} From 7333d7b8d4c128092b39c1e5616977c0a6aff015 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 5 Jun 2018 10:27:29 +0200 Subject: [PATCH 058/310] alerting: invert sendOnce to sendReminder --- pkg/api/dtos/alerting.go | 46 +++++++------- pkg/api/dtos/alerting_test.go | 1 + pkg/models/alert_notifications.go | 46 +++++++------- pkg/services/alerting/interfaces.go | 2 +- .../alerting/notifiers/alertmanager.go | 2 +- pkg/services/alerting/notifiers/base.go | 54 ++++++++-------- pkg/services/alerting/notifiers/base_test.go | 13 +++- pkg/services/alerting/notifiers/dingding.go | 2 +- pkg/services/alerting/notifiers/discord.go | 2 +- pkg/services/alerting/notifiers/email.go | 2 +- pkg/services/alerting/notifiers/hipchat.go | 2 +- pkg/services/alerting/notifiers/kafka.go | 2 +- pkg/services/alerting/notifiers/line.go | 2 +- pkg/services/alerting/notifiers/opsgenie.go | 2 +- pkg/services/alerting/notifiers/pagerduty.go | 2 +- pkg/services/alerting/notifiers/pushover.go | 2 +- pkg/services/alerting/notifiers/sensu.go | 2 +- pkg/services/alerting/notifiers/slack.go | 2 +- pkg/services/alerting/notifiers/teams.go | 2 +- pkg/services/alerting/notifiers/telegram.go | 2 +- pkg/services/alerting/notifiers/threema.go | 2 +- pkg/services/alerting/notifiers/victorops.go | 2 +- pkg/services/alerting/notifiers/webhook.go | 2 +- pkg/services/sqlstore/alert_notification.go | 32 +++++----- .../sqlstore/alert_notification_test.go | 63 ++++++++++--------- pkg/services/sqlstore/migrations/alert_mig.go | 5 +- .../alerting/notification_edit_ctrl.ts | 2 +- .../alerting/partials/notification_edit.html | 23 +++++-- 28 files changed, 173 insertions(+), 148 deletions(-) diff --git a/pkg/api/dtos/alerting.go b/pkg/api/dtos/alerting.go index f8671978148..697d0a35a08 100644 --- a/pkg/api/dtos/alerting.go +++ b/pkg/api/dtos/alerting.go @@ -49,28 +49,28 @@ func formatShort(interval time.Duration) string { func NewAlertNotification(notification *models.AlertNotification) *AlertNotification { return &AlertNotification{ - Id: notification.Id, - Name: notification.Name, - Type: notification.Type, - IsDefault: notification.IsDefault, - Created: notification.Created, - Updated: notification.Updated, - Frequency: formatShort(notification.Frequency), - NotifyOnce: notification.NotifyOnce, - Settings: notification.Settings, + Id: notification.Id, + Name: notification.Name, + Type: notification.Type, + IsDefault: notification.IsDefault, + Created: notification.Created, + Updated: notification.Updated, + Frequency: formatShort(notification.Frequency), + SendReminder: notification.SendReminder, + Settings: notification.Settings, } } type AlertNotification struct { - Id int64 `json:"id"` - Name string `json:"name"` - Type string `json:"type"` - IsDefault bool `json:"isDefault"` - NotifyOnce bool `json:"notifyOnce"` - Frequency string `json:"frequency"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` - Settings *simplejson.Json `json:"settings"` + Id int64 `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + IsDefault bool `json:"isDefault"` + SendReminder bool `json:"sendReminder"` + Frequency string `json:"frequency"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` + Settings *simplejson.Json `json:"settings"` } type AlertTestCommand struct { @@ -100,11 +100,11 @@ type EvalMatch struct { } type NotificationTestCommand struct { - Name string `json:"name"` - Type string `json:"type"` - NotifyOnce bool `json:"notifyOnce"` - Frequency string `json:"frequency"` - Settings *simplejson.Json `json:"settings"` + Name string `json:"name"` + Type string `json:"type"` + SendReminder bool `json:"sendReminder"` + Frequency string `json:"frequency"` + Settings *simplejson.Json `json:"settings"` } type PauseAlertCommand struct { diff --git a/pkg/api/dtos/alerting_test.go b/pkg/api/dtos/alerting_test.go index ea4c97fb4cc..bd0d9ff8feb 100644 --- a/pkg/api/dtos/alerting_test.go +++ b/pkg/api/dtos/alerting_test.go @@ -14,6 +14,7 @@ func TestFormatShort(t *testing.T) { {interval: time.Duration(time.Hour + time.Minute), expected: "1h1m"}, {interval: time.Duration((time.Hour * 10) + time.Minute), expected: "10h1m"}, {interval: time.Duration((time.Hour * 10) + (time.Minute * 10) + time.Second), expected: "10h10m1s"}, + {interval: time.Duration(time.Minute * 10), expected: "10m"}, } for _, tc := range tcs { diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index ed6b8f372d1..c17124dd6ef 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -12,38 +12,38 @@ var ( ) type AlertNotification struct { - Id int64 `json:"id"` - OrgId int64 `json:"-"` - Name string `json:"name"` - Type string `json:"type"` - NotifyOnce bool `json:"notifyOnce"` - Frequency time.Duration `json:"frequency"` - IsDefault bool `json:"isDefault"` - Settings *simplejson.Json `json:"settings"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` + Id int64 `json:"id"` + OrgId int64 `json:"-"` + Name string `json:"name"` + Type string `json:"type"` + SendReminder bool `json:"sendReminder"` + Frequency time.Duration `json:"frequency"` + IsDefault bool `json:"isDefault"` + Settings *simplejson.Json `json:"settings"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` } type CreateAlertNotificationCommand struct { - Name string `json:"name" binding:"Required"` - Type string `json:"type" binding:"Required"` - NotifyOnce bool `json:"notifyOnce"` - Frequency string `json:"frequency"` - IsDefault bool `json:"isDefault"` - Settings *simplejson.Json `json:"settings"` + Name string `json:"name" binding:"Required"` + Type string `json:"type" binding:"Required"` + SendReminder bool `json:"sendReminder"` + Frequency string `json:"frequency"` + IsDefault bool `json:"isDefault"` + Settings *simplejson.Json `json:"settings"` OrgId int64 `json:"-"` Result *AlertNotification } type UpdateAlertNotificationCommand struct { - Id int64 `json:"id" binding:"Required"` - Name string `json:"name" binding:"Required"` - Type string `json:"type" binding:"Required"` - NotifyOnce bool `json:"notifyOnce"` - Frequency string `json:"frequency"` - IsDefault bool `json:"isDefault"` - Settings *simplejson.Json `json:"settings" binding:"Required"` + Id int64 `json:"id" binding:"Required"` + Name string `json:"name" binding:"Required"` + Type string `json:"type" binding:"Required"` + SendReminder bool `json:"sendReminder"` + Frequency string `json:"frequency"` + IsDefault bool `json:"isDefault"` + Settings *simplejson.Json `json:"settings" binding:"Required"` OrgId int64 `json:"-"` Result *AlertNotification diff --git a/pkg/services/alerting/interfaces.go b/pkg/services/alerting/interfaces.go index 8842b35fba2..95fd4b5d04e 100644 --- a/pkg/services/alerting/interfaces.go +++ b/pkg/services/alerting/interfaces.go @@ -19,7 +19,7 @@ type Notifier interface { GetNotifierId() int64 GetIsDefault() bool - GetNotifyOnce() bool + GetSendReminder() bool GetFrequency() time.Duration } diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go index 3eeb25986e0..42ffa9b2d6e 100644 --- a/pkg/services/alerting/notifiers/alertmanager.go +++ b/pkg/services/alerting/notifiers/alertmanager.go @@ -33,7 +33,7 @@ func NewAlertmanagerNotifier(model *m.AlertNotification) (alerting.Notifier, err } return &AlertmanagerNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), + NotifierBase: NewNotifierBase(model), Url: url, log: log.New("alerting.notifier.prometheus-alertmanager"), }, nil diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 7672d397491..1d0d904457f 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -3,54 +3,56 @@ package notifiers import ( "time" - "github.com/grafana/grafana/pkg/components/simplejson" - m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting" ) type NotifierBase struct { - Name string - Type string - Id int64 - IsDeault bool - UploadImage bool - NotifyOnce bool - Frequency time.Duration + Name string + Type string + Id int64 + IsDeault bool + UploadImage bool + SendReminder bool + Frequency time.Duration } -func NewNotifierBase(id int64, isDefault bool, name, notifierType string, notifyOnce bool, frequency time.Duration, model *simplejson.Json) NotifierBase { +func NewNotifierBase(model *models.AlertNotification) NotifierBase { uploadImage := true - value, exist := model.CheckGet("uploadImage") + value, exist := model.Settings.CheckGet("uploadImage") if exist { uploadImage = value.MustBool() } return NotifierBase{ - Id: id, - Name: name, - IsDeault: isDefault, - Type: notifierType, - UploadImage: uploadImage, - NotifyOnce: notifyOnce, - Frequency: frequency, + Id: model.Id, + Name: model.Name, + IsDeault: model.IsDefault, + Type: model.Type, + UploadImage: uploadImage, + SendReminder: model.SendReminder, + Frequency: model.Frequency, } } -func defaultShouldNotify(context *alerting.EvalContext, notifyOnce bool, frequency time.Duration, lastNotify *time.Time) bool { +func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequency time.Duration, lastNotify *time.Time) bool { // Only notify on state change. - if context.PrevAlertState == context.Rule.State && notifyOnce { + if context.PrevAlertState == context.Rule.State && !sendReminder { return false } + // Do not notify if interval has not elapsed - if !notifyOnce && lastNotify != nil && lastNotify.Add(frequency).After(time.Now()) { + if sendReminder && lastNotify != nil && lastNotify.Add(frequency).After(time.Now()) { return false } + // Do not notify if alert state if OK or pending even on repeated notify - if !notifyOnce && (context.Rule.State == m.AlertStateOK || context.Rule.State == m.AlertStatePending) { + if sendReminder && (context.Rule.State == models.AlertStateOK || context.Rule.State == models.AlertStatePending) { return false } + // Do not notify when we become OK for the first time. - if (context.PrevAlertState == m.AlertStatePending) && (context.Rule.State == m.AlertStateOK) { + if (context.PrevAlertState == models.AlertStatePending) && (context.Rule.State == models.AlertStateOK) { return false } return true @@ -58,7 +60,7 @@ func defaultShouldNotify(context *alerting.EvalContext, notifyOnce bool, frequen func (n *NotifierBase) ShouldNotify(context *alerting.EvalContext) bool { lastNotify := context.LastNotify(n.Id) - return defaultShouldNotify(context, n.NotifyOnce, n.Frequency, lastNotify) + return defaultShouldNotify(context, n.SendReminder, n.Frequency, lastNotify) } func (n *NotifierBase) GetType() string { @@ -77,8 +79,8 @@ func (n *NotifierBase) GetIsDefault() bool { return n.IsDeault } -func (n *NotifierBase) GetNotifyOnce() bool { - return n.NotifyOnce +func (n *NotifierBase) GetSendReminder() bool { + return n.SendReminder } func (n *NotifierBase) GetFrequency() time.Duration { diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go index 5f2d4989063..28e2ff9f024 100644 --- a/pkg/services/alerting/notifiers/base_test.go +++ b/pkg/services/alerting/notifiers/base_test.go @@ -16,22 +16,29 @@ func TestBaseNotifier(t *testing.T) { Convey("default constructor for notifiers", func() { bJson := simplejson.New() + model := &m.AlertNotification{ + Id: 1, + Name: "name", + Type: "email", + Settings: bJson, + } + Convey("can parse false value", func() { bJson.Set("uploadImage", false) - base := NewNotifierBase(1, false, "name", "email", true, 0, bJson) + base := NewNotifierBase(model) So(base.UploadImage, ShouldBeFalse) }) Convey("can parse true value", func() { bJson.Set("uploadImage", true) - base := NewNotifierBase(1, false, "name", "email", true, 0, bJson) + base := NewNotifierBase(model) So(base.UploadImage, ShouldBeTrue) }) Convey("default value should be true for backwards compatibility", func() { - base := NewNotifierBase(1, false, "name", "email", true, 0, bJson) + base := NewNotifierBase(model) So(base.UploadImage, ShouldBeTrue) }) }) diff --git a/pkg/services/alerting/notifiers/dingding.go b/pkg/services/alerting/notifiers/dingding.go index 78446c56f88..738e43af2d2 100644 --- a/pkg/services/alerting/notifiers/dingding.go +++ b/pkg/services/alerting/notifiers/dingding.go @@ -32,7 +32,7 @@ func NewDingDingNotifier(model *m.AlertNotification) (alerting.Notifier, error) } return &DingDingNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), + NotifierBase: NewNotifierBase(model), Url: url, log: log.New("alerting.notifier.dingding"), }, nil diff --git a/pkg/services/alerting/notifiers/discord.go b/pkg/services/alerting/notifiers/discord.go index 693ed31e206..57d9d438fa2 100644 --- a/pkg/services/alerting/notifiers/discord.go +++ b/pkg/services/alerting/notifiers/discord.go @@ -39,7 +39,7 @@ func NewDiscordNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &DiscordNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), + NotifierBase: NewNotifierBase(model), WebhookURL: url, log: log.New("alerting.notifier.discord"), }, nil diff --git a/pkg/services/alerting/notifiers/email.go b/pkg/services/alerting/notifiers/email.go index 234a4f8e756..17b88f7d97f 100644 --- a/pkg/services/alerting/notifiers/email.go +++ b/pkg/services/alerting/notifiers/email.go @@ -52,7 +52,7 @@ func NewEmailNotifier(model *m.AlertNotification) (alerting.Notifier, error) { }) return &EmailNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), + NotifierBase: NewNotifierBase(model), Addresses: addresses, log: log.New("alerting.notifier.email"), }, nil diff --git a/pkg/services/alerting/notifiers/hipchat.go b/pkg/services/alerting/notifiers/hipchat.go index 4eb5b78811e..1c284ec3d2b 100644 --- a/pkg/services/alerting/notifiers/hipchat.go +++ b/pkg/services/alerting/notifiers/hipchat.go @@ -59,7 +59,7 @@ func NewHipChatNotifier(model *models.AlertNotification) (alerting.Notifier, err roomId := model.Settings.Get("roomid").MustString() return &HipChatNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), + NotifierBase: NewNotifierBase(model), Url: url, ApiKey: apikey, RoomId: roomId, diff --git a/pkg/services/alerting/notifiers/kafka.go b/pkg/services/alerting/notifiers/kafka.go index 0dab556d5e1..d8d19fc5dae 100644 --- a/pkg/services/alerting/notifiers/kafka.go +++ b/pkg/services/alerting/notifiers/kafka.go @@ -43,7 +43,7 @@ func NewKafkaNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &KafkaNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), + NotifierBase: NewNotifierBase(model), Endpoint: endpoint, Topic: topic, log: log.New("alerting.notifier.kafka"), diff --git a/pkg/services/alerting/notifiers/line.go b/pkg/services/alerting/notifiers/line.go index 0ee252e6447..9e3888b8f95 100644 --- a/pkg/services/alerting/notifiers/line.go +++ b/pkg/services/alerting/notifiers/line.go @@ -39,7 +39,7 @@ func NewLINENotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &LineNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), + NotifierBase: NewNotifierBase(model), Token: token, log: log.New("alerting.notifier.line"), }, nil diff --git a/pkg/services/alerting/notifiers/opsgenie.go b/pkg/services/alerting/notifiers/opsgenie.go index 991afd5ce9b..84148a0d99c 100644 --- a/pkg/services/alerting/notifiers/opsgenie.go +++ b/pkg/services/alerting/notifiers/opsgenie.go @@ -56,7 +56,7 @@ func NewOpsGenieNotifier(model *m.AlertNotification) (alerting.Notifier, error) } return &OpsGenieNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), + NotifierBase: NewNotifierBase(model), ApiKey: apiKey, ApiUrl: apiUrl, AutoClose: autoClose, diff --git a/pkg/services/alerting/notifiers/pagerduty.go b/pkg/services/alerting/notifiers/pagerduty.go index afa0ba63eca..bf85466388f 100644 --- a/pkg/services/alerting/notifiers/pagerduty.go +++ b/pkg/services/alerting/notifiers/pagerduty.go @@ -51,7 +51,7 @@ func NewPagerdutyNotifier(model *m.AlertNotification) (alerting.Notifier, error) } return &PagerdutyNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), + NotifierBase: NewNotifierBase(model), Key: key, AutoResolve: autoResolve, log: log.New("alerting.notifier.pagerduty"), diff --git a/pkg/services/alerting/notifiers/pushover.go b/pkg/services/alerting/notifiers/pushover.go index 09dfd6f0f9b..55dc02c5f4a 100644 --- a/pkg/services/alerting/notifiers/pushover.go +++ b/pkg/services/alerting/notifiers/pushover.go @@ -99,7 +99,7 @@ func NewPushoverNotifier(model *m.AlertNotification) (alerting.Notifier, error) return nil, alerting.ValidationError{Reason: "API token not given"} } return &PushoverNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), + NotifierBase: NewNotifierBase(model), UserKey: userKey, ApiToken: apiToken, Priority: priority, diff --git a/pkg/services/alerting/notifiers/sensu.go b/pkg/services/alerting/notifiers/sensu.go index e6b94d3223e..21d5d3d9d9e 100644 --- a/pkg/services/alerting/notifiers/sensu.go +++ b/pkg/services/alerting/notifiers/sensu.go @@ -51,7 +51,7 @@ func NewSensuNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &SensuNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), + NotifierBase: NewNotifierBase(model), Url: url, User: model.Settings.Get("username").MustString(), Source: model.Settings.Get("source").MustString(), diff --git a/pkg/services/alerting/notifiers/slack.go b/pkg/services/alerting/notifiers/slack.go index fbbe4b3e59d..93c9a0accc0 100644 --- a/pkg/services/alerting/notifiers/slack.go +++ b/pkg/services/alerting/notifiers/slack.go @@ -78,7 +78,7 @@ func NewSlackNotifier(model *m.AlertNotification) (alerting.Notifier, error) { uploadImage := model.Settings.Get("uploadImage").MustBool(true) return &SlackNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), + NotifierBase: NewNotifierBase(model), Url: url, Recipient: recipient, Mention: mention, diff --git a/pkg/services/alerting/notifiers/teams.go b/pkg/services/alerting/notifiers/teams.go index 362a367e1f2..58dd4b22bb7 100644 --- a/pkg/services/alerting/notifiers/teams.go +++ b/pkg/services/alerting/notifiers/teams.go @@ -33,7 +33,7 @@ func NewTeamsNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &TeamsNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), + NotifierBase: NewNotifierBase(model), Url: url, log: log.New("alerting.notifier.teams"), }, nil diff --git a/pkg/services/alerting/notifiers/telegram.go b/pkg/services/alerting/notifiers/telegram.go index 97696b2290c..b03f7ca38c5 100644 --- a/pkg/services/alerting/notifiers/telegram.go +++ b/pkg/services/alerting/notifiers/telegram.go @@ -78,7 +78,7 @@ func NewTelegramNotifier(model *m.AlertNotification) (alerting.Notifier, error) } return &TelegramNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), + NotifierBase: NewNotifierBase(model), BotToken: botToken, ChatID: chatId, UploadImage: uploadImage, diff --git a/pkg/services/alerting/notifiers/threema.go b/pkg/services/alerting/notifiers/threema.go index e7fb39f27db..28a62fade17 100644 --- a/pkg/services/alerting/notifiers/threema.go +++ b/pkg/services/alerting/notifiers/threema.go @@ -106,7 +106,7 @@ func NewThreemaNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &ThreemaNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), + NotifierBase: NewNotifierBase(model), GatewayID: gatewayID, RecipientID: recipientID, APISecret: apiSecret, diff --git a/pkg/services/alerting/notifiers/victorops.go b/pkg/services/alerting/notifiers/victorops.go index c6c1cf76047..3093aec9957 100644 --- a/pkg/services/alerting/notifiers/victorops.go +++ b/pkg/services/alerting/notifiers/victorops.go @@ -51,7 +51,7 @@ func NewVictoropsNotifier(model *models.AlertNotification) (alerting.Notifier, e } return &VictoropsNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), + NotifierBase: NewNotifierBase(model), URL: url, AutoResolve: autoResolve, log: log.New("alerting.notifier.victorops"), diff --git a/pkg/services/alerting/notifiers/webhook.go b/pkg/services/alerting/notifiers/webhook.go index 26989873e9e..4045e496af9 100644 --- a/pkg/services/alerting/notifiers/webhook.go +++ b/pkg/services/alerting/notifiers/webhook.go @@ -47,7 +47,7 @@ func NewWebHookNotifier(model *m.AlertNotification) (alerting.Notifier, error) { } return &WebhookNotifier{ - NotifierBase: NewNotifierBase(model.Id, model.IsDefault, model.Name, model.Type, model.NotifyOnce, model.Frequency, model.Settings), + NotifierBase: NewNotifierBase(model), Url: url, User: model.Settings.Get("username").MustString(), Password: model.Settings.Get("password").MustString(), diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index ff36c38b1a5..8c136bd3887 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -57,7 +57,7 @@ func GetAlertNotificationsToSend(query *m.GetAlertNotificationsToSendQuery) erro alert_notification.updated, alert_notification.settings, alert_notification.is_default, - alert_notification.notify_once, + alert_notification.send_reminder, alert_notification.frequency FROM alert_notification `) @@ -97,7 +97,7 @@ func getAlertNotificationInternal(query *m.GetAlertNotificationsQuery, sess *DBS alert_notification.updated, alert_notification.settings, alert_notification.is_default, - alert_notification.notify_once, + alert_notification.send_reminder, alert_notification.frequency FROM alert_notification `) @@ -145,7 +145,7 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error } var frequency time.Duration - if !cmd.NotifyOnce { + if cmd.SendReminder { if cmd.Frequency == "" { return m.ErrNotificationFrequencyNotFound } @@ -157,18 +157,18 @@ func CreateAlertNotificationCommand(cmd *m.CreateAlertNotificationCommand) error } alertNotification := &m.AlertNotification{ - OrgId: cmd.OrgId, - Name: cmd.Name, - Type: cmd.Type, - Settings: cmd.Settings, - NotifyOnce: cmd.NotifyOnce, - Frequency: frequency, - Created: time.Now(), - Updated: time.Now(), - IsDefault: cmd.IsDefault, + OrgId: cmd.OrgId, + Name: cmd.Name, + Type: cmd.Type, + Settings: cmd.Settings, + SendReminder: cmd.SendReminder, + Frequency: frequency, + Created: time.Now(), + Updated: time.Now(), + IsDefault: cmd.IsDefault, } - if _, err = sess.MustCols("notify_once").Insert(alertNotification); err != nil { + if _, err = sess.MustCols("send_reminder").Insert(alertNotification); err != nil { return err } @@ -200,9 +200,9 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { current.Name = cmd.Name current.Type = cmd.Type current.IsDefault = cmd.IsDefault - current.NotifyOnce = cmd.NotifyOnce + current.SendReminder = cmd.SendReminder - if !current.NotifyOnce { + if current.SendReminder { if cmd.Frequency == "" { return m.ErrNotificationFrequencyNotFound } @@ -215,7 +215,7 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { current.Frequency = frequency } - sess.UseBool("is_default", "notify_once") + sess.UseBool("is_default", "send_reminder") if affected, err := sess.ID(cmd.Id).Update(current); err != nil { return err diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index 578a53f34ad..fe7f02b22b0 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -23,13 +23,13 @@ func TestAlertNotificationSQLAccess(t *testing.T) { So(cmd.Result, ShouldBeNil) }) - Convey("Cannot save alert notifier with notitfyonce = false", func() { + Convey("Cannot save alert notifier with send reminder = true", func() { cmd := &m.CreateAlertNotificationCommand{ - Name: "ops", - Type: "email", - OrgId: 1, - NotifyOnce: false, - Settings: simplejson.New(), + Name: "ops", + Type: "email", + OrgId: 1, + SendReminder: true, + Settings: simplejson.New(), } Convey("and missing frequency", func() { @@ -47,19 +47,19 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Cannot update alert notifier with notitfyonce = false", func() { cmd := &m.CreateAlertNotificationCommand{ - Name: "ops update", - Type: "email", - OrgId: 1, - NotifyOnce: true, - Settings: simplejson.New(), + Name: "ops update", + Type: "email", + OrgId: 1, + SendReminder: false, + Settings: simplejson.New(), } err := CreateAlertNotificationCommand(cmd) So(err, ShouldBeNil) updateCmd := &m.UpdateAlertNotificationCommand{ - Id: cmd.Result.Id, - NotifyOnce: false, + Id: cmd.Result.Id, + SendReminder: true, } Convey("and missing frequency", func() { @@ -71,18 +71,19 @@ func TestAlertNotificationSQLAccess(t *testing.T) { updateCmd.Frequency = "invalid duration" err := UpdateAlertNotification(updateCmd) + So(err, ShouldNotBeNil) So(err.Error(), ShouldEqual, "time: invalid duration invalid duration") }) }) Convey("Can save Alert Notification", func() { cmd := &m.CreateAlertNotificationCommand{ - Name: "ops", - Type: "email", - OrgId: 1, - NotifyOnce: true, - Frequency: "10s", - Settings: simplejson.New(), + Name: "ops", + Type: "email", + OrgId: 1, + SendReminder: true, + Frequency: "10s", + Settings: simplejson.New(), } err := CreateAlertNotificationCommand(cmd) @@ -98,13 +99,13 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Can update alert notification", func() { newCmd := &m.UpdateAlertNotificationCommand{ - Name: "NewName", - Type: "webhook", - OrgId: cmd.Result.OrgId, - NotifyOnce: true, - Frequency: "10s", - Settings: simplejson.New(), - Id: cmd.Result.Id, + Name: "NewName", + Type: "webhook", + OrgId: cmd.Result.OrgId, + SendReminder: true, + Frequency: "10s", + Settings: simplejson.New(), + Id: cmd.Result.Id, } err := UpdateAlertNotification(newCmd) So(err, ShouldBeNil) @@ -113,12 +114,12 @@ func TestAlertNotificationSQLAccess(t *testing.T) { }) Convey("Can search using an array of ids", func() { - cmd1 := m.CreateAlertNotificationCommand{Name: "nagios", Type: "webhook", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()} - cmd2 := m.CreateAlertNotificationCommand{Name: "slack", Type: "webhook", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()} - cmd3 := m.CreateAlertNotificationCommand{Name: "ops2", Type: "email", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()} - cmd4 := m.CreateAlertNotificationCommand{IsDefault: true, Name: "default", Type: "email", OrgId: 1, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()} + cmd1 := m.CreateAlertNotificationCommand{Name: "nagios", Type: "webhook", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} + cmd2 := m.CreateAlertNotificationCommand{Name: "slack", Type: "webhook", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} + cmd3 := m.CreateAlertNotificationCommand{Name: "ops2", Type: "email", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} + cmd4 := m.CreateAlertNotificationCommand{IsDefault: true, Name: "default", Type: "email", OrgId: 1, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} - otherOrg := m.CreateAlertNotificationCommand{Name: "default", Type: "email", OrgId: 2, NotifyOnce: true, Frequency: "10s", Settings: simplejson.New()} + otherOrg := m.CreateAlertNotificationCommand{Name: "default", Type: "email", OrgId: 2, SendReminder: true, Frequency: "10s", Settings: simplejson.New()} So(CreateAlertNotificationCommand(&cmd1), ShouldBeNil) So(CreateAlertNotificationCommand(&cmd2), ShouldBeNil) diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index d045f611fb2..51509099c7d 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -68,9 +68,10 @@ func addAlertMigrations(mg *Migrator) { mg.AddMigration("Add column frequency", NewAddColumnMigration(alert_notification, &Column{ Name: "frequency", Type: DB_BigInt, Nullable: true, })) - mg.AddMigration("Add column notify_once", NewAddColumnMigration(alert_notification, &Column{ - Name: "notify_once", Type: DB_Bool, Nullable: false, Default: "1", + mg.AddMigration("Add column send_reminder", NewAddColumnMigration(alert_notification, &Column{ + Name: "send_reminder", Type: DB_Bool, Nullable: true, Default: "0", })) + mg.AddMigration("add index alert_notification org_id & name", NewAddIndexMigration(alert_notification, alert_notification.Indices[0])) notification_journal := Table{ diff --git a/public/app/features/alerting/notification_edit_ctrl.ts b/public/app/features/alerting/notification_edit_ctrl.ts index 9d20e871c7c..e066406bc43 100644 --- a/public/app/features/alerting/notification_edit_ctrl.ts +++ b/public/app/features/alerting/notification_edit_ctrl.ts @@ -11,7 +11,7 @@ export class AlertNotificationEditCtrl { model: any; defaults: any = { type: 'email', - notifyOnce: true, + sendReminder: false, frequency: '15m', settings: { httpMethod: 'POST', diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html index 48d44b74581..2a2fbb131b8 100644 --- a/public/app/features/alerting/partials/notification_edit.html +++ b/public/app/features/alerting/partials/notification_edit.html @@ -34,14 +34,27 @@ -
- Notify every - +
+
+ Send reminder every + + + Specify at what interval you want reminder's about this alerting beeing triggered. + Ex. 60s, 10m, 30m, 1h + +
From bcbae7aa6240e974d47f6c55d83195e4ef2348ad Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 5 Jun 2018 12:07:02 +0200 Subject: [PATCH 059/310] alerting: move queries from evalcontext to notifier base --- pkg/services/alerting/eval_context.go | 15 --------------- pkg/services/alerting/interfaces.go | 2 ++ pkg/services/alerting/notifier.go | 1 + pkg/services/alerting/notifiers/base.go | 23 ++++++++++++++++++++--- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go index 3817f4b4a3c..d0441d379b7 100644 --- a/pkg/services/alerting/eval_context.go +++ b/pkg/services/alerting/eval_context.go @@ -143,18 +143,3 @@ func (c *EvalContext) GetNewState() m.AlertStateType { return m.AlertStateOK } - -func (c *EvalContext) LastNotify(notifierId int64) *time.Time { - cmd := &m.GetLatestNotificationQuery{ - OrgId: c.Rule.OrgId, - AlertId: c.Rule.Id, - NotifierId: notifierId, - } - if err := bus.Dispatch(cmd); err != nil { - c.log.Warn("Could not determine last time alert notifier fired", - "Alert name", c.Rule.Name, "Error", err) - return nil - } - - return &cmd.Result.SentAt -} diff --git a/pkg/services/alerting/interfaces.go b/pkg/services/alerting/interfaces.go index 95fd4b5d04e..b4376191df0 100644 --- a/pkg/services/alerting/interfaces.go +++ b/pkg/services/alerting/interfaces.go @@ -15,6 +15,8 @@ type Notifier interface { Notify(evalContext *EvalContext) error GetType() string NeedsImage() bool + + // ShouldNotify checks this evaluation should send an alert notification ShouldNotify(evalContext *EvalContext) bool GetNotifierId() int64 diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 53923a420fe..363a156c5ec 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -131,6 +131,7 @@ func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds [] if err != nil { return nil, err } + if not.ShouldNotify(context) { result = append(result, not) } diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 1d0d904457f..fc00bd5265e 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -3,6 +3,8 @@ package notifiers import ( "time" + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting" ) @@ -15,6 +17,8 @@ type NotifierBase struct { UploadImage bool SendReminder bool Frequency time.Duration + + log log.Logger } func NewNotifierBase(model *models.AlertNotification) NotifierBase { @@ -32,6 +36,7 @@ func NewNotifierBase(model *models.AlertNotification) NotifierBase { UploadImage: uploadImage, SendReminder: model.SendReminder, Frequency: model.Frequency, + log: log.New("alerting.notifier." + model.Name), } } @@ -55,12 +60,24 @@ func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequ if (context.PrevAlertState == models.AlertStatePending) && (context.Rule.State == models.AlertStateOK) { return false } + return true } -func (n *NotifierBase) ShouldNotify(context *alerting.EvalContext) bool { - lastNotify := context.LastNotify(n.Id) - return defaultShouldNotify(context, n.SendReminder, n.Frequency, lastNotify) +// ShouldNotify checks this evaluation should send an alert notification +func (n *NotifierBase) ShouldNotify(c *alerting.EvalContext) bool { + cmd := &models.GetLatestNotificationQuery{ + OrgId: c.Rule.OrgId, + AlertId: c.Rule.Id, + NotifierId: n.Id, + } + + if err := bus.Dispatch(cmd); err != nil { + n.log.Error("Could not determine last time alert notifier fired", "Alert name", c.Rule.Name, "Error", err) + return false + } + + return defaultShouldNotify(c, n.SendReminder, n.Frequency, &cmd.Result.SentAt) } func (n *NotifierBase) GetType() string { From ab70ead5e4d7ab1dabb204e02a4e27e453d6bd67 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 5 Jun 2018 13:10:38 +0200 Subject: [PATCH 060/310] alerting: renames journal table to alert_notification_journal --- pkg/services/sqlstore/migrations/alert_mig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index 51509099c7d..e58959929d1 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -75,7 +75,7 @@ func addAlertMigrations(mg *Migrator) { mg.AddMigration("add index alert_notification org_id & name", NewAddIndexMigration(alert_notification, alert_notification.Indices[0])) notification_journal := Table{ - Name: "notification_journal", + Name: "alert_notification_journal", Columns: []*Column{ {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, {Name: "org_id", Type: DB_BigInt, Nullable: false}, From 171a38df999f919d2bbd59344483759fc83c7af0 Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 5 Jun 2018 14:29:48 +0200 Subject: [PATCH 061/310] alerting: fixes broken table rename --- pkg/models/alert_notifications.go | 4 +-- pkg/services/sqlstore/alert_notification.go | 40 ++++++++++++++++++--- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index c17124dd6ef..8df7a830d8b 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -75,7 +75,7 @@ type GetAllAlertNotificationsQuery struct { Result []*AlertNotification } -type NotificationJournal struct { +type AlertNotificationJournal struct { Id int64 OrgId int64 AlertId int64 @@ -97,7 +97,7 @@ type GetLatestNotificationQuery struct { AlertId int64 NotifierId int64 - Result *NotificationJournal + Result *AlertNotificationJournal } type CleanNotificationJournalCommand struct { diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 8c136bd3887..0223636bd9a 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -2,10 +2,12 @@ package sqlstore import ( "bytes" + "context" "fmt" "strings" "time" + "github.com/go-xorm/xorm" "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" ) @@ -20,6 +22,8 @@ func init() { bus.AddHandler("sql", RecordNotificationJournal) bus.AddHandler("sql", GetLatestNotification) bus.AddHandler("sql", CleanNotificationJournal) + + bus.AddCtxHandler("sql", GetLastestNotification2) } func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error { @@ -230,7 +234,7 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { func RecordNotificationJournal(cmd *m.RecordNotificationJournalCommand) error { return inTransaction(func(sess *DBSession) error { - journalEntry := &m.NotificationJournal{ + journalEntry := &m.AlertNotificationJournal{ OrgId: cmd.OrgId, AlertId: cmd.AlertId, NotifierId: cmd.NotifierId, @@ -246,10 +250,38 @@ func RecordNotificationJournal(cmd *m.RecordNotificationJournalCommand) error { }) } +func startSession(ctx context.Context) *DBSession { + value := ctx.Value("db-session") + var sess *xorm.Session + sess, ok := value.(*xorm.Session) + + if !ok { + return newSession() + } + + old := newSession() + old.Session = sess + + return old +} + +func GetLastestNotification2(ctx context.Context, cmd *m.GetLatestNotificationQuery) error { + sess := startSession(ctx) + + notificationJournal := &m.AlertNotificationJournal{} + _, err := sess.Desc("alert_notification_journal.sent_at").Limit(1).Where("alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal) + if err != nil { + return err + } + + cmd.Result = notificationJournal + return nil +} + func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error { return inTransaction(func(sess *DBSession) error { - notificationJournal := &m.NotificationJournal{} - _, err := sess.Desc("notification_journal.sent_at").Limit(1).Where("notification_journal.org_id = ? AND notification_journal.alert_id = ? AND notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal) + notificationJournal := &m.AlertNotificationJournal{} + _, err := sess.Desc("alert_notification_journal.sent_at").Limit(1).Where("alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal) if err != nil { return err } @@ -261,7 +293,7 @@ func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error { func CleanNotificationJournal(cmd *m.CleanNotificationJournalCommand) error { return inTransaction(func(sess *DBSession) error { - sql := "DELETE FROM notification_journal WHERE notification_journal.org_id = ? AND notification_journal.alert_id = ? AND notification_journal.notifier_id = ?" + sql := "DELETE FROM alert_notification_journal WHERE notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?" _, err := sess.Exec(sql, cmd.OrgId, cmd.AlertId, cmd.NotifierId) return err }) From 850aa21d451618590a8530b02d4ffd68e02bbeb6 Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 7 Jun 2018 07:06:13 +0200 Subject: [PATCH 062/310] removes unused code --- pkg/services/sqlstore/alert_notification.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 0223636bd9a..a2f3e629ae6 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -22,8 +22,6 @@ func init() { bus.AddHandler("sql", RecordNotificationJournal) bus.AddHandler("sql", GetLatestNotification) bus.AddHandler("sql", CleanNotificationJournal) - - bus.AddCtxHandler("sql", GetLastestNotification2) } func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error { @@ -265,19 +263,6 @@ func startSession(ctx context.Context) *DBSession { return old } -func GetLastestNotification2(ctx context.Context, cmd *m.GetLatestNotificationQuery) error { - sess := startSession(ctx) - - notificationJournal := &m.AlertNotificationJournal{} - _, err := sess.Desc("alert_notification_journal.sent_at").Limit(1).Where("alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal) - if err != nil { - return err - } - - cmd.Result = notificationJournal - return nil -} - func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error { return inTransaction(func(sess *DBSession) error { notificationJournal := &m.AlertNotificationJournal{} From 4a8e9cf93f9a04a30be8269ac68653889394a53d Mon Sep 17 00:00:00 2001 From: bergquist Date: Thu, 7 Jun 2018 07:15:50 +0200 Subject: [PATCH 063/310] removes more unused code --- pkg/services/sqlstore/alert_notification.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index a2f3e629ae6..167b9b3bd61 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -2,12 +2,10 @@ package sqlstore import ( "bytes" - "context" "fmt" "strings" "time" - "github.com/go-xorm/xorm" "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" ) @@ -248,21 +246,6 @@ func RecordNotificationJournal(cmd *m.RecordNotificationJournalCommand) error { }) } -func startSession(ctx context.Context) *DBSession { - value := ctx.Value("db-session") - var sess *xorm.Session - sess, ok := value.(*xorm.Session) - - if !ok { - return newSession() - } - - old := newSession() - old.Session = sess - - return old -} - func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error { return inTransaction(func(sess *DBSession) error { notificationJournal := &m.AlertNotificationJournal{} From 7632983c627b9e5f36e9b13455dc6a45031629eb Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 8 Jun 2018 15:51:26 +0200 Subject: [PATCH 064/310] notifications: gather actions in one transaction --- pkg/services/alerting/notifier.go | 47 +++++++++++++++++-------- pkg/services/alerting/notifiers/base.go | 6 +++- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 363a156c5ec..a3d016211c3 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -1,6 +1,7 @@ package alerting import ( + "context" "errors" "fmt" "time" @@ -59,23 +60,39 @@ func (n *notificationService) SendIfNeeded(context *EvalContext) error { return n.sendNotifications(context, notifiers) } -func (n *notificationService) sendNotifications(context *EvalContext, notifiers []Notifier) error { - g, _ := errgroup.WithContext(context.Ctx) +func (n *notificationService) sendNotifications(evalContext *EvalContext, notifiers []Notifier) error { + g, _ := errgroup.WithContext(evalContext.Ctx) for _, notifier := range notifiers { not := notifier //avoid updating scope variable in go routine - n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault()) - metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc() + g.Go(func() error { - success := not.Notify(context) == nil - cmd := &m.RecordNotificationJournalCommand{ - OrgId: context.Rule.OrgId, - AlertId: context.Rule.Id, - NotifierId: not.GetNotifierId(), - SentAt: time.Now(), - Success: success, - } - return bus.Dispatch(cmd) + return bus.InTransaction(evalContext.Ctx, func(ctx context.Context) error { + n.log.Debug("trying to send notification", "id", not.GetNotifierId()) + + // Verify that we can send the notification again + // but this time within the same transaction. + if !evalContext.IsTestRun && !not.ShouldNotify(evalContext) { + return nil + } + + n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault()) + metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc() + + //send notification + success := not.Notify(evalContext) == nil + + //write result to db. + cmd := &m.RecordNotificationJournalCommand{ + OrgId: evalContext.Rule.OrgId, + AlertId: evalContext.Rule.Id, + NotifierId: not.GetNotifierId(), + SentAt: time.Now(), + Success: success, + } + + return bus.DispatchCtx(evalContext.Ctx, cmd) + }) }) } @@ -118,7 +135,7 @@ func (n *notificationService) uploadImage(context *EvalContext) (err error) { return nil } -func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds []int64, context *EvalContext) (NotifierSlice, error) { +func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds []int64, evalContext *EvalContext) (NotifierSlice, error) { query := &m.GetAlertNotificationsToSendQuery{OrgId: orgId, Ids: notificationIds} if err := bus.Dispatch(query); err != nil { @@ -132,7 +149,7 @@ func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds [] return nil, err } - if not.ShouldNotify(context) { + if not.ShouldNotify(evalContext) { result = append(result, not) } } diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index d8cb740daa1..8450816f97e 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -73,11 +73,15 @@ func (n *NotifierBase) ShouldNotify(c *alerting.EvalContext) bool { NotifierId: n.Id, } - if err := bus.Dispatch(cmd); err != nil { + if err := bus.DispatchCtx(c.Ctx, cmd); err != nil { n.log.Error("Could not determine last time alert notifier fired", "Alert name", c.Rule.Name, "Error", err) return false } + if !cmd.Result.Success { + return true + } + return defaultShouldNotify(c, n.SendReminder, n.Frequency, &cmd.Result.SentAt) } From f4b089d5519fbd352874282f771c8dc66731dde2 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 15 Jun 2018 15:30:17 +0200 Subject: [PATCH 065/310] notifications: make journaling ctx aware --- pkg/services/alerting/notifiers/base_test.go | 94 +++++++++++--------- pkg/services/alerting/result_handler.go | 2 +- pkg/services/sqlstore/alert_notification.go | 19 ++-- pkg/services/sqlstore/transactions.go | 4 + 4 files changed, 66 insertions(+), 53 deletions(-) diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go index 28e2ff9f024..b7395030e5b 100644 --- a/pkg/services/alerting/notifiers/base_test.go +++ b/pkg/services/alerting/notifiers/base_test.go @@ -11,56 +11,64 @@ import ( . "github.com/smartystreets/goconvey/convey" ) +func TestShouldSendAlertNotification(t *testing.T) { + tcs := []struct { + prevState m.AlertStateType + newState m.AlertStateType + expected bool + }{ + { + newState: m.AlertStatePending, + prevState: m.AlertStateOK, + expected: false, + }, + { + newState: m.AlertStateOK, + prevState: m.AlertStateAlerting, + expected: true, + }, + } + + for _, tc := range tcs { + context := alerting.NewEvalContext(context.TODO(), &alerting.Rule{ + State: tc.newState, + }) + context.Rule.State = tc.prevState + timeNow := time.Now() + if defaultShouldNotify(context, true, 0, &timeNow) != tc.expected { + t.Errorf("expected %v to return %v", tc, tc.expected) + } + } +} + func TestBaseNotifier(t *testing.T) { - Convey("Base notifier tests", t, func() { - Convey("default constructor for notifiers", func() { - bJson := simplejson.New() + Convey("default constructor for notifiers", t, func() { + bJson := simplejson.New() - model := &m.AlertNotification{ - Id: 1, - Name: "name", - Type: "email", - Settings: bJson, - } + model := &m.AlertNotification{ + Id: 1, + Name: "name", + Type: "email", + Settings: bJson, + } - Convey("can parse false value", func() { - bJson.Set("uploadImage", false) + Convey("can parse false value", func() { + bJson.Set("uploadImage", false) - base := NewNotifierBase(model) - So(base.UploadImage, ShouldBeFalse) - }) - - Convey("can parse true value", func() { - bJson.Set("uploadImage", true) - - base := NewNotifierBase(model) - So(base.UploadImage, ShouldBeTrue) - }) - - Convey("default value should be true for backwards compatibility", func() { - base := NewNotifierBase(model) - So(base.UploadImage, ShouldBeTrue) - }) + base := NewNotifierBase(model) + So(base.UploadImage, ShouldBeFalse) }) - Convey("should notify", func() { - Convey("pending -> ok", func() { - context := alerting.NewEvalContext(context.TODO(), &alerting.Rule{ - State: m.AlertStatePending, - }) - context.Rule.State = m.AlertStateOK - timeNow := time.Now() - So(defaultShouldNotify(context, true, 0, &timeNow), ShouldBeFalse) - }) + Convey("can parse true value", func() { + bJson.Set("uploadImage", true) - Convey("ok -> alerting", func() { - context := alerting.NewEvalContext(context.TODO(), &alerting.Rule{ - State: m.AlertStateOK, - }) - context.Rule.State = m.AlertStateAlerting - timeNow := time.Now() - So(defaultShouldNotify(context, true, 0, &timeNow), ShouldBeTrue) - }) + base := NewNotifierBase(model) + So(base.UploadImage, ShouldBeTrue) + }) + + Convey("default value should be true for backwards compatibility", func() { + base := NewNotifierBase(model) + So(base.UploadImage, ShouldBeTrue) }) }) } diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go index c4c20bd8beb..363d06d1132 100644 --- a/pkg/services/alerting/result_handler.go +++ b/pkg/services/alerting/result_handler.go @@ -95,7 +95,7 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error { NotifierId: notifierId, OrgId: evalContext.Rule.OrgId, } - if err := bus.Dispatch(cmd); err != nil { + if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { handler.log.Error("Failed to clean up old notification records", "notifier", notifierId, "alert", evalContext.Rule.Id, "Error", err) } } diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 167b9b3bd61..26362dcb750 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -2,6 +2,7 @@ package sqlstore import ( "bytes" + "context" "fmt" "strings" "time" @@ -17,9 +18,9 @@ func init() { bus.AddHandler("sql", DeleteAlertNotification) bus.AddHandler("sql", GetAlertNotificationsToSend) bus.AddHandler("sql", GetAllAlertNotifications) - bus.AddHandler("sql", RecordNotificationJournal) - bus.AddHandler("sql", GetLatestNotification) - bus.AddHandler("sql", CleanNotificationJournal) + bus.AddHandlerCtx("sql", RecordNotificationJournal) + bus.AddHandlerCtx("sql", GetLatestNotification) + bus.AddHandlerCtx("sql", CleanNotificationJournal) } func DeleteAlertNotification(cmd *m.DeleteAlertNotificationCommand) error { @@ -228,8 +229,8 @@ func UpdateAlertNotification(cmd *m.UpdateAlertNotificationCommand) error { }) } -func RecordNotificationJournal(cmd *m.RecordNotificationJournalCommand) error { - return inTransaction(func(sess *DBSession) error { +func RecordNotificationJournal(ctx context.Context, cmd *m.RecordNotificationJournalCommand) error { + return inTransactionCtx(ctx, func(sess *DBSession) error { journalEntry := &m.AlertNotificationJournal{ OrgId: cmd.OrgId, AlertId: cmd.AlertId, @@ -246,8 +247,8 @@ func RecordNotificationJournal(cmd *m.RecordNotificationJournalCommand) error { }) } -func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error { - return inTransaction(func(sess *DBSession) error { +func GetLatestNotification(ctx context.Context, cmd *m.GetLatestNotificationQuery) error { + return inTransactionCtx(ctx, func(sess *DBSession) error { notificationJournal := &m.AlertNotificationJournal{} _, err := sess.Desc("alert_notification_journal.sent_at").Limit(1).Where("alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal) if err != nil { @@ -259,8 +260,8 @@ func GetLatestNotification(cmd *m.GetLatestNotificationQuery) error { }) } -func CleanNotificationJournal(cmd *m.CleanNotificationJournalCommand) error { - return inTransaction(func(sess *DBSession) error { +func CleanNotificationJournal(ctx context.Context, cmd *m.CleanNotificationJournalCommand) error { + return inTransactionCtx(ctx, func(sess *DBSession) error { sql := "DELETE FROM alert_notification_journal WHERE notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?" _, err := sess.Exec(sql, cmd.OrgId, cmd.AlertId, cmd.NotifierId) return err diff --git a/pkg/services/sqlstore/transactions.go b/pkg/services/sqlstore/transactions.go index f72b0bb8500..59290f83121 100644 --- a/pkg/services/sqlstore/transactions.go +++ b/pkg/services/sqlstore/transactions.go @@ -103,3 +103,7 @@ func inTransactionWithRetryCtx(ctx context.Context, callback dbTransactionFunc, func inTransaction(callback dbTransactionFunc) error { return inTransactionWithRetry(callback, 0) } + +func inTransactionCtx(ctx context.Context, callback dbTransactionFunc) error { + return inTransactionWithRetryCtx(ctx, callback, 0) +} From 12bf5c225a98121a9198ffd54a80b609e8364657 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 15 Jun 2018 16:27:20 +0200 Subject: [PATCH 066/310] tests for defaultShouldNotify --- pkg/services/alerting/notifiers/base.go | 4 ++ pkg/services/alerting/notifiers/base_test.go | 45 +++++++++++++++++--- pkg/services/sqlstore/alert_notification.go | 2 +- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 8450816f97e..8178054f4d3 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -78,6 +78,10 @@ func (n *NotifierBase) ShouldNotify(c *alerting.EvalContext) bool { return false } + // this currently serves two purposes. + // 1. make sure failed notifications try again + // 2. make sure we send notifications if no previous exist + // this should be refactored //Carl Bergquist if !cmd.Result.Success { return true } diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go index b7395030e5b..96c80cf03bc 100644 --- a/pkg/services/alerting/notifiers/base_test.go +++ b/pkg/services/alerting/notifiers/base_test.go @@ -13,30 +13,61 @@ import ( func TestShouldSendAlertNotification(t *testing.T) { tcs := []struct { - prevState m.AlertStateType - newState m.AlertStateType - expected bool + name string + prevState m.AlertStateType + newState m.AlertStateType + expected bool + sendReminder bool }{ { + name: "pending -> ok should not trigger an notification", newState: m.AlertStatePending, prevState: m.AlertStateOK, expected: false, }, { + name: "ok -> alerting should trigger an notification", newState: m.AlertStateOK, prevState: m.AlertStateAlerting, expected: true, }, + { + name: "ok -> pending should not trigger an notification", + newState: m.AlertStateOK, + prevState: m.AlertStatePending, + expected: false, + }, + { + name: "ok -> ok should not trigger an notification", + newState: m.AlertStateOK, + prevState: m.AlertStateOK, + expected: false, + sendReminder: false, + }, + { + name: "ok -> alerting should not trigger an notification", + newState: m.AlertStateOK, + prevState: m.AlertStateAlerting, + expected: true, + sendReminder: true, + }, + { + name: "ok -> ok with reminder should not trigger an notification", + newState: m.AlertStateOK, + prevState: m.AlertStateOK, + expected: false, + sendReminder: true, + }, } for _, tc := range tcs { - context := alerting.NewEvalContext(context.TODO(), &alerting.Rule{ + evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{ State: tc.newState, }) - context.Rule.State = tc.prevState + evalContext.Rule.State = tc.prevState timeNow := time.Now() - if defaultShouldNotify(context, true, 0, &timeNow) != tc.expected { - t.Errorf("expected %v to return %v", tc, tc.expected) + if defaultShouldNotify(evalContext, true, 0, &timeNow) != tc.expected { + t.Errorf("failed %s. expected %+v to return %v", tc.name, tc, tc.expected) } } } diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 26362dcb750..5f08a70eff3 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -262,7 +262,7 @@ func GetLatestNotification(ctx context.Context, cmd *m.GetLatestNotificationQuer func CleanNotificationJournal(ctx context.Context, cmd *m.CleanNotificationJournalCommand) error { return inTransactionCtx(ctx, func(sess *DBSession) error { - sql := "DELETE FROM alert_notification_journal WHERE notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?" + sql := "DELETE FROM alert_notification_journal WHERE alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?" _, err := sess.Exec(sql, cmd.OrgId, cmd.AlertId, cmd.NotifierId) return err }) From 72224dbe377e5c065624065724ab9ebfb6011ea3 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 15 Jun 2018 16:53:35 +0200 Subject: [PATCH 067/310] adds info about eval/reminder interval --- .../features/alerting/partials/notification_edit.html | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/public/app/features/alerting/partials/notification_edit.html b/public/app/features/alerting/partials/notification_edit.html index 2a2fbb131b8..7132ed41f3c 100644 --- a/public/app/features/alerting/partials/notification_edit.html +++ b/public/app/features/alerting/partials/notification_edit.html @@ -39,6 +39,11 @@ checked="ctrl.model.sendReminder" tooltip="Choose to either notify on state change or at every interval"> +
+ + Alert reminders are sent after rules are evaluated. Therefore the alert rule interval has to be lower than the reminder frequency + +
Send reminder every @@ -50,8 +55,8 @@ ng-if="ctrl.model.sendReminder" spellcheck='false' placeholder='15m'> - - Specify at what interval you want reminder's about this alerting beeing triggered. + + Specify at what interval you want reminder's about this alerting being triggered. Ex. 60s, 10m, 30m, 1h
From c21938d4c44367a35d7004f5919023f50b467f73 Mon Sep 17 00:00:00 2001 From: bergquist Date: Sat, 16 Jun 2018 00:03:13 +0200 Subject: [PATCH 068/310] use epoch to compare timestamp --- pkg/models/alert_notifications.go | 4 +-- pkg/services/alerting/notifier.go | 2 +- pkg/services/alerting/notifiers/base.go | 6 ++-- pkg/services/alerting/notifiers/base_test.go | 3 +- pkg/services/sqlstore/migrations/alert_mig.go | 36 +++++++++---------- 5 files changed, 25 insertions(+), 26 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index 8df7a830d8b..6be2b02c96f 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -80,7 +80,7 @@ type AlertNotificationJournal struct { OrgId int64 AlertId int64 NotifierId int64 - SentAt time.Time + SentAt int64 Success bool } @@ -88,7 +88,7 @@ type RecordNotificationJournalCommand struct { OrgId int64 AlertId int64 NotifierId int64 - SentAt time.Time + SentAt int64 Success bool } diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index a3d016211c3..61526ed642c 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -87,7 +87,7 @@ func (n *notificationService) sendNotifications(evalContext *EvalContext, notifi OrgId: evalContext.Rule.OrgId, AlertId: evalContext.Rule.Id, NotifierId: not.GetNotifierId(), - SentAt: time.Now(), + SentAt: time.Now().Unix(), Success: success, } diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 8178054f4d3..6245650ab4e 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -41,14 +41,14 @@ func NewNotifierBase(model *models.AlertNotification) NotifierBase { } } -func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequency time.Duration, lastNotify *time.Time) bool { +func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequency time.Duration, lastNotify time.Time) bool { // Only notify on state change. if context.PrevAlertState == context.Rule.State && !sendReminder { return false } // Do not notify if interval has not elapsed - if sendReminder && lastNotify != nil && lastNotify.Add(frequency).After(time.Now()) { + if sendReminder && !lastNotify.IsZero() && lastNotify.Add(frequency).After(time.Now()) { return false } @@ -86,7 +86,7 @@ func (n *NotifierBase) ShouldNotify(c *alerting.EvalContext) bool { return true } - return defaultShouldNotify(c, n.SendReminder, n.Frequency, &cmd.Result.SentAt) + return defaultShouldNotify(c, n.SendReminder, n.Frequency, time.Unix(cmd.Result.SentAt, 0)) } func (n *NotifierBase) GetType() string { diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go index 96c80cf03bc..5b75ea4d59b 100644 --- a/pkg/services/alerting/notifiers/base_test.go +++ b/pkg/services/alerting/notifiers/base_test.go @@ -65,8 +65,7 @@ func TestShouldSendAlertNotification(t *testing.T) { State: tc.newState, }) evalContext.Rule.State = tc.prevState - timeNow := time.Now() - if defaultShouldNotify(evalContext, true, 0, &timeNow) != tc.expected { + if defaultShouldNotify(evalContext, true, 0, time.Now()) != tc.expected { t.Errorf("failed %s. expected %+v to return %v", tc.name, tc, tc.expected) } } diff --git a/pkg/services/sqlstore/migrations/alert_mig.go b/pkg/services/sqlstore/migrations/alert_mig.go index e58959929d1..e27e64c6124 100644 --- a/pkg/services/sqlstore/migrations/alert_mig.go +++ b/pkg/services/sqlstore/migrations/alert_mig.go @@ -74,24 +74,6 @@ func addAlertMigrations(mg *Migrator) { mg.AddMigration("add index alert_notification org_id & name", NewAddIndexMigration(alert_notification, alert_notification.Indices[0])) - notification_journal := Table{ - Name: "alert_notification_journal", - Columns: []*Column{ - {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, - {Name: "org_id", Type: DB_BigInt, Nullable: false}, - {Name: "alert_id", Type: DB_BigInt, Nullable: false}, - {Name: "notifier_id", Type: DB_BigInt, Nullable: false}, - {Name: "sent_at", Type: DB_DateTime, Nullable: false}, - {Name: "success", Type: DB_Bool, Nullable: false}, - }, - Indices: []*Index{ - {Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: IndexType}, - }, - } - - mg.AddMigration("create notification_journal table v1", NewAddTableMigration(notification_journal)) - mg.AddMigration("add index notification_journal org_id & alert_id & notifier_id", NewAddIndexMigration(notification_journal, notification_journal.Indices[0])) - mg.AddMigration("Update alert table charset", NewTableCharsetMigration("alert", []*Column{ {Name: "name", Type: DB_NVarchar, Length: 255, Nullable: false}, {Name: "message", Type: DB_Text, Nullable: false}, @@ -107,4 +89,22 @@ func addAlertMigrations(mg *Migrator) { {Name: "type", Type: DB_NVarchar, Length: 255, Nullable: false}, {Name: "settings", Type: DB_Text, Nullable: false}, })) + + notification_journal := Table{ + Name: "alert_notification_journal", + Columns: []*Column{ + {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, + {Name: "org_id", Type: DB_BigInt, Nullable: false}, + {Name: "alert_id", Type: DB_BigInt, Nullable: false}, + {Name: "notifier_id", Type: DB_BigInt, Nullable: false}, + {Name: "sent_at", Type: DB_BigInt, Nullable: false}, + {Name: "success", Type: DB_Bool, Nullable: false}, + }, + Indices: []*Index{ + {Cols: []string{"org_id", "alert_id", "notifier_id"}, Type: IndexType}, + }, + } + + mg.AddMigration("create notification_journal table v1", NewAddTableMigration(notification_journal)) + mg.AddMigration("add index notification_journal org_id & alert_id & notifier_id", NewAddIndexMigration(notification_journal, notification_journal.Indices[0])) } From 83a12afc07ef0d6891e9d9f76736359ec067c024 Mon Sep 17 00:00:00 2001 From: bergquist Date: Sat, 16 Jun 2018 11:27:04 +0200 Subject: [PATCH 069/310] adds tests for journaling sql operations --- pkg/models/alert_notifications.go | 1 + pkg/services/alerting/notifiers/base.go | 11 ++--- pkg/services/sqlstore/alert_notification.go | 13 ++++-- .../sqlstore/alert_notification_test.go | 43 +++++++++++++++++++ 4 files changed, 60 insertions(+), 8 deletions(-) diff --git a/pkg/models/alert_notifications.go b/pkg/models/alert_notifications.go index 6be2b02c96f..42d33d5ed22 100644 --- a/pkg/models/alert_notifications.go +++ b/pkg/models/alert_notifications.go @@ -9,6 +9,7 @@ import ( var ( ErrNotificationFrequencyNotFound = errors.New("Notification frequency not specified") + ErrJournalingNotFound = errors.New("alert notification journaling not found") ) type AlertNotification struct { diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 6245650ab4e..4869c40f436 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -73,15 +73,16 @@ func (n *NotifierBase) ShouldNotify(c *alerting.EvalContext) bool { NotifierId: n.Id, } - if err := bus.DispatchCtx(c.Ctx, cmd); err != nil { + err := bus.DispatchCtx(c.Ctx, cmd) + if err != nil { n.log.Error("Could not determine last time alert notifier fired", "Alert name", c.Rule.Name, "Error", err) return false } - // this currently serves two purposes. - // 1. make sure failed notifications try again - // 2. make sure we send notifications if no previous exist - // this should be refactored //Carl Bergquist + if err == models.ErrJournalingNotFound { + return true + } + if !cmd.Result.Success { return true } diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 5f08a70eff3..3f2ca109c1a 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -249,13 +249,20 @@ func RecordNotificationJournal(ctx context.Context, cmd *m.RecordNotificationJou func GetLatestNotification(ctx context.Context, cmd *m.GetLatestNotificationQuery) error { return inTransactionCtx(ctx, func(sess *DBSession) error { - notificationJournal := &m.AlertNotificationJournal{} - _, err := sess.Desc("alert_notification_journal.sent_at").Limit(1).Where("alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(notificationJournal) + nj := &m.AlertNotificationJournal{} + _, err := sess.Desc("alert_notification_journal.sent_at"). + Limit(1). + Where("alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(nj) + if err != nil { return err } - cmd.Result = notificationJournal + if nj.AlertId == 0 && nj.Id == 0 && nj.NotifierId == 0 && nj.OrgId == 0 { + return m.ErrJournalingNotFound + } + + cmd.Result = nj return nil }) } diff --git a/pkg/services/sqlstore/alert_notification_test.go b/pkg/services/sqlstore/alert_notification_test.go index fe7f02b22b0..aba437f427e 100644 --- a/pkg/services/sqlstore/alert_notification_test.go +++ b/pkg/services/sqlstore/alert_notification_test.go @@ -1,6 +1,7 @@ package sqlstore import ( + "context" "testing" "github.com/grafana/grafana/pkg/components/simplejson" @@ -12,6 +13,48 @@ func TestAlertNotificationSQLAccess(t *testing.T) { Convey("Testing Alert notification sql access", t, func() { InitTestDB(t) + Convey("Alert notification journal", func() { + var alertId int64 = 5 + var orgId int64 = 5 + var notifierId int64 = 5 + + Convey("Getting last journal should raise error if no one exists", func() { + query := &m.GetLatestNotificationQuery{AlertId: alertId, OrgId: orgId, NotifierId: notifierId} + err := GetLatestNotification(context.Background(), query) + So(err, ShouldEqual, m.ErrJournalingNotFound) + + Convey("shoulbe be able to record two journaling events", func() { + createCmd := &m.RecordNotificationJournalCommand{AlertId: alertId, NotifierId: notifierId, OrgId: orgId, Success: true, SentAt: 1} + + err := RecordNotificationJournal(context.Background(), createCmd) + So(err, ShouldBeNil) + + createCmd.SentAt += 1000 //increase epoch + + err = RecordNotificationJournal(context.Background(), createCmd) + So(err, ShouldBeNil) + + Convey("get last journaling event", func() { + err := GetLatestNotification(context.Background(), query) + So(err, ShouldBeNil) + So(query.Result.SentAt, ShouldEqual, 1001) + + Convey("be able to clear all journaling for an notifier", func() { + cmd := &m.CleanNotificationJournalCommand{AlertId: alertId, NotifierId: notifierId, OrgId: orgId} + err := CleanNotificationJournal(context.Background(), cmd) + So(err, ShouldBeNil) + + Convey("querying for last junaling should raise error", func() { + query := &m.GetLatestNotificationQuery{AlertId: alertId, OrgId: orgId, NotifierId: notifierId} + err := GetLatestNotification(context.Background(), query) + So(err, ShouldEqual, m.ErrJournalingNotFound) + }) + }) + }) + }) + }) + }) + Convey("Alert notifications should be empty", func() { cmd := &m.GetAlertNotificationsQuery{ OrgId: 2, From 8ff538be074f228239e1694c617fc57c89d5d5cf Mon Sep 17 00:00:00 2001 From: bergquist Date: Tue, 26 Jun 2018 14:13:45 +0200 Subject: [PATCH 070/310] notifier: handle known error first --- pkg/services/alerting/notifiers/base.go | 8 ++--- pkg/services/alerting/notifiers/base_test.go | 38 ++++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 4869c40f436..31ec77cbf25 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -74,15 +74,15 @@ func (n *NotifierBase) ShouldNotify(c *alerting.EvalContext) bool { } err := bus.DispatchCtx(c.Ctx, cmd) + if err == models.ErrJournalingNotFound { + return true + } + if err != nil { n.log.Error("Could not determine last time alert notifier fired", "Alert name", c.Rule.Name, "Error", err) return false } - if err == models.ErrJournalingNotFound { - return true - } - if !cmd.Result.Success { return true } diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go index 5b75ea4d59b..3fd23b69c6e 100644 --- a/pkg/services/alerting/notifiers/base_test.go +++ b/pkg/services/alerting/notifiers/base_test.go @@ -2,9 +2,12 @@ package notifiers import ( "context" + "errors" "testing" "time" + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/components/simplejson" m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/services/alerting" @@ -64,6 +67,7 @@ func TestShouldSendAlertNotification(t *testing.T) { evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{ State: tc.newState, }) + evalContext.Rule.State = tc.prevState if defaultShouldNotify(evalContext, true, 0, time.Now()) != tc.expected { t.Errorf("failed %s. expected %+v to return %v", tc.name, tc, tc.expected) @@ -71,6 +75,40 @@ func TestShouldSendAlertNotification(t *testing.T) { } } +func TestShouldNotifyWhenNoJournalingIsFound(t *testing.T) { + Convey("base notifier", t, func() { + bus.ClearBusHandlers() + + notifier := NewNotifierBase(&m.AlertNotification{ + Id: 1, + Name: "name", + Type: "email", + Settings: simplejson.New(), + }) + evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{}) + + Convey("should notify if no journaling is found", func() { + bus.AddHandlerCtx("", func(ctx context.Context, q *m.GetLatestNotificationQuery) error { + return m.ErrJournalingNotFound + }) + + if !notifier.ShouldNotify(evalContext) { + t.Errorf("should send notifications when ErrJournalingNotFound is returned") + } + }) + + Convey("should not notify query returns error", func() { + bus.AddHandlerCtx("", func(ctx context.Context, q *m.GetLatestNotificationQuery) error { + return errors.New("some kind of error unknown error") + }) + + if notifier.ShouldNotify(evalContext) { + t.Errorf("should not send notifications when query returns error") + } + }) + }) +} + func TestBaseNotifier(t *testing.T) { Convey("default constructor for notifiers", t, func() { bJson := simplejson.New() From 396f8e6464a38e6ac925ecb8465a71e5118b79c5 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 29 Jun 2018 15:15:31 +0200 Subject: [PATCH 071/310] notifications: read without tran, write with tran --- pkg/services/alerting/interfaces.go | 7 +++++-- pkg/services/alerting/notifier.go | 6 +++--- pkg/services/alerting/notifiers/alertmanager.go | 3 ++- pkg/services/alerting/notifiers/base.go | 5 +++-- pkg/services/alerting/notifiers/base_test.go | 4 ++-- pkg/services/sqlstore/alert_notification.go | 1 + 6 files changed, 16 insertions(+), 10 deletions(-) diff --git a/pkg/services/alerting/interfaces.go b/pkg/services/alerting/interfaces.go index b4376191df0..46f8b3c769c 100644 --- a/pkg/services/alerting/interfaces.go +++ b/pkg/services/alerting/interfaces.go @@ -1,6 +1,9 @@ package alerting -import "time" +import ( + "context" + "time" +) type EvalHandler interface { Eval(evalContext *EvalContext) @@ -17,7 +20,7 @@ type Notifier interface { NeedsImage() bool // ShouldNotify checks this evaluation should send an alert notification - ShouldNotify(evalContext *EvalContext) bool + ShouldNotify(ctx context.Context, evalContext *EvalContext) bool GetNotifierId() int64 GetIsDefault() bool diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 61526ed642c..a19e44a8f99 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -72,7 +72,7 @@ func (n *notificationService) sendNotifications(evalContext *EvalContext, notifi // Verify that we can send the notification again // but this time within the same transaction. - if !evalContext.IsTestRun && !not.ShouldNotify(evalContext) { + if !evalContext.IsTestRun && !not.ShouldNotify(context.Background(), evalContext) { return nil } @@ -91,7 +91,7 @@ func (n *notificationService) sendNotifications(evalContext *EvalContext, notifi Success: success, } - return bus.DispatchCtx(evalContext.Ctx, cmd) + return bus.DispatchCtx(ctx, cmd) }) }) } @@ -149,7 +149,7 @@ func (n *notificationService) getNeededNotifiers(orgId int64, notificationIds [] return nil, err } - if not.ShouldNotify(evalContext) { + if not.ShouldNotify(evalContext.Ctx, evalContext) { result = append(result, not) } } diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go index 42ffa9b2d6e..9826dd1dffb 100644 --- a/pkg/services/alerting/notifiers/alertmanager.go +++ b/pkg/services/alerting/notifiers/alertmanager.go @@ -1,6 +1,7 @@ package notifiers import ( + "context" "time" "github.com/grafana/grafana/pkg/bus" @@ -45,7 +46,7 @@ type AlertmanagerNotifier struct { log log.Logger } -func (this *AlertmanagerNotifier) ShouldNotify(evalContext *alerting.EvalContext) bool { +func (this *AlertmanagerNotifier) ShouldNotify(ctx context.Context, evalContext *alerting.EvalContext) bool { this.log.Debug("Should notify", "ruleId", evalContext.Rule.Id, "state", evalContext.Rule.State, "previousState", evalContext.PrevAlertState) // Do not notify when we become OK for the first time. diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 31ec77cbf25..ca011356247 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -1,6 +1,7 @@ package notifiers import ( + "context" "time" "github.com/grafana/grafana/pkg/bus" @@ -66,14 +67,14 @@ func defaultShouldNotify(context *alerting.EvalContext, sendReminder bool, frequ } // ShouldNotify checks this evaluation should send an alert notification -func (n *NotifierBase) ShouldNotify(c *alerting.EvalContext) bool { +func (n *NotifierBase) ShouldNotify(ctx context.Context, c *alerting.EvalContext) bool { cmd := &models.GetLatestNotificationQuery{ OrgId: c.Rule.OrgId, AlertId: c.Rule.Id, NotifierId: n.Id, } - err := bus.DispatchCtx(c.Ctx, cmd) + err := bus.DispatchCtx(ctx, cmd) if err == models.ErrJournalingNotFound { return true } diff --git a/pkg/services/alerting/notifiers/base_test.go b/pkg/services/alerting/notifiers/base_test.go index 3fd23b69c6e..57b82f32466 100644 --- a/pkg/services/alerting/notifiers/base_test.go +++ b/pkg/services/alerting/notifiers/base_test.go @@ -92,7 +92,7 @@ func TestShouldNotifyWhenNoJournalingIsFound(t *testing.T) { return m.ErrJournalingNotFound }) - if !notifier.ShouldNotify(evalContext) { + if !notifier.ShouldNotify(context.Background(), evalContext) { t.Errorf("should send notifications when ErrJournalingNotFound is returned") } }) @@ -102,7 +102,7 @@ func TestShouldNotifyWhenNoJournalingIsFound(t *testing.T) { return errors.New("some kind of error unknown error") }) - if notifier.ShouldNotify(evalContext) { + if notifier.ShouldNotify(context.Background(), evalContext) { t.Errorf("should not send notifications when query returns error") } }) diff --git a/pkg/services/sqlstore/alert_notification.go b/pkg/services/sqlstore/alert_notification.go index 3f2ca109c1a..8fb1e2212a9 100644 --- a/pkg/services/sqlstore/alert_notification.go +++ b/pkg/services/sqlstore/alert_notification.go @@ -250,6 +250,7 @@ func RecordNotificationJournal(ctx context.Context, cmd *m.RecordNotificationJou func GetLatestNotification(ctx context.Context, cmd *m.GetLatestNotificationQuery) error { return inTransactionCtx(ctx, func(sess *DBSession) error { nj := &m.AlertNotificationJournal{} + _, err := sess.Desc("alert_notification_journal.sent_at"). Limit(1). Where("alert_notification_journal.org_id = ? AND alert_notification_journal.alert_id = ? AND alert_notification_journal.notifier_id = ?", cmd.OrgId, cmd.AlertId, cmd.NotifierId).Get(nj) From e91e3ea771228af267178f6fd927b4afcf3b6e75 Mon Sep 17 00:00:00 2001 From: bergquist Date: Fri, 29 Jun 2018 16:16:09 +0200 Subject: [PATCH 072/310] notifications: send notifications synchronous --- pkg/services/alerting/notifier.go | 54 +++++++++++++++---------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index 7fe97596494..fb2933f6e26 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -6,8 +6,6 @@ import ( "fmt" "time" - "golang.org/x/sync/errgroup" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/components/imguploader" "github.com/grafana/grafana/pkg/log" @@ -61,42 +59,42 @@ func (n *notificationService) SendIfNeeded(context *EvalContext) error { } func (n *notificationService) sendNotifications(evalContext *EvalContext, notifiers []Notifier) error { - g, _ := errgroup.WithContext(evalContext.Ctx) - for _, notifier := range notifiers { - not := notifier //avoid updating scope variable in go routine + not := notifier - g.Go(func() error { - return bus.InTransaction(evalContext.Ctx, func(ctx context.Context) error { - n.log.Debug("trying to send notification", "id", not.GetNotifierId()) + err := bus.InTransaction(evalContext.Ctx, func(ctx context.Context) error { + n.log.Debug("trying to send notification", "id", not.GetNotifierId()) - // Verify that we can send the notification again - // but this time within the same transaction. - if !evalContext.IsTestRun && !not.ShouldNotify(context.Background(), evalContext) { - return nil - } + // Verify that we can send the notification again + // but this time within the same transaction. + if !evalContext.IsTestRun && !not.ShouldNotify(context.Background(), evalContext) { + return nil + } - n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault()) - metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc() + n.log.Debug("Sending notification", "type", not.GetType(), "id", not.GetNotifierId(), "isDefault", not.GetIsDefault()) + metrics.M_Alerting_Notification_Sent.WithLabelValues(not.GetType()).Inc() - //send notification - success := not.Notify(evalContext) == nil + //send notification + success := not.Notify(evalContext) == nil - //write result to db. - cmd := &m.RecordNotificationJournalCommand{ - OrgId: evalContext.Rule.OrgId, - AlertId: evalContext.Rule.Id, - NotifierId: not.GetNotifierId(), - SentAt: time.Now().Unix(), - Success: success, - } + //write result to db. + cmd := &m.RecordNotificationJournalCommand{ + OrgId: evalContext.Rule.OrgId, + AlertId: evalContext.Rule.Id, + NotifierId: not.GetNotifierId(), + SentAt: time.Now().Unix(), + Success: success, + } - return bus.DispatchCtx(ctx, cmd) - }) + return bus.DispatchCtx(ctx, cmd) }) + + if err != nil { + return err + } } - return g.Wait() + return nil } func (n *notificationService) uploadImage(context *EvalContext) (err error) { From 410449b5e75699e439a19c3112732e42821b1198 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 30 Jun 2018 11:11:34 +0200 Subject: [PATCH 073/310] use sqlPart for ui parts --- .../postgres/partials/query.editor.html | 9 +- .../plugins/datasource/postgres/query_ctrl.ts | 202 +++++++----------- .../plugins/datasource/postgres/query_part.ts | 29 ++- 3 files changed, 106 insertions(+), 134 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 220f11fdfef..ea5b4b4e184 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -18,8 +18,13 @@
-
- +
+ + +
+ +
+
diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index dd1da1c75cf..1eb8b3ef80a 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -1,8 +1,8 @@ -import angular from 'angular'; import _ from 'lodash'; import { PostgresQueryBuilder } from './query_builder'; import { QueryCtrl } from 'app/plugins/sdk'; import PostgresQuery from './postgres_query'; +import sqlPart from './query_part'; export interface QueryMeta { sql: string; @@ -30,11 +30,11 @@ export class PostgresQueryCtrl extends QueryCtrl { schemaSegment: any; tableSegment: any; whereSegments: any; + whereAdd: any; timeColumnSegment: any; metricColumnSegment: any; selectMenu: any; groupBySegment: any; - removeWhereFilterSegment: any; /** @ngInject **/ constructor($scope, $injector, private templateSrv, private $q, private uiSegmentSrv) { @@ -55,12 +55,12 @@ export class PostgresQueryCtrl extends QueryCtrl { } } - this.schemaSegment= uiSegmentSrv.newSegment(this.target.schema); + this.schemaSegment = uiSegmentSrv.newSegment(this.target.schema); if (!this.target.table) { - this.tableSegment = uiSegmentSrv.newSegment({value: 'select table',fake: true}); + this.tableSegment = uiSegmentSrv.newSegment({ value: 'select table', fake: true }); } else { - this.tableSegment= uiSegmentSrv.newSegment(this.target.table); + this.tableSegment = uiSegmentSrv.newSegment(this.target.table); } this.timeColumnSegment = uiSegmentSrv.newSegment(this.target.timeColumn); @@ -68,23 +68,19 @@ export class PostgresQueryCtrl extends QueryCtrl { this.buildSelectMenu(); this.buildWhereSegments(); + this.whereAdd = this.uiSegmentSrv.newPlusButton(); this.groupBySegment = this.uiSegmentSrv.newPlusButton(); - this.removeWhereFilterSegment = uiSegmentSrv.newSegment({ - fake: true, - value: '-- remove filter --', - }); this.panelCtrl.events.on('data-received', this.onDataReceived.bind(this), $scope); this.panelCtrl.events.on('data-error', this.onDataError.bind(this), $scope); - } buildSelectMenu() { this.selectMenu = [ - {text: "aggregate", value: "aggregate"}, - {text: "math", value: "math"}, - {text: "alias", value: "alias"}, - {text: "column", value: "column"}, + { text: 'Aggregate', value: 'aggregate' }, + { text: 'Math', value: 'math' }, + { text: 'Alias', value: 'alias' }, + { text: 'Column', value: 'column' }, ]; } @@ -108,14 +104,14 @@ export class PostgresQueryCtrl extends QueryCtrl { getTimeColumnSegments() { return this.datasource - .metricFindQuery(this.queryBuilder.buildColumnQuery("time")) + .metricFindQuery(this.queryBuilder.buildColumnQuery('time')) .then(this.transformToSegments(true)) .catch(this.handleQueryError.bind(this)); } getMetricColumnSegments() { return this.datasource - .metricFindQuery(this.queryBuilder.buildColumnQuery("metric")) + .metricFindQuery(this.queryBuilder.buildColumnQuery('metric')) .then(this.transformToSegments(true)) .catch(this.handleQueryError.bind(this)); } @@ -186,7 +182,7 @@ export class PostgresQueryCtrl extends QueryCtrl { } addSelectPart(selectParts, cat, subitem) { - if ("submenu" in cat) { + if ('submenu' in cat) { this.queryModel.addSelectPart(selectParts, subitem.value); } else { this.queryModel.addSelectPart(selectParts, cat.value); @@ -198,18 +194,17 @@ export class PostgresQueryCtrl extends QueryCtrl { switch (evt.name) { case 'get-param-options': { switch (part.def.type) { - case "aggregate": + case 'aggregate': return this.datasource .metricFindQuery(this.queryBuilder.buildAggregateQuery()) .then(this.transformToSegments(false)) .catch(this.handleQueryError.bind(this)); - case "column": + case 'column': return this.datasource - .metricFindQuery(this.queryBuilder.buildColumnQuery("value")) + .metricFindQuery(this.queryBuilder.buildColumnQuery('value')) .then(this.transformToSegments(true)) .catch(this.handleQueryError.bind(this)); } - } case 'part-param-changed': { this.panelCtrl.refresh(); @@ -251,125 +246,75 @@ export class PostgresQueryCtrl extends QueryCtrl { buildWhereSegments() { this.whereSegments = []; - for (let constraint of this.target.where) { - - if (constraint.condition) { - this.whereSegments.push(this.uiSegmentSrv.newCondition(constraint.condition)); - } - this.whereSegments.push(this.uiSegmentSrv.newKey(constraint.key)); - this.whereSegments.push(this.uiSegmentSrv.newOperator(constraint.operator)); - this.whereSegments.push(this.uiSegmentSrv.newKeyValue(constraint.value)); - } - - var count = this.whereSegments.length; - var lastSegment = this.whereSegments[Math.max(count - 1, 0)]; - - if (!lastSegment || lastSegment.type !== 'plus-button') { - this.whereSegments.push(this.uiSegmentSrv.newPlusButton()); - } + this.whereSegments.push(sqlPart.create({ type: 'expression', params: ['value', '=', 'value'] })); + // for (let constraint of this.target.where) { + // + // this.whereSegments.push(sqlPart.create({type: 'column',params: ['1']})); + // if (constraint.condition) { + // this.whereSegments.push(this.uiSegmentSrv.newCondition(constraint.condition)); + // } + // this.whereSegments.push(this.uiSegmentSrv.newKey(constraint.key)); + // this.whereSegments.push(this.uiSegmentSrv.newOperator(constraint.operator)); + // this.whereSegments.push(this.uiSegmentSrv.newKeyValue(constraint.value)); + // } } - getWhereSegments(segment, index) { - var query, addTemplateVars; - - if (segment.type === 'condition') { - return this.$q.when([this.uiSegmentSrv.newSegment('AND'), this.uiSegmentSrv.newSegment('OR')]); - } - if (segment.type === 'operator') { - var columnName = this.whereSegments[index - 1].value; - query = this.queryBuilder.buildDatatypeQuery(columnName); - return this.datasource.metricFindQuery(query) - .then(results => { - var datatype = results[0].text; - switch (datatype) { - case "text": - case "character": - case "character varying": - return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '~', '~*','!~','!~*','IN'])); + handleWherePartEvent(whereParts, part, evt, index) { + switch (evt.name) { + case 'get-param-options': { + switch (evt.param.name) { + case 'left': + return this.datasource + .metricFindQuery(this.queryBuilder.buildColumnQuery()) + .then(this.transformToSegments(false)) + .catch(this.handleQueryError.bind(this)); + case 'right': + return this.datasource + .metricFindQuery(this.queryBuilder.buildValueQuery(part.params[0])) + .then(this.transformToSegments(true)) + .catch(this.handleQueryError.bind(this)); + case 'op': + return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '<', '<=', '>', '>=', 'IN'])); default: - return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '<', '<=', '>', '>='])); + return Promise.resolve([]); } - }) - .catch(this.handleQueryError.bind(this)); + } + case 'part-param-changed': { + this.panelCtrl.refresh(); + break; + } + case 'action': { + whereParts.splice(whereParts.indexOf(part), 1); + this.panelCtrl.refresh(); + break; + } + case 'get-part-actions': { + return this.$q.when([{ text: 'Remove', value: 'remove-part' }]); + } } - - if (segment.type === 'key' || segment.type === 'plus-button') { - query = this.queryBuilder.buildColumnQuery(); - - addTemplateVars = false; - } else if (segment.type === 'value') { - query = this.queryBuilder.buildValueQuery(this.whereSegments[index -2].value); - addTemplateVars = true; - } - - return this.datasource - .metricFindQuery(query) - .then(this.transformToSegments(addTemplateVars)) - .then(results => { - if (segment.type === 'key') { - results.splice(0, 0, angular.copy(this.removeWhereFilterSegment)); - } - return results; - }) - .catch(this.handleQueryError.bind(this)); } - whereSegmentUpdated(segment, index) { - this.whereSegments[index] = segment; + getWhereOptions() { + var options = []; + options.push(this.uiSegmentSrv.newSegment({ type: 'function', value: '$__timeFilter' })); + options.push(this.uiSegmentSrv.newSegment({ type: 'function', value: '$__unixEpochFilter' })); + options.push(this.uiSegmentSrv.newSegment({ type: 'function', value: 'Expression' })); + return Promise.resolve(options); + } - // handle remove where condition - if (segment.value === this.removeWhereFilterSegment.value) { - this.whereSegments.splice(index, 3); - if (this.whereSegments.length === 0) { - this.whereSegments.push(this.uiSegmentSrv.newPlusButton()); - } else if (this.whereSegments.length > 2) { - this.whereSegments.splice(Math.max(index - 1, 0), 1); - if (this.whereSegments[this.whereSegments.length - 1].type !== 'plus-button') { - this.whereSegments.push(this.uiSegmentSrv.newPlusButton()); - } + whereAddAction(part, index) { + switch (this.whereAdd.type) { + case 'macro': { + this.whereSegments.push( + sqlPart.create({ type: 'function', name: this.whereAdd.value, params: ['value', '=', 'value'] }) + ); } - } else { - if (segment.type === 'plus-button') { - if (index > 2) { - this.whereSegments.splice(index, 0, this.uiSegmentSrv.newCondition('AND')); - } - this.whereSegments.push(this.uiSegmentSrv.newOperator('=')); - this.whereSegments.push(this.uiSegmentSrv.newFake('select value', 'value', 'query-segment-value')); - segment.type = 'key'; - segment.cssClass = 'query-segment-key'; - } - - if (index + 1 === this.whereSegments.length) { - this.whereSegments.push(this.uiSegmentSrv.newPlusButton()); + default: { + this.whereSegments.push(sqlPart.create({ type: 'expression', params: ['value', '=', 'value'] })); } } - this.rebuildTargetWhereConditions(); - } - - rebuildTargetWhereConditions() { - var where = []; - var tagIndex = 0; - - _.each(this.whereSegments, (segment2, index) => { - if (segment2.type === 'key') { - if (where.length === 0) { - where.push({}); - } - where[tagIndex].key = segment2.value; - } else if (segment2.type === 'value') { - where[tagIndex].value = segment2.value; - } else if (segment2.type === 'template') { - where[tagIndex].value = segment2.value; - } else if (segment2.type === 'condition') { - where.push({ condition: segment2.value }); - tagIndex += 1; - } else if (segment2.type === 'operator') { - where[tagIndex].operator = segment2.value; - } - }); - - this.target.where = where; + this.whereAdd = this.uiSegmentSrv.newPlusButton(); this.panelCtrl.refresh(); } @@ -406,5 +351,4 @@ export class PostgresQueryCtrl extends QueryCtrl { this.error = err.message || 'Failed to issue metric query'; return []; } - } diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/query_part.ts index d672328c93e..4cda0d47d0d 100644 --- a/public/app/plugins/datasource/postgres/query_part.ts +++ b/public/app/plugins/datasource/postgres/query_part.ts @@ -32,7 +32,7 @@ function replaceAggregationAddStrategy(selectParts, partModel) { // look for existing aggregation for (var i = 0; i < selectParts.length; i++) { var part = selectParts[i]; - if (part.def.type === "aggregate") { + if (part.def.type === 'aggregate') { selectParts[i] = partModel; return; } @@ -83,6 +83,15 @@ function addColumnStrategy(selectParts, partModel, query) { query.selectModels.push(parts); } +function addExpressionStrategy(selectParts, partModel, query) { + // copy all parts + var parts = _.map(selectParts, function(part: any) { + return createPart({ type: part.def.type, params: _.clone(part.params) }); + }); + + query.selectModels.push(parts); +} + register({ type: 'column', style: 'label', @@ -92,11 +101,25 @@ register({ renderer: columnRenderer, }); +register({ + type: 'expression', + style: 'expression', + label: 'Expr:', + addStrategy: addExpressionStrategy, + params: [ + { name: 'left', type: 'string', dynamicLookup: true }, + { name: 'op', type: 'string', dynamicLookup: true }, + { name: 'right', type: 'string', dynamicLookup: true }, + ], + defaultParams: ['value', '=', 'value'], + renderer: columnRenderer, +}); + register({ type: 'aggregate', style: 'label', addStrategy: replaceAggregationAddStrategy, - params: [{name: 'name', type: 'string', dynamicLookup: true}], + params: [{ name: 'name', type: 'string', dynamicLookup: true }], defaultParams: ['avg'], renderer: aggregateRenderer, }); @@ -136,7 +159,7 @@ register({ options: ['none', 'NULL', '0'], }, ], - defaultParams: ['$__interval','none'], + defaultParams: ['$__interval', 'none'], renderer: functionRenderer, }); From 66c56e7bbebd188a232ae6ac7a444c6463a7e9ef Mon Sep 17 00:00:00 2001 From: bergquist Date: Sat, 30 Jun 2018 23:14:18 +0200 Subject: [PATCH 074/310] notifications: dont return error if one notifer failed --- pkg/services/alerting/notifier.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/services/alerting/notifier.go b/pkg/services/alerting/notifier.go index fb2933f6e26..41c4e79445c 100644 --- a/pkg/services/alerting/notifier.go +++ b/pkg/services/alerting/notifier.go @@ -90,7 +90,7 @@ func (n *notificationService) sendNotifications(evalContext *EvalContext, notifi }) if err != nil { - return err + n.log.Error("failed to send notification", "id", not.GetNotifierId()) } } From 9847c2186f551b317fb24a91653d047ede1bc167 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 4 Jul 2018 11:39:58 +0200 Subject: [PATCH 075/310] mv query_part to sql_part --- .../plugins/datasource/postgres/{query_part.ts => sql_part.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename public/app/plugins/datasource/postgres/{query_part.ts => sql_part.ts} (100%) diff --git a/public/app/plugins/datasource/postgres/query_part.ts b/public/app/plugins/datasource/postgres/sql_part.ts similarity index 100% rename from public/app/plugins/datasource/postgres/query_part.ts rename to public/app/plugins/datasource/postgres/sql_part.ts From ced0a5828f0f5f08b84c01bdbc7e25cd2d963983 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 4 Jul 2018 11:56:26 +0200 Subject: [PATCH 076/310] rearrange elements of query builder --- .../postgres/partials/query.editor.html | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index ea5b4b4e184..4f7bed3e2c4 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -12,31 +12,13 @@
- + - -
-
- - -
- -
- -
- -
-
-
- -
- -
-
+
@@ -49,7 +31,7 @@
-
@@ -73,7 +55,27 @@
-
+ +
+ + +
+ +
+ +
+ +
+
+
+ +
+ +
+
+ From fee36b2b3530eccd3821d41b0a51e0b3046b2544 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 4 Jul 2018 12:22:45 +0200 Subject: [PATCH 077/310] include where constraints in query generation --- .../datasource/postgres/postgres_query.ts | 60 +++++++++---------- .../plugins/datasource/postgres/query_ctrl.ts | 28 +++------ 2 files changed, 35 insertions(+), 53 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index cdb51c00561..4e021d37591 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -1,11 +1,12 @@ import _ from 'lodash'; -import queryPart from './query_part'; +import sqlPart from './sql_part'; export default class PostgresQuery { target: any; selectModels: any[]; queryBuilder: any; - groupByParts: any; + groupByParts: any[]; + whereParts: any[]; templateSrv: any; scopedVars: any; @@ -31,18 +32,19 @@ export default class PostgresQuery { } quoteIdentifier(value) { - return '"' + value.replace('"','""') + '"'; + return '"' + value.replace('"', '""') + '"'; } quoteLiteral(value) { - return "'" + value.replace("'","''") + "'"; + return "'" + value.replace("'", "''") + "'"; } updateProjection() { this.selectModels = _.map(this.target.select, function(parts: any) { - return _.map(parts, queryPart.create); + return _.map(parts, sqlPart.create); }); - this.groupByParts = _.map(this.target.groupBy, queryPart.create); + this.whereParts = _.map(this.target.where, sqlPart.create); + this.groupByParts = _.map(this.target.groupBy, sqlPart.create); } updatePersistedParts() { @@ -51,6 +53,9 @@ export default class PostgresQuery { return { type: part.def.type, params: part.params }; }); }); + this.target.where = _.map(this.whereParts, function(part: any) { + return { type: part.def.type, params: part.params }; + }); } hasGroupByTime() { @@ -60,8 +65,8 @@ export default class PostgresQuery { addGroupBy(value) { var stringParts = value.match(/^(\w+)(\((.*)\))?$/); var typePart = stringParts[1]; - var args = stringParts[3].split(","); - var partModel = queryPart.create({ type: typePart, params: args }); + var args = stringParts[3].split(','); + var partModel = sqlPart.create({ type: typePart, params: args }); var partCount = this.target.groupBy.length; if (partCount === 0) { @@ -86,7 +91,7 @@ export default class PostgresQuery { // remove aggregations this.target.select = _.map(this.target.select, (s: any) => { return _.filter(s, (part: any) => { - if (part.type === "aggregate") { + if (part.type === 'aggregate') { return false; } return true; @@ -118,25 +123,15 @@ export default class PostgresQuery { this.updatePersistedParts(); } - addSelectPart(selectParts, type) { - var partModel = queryPart.create({ type: type }); - partModel.def.addStrategy(selectParts, partModel, this); - this.updatePersistedParts(); + removeWherePart(whereParts, part) { + var partIndex = _.indexOf(whereParts, part); + whereParts.splice(partIndex, 1); } - private renderWhereConstraint(constraint, index, interpolate) { - var str = ''; - var operator = constraint.operator; - var value = constraint.value; - if (index > 0) { - str = (constraint.condition || 'AND') + ' '; - } - - if (interpolate) { - value = this.templateSrv.replace(value, this.scopedVars); - } - - return str + constraint.key + ' ' + operator + ' ' + value; + addSelectPart(selectParts, type) { + var partModel = sqlPart.create({ type: type }); + partModel.def.addStrategy(selectParts, partModel, this); + this.updatePersistedParts(); } interpolateQueryStr(value, variable, defaultFormatFn) { @@ -170,8 +165,8 @@ export default class PostgresQuery { if (timeGroup) { var args; - if (timeGroup.params.length > 1 && timeGroup.params[1] !== "none") { - args = timeGroup.params.join(","); + if (timeGroup.params.length > 1 && timeGroup.params[1] !== 'none') { + args = timeGroup.params.join(','); } else { args = timeGroup.params[0]; } @@ -181,7 +176,7 @@ export default class PostgresQuery { } if (this.target.metricColumn !== 'None') { - query += "," + this.quoteIdentifier(this.target.metricColumn) + " AS metric"; + query += ',' + this.quoteIdentifier(this.target.metricColumn) + ' AS metric'; } var i, y; @@ -198,7 +193,7 @@ export default class PostgresQuery { query += ' FROM ' + this.quoteIdentifier(target.schema) + '.' + this.quoteIdentifier(target.table) + ' WHERE '; var conditions = _.map(target.where, (tag, index) => { - return this.renderWhereConstraint(tag, index, false); + return tag.params.join(' '); }); if (conditions.length > 0) { @@ -222,8 +217,8 @@ export default class PostgresQuery { if (groupBySection.length) { query += ' GROUP BY ' + groupBySection; - if (this.target.metricColumn !== "None") { - query += ",2"; + if (this.target.metricColumn !== 'None') { + query += ',2'; } } @@ -235,5 +230,4 @@ export default class PostgresQuery { } return query; } - } diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 1eb8b3ef80a..ecec978ae58 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -2,7 +2,7 @@ import _ from 'lodash'; import { PostgresQueryBuilder } from './query_builder'; import { QueryCtrl } from 'app/plugins/sdk'; import PostgresQuery from './postgres_query'; -import sqlPart from './query_part'; +import sqlPart from './sql_part'; export interface QueryMeta { sql: string; @@ -29,7 +29,6 @@ export class PostgresQueryCtrl extends QueryCtrl { showHelp: boolean; schemaSegment: any; tableSegment: any; - whereSegments: any; whereAdd: any; timeColumnSegment: any; metricColumnSegment: any; @@ -245,18 +244,8 @@ export class PostgresQueryCtrl extends QueryCtrl { } buildWhereSegments() { - this.whereSegments = []; - this.whereSegments.push(sqlPart.create({ type: 'expression', params: ['value', '=', 'value'] })); - // for (let constraint of this.target.where) { - // - // this.whereSegments.push(sqlPart.create({type: 'column',params: ['1']})); - // if (constraint.condition) { - // this.whereSegments.push(this.uiSegmentSrv.newCondition(constraint.condition)); - // } - // this.whereSegments.push(this.uiSegmentSrv.newKey(constraint.key)); - // this.whereSegments.push(this.uiSegmentSrv.newOperator(constraint.operator)); - // this.whereSegments.push(this.uiSegmentSrv.newKeyValue(constraint.value)); - // } + // this.whereSegments = []; + // this.whereSegments.push(sqlPart.create({ type: 'expression', params: ['value', '=', 'value'] })); } handleWherePartEvent(whereParts, part, evt, index) { @@ -276,7 +265,7 @@ export class PostgresQueryCtrl extends QueryCtrl { case 'op': return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '<', '<=', '>', '>=', 'IN'])); default: - return Promise.resolve([]); + return this.$q.when([]); } } case 'part-param-changed': { @@ -284,7 +273,7 @@ export class PostgresQueryCtrl extends QueryCtrl { break; } case 'action': { - whereParts.splice(whereParts.indexOf(part), 1); + this.queryModel.removeWherePart(part, index); this.panelCtrl.refresh(); break; } @@ -299,21 +288,20 @@ export class PostgresQueryCtrl extends QueryCtrl { options.push(this.uiSegmentSrv.newSegment({ type: 'function', value: '$__timeFilter' })); options.push(this.uiSegmentSrv.newSegment({ type: 'function', value: '$__unixEpochFilter' })); options.push(this.uiSegmentSrv.newSegment({ type: 'function', value: 'Expression' })); - return Promise.resolve(options); + return this.$q.when(options); } whereAddAction(part, index) { switch (this.whereAdd.type) { case 'macro': { - this.whereSegments.push( + this.queryModel.whereParts.push( sqlPart.create({ type: 'function', name: this.whereAdd.value, params: ['value', '=', 'value'] }) ); } default: { - this.whereSegments.push(sqlPart.create({ type: 'expression', params: ['value', '=', 'value'] })); + this.queryModel.whereParts.push(sqlPart.create({ type: 'expression', params: ['value', '=', 'value'] })); } } - this.whereAdd = this.uiSegmentSrv.newPlusButton(); this.panelCtrl.refresh(); } From 7d30ca04dec0a9ba9e434e8359e22a5117c802d3 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 4 Jul 2018 15:37:06 +0200 Subject: [PATCH 079/310] remove dead code from sql_part fix where clause query generation --- .../app/core/components/sql_part/sql_part.ts | 33 +++++-------------- .../datasource/postgres/postgres_query.ts | 5 ++- 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/public/app/core/components/sql_part/sql_part.ts b/public/app/core/components/sql_part/sql_part.ts index e74d4d860c6..f83158177eb 100644 --- a/public/app/core/components/sql_part/sql_part.ts +++ b/public/app/core/components/sql_part/sql_part.ts @@ -21,14 +21,14 @@ export class SqlPartDef { this.label = this.type[0].toUpperCase() + this.type.substring(1) + ':'; } this.style = options.style; - if (this.style === "function") { - this.wrapOpen = "("; - this.wrapClose = ")"; - this.separator = ", "; + if (this.style === 'function') { + this.wrapOpen = '('; + this.wrapClose = ')'; + this.separator = ', '; } else { - this.wrapOpen = " "; - this.wrapClose = " "; - this.separator = " "; + this.wrapOpen = ' '; + this.wrapClose = ' '; + this.separator = ' '; } this.params = options.params; this.defaultParams = options.defaultParams; @@ -60,25 +60,9 @@ export class SqlPart { return this.def.renderer(this, innerExpr); } - hasMultipleParamsInString(strValue, index) { - if (strValue.indexOf(',') === -1) { - return false; - } - - return this.def.params[index + 1] && this.def.params[index + 1].optional; - } - updateParam(strValue, index) { - // handle optional parameters - // if string contains ',' and next param is optional, split and update both - if (this.hasMultipleParamsInString(strValue, index)) { - _.each(strValue.split(','), (partVal, idx) => { - this.updateParam(partVal.trim(), idx); - }); - return; - } - if (strValue === '' && this.def.params[index].optional) { + // XXX check if this is still required this.params.splice(index, 1); } else { this.params[index] = strValue; @@ -132,4 +116,3 @@ export function suffixRenderer(part, innerExpr) { export function identityRenderer(part, innerExpr) { return part.params[0]; } - diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 4e021d37591..78429739c97 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -56,6 +56,9 @@ export default class PostgresQuery { this.target.where = _.map(this.whereParts, function(part: any) { return { type: part.def.type, params: part.params }; }); + this.target.groupBy = _.map(this.groupByParts, function(part: any) { + return { type: part.def.type, params: part.params }; + }); } hasGroupByTime() { @@ -197,7 +200,7 @@ export default class PostgresQuery { }); if (conditions.length > 0) { - query += '(' + conditions.join(' ') + ') AND '; + query += '(' + conditions.join(' AND ') + ') AND '; } query += '$__timeFilter(' + this.quoteIdentifier(target.timeColumn) + ')'; From 3595436614fac1430a7d5ebd50e21fb0a7fe6b13 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 4 Jul 2018 18:57:55 +0200 Subject: [PATCH 080/310] fix where constraint handling --- public/app/plugins/datasource/postgres/query_ctrl.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index ecec978ae58..d2b7631dfdc 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -303,6 +303,7 @@ export class PostgresQueryCtrl extends QueryCtrl { } } this.whereAdd = this.uiSegmentSrv.newPlusButton(); + this.queryModel.updatePersistedParts(); this.panelCtrl.refresh(); } From c6046510924433e0ba9f2c0d5be8779eb22bd13c Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 5 Jul 2018 10:19:22 +0200 Subject: [PATCH 081/310] fix group by ui --- .../postgres/partials/query.editor.html | 2 +- .../datasource/postgres/postgres_query.ts | 25 ++++++------------- .../plugins/datasource/postgres/query_ctrl.ts | 20 +++++++++------ 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 4f7bed3e2c4..28f8af52930 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -86,7 +86,7 @@
- +
diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 78429739c97..aad36cee569 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -65,27 +65,23 @@ export default class PostgresQuery { return _.find(this.target.groupBy, (g: any) => g.type === 'time'); } - addGroupBy(value) { - var stringParts = value.match(/^(\w+)(\((.*)\))?$/); - var typePart = stringParts[1]; - var args = stringParts[3].split(','); - var partModel = sqlPart.create({ type: typePart, params: args }); + addGroupBy(partType, value) { + var partModel = sqlPart.create({ type: partType, params: [value] }); var partCount = this.target.groupBy.length; if (partCount === 0) { this.target.groupBy.push(partModel.part); - } else if (typePart === 'time') { + } else if (partType === 'time') { + // put timeGroup at start this.target.groupBy.splice(0, 0, partModel.part); - } else if (typePart === 'column') { - if (this.target.groupBy[partCount - 1].type === 'fill') { - this.target.groupBy.splice(partCount - 1, 0, partModel.part); - } else { - this.target.groupBy.push(partModel.part); - } } else { this.target.groupBy.push(partModel.part); } + if (partType === 'time') { + partModel.part.params = ['1m', 'none']; + } + this.updateProjection(); } @@ -126,11 +122,6 @@ export default class PostgresQuery { this.updatePersistedParts(); } - removeWherePart(whereParts, part) { - var partIndex = _.indexOf(whereParts, part); - whereParts.splice(partIndex, 1); - } - addSelectPart(selectParts, type) { var partModel = sqlPart.create({ type: type }); partModel.def.addStrategy(selectParts, partModel, this); diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index d2b7631dfdc..37e5aa13bd0 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -33,7 +33,7 @@ export class PostgresQueryCtrl extends QueryCtrl { timeColumnSegment: any; metricColumnSegment: any; selectMenu: any; - groupBySegment: any; + groupByAdd: any; /** @ngInject **/ constructor($scope, $injector, private templateSrv, private $q, private uiSegmentSrv) { @@ -68,7 +68,7 @@ export class PostgresQueryCtrl extends QueryCtrl { this.buildSelectMenu(); this.buildWhereSegments(); this.whereAdd = this.uiSegmentSrv.newPlusButton(); - this.groupBySegment = this.uiSegmentSrv.newPlusButton(); + this.groupByAdd = this.uiSegmentSrv.newPlusButton(); this.panelCtrl.events.on('data-received', this.onDataReceived.bind(this), $scope); this.panelCtrl.events.on('data-error', this.onDataError.bind(this), $scope); @@ -273,7 +273,7 @@ export class PostgresQueryCtrl extends QueryCtrl { break; } case 'action': { - this.queryModel.removeWherePart(part, index); + whereParts.splice(index, 1); this.panelCtrl.refresh(); break; } @@ -302,7 +302,11 @@ export class PostgresQueryCtrl extends QueryCtrl { this.queryModel.whereParts.push(sqlPart.create({ type: 'expression', params: ['value', '=', 'value'] })); } } - this.whereAdd = this.uiSegmentSrv.newPlusButton(); + + var plusButton = this.uiSegmentSrv.newPlusButton(); + this.whereAdd.html = plusButton.html; + this.whereAdd.value = plusButton.value; + this.queryModel.updatePersistedParts(); this.panelCtrl.refresh(); } @@ -324,15 +328,15 @@ export class PostgresQueryCtrl extends QueryCtrl { } groupByAction() { - switch (this.groupBySegment.value) { + switch (this.groupByAdd.value) { default: { - this.queryModel.addGroupBy(this.groupBySegment.value); + this.queryModel.addGroupBy(this.groupByAdd.type, this.groupByAdd.value); } } var plusButton = this.uiSegmentSrv.newPlusButton(); - this.groupBySegment.value = plusButton.value; - this.groupBySegment.html = plusButton.html; + this.groupByAdd.html = plusButton.html; + this.groupByAdd.value = plusButton.value; this.panelCtrl.refresh(); } From 3f614e635bbbfca9646bb0720d76916e781fa630 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 5 Jul 2018 11:27:19 +0200 Subject: [PATCH 082/310] do not autoquote identifiers --- .../app/plugins/datasource/postgres/postgres_query.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index aad36cee569..7e6f9432c98 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -164,13 +164,13 @@ export default class PostgresQuery { } else { args = timeGroup.params[0]; } - query += '$__timeGroup(' + this.quoteIdentifier(target.timeColumn) + ',' + args + ')'; + query += '$__timeGroup(' + target.timeColumn + ',' + args + ')'; } else { - query += this.quoteIdentifier(target.timeColumn) + ' AS time'; + query += target.timeColumn + ' AS "time"'; } if (this.target.metricColumn !== 'None') { - query += ',' + this.quoteIdentifier(this.target.metricColumn) + ' AS metric'; + query += ',' + this.target.metricColumn + ' AS metric'; } var i, y; @@ -185,7 +185,7 @@ export default class PostgresQuery { query += ', ' + selectText; } - query += ' FROM ' + this.quoteIdentifier(target.schema) + '.' + this.quoteIdentifier(target.table) + ' WHERE '; + query += ' FROM ' + target.schema + '.' + target.table + ' WHERE '; var conditions = _.map(target.where, (tag, index) => { return tag.params.join(' '); }); @@ -194,7 +194,7 @@ export default class PostgresQuery { query += '(' + conditions.join(' AND ') + ') AND '; } - query += '$__timeFilter(' + this.quoteIdentifier(target.timeColumn) + ')'; + query += '$__timeFilter(' + target.timeColumn + ')'; var groupBySection = ''; for (i = 0; i < this.groupByParts.length; i++) { From d8c7756489bd26a5e80c3cef22dd545323cb6308 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 5 Jul 2018 21:36:39 +0200 Subject: [PATCH 083/310] dont autoquote, suggest quoted values if requried --- .../datasource/postgres/postgres_query.ts | 8 ++++ .../datasource/postgres/query_builder.ts | 48 ++++++++++--------- .../plugins/datasource/postgres/sql_part.ts | 2 +- 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 7e6f9432c98..9cc8de7b491 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -31,6 +31,14 @@ export default class PostgresQuery { this.updateProjection(); } + unquoteIdentifier(value) { + if (value[0] === '"') { + return value.substring(1, value.length - 1).replace('""', '"'); + } else { + return value; + } + } + quoteIdentifier(value) { return '"' + value.replace('"', '""') + '"'; } diff --git a/public/app/plugins/datasource/postgres/query_builder.ts b/public/app/plugins/datasource/postgres/query_builder.ts index dd6f95b550c..843f18b0c17 100644 --- a/public/app/plugins/datasource/postgres/query_builder.ts +++ b/public/app/plugins/datasource/postgres/query_builder.ts @@ -1,35 +1,39 @@ - export class PostgresQueryBuilder { constructor(private target, private queryModel) {} buildSchemaQuery() { - var query = "SELECT schema_name FROM information_schema.schemata WHERE"; + var query = 'SELECT quote_ident(schema_name) FROM information_schema.schemata WHERE'; query += " schema_name NOT LIKE 'pg_%' AND schema_name NOT LIKE '\\_%' AND schema_name <> 'information_schema';"; return query; } buildTableQuery() { - var query = "SELECT table_name FROM information_schema.tables WHERE "; - query += "table_schema = " + this.queryModel.quoteLiteral(this.target.schema); + var query = 'SELECT quote_ident(table_name) FROM information_schema.tables WHERE '; + query += 'table_schema = ' + this.quoteLiteral(this.target.schema); return query; } + quoteLiteral(value) { + return this.queryModel.quoteLiteral(this.queryModel.unquoteIdentifier(value)); + } + buildColumnQuery(type?: string) { - var query = "SELECT column_name FROM information_schema.columns WHERE "; - query += "table_schema = " + this.queryModel.quoteLiteral(this.target.schema); - query += " AND table_name = " + this.queryModel.quoteLiteral(this.target.table); + var query = 'SELECT quote_ident(column_name) FROM information_schema.columns WHERE '; + query += 'table_schema = ' + this.quoteLiteral(this.target.schema); + query += ' AND table_name = ' + this.quoteLiteral(this.target.table); switch (type) { - case "time": { - query += " AND data_type IN ('timestamp without time zone','timestamp with time zone','bigint','integer','double precision','real')"; + case 'time': { + query += + " AND data_type IN ('timestamp without time zone','timestamp with time zone','bigint','integer','double precision','real')"; break; } - case "metric": { + case 'metric': { query += " AND data_type IN ('text','char','varchar')"; break; } - case "value": { + case 'value': { query += " AND data_type IN ('bigint','integer','double precision','real')"; break; } @@ -39,27 +43,25 @@ export class PostgresQueryBuilder { } buildValueQuery(column: string) { - var query = "SELECT DISTINCT quote_literal(" + this.queryModel.quoteIdentifier(column) + ")"; - query += " FROM " + this.queryModel.quoteIdentifier(this.target.schema); - query += "." + this.queryModel.quoteIdentifier(this.target.table); - query += " ORDER BY 1 LIMIT 100"; + var query = 'SELECT DISTINCT quote_literal(' + column + ')'; + query += ' FROM ' + this.target.schema + '.' + this.target.table; + query += ' ORDER BY 1 LIMIT 100'; return query; } buildDatatypeQuery(column: string) { - var query = "SELECT data_type FROM information_schema.columns WHERE "; - query += " table_schema = " + this.queryModel.quoteLiteral(this.target.schema); - query += " AND table_name = " + this.queryModel.quoteLiteral(this.target.table); - query += " AND column_name = " + this.queryModel.quoteLiteral(column); + var query = 'SELECT data_type FROM information_schema.columns WHERE '; + query += ' table_schema = ' + this.quoteLiteral(this.target.schema); + query += ' AND table_name = ' + this.quoteLiteral(this.target.table); + query += ' AND column_name = ' + this.quoteLiteral(column); return query; } buildAggregateQuery() { - var query = "SELECT DISTINCT proname FROM pg_aggregate "; - query += "INNER JOIN pg_proc ON pg_aggregate.aggfnoid = pg_proc.oid "; - query += "INNER JOIN pg_type ON pg_type.oid=pg_proc.prorettype "; + var query = 'SELECT DISTINCT proname FROM pg_aggregate '; + query += 'INNER JOIN pg_proc ON pg_aggregate.aggfnoid = pg_proc.oid '; + query += 'INNER JOIN pg_type ON pg_type.oid=pg_proc.prorettype '; query += "WHERE pronargs=1 AND typname IN ('int8','float8') AND aggkind='n' ORDER BY 1"; return query; } - } diff --git a/public/app/plugins/datasource/postgres/sql_part.ts b/public/app/plugins/datasource/postgres/sql_part.ts index 4cda0d47d0d..bb29fde9fb4 100644 --- a/public/app/plugins/datasource/postgres/sql_part.ts +++ b/public/app/plugins/datasource/postgres/sql_part.ts @@ -25,7 +25,7 @@ function aggregateRenderer(part, innerExpr) { } function columnRenderer(part, innerExpr) { - return '"' + part.params[0] + '"'; + return part.params[0]; } function replaceAggregationAddStrategy(selectParts, partModel) { From 85ab1cfa8f9723c44d8c79f9777a02b0423be2e9 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 6 Jul 2018 09:28:34 +0200 Subject: [PATCH 084/310] fix constraint removal --- public/app/plugins/datasource/postgres/postgres_query.ts | 3 ++- public/app/plugins/datasource/postgres/query_ctrl.ts | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 9cc8de7b491..4d90ad83e3b 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -31,8 +31,9 @@ export default class PostgresQuery { this.updateProjection(); } + // remove identifier quoting from identifier to use in metadata queries unquoteIdentifier(value) { - if (value[0] === '"') { + if (value[0] === '"' && value[value.length - 1] === '"') { return value.substring(1, value.length - 1).replace('""', '"'); } else { return value; diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 37e5aa13bd0..4aa2db90c1d 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -273,7 +273,9 @@ export class PostgresQueryCtrl extends QueryCtrl { break; } case 'action': { + // remove element whereParts.splice(index, 1); + this.queryModel.updatePersistedParts(); this.panelCtrl.refresh(); break; } From 8ed210c8d5a130e46745c177a911b9254d022de1 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 6 Jul 2018 10:38:18 +0200 Subject: [PATCH 085/310] remove dead code, make label more flexible --- .../app/core/components/sql_part/sql_part.ts | 36 ++++++------------- .../components/sql_part/sql_part_editor.ts | 2 +- 2 files changed, 11 insertions(+), 27 deletions(-) diff --git a/public/app/core/components/sql_part/sql_part.ts b/public/app/core/components/sql_part/sql_part.ts index f83158177eb..0cc4c31ffbf 100644 --- a/public/app/core/components/sql_part/sql_part.ts +++ b/public/app/core/components/sql_part/sql_part.ts @@ -42,7 +42,8 @@ export class SqlPart { part: any; def: SqlPartDef; params: any[]; - text: string; + label: string; + name: string; constructor(part: any, def: any) { this.part = part; @@ -51,38 +52,21 @@ export class SqlPart { throw { message: 'Could not find sql part ' + part.type }; } + if (part.name) { + this.name = part.name; + this.label = def.label + ' ' + part.name; + } else { + this.name = ''; + this.label = def.label; + } + part.params = part.params || _.clone(this.def.defaultParams); this.params = part.params; - this.updateText(); } render(innerExpr: string) { return this.def.renderer(this, innerExpr); } - - updateParam(strValue, index) { - if (strValue === '' && this.def.params[index].optional) { - // XXX check if this is still required - this.params.splice(index, 1); - } else { - this.params[index] = strValue; - } - - this.part.params = this.params; - this.updateText(); - } - - updateText() { - if (this.params.length === 0) { - this.text = this.def.type + '()'; - return; - } - - var text = this.def.type + '('; - text += this.params.join(', '); - text += ')'; - this.text = text; - } } export function functionRenderer(part, innerExpr) { diff --git a/public/app/core/components/sql_part/sql_part_editor.ts b/public/app/core/components/sql_part/sql_part_editor.ts index e2222abf18f..56329d58a6c 100644 --- a/public/app/core/components/sql_part/sql_part_editor.ts +++ b/public/app/core/components/sql_part/sql_part_editor.ts @@ -4,7 +4,7 @@ import coreModule from 'app/core/core_module'; var template = `
-{{part.def.label}} +{{part.label}} {{part.def.wrapOpen}}{{part.def.wrapClose}}
  • From 7600c6efcbeb0b5a103badd86806fe96aa756f34 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 6 Jul 2018 10:38:52 +0200 Subject: [PATCH 086/310] remove hardcoded $__timeFilter, make macros functional in where clause --- .../plugins/datasource/postgres/postgres_query.ts | 15 ++++++++++----- .../app/plugins/datasource/postgres/query_ctrl.ts | 11 +++++------ .../app/plugins/datasource/postgres/sql_part.ts | 10 ++++++++++ 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 4d90ad83e3b..36660ecdd42 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -63,7 +63,7 @@ export default class PostgresQuery { }); }); this.target.where = _.map(this.whereParts, function(part: any) { - return { type: part.def.type, params: part.params }; + return { type: part.def.type, name: part.name, params: part.params }; }); this.target.groupBy = _.map(this.groupByParts, function(part: any) { return { type: part.def.type, params: part.params }; @@ -196,15 +196,20 @@ export default class PostgresQuery { query += ' FROM ' + target.schema + '.' + target.table + ' WHERE '; var conditions = _.map(target.where, (tag, index) => { - return tag.params.join(' '); + switch (tag.type) { + case 'macro': + return tag.name + '(' + target.timeColumn + ')'; + break; + case 'expression': + return tag.params.join(' '); + break; + } }); if (conditions.length > 0) { - query += '(' + conditions.join(' AND ') + ') AND '; + query += conditions.join(' AND '); } - query += '$__timeFilter(' + target.timeColumn + ')'; - var groupBySection = ''; for (i = 0; i < this.groupByParts.length; i++) { var part = this.groupByParts[i]; diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 4aa2db90c1d..ab365561b79 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -287,18 +287,17 @@ export class PostgresQueryCtrl extends QueryCtrl { getWhereOptions() { var options = []; - options.push(this.uiSegmentSrv.newSegment({ type: 'function', value: '$__timeFilter' })); - options.push(this.uiSegmentSrv.newSegment({ type: 'function', value: '$__unixEpochFilter' })); - options.push(this.uiSegmentSrv.newSegment({ type: 'function', value: 'Expression' })); + options.push(this.uiSegmentSrv.newSegment({ type: 'macro', value: '$__timeFilter' })); + options.push(this.uiSegmentSrv.newSegment({ type: 'macro', value: '$__unixEpochFilter' })); + options.push(this.uiSegmentSrv.newSegment({ type: 'expression', value: 'Expression' })); return this.$q.when(options); } whereAddAction(part, index) { switch (this.whereAdd.type) { case 'macro': { - this.queryModel.whereParts.push( - sqlPart.create({ type: 'function', name: this.whereAdd.value, params: ['value', '=', 'value'] }) - ); + this.queryModel.whereParts.push(sqlPart.create({ type: 'macro', name: this.whereAdd.value, params: [] })); + break; } default: { this.queryModel.whereParts.push(sqlPart.create({ type: 'expression', params: ['value', '=', 'value'] })); diff --git a/public/app/plugins/datasource/postgres/sql_part.ts b/public/app/plugins/datasource/postgres/sql_part.ts index bb29fde9fb4..598de56d4b1 100644 --- a/public/app/plugins/datasource/postgres/sql_part.ts +++ b/public/app/plugins/datasource/postgres/sql_part.ts @@ -115,6 +115,16 @@ register({ renderer: columnRenderer, }); +register({ + type: 'macro', + style: 'function', + label: 'Macro:', + addStrategy: addExpressionStrategy, + params: [], + defaultParams: [], + renderer: columnRenderer, +}); + register({ type: 'aggregate', style: 'label', From 6143924ec67622192a60866d27e86f3eabc070cc Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 6 Jul 2018 10:56:47 +0200 Subject: [PATCH 087/310] fix where clause generation --- public/app/plugins/datasource/postgres/postgres_query.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 36660ecdd42..6a5a2925625 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -194,7 +194,7 @@ export default class PostgresQuery { query += ', ' + selectText; } - query += ' FROM ' + target.schema + '.' + target.table + ' WHERE '; + query += ' FROM ' + target.schema + '.' + target.table; var conditions = _.map(target.where, (tag, index) => { switch (tag.type) { case 'macro': @@ -207,7 +207,7 @@ export default class PostgresQuery { }); if (conditions.length > 0) { - query += conditions.join(' AND '); + query += ' WHERE ' + conditions.join(' AND '); } var groupBySection = ''; From d9db3d14022dc1546f5a4697ed43757f459376b0 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 6 Jul 2018 11:24:49 +0200 Subject: [PATCH 088/310] put updateParam back in --- public/app/core/components/sql_part/sql_part.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/public/app/core/components/sql_part/sql_part.ts b/public/app/core/components/sql_part/sql_part.ts index 0cc4c31ffbf..5b049237881 100644 --- a/public/app/core/components/sql_part/sql_part.ts +++ b/public/app/core/components/sql_part/sql_part.ts @@ -64,6 +64,17 @@ export class SqlPart { this.params = part.params; } + updateParam(strValue, index) { + // handle optional parameters + if (strValue === '' && this.def.params[index].optional) { + this.params.splice(index, 1); + } else { + this.params[index] = strValue; + } + + this.part.params = this.params; + } + render(innerExpr: string) { return this.def.renderer(this, innerExpr); } From a63298bca2d1c6aba7a9341bf202bbc007cbd2e5 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 6 Jul 2018 11:25:20 +0200 Subject: [PATCH 089/310] remove unused function removeSelect --- public/app/plugins/datasource/postgres/postgres_query.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 6a5a2925625..ef903ea3259 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -111,11 +111,6 @@ export default class PostgresQuery { this.updateProjection(); } - removeSelect(index: number) { - this.target.select.splice(index, 1); - this.updateProjection(); - } - removeSelectPart(selectParts, part) { // if we remove the field remove the whole statement if (part.def.type === 'column') { From e4b4b3a772d58aa590ce012f56644c7e2c58d2d0 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 6 Jul 2018 12:38:43 +0200 Subject: [PATCH 090/310] rename quoteLiteral to quoteIdentAsLiteral --- .../plugins/datasource/postgres/query_builder.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_builder.ts b/public/app/plugins/datasource/postgres/query_builder.ts index 843f18b0c17..852fb91a706 100644 --- a/public/app/plugins/datasource/postgres/query_builder.ts +++ b/public/app/plugins/datasource/postgres/query_builder.ts @@ -10,18 +10,19 @@ export class PostgresQueryBuilder { buildTableQuery() { var query = 'SELECT quote_ident(table_name) FROM information_schema.tables WHERE '; - query += 'table_schema = ' + this.quoteLiteral(this.target.schema); + query += 'table_schema = ' + this.quoteIdentAsLiteral(this.target.schema); return query; } - quoteLiteral(value) { + // quote identifier as literal to use in metadata queries + quoteIdentAsLiteral(value) { return this.queryModel.quoteLiteral(this.queryModel.unquoteIdentifier(value)); } buildColumnQuery(type?: string) { var query = 'SELECT quote_ident(column_name) FROM information_schema.columns WHERE '; - query += 'table_schema = ' + this.quoteLiteral(this.target.schema); - query += ' AND table_name = ' + this.quoteLiteral(this.target.table); + query += 'table_schema = ' + this.quoteIdentAsLiteral(this.target.schema); + query += ' AND table_name = ' + this.quoteIdentAsLiteral(this.target.table); switch (type) { case 'time': { @@ -51,9 +52,9 @@ export class PostgresQueryBuilder { buildDatatypeQuery(column: string) { var query = 'SELECT data_type FROM information_schema.columns WHERE '; - query += ' table_schema = ' + this.quoteLiteral(this.target.schema); - query += ' AND table_name = ' + this.quoteLiteral(this.target.table); - query += ' AND column_name = ' + this.quoteLiteral(column); + query += ' table_schema = ' + this.quoteIdentAsLiteral(this.target.schema); + query += ' AND table_name = ' + this.quoteIdentAsLiteral(this.target.table); + query += ' AND column_name = ' + this.quoteIdentAsLiteral(column); return query; } From b2b4860bdb718c17babb2ed76ff8cd9098acd17b Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 7 Jul 2018 19:43:21 +0200 Subject: [PATCH 091/310] refactor transformToSegments --- .../plugins/datasource/postgres/query_ctrl.ts | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index ab365561b79..8f032cc0917 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -90,28 +90,28 @@ export class PostgresQueryCtrl extends QueryCtrl { getSchemaSegments() { return this.datasource .metricFindQuery(this.queryBuilder.buildSchemaQuery()) - .then(this.transformToSegments(true)) + .then(this.transformToSegments({})) .catch(this.handleQueryError.bind(this)); } getTableSegments() { return this.datasource .metricFindQuery(this.queryBuilder.buildTableQuery()) - .then(this.transformToSegments(true)) + .then(this.transformToSegments({})) .catch(this.handleQueryError.bind(this)); } getTimeColumnSegments() { return this.datasource .metricFindQuery(this.queryBuilder.buildColumnQuery('time')) - .then(this.transformToSegments(true)) + .then(this.transformToSegments({})) .catch(this.handleQueryError.bind(this)); } getMetricColumnSegments() { return this.datasource .metricFindQuery(this.queryBuilder.buildColumnQuery('metric')) - .then(this.transformToSegments(true)) + .then(this.transformToSegments({})) .catch(this.handleQueryError.bind(this)); } @@ -155,7 +155,7 @@ export class PostgresQueryCtrl extends QueryCtrl { } } - transformToSegments(addTemplateVars) { + transformToSegments(config) { return results => { var segments = _.map(results, segment => { return this.uiSegmentSrv.newSegment({ @@ -164,12 +164,18 @@ export class PostgresQueryCtrl extends QueryCtrl { }); }); - if (addTemplateVars) { + if (config.addTemplateVars) { for (let variable of this.templateSrv.variables) { + var value; + value = '$' + variable.name; + if (config.templateQuoter && variable.multi === false) { + value = config.templateQuoter(value); + } + segments.unshift( this.uiSegmentSrv.newSegment({ type: 'template', - value: '$' + variable.name, + value: value, expandable: true, }) ); @@ -196,12 +202,12 @@ export class PostgresQueryCtrl extends QueryCtrl { case 'aggregate': return this.datasource .metricFindQuery(this.queryBuilder.buildAggregateQuery()) - .then(this.transformToSegments(false)) + .then(this.transformToSegments({})) .catch(this.handleQueryError.bind(this)); case 'column': return this.datasource .metricFindQuery(this.queryBuilder.buildColumnQuery('value')) - .then(this.transformToSegments(true)) + .then(this.transformToSegments({})) .catch(this.handleQueryError.bind(this)); } } @@ -225,7 +231,7 @@ export class PostgresQueryCtrl extends QueryCtrl { case 'get-param-options': { return this.datasource .metricFindQuery(this.queryBuilder.buildColumnQuery()) - .then(this.transformToSegments(true)) + .then(this.transformToSegments({})) .catch(this.handleQueryError.bind(this)); } case 'part-param-changed': { @@ -255,12 +261,12 @@ export class PostgresQueryCtrl extends QueryCtrl { case 'left': return this.datasource .metricFindQuery(this.queryBuilder.buildColumnQuery()) - .then(this.transformToSegments(false)) + .then(this.transformToSegments({})) .catch(this.handleQueryError.bind(this)); case 'right': return this.datasource .metricFindQuery(this.queryBuilder.buildValueQuery(part.params[0])) - .then(this.transformToSegments(true)) + .then(this.transformToSegments({ addTemplateVars: true, templateQuoter: this.queryModel.quoteLiteral })) .catch(this.handleQueryError.bind(this)); case 'op': return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '<', '<=', '>', '>=', 'IN'])); From eb170e741b8b08f3680b289795f851fa3e859461 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 7 Jul 2018 19:59:25 +0200 Subject: [PATCH 092/310] dont break on panels that dont have rawQuery set --- public/app/plugins/datasource/postgres/postgres_query.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index ef903ea3259..b8d3d107bb6 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -149,7 +149,7 @@ export default class PostgresQuery { render(interpolate?) { var target = this.target; - if (target.rawQuery) { + if (target.rawQuery || !('rawQuery' in target)) { if (interpolate) { return this.templateSrv.replace(target.rawSql, this.scopedVars, this.interpolateQueryStr); } else { From 8490ae41b69f8fa3cecca33192578263aa0f7d50 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 7 Jul 2018 20:41:24 +0200 Subject: [PATCH 093/310] handle pre query gui queries gracefully --- public/app/plugins/datasource/postgres/postgres_query.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index b8d3d107bb6..71cf1ae7fce 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -25,6 +25,11 @@ export default class PostgresQuery { target.where = target.where || []; target.select = target.select || [[{ type: 'column', params: ['value'] }]]; + // handle pre query gui panels gracefully + if (!('rawQuery' in this.target)) { + target.rawQuery = true; + } + // give interpolateQueryStr access to this this.interpolateQueryStr = this.interpolateQueryStr.bind(this); @@ -149,7 +154,7 @@ export default class PostgresQuery { render(interpolate?) { var target = this.target; - if (target.rawQuery || !('rawQuery' in target)) { + if (target.rawQuery) { if (interpolate) { return this.templateSrv.replace(target.rawSql, this.scopedVars, this.interpolateQueryStr); } else { From 5b5dcabd22f03bacfb555cdf297bc245c67d416c Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 7 Jul 2018 21:21:11 +0200 Subject: [PATCH 094/310] add None to metric column suggestions --- public/app/plugins/datasource/postgres/query_ctrl.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 8f032cc0917..b44692ed673 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -111,7 +111,7 @@ export class PostgresQueryCtrl extends QueryCtrl { getMetricColumnSegments() { return this.datasource .metricFindQuery(this.queryBuilder.buildColumnQuery('metric')) - .then(this.transformToSegments({})) + .then(this.transformToSegments({ addNone: true })) .catch(this.handleQueryError.bind(this)); } @@ -182,6 +182,10 @@ export class PostgresQueryCtrl extends QueryCtrl { } } + if (config.addNone) { + segments.unshift(this.uiSegmentSrv.newSegment({ type: 'template', value: 'None', expandable: true })); + } + return segments; }; } From 76275cae373a4114afdd2086f0f39968d5d99585 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 9 Jul 2018 11:22:03 +0200 Subject: [PATCH 096/310] show current value in dropdown when its not part of list --- .../components/sql_part/sql_part_editor.ts | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/public/app/core/components/sql_part/sql_part_editor.ts b/public/app/core/components/sql_part/sql_part_editor.ts index 56329d58a6c..8bac51680ad 100644 --- a/public/app/core/components/sql_part/sql_part_editor.ts +++ b/public/app/core/components/sql_part/sql_part_editor.ts @@ -15,7 +15,7 @@ var template = ` /** @ngInject */ export function sqlPartEditorDirective($compile, templateSrv) { - var paramTemplate = ''; + var paramTemplate = ''; return { restrict: 'E', @@ -30,6 +30,7 @@ export function sqlPartEditorDirective($compile, templateSrv) { var partDef = part.def; var $paramsContainer = elem.find('.query-part-parameters'); var debounceLookup = $scope.debounce; + var cancelBlur = null; $scope.partActions = []; @@ -53,7 +54,23 @@ export function sqlPartEditorDirective($compile, templateSrv) { } } - function inputBlur(paramIndex) { + $scope.inputBlur = function(paramIndex) { + cancelBlur = setTimeout($scope.switchToLink, 200); + }; + + $scope.updater = function(value) { + if (value === part.params[paramIndex]) { + clearTimeout(cancelBlur); + $input.focus(); + } + return value; + setTimeout(function() { + $scope.inputBlur.call($input[0], paramIndex); + }, 0); + return value; + }; + + $scope.switchToLink = function(paramIndex) { /*jshint validthis:true */ var $input = $(this); var $link = $input.prev(); @@ -70,12 +87,12 @@ export function sqlPartEditorDirective($compile, templateSrv) { $input.hide(); $link.show(); - } + }; function inputKeyPress(paramIndex, e) { /*jshint validthis:true */ if (e.which === 13) { - inputBlur.call(this, paramIndex); + $scope.inputBlur.call(this, paramIndex); } } @@ -105,6 +122,12 @@ export function sqlPartEditorDirective($compile, templateSrv) { var dynamicOptions = _.map(result, function(op) { return op.value; }); + + // add current value to dropdown if its not in resultset + if (_.indexOf(dynamicOptions, part.params[paramIndex]) === -1) { + dynamicOptions.unshift(part.params[paramIndex]); + } + callback(dynamicOptions); }); }); @@ -116,12 +139,7 @@ export function sqlPartEditorDirective($compile, templateSrv) { source: typeaheadSource, minLength: 0, items: 1000, - updater: function(value) { - setTimeout(function() { - inputBlur.call($input[0], paramIndex); - }, 0); - return value; - }, + updater: $scope.updater, }); var typeahead = $input.data('typeahead'); @@ -157,13 +175,13 @@ export function sqlPartEditorDirective($compile, templateSrv) { } var paramValue = templateSrv.highlightVariablesAsHtml(part.params[index]); - var $paramLink = $('' + paramValue + ''); + var $paramLink = $('' + paramValue + ''); var $input = $(paramTemplate); $paramLink.appendTo($paramsContainer); $input.appendTo($paramsContainer); - $input.blur(_.partial(inputBlur, index)); + $input.blur(_.partial($scope.inputBlur, index)); $input.keyup(inputKeyDown); $input.keypress(_.partial(inputKeyPress, index)); $paramLink.click(_.partial(clickFuncParam, index)); From 6a981d39d8d3c7a6869db5e7ae0eae72d74fdf36 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 9 Jul 2018 13:04:55 +0200 Subject: [PATCH 097/310] Revert "show current value in dropdown when its not part of list" This reverts commit 76275cae373a4114afdd2086f0f39968d5d99585. --- .../components/sql_part/sql_part_editor.ts | 42 ++++++------------- 1 file changed, 12 insertions(+), 30 deletions(-) diff --git a/public/app/core/components/sql_part/sql_part_editor.ts b/public/app/core/components/sql_part/sql_part_editor.ts index 8bac51680ad..56329d58a6c 100644 --- a/public/app/core/components/sql_part/sql_part_editor.ts +++ b/public/app/core/components/sql_part/sql_part_editor.ts @@ -15,7 +15,7 @@ var template = ` /** @ngInject */ export function sqlPartEditorDirective($compile, templateSrv) { - var paramTemplate = ''; + var paramTemplate = ''; return { restrict: 'E', @@ -30,7 +30,6 @@ export function sqlPartEditorDirective($compile, templateSrv) { var partDef = part.def; var $paramsContainer = elem.find('.query-part-parameters'); var debounceLookup = $scope.debounce; - var cancelBlur = null; $scope.partActions = []; @@ -54,23 +53,7 @@ export function sqlPartEditorDirective($compile, templateSrv) { } } - $scope.inputBlur = function(paramIndex) { - cancelBlur = setTimeout($scope.switchToLink, 200); - }; - - $scope.updater = function(value) { - if (value === part.params[paramIndex]) { - clearTimeout(cancelBlur); - $input.focus(); - } - return value; - setTimeout(function() { - $scope.inputBlur.call($input[0], paramIndex); - }, 0); - return value; - }; - - $scope.switchToLink = function(paramIndex) { + function inputBlur(paramIndex) { /*jshint validthis:true */ var $input = $(this); var $link = $input.prev(); @@ -87,12 +70,12 @@ export function sqlPartEditorDirective($compile, templateSrv) { $input.hide(); $link.show(); - }; + } function inputKeyPress(paramIndex, e) { /*jshint validthis:true */ if (e.which === 13) { - $scope.inputBlur.call(this, paramIndex); + inputBlur.call(this, paramIndex); } } @@ -122,12 +105,6 @@ export function sqlPartEditorDirective($compile, templateSrv) { var dynamicOptions = _.map(result, function(op) { return op.value; }); - - // add current value to dropdown if its not in resultset - if (_.indexOf(dynamicOptions, part.params[paramIndex]) === -1) { - dynamicOptions.unshift(part.params[paramIndex]); - } - callback(dynamicOptions); }); }); @@ -139,7 +116,12 @@ export function sqlPartEditorDirective($compile, templateSrv) { source: typeaheadSource, minLength: 0, items: 1000, - updater: $scope.updater, + updater: function(value) { + setTimeout(function() { + inputBlur.call($input[0], paramIndex); + }, 0); + return value; + }, }); var typeahead = $input.data('typeahead'); @@ -175,13 +157,13 @@ export function sqlPartEditorDirective($compile, templateSrv) { } var paramValue = templateSrv.highlightVariablesAsHtml(part.params[index]); - var $paramLink = $('' + paramValue + ''); + var $paramLink = $('' + paramValue + ''); var $input = $(paramTemplate); $paramLink.appendTo($paramsContainer); $input.appendTo($paramsContainer); - $input.blur(_.partial($scope.inputBlur, index)); + $input.blur(_.partial(inputBlur, index)); $input.keyup(inputKeyDown); $input.keypress(_.partial(inputKeyPress, index)); $paramLink.click(_.partial(clickFuncParam, index)); From 2d951ba611b7af44931652149327e677bb36ec53 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 9 Jul 2018 14:27:10 +0200 Subject: [PATCH 098/310] add current value to dropdown if its not in resultset --- public/app/core/components/sql_part/sql_part_editor.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/public/app/core/components/sql_part/sql_part_editor.ts b/public/app/core/components/sql_part/sql_part_editor.ts index 56329d58a6c..c30cf76b685 100644 --- a/public/app/core/components/sql_part/sql_part_editor.ts +++ b/public/app/core/components/sql_part/sql_part_editor.ts @@ -105,6 +105,12 @@ export function sqlPartEditorDirective($compile, templateSrv) { var dynamicOptions = _.map(result, function(op) { return op.value; }); + + // add current value to dropdown if its not in dynamicOptions + if (_.indexOf(dynamicOptions, part.params[paramIndex]) === -1) { + dynamicOptions.unshift(part.params[paramIndex]); + } + callback(dynamicOptions); }); }); From 02abda7d922da623cdffaf363cce2ef8fd9bf36d Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 9 Jul 2018 14:47:59 +0200 Subject: [PATCH 099/310] fix spelling --- public/app/core/components/sql_part/sql_part_editor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/core/components/sql_part/sql_part_editor.ts b/public/app/core/components/sql_part/sql_part_editor.ts index c30cf76b685..a564a521664 100644 --- a/public/app/core/components/sql_part/sql_part_editor.ts +++ b/public/app/core/components/sql_part/sql_part_editor.ts @@ -106,7 +106,7 @@ export function sqlPartEditorDirective($compile, templateSrv) { return op.value; }); - // add current value to dropdown if its not in dynamicOptions + // add current value to dropdown if it's not in dynamicOptions if (_.indexOf(dynamicOptions, part.params[paramIndex]) === -1) { dynamicOptions.unshift(part.params[paramIndex]); } From f140967423301f770f2da761cc84b7973474d571 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 9 Jul 2018 22:00:50 +0200 Subject: [PATCH 100/310] rename inputBlur to switchToLink --- public/app/core/components/sql_part/sql_part_editor.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/public/app/core/components/sql_part/sql_part_editor.ts b/public/app/core/components/sql_part/sql_part_editor.ts index a564a521664..cba2a2765a7 100644 --- a/public/app/core/components/sql_part/sql_part_editor.ts +++ b/public/app/core/components/sql_part/sql_part_editor.ts @@ -53,7 +53,7 @@ export function sqlPartEditorDirective($compile, templateSrv) { } } - function inputBlur(paramIndex) { + function switchToLink(paramIndex) { /*jshint validthis:true */ var $input = $(this); var $link = $input.prev(); @@ -75,7 +75,7 @@ export function sqlPartEditorDirective($compile, templateSrv) { function inputKeyPress(paramIndex, e) { /*jshint validthis:true */ if (e.which === 13) { - inputBlur.call(this, paramIndex); + switchToLink.call(this, paramIndex); } } @@ -124,7 +124,7 @@ export function sqlPartEditorDirective($compile, templateSrv) { items: 1000, updater: function(value) { setTimeout(function() { - inputBlur.call($input[0], paramIndex); + switchToLink.call($input[0], paramIndex); }, 0); return value; }, @@ -169,7 +169,7 @@ export function sqlPartEditorDirective($compile, templateSrv) { $paramLink.appendTo($paramsContainer); $input.appendTo($paramsContainer); - $input.blur(_.partial(inputBlur, index)); + $input.blur(_.partial(switchToLink, index)); $input.keyup(inputKeyDown); $input.keypress(_.partial(inputKeyPress, index)); $paramLink.click(_.partial(clickFuncParam, index)); From bfc751542cfc78d88f4943a61241edb7c859cbf0 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 9 Jul 2018 22:38:29 +0200 Subject: [PATCH 101/310] fix editing expression parts --- .../components/sql_part/sql_part_editor.ts | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/public/app/core/components/sql_part/sql_part_editor.ts b/public/app/core/components/sql_part/sql_part_editor.ts index cba2a2765a7..b002e19763a 100644 --- a/public/app/core/components/sql_part/sql_part_editor.ts +++ b/public/app/core/components/sql_part/sql_part_editor.ts @@ -30,6 +30,7 @@ export function sqlPartEditorDirective($compile, templateSrv) { var partDef = part.def; var $paramsContainer = elem.find('.query-part-parameters'); var debounceLookup = $scope.debounce; + var cancelBlur = null; $scope.partActions = []; @@ -53,9 +54,14 @@ export function sqlPartEditorDirective($compile, templateSrv) { } } - function switchToLink(paramIndex) { + function inputBlur($input, paramIndex) { + cancelBlur = setTimeout(function() { + switchToLink.call(this, $input, paramIndex); + }, 200); + } + + function switchToLink($input, paramIndex) { /*jshint validthis:true */ - var $input = $(this); var $link = $input.prev(); var newValue = $input.val(); @@ -123,9 +129,11 @@ export function sqlPartEditorDirective($compile, templateSrv) { minLength: 0, items: 1000, updater: function(value) { - setTimeout(function() { - switchToLink.call($input[0], paramIndex); - }, 0); + if (value === part.params[paramIndex]) { + clearTimeout(cancelBlur); + $input.focus(); + return value; + } return value; }, }); @@ -169,7 +177,7 @@ export function sqlPartEditorDirective($compile, templateSrv) { $paramLink.appendTo($paramsContainer); $input.appendTo($paramsContainer); - $input.blur(_.partial(switchToLink, index)); + $input.blur(_.partial(inputBlur, $input, index)); $input.keyup(inputKeyDown); $input.keypress(_.partial(inputKeyPress, index)); $paramLink.click(_.partial(clickFuncParam, index)); From 5a307e80aaea980035ea1c77c88d77969f97fb61 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 9 Jul 2018 22:43:50 +0200 Subject: [PATCH 102/310] fix enter in sql_part_editor --- public/app/core/components/sql_part/sql_part_editor.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/app/core/components/sql_part/sql_part_editor.ts b/public/app/core/components/sql_part/sql_part_editor.ts index b002e19763a..1b0210af1e7 100644 --- a/public/app/core/components/sql_part/sql_part_editor.ts +++ b/public/app/core/components/sql_part/sql_part_editor.ts @@ -56,7 +56,7 @@ export function sqlPartEditorDirective($compile, templateSrv) { function inputBlur($input, paramIndex) { cancelBlur = setTimeout(function() { - switchToLink.call(this, $input, paramIndex); + switchToLink($input, paramIndex); }, 200); } @@ -81,7 +81,7 @@ export function sqlPartEditorDirective($compile, templateSrv) { function inputKeyPress(paramIndex, e) { /*jshint validthis:true */ if (e.which === 13) { - switchToLink.call(this, paramIndex); + switchToLink($(this), paramIndex); } } From dc93055a109a21076dcb2c3961215e9551338a92 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 10 Jul 2018 10:19:30 +0200 Subject: [PATCH 103/310] add column alias when add aggregate function --- public/app/plugins/datasource/postgres/sql_part.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/sql_part.ts b/public/app/plugins/datasource/postgres/sql_part.ts index 598de56d4b1..8de764c45e9 100644 --- a/public/app/plugins/datasource/postgres/sql_part.ts +++ b/public/app/plugins/datasource/postgres/sql_part.ts @@ -29,6 +29,8 @@ function columnRenderer(part, innerExpr) { } function replaceAggregationAddStrategy(selectParts, partModel) { + var hasAlias = false; + // look for existing aggregation for (var i = 0; i < selectParts.length; i++) { var part = selectParts[i]; @@ -36,6 +38,15 @@ function replaceAggregationAddStrategy(selectParts, partModel) { selectParts[i] = partModel; return; } + if (part.def.type === 'alias') { + hasAlias = true; + } + } + + // add alias if none exists yet + if (!hasAlias) { + var aliasModel = createPart({ type: 'alias', params: [selectParts[0].params[0]] }); + selectParts.push(aliasModel); } selectParts.splice(1, 0, partModel); @@ -117,7 +128,7 @@ register({ register({ type: 'macro', - style: 'function', + style: 'label', label: 'Macro:', addStrategy: addExpressionStrategy, params: [], From 6e7161f238551efbb9bf72c61ce0765c52966699 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 10 Jul 2018 11:14:00 +0200 Subject: [PATCH 105/310] add aggregates when adding group by --- public/app/plugins/datasource/postgres/postgres_query.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 71cf1ae7fce..2c01417ad12 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -96,6 +96,14 @@ export default class PostgresQuery { partModel.part.params = ['1m', 'none']; } + // add aggregates when adding group by + for (let i = 0; i < this.target.select.length; i++) { + var selectParts = this.target.select[i]; + if (!selectParts.some(part => part.type === 'aggregate')) { + selectParts.splice(1, 0, { type: 'aggregate', params: ['avg'] }); + } + } + this.updateProjection(); } From e8e24f6139f6f5589217ddf05c4369932df7a51f Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 10 Jul 2018 11:32:23 +0200 Subject: [PATCH 106/310] add alias when adding group by --- public/app/plugins/datasource/postgres/postgres_query.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 2c01417ad12..55ef242d1e2 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -101,6 +101,9 @@ export default class PostgresQuery { var selectParts = this.target.select[i]; if (!selectParts.some(part => part.type === 'aggregate')) { selectParts.splice(1, 0, { type: 'aggregate', params: ['avg'] }); + if (!selectParts.some(part => part.type === 'alias')) { + selectParts.push({ type: 'alias', params: [selectParts[0].params[0]] }); + } } } From 9d5b4cb91d28bd428a5f8c5e5de04e7324348d30 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 10 Jul 2018 14:45:36 +0200 Subject: [PATCH 107/310] dont throw exception for unknown types --- public/app/plugins/datasource/postgres/sql_part.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/sql_part.ts b/public/app/plugins/datasource/postgres/sql_part.ts index 8de764c45e9..e39a339c9da 100644 --- a/public/app/plugins/datasource/postgres/sql_part.ts +++ b/public/app/plugins/datasource/postgres/sql_part.ts @@ -6,7 +6,7 @@ var index = []; function createPart(part): any { var def = index[part.type]; if (!def) { - throw { message: 'Could not find query part ' + part.type }; + return null; } return new SqlPart(part, def); From 0e610cad1b7069ac008393dbf36f7f281789c92d Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 10 Jul 2018 21:40:59 +0200 Subject: [PATCH 108/310] set query gui as default handle old panels gracefully --- .../plugins/datasource/postgres/postgres_query.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 55ef242d1e2..665393c8ac8 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -27,7 +27,13 @@ export default class PostgresQuery { // handle pre query gui panels gracefully if (!('rawQuery' in this.target)) { - target.rawQuery = true; + if ('rawSql' in target) { + // pre query gui panel + target.rawQuery = true; + } else { + // new panel + target.rawQuery = false; + } } // give interpolateQueryStr access to this @@ -55,10 +61,10 @@ export default class PostgresQuery { updateProjection() { this.selectModels = _.map(this.target.select, function(parts: any) { - return _.map(parts, sqlPart.create); + return _.map(parts, sqlPart.create).filter(n => n); }); - this.whereParts = _.map(this.target.where, sqlPart.create); - this.groupByParts = _.map(this.target.groupBy, sqlPart.create); + this.whereParts = _.map(this.target.where, sqlPart.create).filter(n => n); + this.groupByParts = _.map(this.target.groupBy, sqlPart.create).filter(n => n); } updatePersistedParts() { From c8a11d597f88d357024ae22ebec89c105cf8a6ab Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 10 Jul 2018 21:43:10 +0200 Subject: [PATCH 109/310] prepare sql part for special functions --- .../plugins/datasource/postgres/query_ctrl.ts | 1 + .../plugins/datasource/postgres/sql_part.ts | 39 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index b44692ed673..e95e4132e23 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -78,6 +78,7 @@ export class PostgresQueryCtrl extends QueryCtrl { this.selectMenu = [ { text: 'Aggregate', value: 'aggregate' }, { text: 'Math', value: 'math' }, + // { text: 'Special', value: 'special' }, { text: 'Alias', value: 'alias' }, { text: 'Column', value: 'column' }, ]; diff --git a/public/app/plugins/datasource/postgres/sql_part.ts b/public/app/plugins/datasource/postgres/sql_part.ts index e39a339c9da..3e4664c0caf 100644 --- a/public/app/plugins/datasource/postgres/sql_part.ts +++ b/public/app/plugins/datasource/postgres/sql_part.ts @@ -52,6 +52,30 @@ function replaceAggregationAddStrategy(selectParts, partModel) { selectParts.splice(1, 0, partModel); } +function replaceSpecialAddStrategy(selectParts, partModel) { + var hasAlias = false; + + // look for existing aggregation + for (var i = 0; i < selectParts.length; i++) { + var part = selectParts[i]; + if (part.def.type === 'special') { + selectParts[i] = partModel; + return; + } + if (part.def.type === 'alias') { + hasAlias = true; + } + } + + // add alias if none exists yet + if (!hasAlias) { + var aliasModel = createPart({ type: 'alias', params: [selectParts[0].params[0]] }); + selectParts.push(aliasModel); + } + + selectParts.splice(1, 0, partModel); +} + function addMathStrategy(selectParts, partModel) { var partCount = selectParts.length; if (partCount > 0) { @@ -184,6 +208,21 @@ register({ renderer: functionRenderer, }); +register({ + type: 'special', + style: 'label', + params: [ + { + name: 'function', + type: 'string', + options: ['increase', 'rate'], + }, + ], + defaultParams: ['increase'], + addStrategy: replaceSpecialAddStrategy, + renderer: aggregateRenderer, +}); + export default { create: createPart, }; From c5bde0ef457312403ac0d8790d623fc831744b10 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 11 Jul 2018 10:50:49 +0200 Subject: [PATCH 110/310] Refactor render function on PostgresQuery --- .../plugins/datasource/postgres/postgres_query.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 665393c8ac8..b6f00d7efbf 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -169,7 +169,8 @@ export default class PostgresQuery { } render(interpolate?) { - var target = this.target; + let target = this.target; + let query; if (target.rawQuery) { if (interpolate) { @@ -179,6 +180,14 @@ export default class PostgresQuery { } } + query = this.buildQuery(target); + if (interpolate) { + query = this.templateSrv.replace(query, this.scopedVars, this.interpolateQueryStr); + } + return query; + } + + buildQuery(target) { var query = 'SELECT '; var timeGroup = this.hasGroupByTime(); @@ -250,9 +259,6 @@ export default class PostgresQuery { query += ' ORDER BY 1'; this.target.rawSql = query; - if (interpolate) { - query = this.templateSrv.replace(query, this.scopedVars, this.interpolateQueryStr); - } return query; } } From 772825ed09ca4e2a976a069359e80c2b4fe67211 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 11 Jul 2018 11:00:12 +0200 Subject: [PATCH 111/310] Refactor time column sql generation --- .../datasource/postgres/postgres_query.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index b6f00d7efbf..46bbad5589e 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -187,10 +187,9 @@ export default class PostgresQuery { return query; } - buildQuery(target) { - var query = 'SELECT '; - - var timeGroup = this.hasGroupByTime(); + buildTimeColumn(target) { + let timeGroup = this.hasGroupByTime(); + let query; if (timeGroup) { var args; @@ -199,11 +198,19 @@ export default class PostgresQuery { } else { args = timeGroup.params[0]; } - query += '$__timeGroup(' + target.timeColumn + ',' + args + ')'; + query = '$__timeGroup(' + target.timeColumn + ',' + args + ')'; } else { - query += target.timeColumn + ' AS "time"'; + query = target.timeColumn + ' AS "time"'; } + return query; + } + + buildQuery(target) { + var query = 'SELECT '; + + query += this.buildTimeColumn(target); + if (this.target.metricColumn !== 'None') { query += ',' + this.target.metricColumn + ' AS metric'; } From 5dd236bf0535167c4d3f9bcd05c649d0f08c8ddc Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 11 Jul 2018 11:09:28 +0200 Subject: [PATCH 112/310] refactor SQL generation for value columns --- .../datasource/postgres/postgres_query.ts | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 46bbad5589e..a242782de76 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -206,6 +206,22 @@ export default class PostgresQuery { return query; } + buildValueColumns(target) { + let query = ''; + for (let i = 0; i < this.selectModels.length; i++) { + let parts = this.selectModels[i]; + var selectText = ''; + for (let y = 0; y < parts.length; y++) { + let part = parts[y]; + selectText = part.render(selectText); + } + + query += ', ' + selectText; + } + + return query; + } + buildQuery(target) { var query = 'SELECT '; @@ -215,17 +231,7 @@ export default class PostgresQuery { query += ',' + this.target.metricColumn + ' AS metric'; } - var i, y; - for (i = 0; i < this.selectModels.length; i++) { - let parts = this.selectModels[i]; - var selectText = ''; - for (y = 0; y < parts.length; y++) { - let part = parts[y]; - selectText = part.render(selectText); - } - - query += ', ' + selectText; - } + query += this.buildValueColumns(target); query += ' FROM ' + target.schema + '.' + target.table; var conditions = _.map(target.where, (tag, index) => { @@ -244,7 +250,7 @@ export default class PostgresQuery { } var groupBySection = ''; - for (i = 0; i < this.groupByParts.length; i++) { + for (let i = 0; i < this.groupByParts.length; i++) { var part = this.groupByParts[i]; if (i > 0) { groupBySection += ', '; From b9edc3c61104498327d1c8b8b0e83bc59cc1b2cc Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 11 Jul 2018 11:50:02 +0200 Subject: [PATCH 113/310] Refactor where clause generation --- .../datasource/postgres/postgres_query.ts | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index a242782de76..f6a069f1ab5 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -222,18 +222,8 @@ export default class PostgresQuery { return query; } - buildQuery(target) { - var query = 'SELECT '; - - query += this.buildTimeColumn(target); - - if (this.target.metricColumn !== 'None') { - query += ',' + this.target.metricColumn + ' AS metric'; - } - - query += this.buildValueColumns(target); - - query += ' FROM ' + target.schema + '.' + target.table; + buildWhereClause(target) { + let query = ''; var conditions = _.map(target.where, (tag, index) => { switch (tag.type) { case 'macro': @@ -246,9 +236,27 @@ export default class PostgresQuery { }); if (conditions.length > 0) { - query += ' WHERE ' + conditions.join(' AND '); + query = ' WHERE ' + conditions.join(' AND '); } + return query; + } + + buildQuery(target) { + var query = 'SELECT '; + + query += this.buildTimeColumn(target); + + if (this.target.metricColumn !== 'None') { + query += ',' + this.target.metricColumn + ' AS metric'; + } + + query += this.buildValueColumns(target); + + query += ' FROM ' + target.schema + '.' + target.table; + + query += this.buildWhereClause(target); + var groupBySection = ''; for (let i = 0; i < this.groupByParts.length; i++) { var part = this.groupByParts[i]; From a86e77fc68dac24a92346394a6ecfacf7be0b867 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 11 Jul 2018 12:20:24 +0200 Subject: [PATCH 114/310] Refactor group by query generation --- .../datasource/postgres/postgres_query.ts | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index f6a069f1ab5..c54b1b7f510 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -242,22 +242,10 @@ export default class PostgresQuery { return query; } - buildQuery(target) { - var query = 'SELECT '; + buildGroupByClause(target) { + let query = ''; + let groupBySection = ''; - query += this.buildTimeColumn(target); - - if (this.target.metricColumn !== 'None') { - query += ',' + this.target.metricColumn + ' AS metric'; - } - - query += this.buildValueColumns(target); - - query += ' FROM ' + target.schema + '.' + target.table; - - query += this.buildWhereClause(target); - - var groupBySection = ''; for (let i = 0; i < this.groupByParts.length; i++) { var part = this.groupByParts[i]; if (i > 0) { @@ -271,11 +259,29 @@ export default class PostgresQuery { } if (groupBySection.length) { - query += ' GROUP BY ' + groupBySection; + query = ' GROUP BY ' + groupBySection; if (this.target.metricColumn !== 'None') { query += ',2'; } } + return query; + } + + buildQuery(target) { + let query = 'SELECT '; + + query += this.buildTimeColumn(target); + + if (this.target.metricColumn !== 'None') { + query += ',' + this.target.metricColumn + ' AS metric'; + } + + query += this.buildValueColumns(target); + + query += ' FROM ' + target.schema + '.' + target.table; + + query += this.buildWhereClause(target); + query += this.buildGroupByClause(target); query += ' ORDER BY 1'; From 9ba42f63f1694c86e47b46f8cfef701ad1ce842c Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 11 Jul 2018 12:33:37 +0200 Subject: [PATCH 115/310] Refactor metric column sql generation --- .../datasource/postgres/postgres_query.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index c54b1b7f510..700875e145d 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -184,6 +184,7 @@ export default class PostgresQuery { if (interpolate) { query = this.templateSrv.replace(query, this.scopedVars, this.interpolateQueryStr); } + this.target.rawSql = query; return query; } @@ -206,6 +207,16 @@ export default class PostgresQuery { return query; } + buildMetricColumn(target) { + let query = ''; + + if (this.target.metricColumn !== 'None') { + query += ',' + this.target.metricColumn + ' AS metric'; + } + + return query; + } + buildValueColumns(target) { let query = ''; for (let i = 0; i < this.selectModels.length; i++) { @@ -271,11 +282,7 @@ export default class PostgresQuery { let query = 'SELECT '; query += this.buildTimeColumn(target); - - if (this.target.metricColumn !== 'None') { - query += ',' + this.target.metricColumn + ' AS metric'; - } - + query += this.buildMetricColumn(target); query += this.buildValueColumns(target); query += ' FROM ' + target.schema + '.' + target.table; @@ -285,7 +292,6 @@ export default class PostgresQuery { query += ' ORDER BY 1'; - this.target.rawSql = query; return query; } } From 2784a2df6b5d51182c6e875a8f7888cce13f5fc9 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 11 Jul 2018 13:03:06 +0200 Subject: [PATCH 116/310] Refactor value column SQL generation --- .../datasource/postgres/postgres_query.ts | 36 ++++++++++--------- .../plugins/datasource/postgres/query_ctrl.ts | 2 +- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 700875e145d..246301984df 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -86,8 +86,8 @@ export default class PostgresQuery { } addGroupBy(partType, value) { - var partModel = sqlPart.create({ type: partType, params: [value] }); - var partCount = this.target.groupBy.length; + let partModel = sqlPart.create({ type: partType, params: [value] }); + let partCount = this.target.groupBy.length; if (partCount === 0) { this.target.groupBy.push(partModel.part); @@ -137,11 +137,11 @@ export default class PostgresQuery { // if we remove the field remove the whole statement if (part.def.type === 'column') { if (this.selectModels.length > 1) { - var modelsIndex = _.indexOf(this.selectModels, selectParts); + let modelsIndex = _.indexOf(this.selectModels, selectParts); this.selectModels.splice(modelsIndex, 1); } } else { - var partIndex = _.indexOf(selectParts, part); + let partIndex = _.indexOf(selectParts, part); selectParts.splice(partIndex, 1); } @@ -149,7 +149,7 @@ export default class PostgresQuery { } addSelectPart(selectParts, type) { - var partModel = sqlPart.create({ type: type }); + let partModel = sqlPart.create({ type: type }); partModel.def.addStrategy(selectParts, partModel, this); this.updatePersistedParts(); } @@ -164,7 +164,7 @@ export default class PostgresQuery { return this.quoteLiteral(value); } - var escapedValues = _.map(value, this.quoteLiteral); + let escapedValues = _.map(value, this.quoteLiteral); return '(' + escapedValues.join(',') + ')'; } @@ -193,7 +193,7 @@ export default class PostgresQuery { let query; if (timeGroup) { - var args; + let args; if (timeGroup.params.length > 1 && timeGroup.params[1] !== 'none') { args = timeGroup.params.join(','); } else { @@ -220,22 +220,24 @@ export default class PostgresQuery { buildValueColumns(target) { let query = ''; for (let i = 0; i < this.selectModels.length; i++) { - let parts = this.selectModels[i]; - var selectText = ''; - for (let y = 0; y < parts.length; y++) { - let part = parts[y]; - selectText = part.render(selectText); - } - - query += ', ' + selectText; + query += ', ' + this.buildValueColumn(target, this.selectModels[i]); } return query; } + buildValueColumn(target, column) { + let selectText = ''; + for (let i = 0; i < column.length; i++) { + let part = column[i]; + selectText = part.render(selectText); + } + return selectText; + } + buildWhereClause(target) { let query = ''; - var conditions = _.map(target.where, (tag, index) => { + let conditions = _.map(target.where, (tag, index) => { switch (tag.type) { case 'macro': return tag.name + '(' + target.timeColumn + ')'; @@ -258,7 +260,7 @@ export default class PostgresQuery { let groupBySection = ''; for (let i = 0; i < this.groupByParts.length; i++) { - var part = this.groupByParts[i]; + let part = this.groupByParts[i]; if (i > 0) { groupBySection += ', '; } diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index e95e4132e23..b9037d6c3f0 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -78,7 +78,7 @@ export class PostgresQueryCtrl extends QueryCtrl { this.selectMenu = [ { text: 'Aggregate', value: 'aggregate' }, { text: 'Math', value: 'math' }, - // { text: 'Special', value: 'special' }, + { text: 'Special', value: 'special' }, { text: 'Alias', value: 'alias' }, { text: 'Column', value: 'column' }, ]; From bbe7b9b90814f1384ed6c72a5fa5500a03734961 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 12 Jul 2018 09:57:04 +0200 Subject: [PATCH 117/310] add rate and increase special functions --- .../datasource/postgres/postgres_query.ts | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 246301984df..9a36a838c82 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -219,20 +219,48 @@ export default class PostgresQuery { buildValueColumns(target) { let query = ''; - for (let i = 0; i < this.selectModels.length; i++) { - query += ', ' + this.buildValueColumn(target, this.selectModels[i]); + for (let i = 0; i < target.select.length; i++) { + query += ', ' + this.buildValueColumn(target, target.select[i]); } return query; } buildValueColumn(target, column) { - let selectText = ''; - for (let i = 0; i < column.length; i++) { - let part = column[i]; - selectText = part.render(selectText); + let query = ''; + + let columnName = _.find(column, (g: any) => g.type === 'column'); + query = columnName.params[0]; + + let aggregate = _.find(column, (g: any) => g.type === 'aggregate'); + if (aggregate) { + query = aggregate.params[0] + '(' + query + ')'; } - return selectText; + + let special = _.find(column, (g: any) => g.type === 'special'); + if (special) { + let over = ''; + if (target.metricColumn) { + over = 'PARTITION BY ' + target.metricColumn; + } + switch (special.params[0]) { + case 'increase': + query = query + ' - lag(' + query + ') OVER ()'; + break; + case 'rate': + let timeColumn = target.timeColumn; + query = '(' + query + ' - lag(' + query + ') OVER (' + over + '))'; + query += '/extract(epoch from ' + timeColumn + ' - lag(' + timeColumn + ') OVER (' + over + '))'; + break; + } + } + + let alias = _.find(column, (g: any) => g.type === 'alias'); + if (alias) { + query += ' AS ' + this.quoteIdentifier(alias.params[0]); + } + + return query; } buildWhereClause(target) { From 0935526d232d716d852566a3aa399ad663cd881d Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 12 Jul 2018 10:13:21 +0200 Subject: [PATCH 118/310] partition by metricColumn when using increase --- public/app/plugins/datasource/postgres/postgres_query.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 9a36a838c82..9e32736b405 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -245,7 +245,7 @@ export default class PostgresQuery { } switch (special.params[0]) { case 'increase': - query = query + ' - lag(' + query + ') OVER ()'; + query = query + ' - lag(' + query + ') OVER (' + over + ')'; break; case 'rate': let timeColumn = target.timeColumn; From a24a6624e339611ea5c43e6b19ae5a4b17ea1f71 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 12 Jul 2018 13:15:19 +0200 Subject: [PATCH 119/310] handle counter overflow and resets in rate --- public/app/plugins/datasource/postgres/postgres_query.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 9e32736b405..b371af3f1b1 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -240,7 +240,7 @@ export default class PostgresQuery { let special = _.find(column, (g: any) => g.type === 'special'); if (special) { let over = ''; - if (target.metricColumn) { + if (target.metricColumn !== 'None') { over = 'PARTITION BY ' + target.metricColumn; } switch (special.params[0]) { @@ -249,7 +249,9 @@ export default class PostgresQuery { break; case 'rate': let timeColumn = target.timeColumn; - query = '(' + query + ' - lag(' + query + ') OVER (' + over + '))'; + let curr = query; + let prev = 'lag(' + curr + ') OVER (' + over + ')'; + query = '(CASE WHEN ' + curr + ' >= ' + prev + ' THEN ' + curr + ' - ' + prev + ' ELSE ' + curr + ' END)'; query += '/extract(epoch from ' + timeColumn + ' - lag(' + timeColumn + ') OVER (' + over + '))'; break; } From bf21f1bf76ac70e6b65466088e7f637d2faf46fa Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 12 Jul 2018 22:36:41 +0200 Subject: [PATCH 120/310] indent generated SQL --- .../plugins/datasource/postgres/postgres_query.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index b371af3f1b1..1cac6835ca1 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -204,14 +204,14 @@ export default class PostgresQuery { query = target.timeColumn + ' AS "time"'; } - return query; + return '\n ' + query; } buildMetricColumn(target) { let query = ''; if (this.target.metricColumn !== 'None') { - query += ',' + this.target.metricColumn + ' AS metric'; + query += ',\n ' + this.target.metricColumn + ' AS metric'; } return query; @@ -220,7 +220,7 @@ export default class PostgresQuery { buildValueColumns(target) { let query = ''; for (let i = 0; i < target.select.length; i++) { - query += ', ' + this.buildValueColumn(target, target.select[i]); + query += ',\n ' + this.buildValueColumn(target, target.select[i]); } return query; @@ -279,7 +279,7 @@ export default class PostgresQuery { }); if (conditions.length > 0) { - query = ' WHERE ' + conditions.join(' AND '); + query = '\nWHERE\n ' + conditions.join(' AND\n '); } return query; @@ -302,7 +302,7 @@ export default class PostgresQuery { } if (groupBySection.length) { - query = ' GROUP BY ' + groupBySection; + query = '\nGROUP BY ' + groupBySection; if (this.target.metricColumn !== 'None') { query += ',2'; } @@ -317,12 +317,12 @@ export default class PostgresQuery { query += this.buildMetricColumn(target); query += this.buildValueColumns(target); - query += ' FROM ' + target.schema + '.' + target.table; + query += '\nFROM ' + target.schema + '.' + target.table; query += this.buildWhereClause(target); query += this.buildGroupByClause(target); - query += ' ORDER BY 1'; + query += '\nORDER BY 1'; return query; } From 693efda4d586a3cedfeca05b82a785b32cbd7d1f Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 13 Jul 2018 09:31:39 +0200 Subject: [PATCH 121/310] send timerange with metricFindQuery --- .../app/plugins/datasource/postgres/datasource.ts | 14 +++++--------- .../plugins/datasource/postgres/query_builder.ts | 1 + 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/public/app/plugins/datasource/postgres/datasource.ts b/public/app/plugins/datasource/postgres/datasource.ts index 3a4bd27bb4c..1324dbadebf 100644 --- a/public/app/plugins/datasource/postgres/datasource.ts +++ b/public/app/plugins/datasource/postgres/datasource.ts @@ -8,7 +8,7 @@ export class PostgresDatasource { responseParser: ResponseParser; /** @ngInject **/ - constructor(instanceSettings, private backendSrv, private $q, private templateSrv) { + constructor(instanceSettings, private backendSrv, private $q, private templateSrv, private timeSrv) { this.name = instanceSettings.name; this.id = instanceSettings.id; this.responseParser = new ResponseParser(this.$q); @@ -106,17 +106,13 @@ export class PostgresDatasource { format: 'table', }; - var data = { + let range = this.timeSrv.timeRange(); + let data = { queries: [interpolatedQuery], + from: range.from.valueOf().toString(), + to: range.to.valueOf().toString(), }; - if (optionalOptions && optionalOptions.range && optionalOptions.range.from) { - data['from'] = optionalOptions.range.from.valueOf().toString(); - } - if (optionalOptions && optionalOptions.range && optionalOptions.range.to) { - data['to'] = optionalOptions.range.to.valueOf().toString(); - } - return this.backendSrv .datasourceRequest({ url: '/api/tsdb/query', diff --git a/public/app/plugins/datasource/postgres/query_builder.ts b/public/app/plugins/datasource/postgres/query_builder.ts index 852fb91a706..cfee317d96f 100644 --- a/public/app/plugins/datasource/postgres/query_builder.ts +++ b/public/app/plugins/datasource/postgres/query_builder.ts @@ -46,6 +46,7 @@ export class PostgresQueryBuilder { buildValueQuery(column: string) { var query = 'SELECT DISTINCT quote_literal(' + column + ')'; query += ' FROM ' + this.target.schema + '.' + this.target.table; + query += ' WHERE $__timeFilter(' + this.target.timeColumn + ')'; query += ' ORDER BY 1 LIMIT 100'; return query; } From 17591ca29f80df8c312ec3d1188ad34071c378f6 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 13 Jul 2018 10:15:55 +0200 Subject: [PATCH 122/310] fix tests for postgres datasource --- .../datasource/postgres/specs/datasource.jest.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/specs/datasource.jest.ts b/public/app/plugins/datasource/postgres/specs/datasource.jest.ts index 107cd76e6c5..526c7d236e3 100644 --- a/public/app/plugins/datasource/postgres/specs/datasource.jest.ts +++ b/public/app/plugins/datasource/postgres/specs/datasource.jest.ts @@ -9,12 +9,23 @@ describe('PostgreSQLDatasource', function() { let templateSrv = { replace: jest.fn(text => text), }; + const raw = { + from: moment.utc('2018-04-25 10:00'), + to: moment.utc('2018-04-25 11:00'), + }; let ctx = { backendSrv, + timeSrvMock: { + timeRange: () => ({ + from: raw.from, + to: raw.to, + raw: raw, + }), + }, }; beforeEach(() => { - ctx.ds = new PostgresDatasource(instanceSettings, backendSrv, {}, templateSrv); + ctx.ds = new PostgresDatasource(instanceSettings, backendSrv, {}, templateSrv, ctx.timeSrvMock); }); describe('When performing annotationQuery', function() { From b745fab190f904af9b538105ac6ac83dba18bedc Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 13 Jul 2018 22:29:10 +0200 Subject: [PATCH 123/310] add tests for query generation --- .../datasource/postgres/postgres_query.ts | 28 ++--- .../plugins/datasource/postgres/query_ctrl.ts | 2 +- .../postgres/specs/postgres_query.jest.ts | 100 ++++++++++++++++++ 3 files changed, 115 insertions(+), 15 deletions(-) create mode 100644 public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 1cac6835ca1..babd7c9d6ec 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -204,17 +204,15 @@ export default class PostgresQuery { query = target.timeColumn + ' AS "time"'; } - return '\n ' + query; + return query; } buildMetricColumn(target) { - let query = ''; - - if (this.target.metricColumn !== 'None') { - query += ',\n ' + this.target.metricColumn + ' AS metric'; + if (target.metricColumn !== 'None') { + return target.metricColumn + ' AS metric'; } - return query; + return ''; } buildValueColumns(target) { @@ -289,21 +287,21 @@ export default class PostgresQuery { let query = ''; let groupBySection = ''; - for (let i = 0; i < this.groupByParts.length; i++) { - let part = this.groupByParts[i]; + for (let i = 0; i < target.groupBy.length; i++) { + let part = target.groupBy[i]; if (i > 0) { groupBySection += ', '; } - if (part.def.type === 'time') { + if (part.type === 'time') { groupBySection += '1'; } else { - groupBySection += part.render(''); + groupBySection += part.params[0]; } } if (groupBySection.length) { query = '\nGROUP BY ' + groupBySection; - if (this.target.metricColumn !== 'None') { + if (target.metricColumn !== 'None') { query += ',2'; } } @@ -311,10 +309,12 @@ export default class PostgresQuery { } buildQuery(target) { - let query = 'SELECT '; + let query = 'SELECT'; - query += this.buildTimeColumn(target); - query += this.buildMetricColumn(target); + query += '\n ' + this.buildTimeColumn(target); + if (target.metricColumn !== 'None') { + query += '\n ' + this.buildMetricColumn(target); + } query += this.buildValueColumns(target); query += '\nFROM ' + target.schema + '.' + target.table; diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index b9037d6c3f0..e10fe25e4ad 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -299,7 +299,7 @@ export class PostgresQueryCtrl extends QueryCtrl { getWhereOptions() { var options = []; options.push(this.uiSegmentSrv.newSegment({ type: 'macro', value: '$__timeFilter' })); - options.push(this.uiSegmentSrv.newSegment({ type: 'macro', value: '$__unixEpochFilter' })); + // options.push(this.uiSegmentSrv.newSegment({ type: 'macro', value: '$__unixEpochFilter' })); options.push(this.uiSegmentSrv.newSegment({ type: 'expression', value: 'Expression' })); return this.$q.when(options); } diff --git a/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts new file mode 100644 index 00000000000..fc05441f985 --- /dev/null +++ b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts @@ -0,0 +1,100 @@ +import PostgresQuery from '../postgres_query'; + +describe('PostgresQuery', function() { + describe('When initializing', function() { + it('should not be in SQL mode', function() { + let query = new PostgresQuery({}, templateSrv); + expect(query.target.rawQuery).toBe(false); + }); + it('should be in SQL mode for pre query builder queries', function() { + let query = new PostgresQuery({ rawSql: 'SELECT 1' }, templateSrv); + expect(query.target.rawQuery).toBe(true); + }); + }); + + describe('When generating time column SQL', function() { + let query = new PostgresQuery({}, templateSrv); + + expect(query.buildTimeColumn({ timeColumn: 'time' })).toBe('time AS "time"'); + expect(query.buildTimeColumn({ timeColumn: '"time"' })).toBe('"time" AS "time"'); + }); + + describe('When generating time column SQL with group by time', function() { + let query = new PostgresQuery( + { timeColumn: 'time', groupBy: [{ type: 'time', params: ['5m', 'none'] }] }, + templateSrv + ); + expect(query.buildTimeColumn(query.target)).toBe('$__timeGroup(time,5m)'); + + query = new PostgresQuery({ timeColumn: 'time', groupBy: [{ type: 'time', params: ['5m', 'NULL'] }] }, templateSrv); + expect(query.buildTimeColumn(query.target)).toBe('$__timeGroup(time,5m,NULL)'); + }); + + describe('When generating metric column SQL', function() { + let query = new PostgresQuery({}, templateSrv); + + expect(query.buildMetricColumn({ metricColumn: 'host' })).toBe('host AS metric'); + expect(query.buildMetricColumn({ metricColumn: '"host"' })).toBe('"host" AS metric'); + }); + + describe('When generating value column SQL', function() { + let query = new PostgresQuery({}, templateSrv); + + let column = [{ type: 'column', params: ['value'] }]; + expect(query.buildValueColumn(query.target, column)).toBe('value'); + column = [{ type: 'column', params: ['value'] }, { type: 'alias', params: ['alias'] }]; + expect(query.buildValueColumn(query.target, column)).toBe('value AS "alias"'); + column = [ + { type: 'column', params: ['v'] }, + { type: 'alias', params: ['a'] }, + { type: 'aggregate', params: ['max'] }, + ]; + expect(query.buildValueColumn(query.target, column)).toBe('max(v) AS "a"'); + column = [ + { type: 'column', params: ['v'] }, + { type: 'alias', params: ['a'] }, + { type: 'special', params: ['increase'] }, + ]; + expect(query.buildValueColumn(query.target, column)).toBe('v - lag(v) OVER () AS "a"'); + }); + + describe('When generating WHERE clause', function() { + let query = new PostgresQuery({ where: [] }, templateSrv); + let target; + + expect(query.buildWhereClause(query.target)).toBe(''); + target = { where: [{ type: 'macro', name: '$__timeFilter' }], timeColumn: 't' }; + expect(query.buildWhereClause(target)).toBe('\nWHERE\n $__timeFilter(t)'); + target = { where: [{ type: 'expression', params: ['v', '=', '1'] }], timeColumn: 't' }; + expect(query.buildWhereClause(target)).toBe('\nWHERE\n v = 1'); + target = { + where: [{ type: 'macro', name: '$__timeFilter' }, { type: 'expression', params: ['v', '=', '1'] }], + timeColumn: 't', + }; + expect(query.buildWhereClause(target)).toBe('\nWHERE\n $__timeFilter(t) AND\n v = 1'); + }); + + describe('When generating GROUP BY clause', function() { + let query = new PostgresQuery({ groupBy: [] }, templateSrv); + let target; + + expect(query.buildGroupByClause(query.target)).toBe(''); + target = { groupBy: [{ type: 'time', params: ['5m'] }], metricColumn: 'None' }; + expect(query.buildGroupByClause(target)).toBe('\nGROUP BY 1'); + target = { groupBy: [{ type: 'time', params: ['5m'] }], metricColumn: 'm' }; + expect(query.buildGroupByClause(target)).toBe('\nGROUP BY 1,2'); + }); + + describe('When generating complete statement', function() { + let target = { + timeColumn: 't', + schema: 'public', + table: 'table', + select: [[{ type: 'column', params: ['value'] }]], + }; + let result = 'SELECT\n t AS "time",\n value\nFROM public.table\nORDER BY 1'; + let query = new PostgresQuery(target, templateSrv); + + expect(query.buildQuery(query.target)).toBe(result); + }); +}); From d0c2b19c51a94e4cb5e8d1393bc8e739e66594a9 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 14 Jul 2018 11:05:35 +0200 Subject: [PATCH 124/310] Add templateSrv to PostgresQuery tests --- .../plugins/datasource/postgres/specs/postgres_query.jest.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts index fc05441f985..ef8fdea951b 100644 --- a/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts +++ b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts @@ -1,6 +1,10 @@ import PostgresQuery from '../postgres_query'; describe('PostgresQuery', function() { + let templateSrv = { + replace: jest.fn(text => text), + }; + describe('When initializing', function() { it('should not be in SQL mode', function() { let query = new PostgresQuery({}, templateSrv); From 569a8567f97c723df40feaee26e517cb7f7f6607 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 14 Jul 2018 11:32:00 +0200 Subject: [PATCH 125/310] use let for variable declaration --- .../plugins/datasource/postgres/query_builder.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_builder.ts b/public/app/plugins/datasource/postgres/query_builder.ts index cfee317d96f..ffa8f6ba270 100644 --- a/public/app/plugins/datasource/postgres/query_builder.ts +++ b/public/app/plugins/datasource/postgres/query_builder.ts @@ -2,14 +2,14 @@ export class PostgresQueryBuilder { constructor(private target, private queryModel) {} buildSchemaQuery() { - var query = 'SELECT quote_ident(schema_name) FROM information_schema.schemata WHERE'; + let query = 'SELECT quote_ident(schema_name) FROM information_schema.schemata WHERE'; query += " schema_name NOT LIKE 'pg_%' AND schema_name NOT LIKE '\\_%' AND schema_name <> 'information_schema';"; return query; } buildTableQuery() { - var query = 'SELECT quote_ident(table_name) FROM information_schema.tables WHERE '; + let query = 'SELECT quote_ident(table_name) FROM information_schema.tables WHERE '; query += 'table_schema = ' + this.quoteIdentAsLiteral(this.target.schema); return query; } @@ -20,7 +20,7 @@ export class PostgresQueryBuilder { } buildColumnQuery(type?: string) { - var query = 'SELECT quote_ident(column_name) FROM information_schema.columns WHERE '; + let query = 'SELECT quote_ident(column_name) FROM information_schema.columns WHERE '; query += 'table_schema = ' + this.quoteIdentAsLiteral(this.target.schema); query += ' AND table_name = ' + this.quoteIdentAsLiteral(this.target.table); @@ -31,7 +31,7 @@ export class PostgresQueryBuilder { break; } case 'metric': { - query += " AND data_type IN ('text','char','varchar')"; + query += " AND data_type IN ('text','char','varchar','integer','bigint')"; break; } case 'value': { @@ -44,7 +44,7 @@ export class PostgresQueryBuilder { } buildValueQuery(column: string) { - var query = 'SELECT DISTINCT quote_literal(' + column + ')'; + let query = 'SELECT DISTINCT quote_literal(' + column + ')'; query += ' FROM ' + this.target.schema + '.' + this.target.table; query += ' WHERE $__timeFilter(' + this.target.timeColumn + ')'; query += ' ORDER BY 1 LIMIT 100'; @@ -52,7 +52,7 @@ export class PostgresQueryBuilder { } buildDatatypeQuery(column: string) { - var query = 'SELECT data_type FROM information_schema.columns WHERE '; + let query = 'SELECT data_type FROM information_schema.columns WHERE '; query += ' table_schema = ' + this.quoteIdentAsLiteral(this.target.schema); query += ' AND table_name = ' + this.quoteIdentAsLiteral(this.target.table); query += ' AND column_name = ' + this.quoteIdentAsLiteral(column); @@ -60,10 +60,10 @@ export class PostgresQueryBuilder { } buildAggregateQuery() { - var query = 'SELECT DISTINCT proname FROM pg_aggregate '; + let query = 'SELECT DISTINCT proname FROM pg_aggregate '; query += 'INNER JOIN pg_proc ON pg_aggregate.aggfnoid = pg_proc.oid '; query += 'INNER JOIN pg_type ON pg_type.oid=pg_proc.prorettype '; - query += "WHERE pronargs=1 AND typname IN ('int8','float8') AND aggkind='n' ORDER BY 1"; + query += "WHERE pronargs=1 AND typname IN ('float8') AND aggkind='n' ORDER BY 1"; return query; } } From 2dc9d4e9600c6c0b3da14fafb4ca6f8f2d24be32 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 14 Jul 2018 16:46:53 +0200 Subject: [PATCH 126/310] refactor addGroupBy and removeGroupByPart --- .../datasource/postgres/postgres_query.ts | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index babd7c9d6ec..f9c44553074 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -86,42 +86,41 @@ export default class PostgresQuery { } addGroupBy(partType, value) { - let partModel = sqlPart.create({ type: partType, params: [value] }); - let partCount = this.target.groupBy.length; - - if (partCount === 0) { - this.target.groupBy.push(partModel.part); - } else if (partType === 'time') { - // put timeGroup at start - this.target.groupBy.splice(0, 0, partModel.part); - } else { - this.target.groupBy.push(partModel.part); + let params = [value]; + if (partType === 'time') { + params = ['1m', 'none']; } + let partModel = sqlPart.create({ type: partType, params: params }); if (partType === 'time') { - partModel.part.params = ['1m', 'none']; + // put timeGroup at start + this.groupByParts.splice(0, 0, partModel); + } else { + this.groupByParts(partModel); } // add aggregates when adding group by - for (let i = 0; i < this.target.select.length; i++) { - var selectParts = this.target.select[i]; - if (!selectParts.some(part => part.type === 'aggregate')) { - selectParts.splice(1, 0, { type: 'aggregate', params: ['avg'] }); - if (!selectParts.some(part => part.type === 'alias')) { - selectParts.push({ type: 'alias', params: [selectParts[0].params[0]] }); + for (let i = 0; i < this.selectModels.length; i++) { + var selectParts = this.selectModels[i]; + if (!selectParts.some(part => part.def.type === 'aggregate')) { + let aggregate = sqlPart.create({ type: 'aggregate', params: ['avg'] }); + selectParts.splice(1, 0, aggregate); + if (!selectParts.some(part => part.def.type === 'alias')) { + let alias = sqlPart.create({ type: 'alias', params: [selectParts[0].part.params[0]] }); + selectParts.push(alias); } } } - this.updateProjection(); + this.updatePersistedParts(); } removeGroupByPart(part, index) { if (part.def.type === 'time') { // remove aggregations - this.target.select = _.map(this.target.select, (s: any) => { + this.selectModels = _.map(this.selectModels, (s: any) => { return _.filter(s, (part: any) => { - if (part.type === 'aggregate') { + if (part.def.type === 'aggregate') { return false; } return true; @@ -129,8 +128,8 @@ export default class PostgresQuery { }); } - this.target.groupBy.splice(index, 1); - this.updateProjection(); + this.groupByParts.splice(index, 1); + this.updatePersistedParts(); } removeSelectPart(selectParts, part) { From d7ed706e125c166d494104c8b5696e664bbb3bf3 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 14 Jul 2018 20:27:01 +0200 Subject: [PATCH 127/310] refactor PostgresQueryCtrl and PostgresQuery --- .../postgres/partials/query.editor.html | 8 +- .../datasource/postgres/postgres_query.ts | 96 ------------- .../plugins/datasource/postgres/query_ctrl.ts | 130 +++++++++++++++--- 3 files changed, 118 insertions(+), 116 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 28f8af52930..f0e15fc1a50 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -29,7 +29,7 @@
-
+
-
- +
+
@@ -79,7 +79,7 @@ GROUP BY - diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index f9c44553074..12323ec9fac 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -1,12 +1,8 @@ import _ from 'lodash'; -import sqlPart from './sql_part'; export default class PostgresQuery { target: any; - selectModels: any[]; queryBuilder: any; - groupByParts: any[]; - whereParts: any[]; templateSrv: any; scopedVars: any; @@ -38,8 +34,6 @@ export default class PostgresQuery { // give interpolateQueryStr access to this this.interpolateQueryStr = this.interpolateQueryStr.bind(this); - - this.updateProjection(); } // remove identifier quoting from identifier to use in metadata queries @@ -59,100 +53,10 @@ export default class PostgresQuery { return "'" + value.replace("'", "''") + "'"; } - updateProjection() { - this.selectModels = _.map(this.target.select, function(parts: any) { - return _.map(parts, sqlPart.create).filter(n => n); - }); - this.whereParts = _.map(this.target.where, sqlPart.create).filter(n => n); - this.groupByParts = _.map(this.target.groupBy, sqlPart.create).filter(n => n); - } - - updatePersistedParts() { - this.target.select = _.map(this.selectModels, function(selectParts) { - return _.map(selectParts, function(part: any) { - return { type: part.def.type, params: part.params }; - }); - }); - this.target.where = _.map(this.whereParts, function(part: any) { - return { type: part.def.type, name: part.name, params: part.params }; - }); - this.target.groupBy = _.map(this.groupByParts, function(part: any) { - return { type: part.def.type, params: part.params }; - }); - } - hasGroupByTime() { return _.find(this.target.groupBy, (g: any) => g.type === 'time'); } - addGroupBy(partType, value) { - let params = [value]; - if (partType === 'time') { - params = ['1m', 'none']; - } - let partModel = sqlPart.create({ type: partType, params: params }); - - if (partType === 'time') { - // put timeGroup at start - this.groupByParts.splice(0, 0, partModel); - } else { - this.groupByParts(partModel); - } - - // add aggregates when adding group by - for (let i = 0; i < this.selectModels.length; i++) { - var selectParts = this.selectModels[i]; - if (!selectParts.some(part => part.def.type === 'aggregate')) { - let aggregate = sqlPart.create({ type: 'aggregate', params: ['avg'] }); - selectParts.splice(1, 0, aggregate); - if (!selectParts.some(part => part.def.type === 'alias')) { - let alias = sqlPart.create({ type: 'alias', params: [selectParts[0].part.params[0]] }); - selectParts.push(alias); - } - } - } - - this.updatePersistedParts(); - } - - removeGroupByPart(part, index) { - if (part.def.type === 'time') { - // remove aggregations - this.selectModels = _.map(this.selectModels, (s: any) => { - return _.filter(s, (part: any) => { - if (part.def.type === 'aggregate') { - return false; - } - return true; - }); - }); - } - - this.groupByParts.splice(index, 1); - this.updatePersistedParts(); - } - - removeSelectPart(selectParts, part) { - // if we remove the field remove the whole statement - if (part.def.type === 'column') { - if (this.selectModels.length > 1) { - let modelsIndex = _.indexOf(this.selectModels, selectParts); - this.selectModels.splice(modelsIndex, 1); - } - } else { - let partIndex = _.indexOf(selectParts, part); - selectParts.splice(partIndex, 1); - } - - this.updatePersistedParts(); - } - - addSelectPart(selectParts, type) { - let partModel = sqlPart.create({ type: type }); - partModel.def.addStrategy(selectParts, partModel, this); - this.updatePersistedParts(); - } - interpolateQueryStr(value, variable, defaultFormatFn) { // if no multi or include all do not regexEscape if (!variable.multi && !variable.includeAll) { diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index e10fe25e4ad..27c3d1e50fa 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -33,6 +33,9 @@ export class PostgresQueryCtrl extends QueryCtrl { timeColumnSegment: any; metricColumnSegment: any; selectMenu: any; + selectModels: any[]; + groupByParts: any[]; + whereParts: any[]; groupByAdd: any; /** @ngInject **/ @@ -41,6 +44,7 @@ export class PostgresQueryCtrl extends QueryCtrl { this.target = this.target; this.queryModel = new PostgresQuery(this.target, templateSrv, this.panel.scopedVars); this.queryBuilder = new PostgresQueryBuilder(this.target, this.queryModel); + this.updateProjection(); this.formats = [{ text: 'Time series', value: 'time_series' }, { text: 'Table', value: 'table' }]; @@ -66,7 +70,6 @@ export class PostgresQueryCtrl extends QueryCtrl { this.metricColumnSegment = uiSegmentSrv.newSegment(this.target.metricColumn); this.buildSelectMenu(); - this.buildWhereSegments(); this.whereAdd = this.uiSegmentSrv.newPlusButton(); this.groupByAdd = this.uiSegmentSrv.newPlusButton(); @@ -74,10 +77,31 @@ export class PostgresQueryCtrl extends QueryCtrl { this.panelCtrl.events.on('data-error', this.onDataError.bind(this), $scope); } + updateProjection() { + this.selectModels = _.map(this.target.select, function(parts: any) { + return _.map(parts, sqlPart.create).filter(n => n); + }); + this.whereParts = _.map(this.target.where, sqlPart.create).filter(n => n); + this.groupByParts = _.map(this.target.groupBy, sqlPart.create).filter(n => n); + } + + updatePersistedParts() { + this.target.select = _.map(this.selectModels, function(selectParts) { + return _.map(selectParts, function(part: any) { + return { type: part.def.type, params: part.params }; + }); + }); + this.target.where = _.map(this.whereParts, function(part: any) { + return { type: part.def.type, name: part.name, params: part.params }; + }); + this.target.groupBy = _.map(this.groupByParts, function(part: any) { + return { type: part.def.type, params: part.params }; + }); + } + buildSelectMenu() { this.selectMenu = [ { text: 'Aggregate', value: 'aggregate' }, - { text: 'Math', value: 'math' }, { text: 'Special', value: 'special' }, { text: 'Alias', value: 'alias' }, { text: 'Column', value: 'column' }, @@ -88,6 +112,10 @@ export class PostgresQueryCtrl extends QueryCtrl { this.target.rawQuery = !this.target.rawQuery; } + resetPlusButton(button) {} + + // schema functions + getSchemaSegments() { return this.datasource .metricFindQuery(this.queryBuilder.buildSchemaQuery()) @@ -95,6 +123,13 @@ export class PostgresQueryCtrl extends QueryCtrl { .catch(this.handleQueryError.bind(this)); } + schemaChanged() { + this.target.schema = this.schemaSegment.value; + this.panelCtrl.refresh(); + } + + // table functions + getTableSegments() { return this.datasource .metricFindQuery(this.queryBuilder.buildTableQuery()) @@ -121,11 +156,6 @@ export class PostgresQueryCtrl extends QueryCtrl { this.panelCtrl.refresh(); } - schemaChanged() { - this.target.schema = this.schemaSegment.value; - this.panelCtrl.refresh(); - } - timeColumnChanged() { this.target.timeColumn = this.timeColumnSegment.value; this.panelCtrl.refresh(); @@ -193,13 +223,34 @@ export class PostgresQueryCtrl extends QueryCtrl { addSelectPart(selectParts, cat, subitem) { if ('submenu' in cat) { - this.queryModel.addSelectPart(selectParts, subitem.value); + this.addSelectPart2(selectParts, subitem.value); } else { - this.queryModel.addSelectPart(selectParts, cat.value); + this.addSelectPart2(selectParts, cat.value); } this.panelCtrl.refresh(); } + removeSelectPart(selectParts, part) { + // if we remove the field remove the whole statement + if (part.def.type === 'column') { + if (this.selectModels.length > 1) { + let modelsIndex = _.indexOf(this.selectModels, selectParts); + this.selectModels.splice(modelsIndex, 1); + } + } else { + let partIndex = _.indexOf(selectParts, part); + selectParts.splice(partIndex, 1); + } + + this.updatePersistedParts(); + } + + addSelectPart2(selectParts, type) { + let partModel = sqlPart.create({ type: type }); + partModel.def.addStrategy(selectParts, partModel, this); + this.updatePersistedParts(); + } + handleSelectPartEvent(selectParts, part, evt) { switch (evt.name) { case 'get-param-options': { @@ -221,7 +272,7 @@ export class PostgresQueryCtrl extends QueryCtrl { break; } case 'action': { - this.queryModel.removeSelectPart(selectParts, part); + this.removeSelectPart(selectParts, part); this.panelCtrl.refresh(); break; } @@ -244,7 +295,7 @@ export class PostgresQueryCtrl extends QueryCtrl { break; } case 'action': { - this.queryModel.removeGroupByPart(part, index); + this.removeGroupBy(part, index); this.panelCtrl.refresh(); break; } @@ -254,6 +305,53 @@ export class PostgresQueryCtrl extends QueryCtrl { } } + addGroupBy(partType, value) { + let params = [value]; + if (partType === 'time') { + params = ['1m', 'none']; + } + let partModel = sqlPart.create({ type: partType, params: params }); + + if (partType === 'time') { + // put timeGroup at start + this.groupByParts.splice(0, 0, partModel); + } else { + this.groupByParts.push(partModel); + } + + // add aggregates when adding group by + for (let i = 0; i < this.selectModels.length; i++) { + var selectParts = this.selectModels[i]; + if (!selectParts.some(part => part.def.type === 'aggregate')) { + let aggregate = sqlPart.create({ type: 'aggregate', params: ['avg'] }); + selectParts.splice(1, 0, aggregate); + if (!selectParts.some(part => part.def.type === 'alias')) { + let alias = sqlPart.create({ type: 'alias', params: [selectParts[0].part.params[0]] }); + selectParts.push(alias); + } + } + } + + this.updatePersistedParts(); + } + + removeGroupBy(part, index) { + if (part.def.type === 'time') { + // remove aggregations + this.selectModels = _.map(this.selectModels, (s: any) => { + return _.filter(s, (part: any) => { + if (part.def.type === 'aggregate') { + return false; + } + return true; + }); + }); + } + + this.groupByParts.splice(index, 1); + this.updatePersistedParts(); + } + buildWhereSegments() { // this.whereSegments = []; // this.whereSegments.push(sqlPart.create({ type: 'expression', params: ['value', '=', 'value'] })); @@ -286,7 +384,7 @@ export class PostgresQueryCtrl extends QueryCtrl { case 'action': { // remove element whereParts.splice(index, 1); - this.queryModel.updatePersistedParts(); + this.updatePersistedParts(); this.panelCtrl.refresh(); break; } @@ -307,11 +405,11 @@ export class PostgresQueryCtrl extends QueryCtrl { whereAddAction(part, index) { switch (this.whereAdd.type) { case 'macro': { - this.queryModel.whereParts.push(sqlPart.create({ type: 'macro', name: this.whereAdd.value, params: [] })); + this.whereParts.push(sqlPart.create({ type: 'macro', name: this.whereAdd.value, params: [] })); break; } default: { - this.queryModel.whereParts.push(sqlPart.create({ type: 'expression', params: ['value', '=', 'value'] })); + this.whereParts.push(sqlPart.create({ type: 'expression', params: ['value', '=', 'value'] })); } } @@ -319,7 +417,7 @@ export class PostgresQueryCtrl extends QueryCtrl { this.whereAdd.html = plusButton.html; this.whereAdd.value = plusButton.value; - this.queryModel.updatePersistedParts(); + this.updatePersistedParts(); this.panelCtrl.refresh(); } @@ -342,7 +440,7 @@ export class PostgresQueryCtrl extends QueryCtrl { groupByAction() { switch (this.groupByAdd.value) { default: { - this.queryModel.addGroupBy(this.groupByAdd.type, this.groupByAdd.value); + this.addGroupBy(this.groupByAdd.type, this.groupByAdd.value); } } From 844beb660d4cfc0928bc5cda6b97247f3b53e29b Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 14 Jul 2018 21:00:06 +0200 Subject: [PATCH 128/310] refactor PostgresQueryCtrl --- .../plugins/datasource/postgres/query_ctrl.ts | 67 +++++++------------ 1 file changed, 24 insertions(+), 43 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 27c3d1e50fa..19e2f8b3a18 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -112,9 +112,11 @@ export class PostgresQueryCtrl extends QueryCtrl { this.target.rawQuery = !this.target.rawQuery; } - resetPlusButton(button) {} - - // schema functions + resetPlusButton(button) { + let plusButton = this.uiSegmentSrv.newPlusButton(); + button.html = plusButton.html; + button.value = plusButton.value; + } getSchemaSegments() { return this.datasource @@ -128,8 +130,6 @@ export class PostgresQueryCtrl extends QueryCtrl { this.panelCtrl.refresh(); } - // table functions - getTableSegments() { return this.datasource .metricFindQuery(this.queryBuilder.buildTableQuery()) @@ -137,6 +137,11 @@ export class PostgresQueryCtrl extends QueryCtrl { .catch(this.handleQueryError.bind(this)); } + tableChanged() { + this.target.table = this.tableSegment.value; + this.panelCtrl.refresh(); + } + getTimeColumnSegments() { return this.datasource .metricFindQuery(this.queryBuilder.buildColumnQuery('time')) @@ -144,6 +149,11 @@ export class PostgresQueryCtrl extends QueryCtrl { .catch(this.handleQueryError.bind(this)); } + timeColumnChanged() { + this.target.timeColumn = this.timeColumnSegment.value; + this.panelCtrl.refresh(); + } + getMetricColumnSegments() { return this.datasource .metricFindQuery(this.queryBuilder.buildColumnQuery('metric')) @@ -151,16 +161,6 @@ export class PostgresQueryCtrl extends QueryCtrl { .catch(this.handleQueryError.bind(this)); } - tableChanged() { - this.target.table = this.tableSegment.value; - this.panelCtrl.refresh(); - } - - timeColumnChanged() { - this.target.timeColumn = this.timeColumnSegment.value; - this.panelCtrl.refresh(); - } - metricColumnChanged() { this.target.metricColumn = this.metricColumnSegment.value; this.panelCtrl.refresh(); @@ -188,7 +188,7 @@ export class PostgresQueryCtrl extends QueryCtrl { transformToSegments(config) { return results => { - var segments = _.map(results, segment => { + let segments = _.map(results, segment => { return this.uiSegmentSrv.newSegment({ value: segment.text, expandable: segment.expandable, @@ -197,7 +197,7 @@ export class PostgresQueryCtrl extends QueryCtrl { if (config.addTemplateVars) { for (let variable of this.templateSrv.variables) { - var value; + let value; value = '$' + variable.name; if (config.templateQuoter && variable.multi === false) { value = config.templateQuoter(value); @@ -222,17 +222,15 @@ export class PostgresQueryCtrl extends QueryCtrl { } addSelectPart(selectParts, cat, subitem) { - if ('submenu' in cat) { - this.addSelectPart2(selectParts, subitem.value); - } else { - this.addSelectPart2(selectParts, cat.value); - } + let partModel = sqlPart.create({ type: cat.value }); + partModel.def.addStrategy(selectParts, partModel, this); + this.updatePersistedParts(); this.panelCtrl.refresh(); } removeSelectPart(selectParts, part) { - // if we remove the field remove the whole statement if (part.def.type === 'column') { + // remove all parts of column unless its last column if (this.selectModels.length > 1) { let modelsIndex = _.indexOf(this.selectModels, selectParts); this.selectModels.splice(modelsIndex, 1); @@ -245,12 +243,6 @@ export class PostgresQueryCtrl extends QueryCtrl { this.updatePersistedParts(); } - addSelectPart2(selectParts, type) { - let partModel = sqlPart.create({ type: type }); - partModel.def.addStrategy(selectParts, partModel, this); - this.updatePersistedParts(); - } - handleSelectPartEvent(selectParts, part, evt) { switch (evt.name) { case 'get-param-options': { @@ -320,8 +312,7 @@ export class PostgresQueryCtrl extends QueryCtrl { } // add aggregates when adding group by - for (let i = 0; i < this.selectModels.length; i++) { - var selectParts = this.selectModels[i]; + for (let selectParts of this.selectModels) { if (!selectParts.some(part => part.def.type === 'aggregate')) { let aggregate = sqlPart.create({ type: 'aggregate', params: ['avg'] }); selectParts.splice(1, 0, aggregate); @@ -352,11 +343,6 @@ export class PostgresQueryCtrl extends QueryCtrl { this.updatePersistedParts(); } - buildWhereSegments() { - // this.whereSegments = []; - // this.whereSegments.push(sqlPart.create({ type: 'expression', params: ['value', '=', 'value'] })); - } - handleWherePartEvent(whereParts, part, evt, index) { switch (evt.name) { case 'get-param-options': { @@ -413,11 +399,8 @@ export class PostgresQueryCtrl extends QueryCtrl { } } - var plusButton = this.uiSegmentSrv.newPlusButton(); - this.whereAdd.html = plusButton.html; - this.whereAdd.value = plusButton.value; - this.updatePersistedParts(); + this.resetPlusButton(this.whereAdd); this.panelCtrl.refresh(); } @@ -444,9 +427,7 @@ export class PostgresQueryCtrl extends QueryCtrl { } } - var plusButton = this.uiSegmentSrv.newPlusButton(); - this.groupByAdd.html = plusButton.html; - this.groupByAdd.value = plusButton.value; + this.resetPlusButton(this.groupByAdd); this.panelCtrl.refresh(); } From fa66645b0f0c4cec7d730aa4930c2474c497a387 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 15 Jul 2018 09:36:49 +0200 Subject: [PATCH 129/310] refactor PostgresQuery --- .../datasource/postgres/postgres_query.ts | 64 ++++++++++--------- .../plugins/datasource/postgres/query_ctrl.ts | 2 +- .../postgres/specs/postgres_query.jest.ts | 61 +++++++++--------- 3 files changed, 67 insertions(+), 60 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 12323ec9fac..7826b455e99 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -15,7 +15,7 @@ export default class PostgresQuery { target.schema = target.schema || 'public'; target.format = target.format || 'time_series'; target.timeColumn = target.timeColumn || 'time'; - target.metricColumn = target.metricColumn || 'None'; + target.metricColumn = target.metricColumn || 'none'; target.groupBy = target.groupBy || []; target.where = target.where || []; @@ -57,6 +57,10 @@ export default class PostgresQuery { return _.find(this.target.groupBy, (g: any) => g.type === 'time'); } + hasMetricColumn() { + return this.target.metricColumn !== 'none'; + } + interpolateQueryStr(value, variable, defaultFormatFn) { // if no multi or include all do not regexEscape if (!variable.multi && !variable.includeAll) { @@ -83,7 +87,7 @@ export default class PostgresQuery { } } - query = this.buildQuery(target); + query = this.buildQuery(); if (interpolate) { query = this.templateSrv.replace(query, this.scopedVars, this.interpolateQueryStr); } @@ -91,7 +95,7 @@ export default class PostgresQuery { return query; } - buildTimeColumn(target) { + buildTimeColumn() { let timeGroup = this.hasGroupByTime(); let query; @@ -102,32 +106,32 @@ export default class PostgresQuery { } else { args = timeGroup.params[0]; } - query = '$__timeGroup(' + target.timeColumn + ',' + args + ')'; + query = '$__timeGroup(' + this.target.timeColumn + ',' + args + ')'; } else { - query = target.timeColumn + ' AS "time"'; + query = this.target.timeColumn + ' AS "time"'; } return query; } - buildMetricColumn(target) { - if (target.metricColumn !== 'None') { - return target.metricColumn + ' AS metric'; + buildMetricColumn() { + if (this.hasMetricColumn()) { + return this.target.metricColumn + ' AS metric'; } return ''; } - buildValueColumns(target) { + buildValueColumns() { let query = ''; - for (let i = 0; i < target.select.length; i++) { - query += ',\n ' + this.buildValueColumn(target, target.select[i]); + for (let column of this.target.select) { + query += ',\n ' + this.buildValueColumn(column); } return query; } - buildValueColumn(target, column) { + buildValueColumn(column) { let query = ''; let columnName = _.find(column, (g: any) => g.type === 'column'); @@ -141,15 +145,15 @@ export default class PostgresQuery { let special = _.find(column, (g: any) => g.type === 'special'); if (special) { let over = ''; - if (target.metricColumn !== 'None') { - over = 'PARTITION BY ' + target.metricColumn; + if (this.hasMetricColumn()) { + over = 'PARTITION BY ' + this.target.metricColumn; } switch (special.params[0]) { case 'increase': query = query + ' - lag(' + query + ') OVER (' + over + ')'; break; case 'rate': - let timeColumn = target.timeColumn; + let timeColumn = this.target.timeColumn; let curr = query; let prev = 'lag(' + curr + ') OVER (' + over + ')'; query = '(CASE WHEN ' + curr + ' >= ' + prev + ' THEN ' + curr + ' - ' + prev + ' ELSE ' + curr + ' END)'; @@ -166,12 +170,12 @@ export default class PostgresQuery { return query; } - buildWhereClause(target) { + buildWhereClause() { let query = ''; - let conditions = _.map(target.where, (tag, index) => { + let conditions = _.map(this.target.where, (tag, index) => { switch (tag.type) { case 'macro': - return tag.name + '(' + target.timeColumn + ')'; + return tag.name + '(' + this.target.timeColumn + ')'; break; case 'expression': return tag.params.join(' '); @@ -186,12 +190,12 @@ export default class PostgresQuery { return query; } - buildGroupByClause(target) { + buildGroupByClause() { let query = ''; let groupBySection = ''; - for (let i = 0; i < target.groupBy.length; i++) { - let part = target.groupBy[i]; + for (let i = 0; i < this.target.groupBy.length; i++) { + let part = this.target.groupBy[i]; if (i > 0) { groupBySection += ', '; } @@ -204,26 +208,26 @@ export default class PostgresQuery { if (groupBySection.length) { query = '\nGROUP BY ' + groupBySection; - if (target.metricColumn !== 'None') { + if (this.hasMetricColumn()) { query += ',2'; } } return query; } - buildQuery(target) { + buildQuery() { let query = 'SELECT'; - query += '\n ' + this.buildTimeColumn(target); - if (target.metricColumn !== 'None') { - query += '\n ' + this.buildMetricColumn(target); + query += '\n ' + this.buildTimeColumn(); + if (this.hasMetricColumn()) { + query += '\n ' + this.buildMetricColumn(); } - query += this.buildValueColumns(target); + query += this.buildValueColumns(); - query += '\nFROM ' + target.schema + '.' + target.table; + query += '\nFROM ' + this.target.schema + '.' + this.target.table; - query += this.buildWhereClause(target); - query += this.buildGroupByClause(target); + query += this.buildWhereClause(); + query += this.buildGroupByClause(); query += '\nORDER BY 1'; diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 19e2f8b3a18..609342423fd 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -214,7 +214,7 @@ export class PostgresQueryCtrl extends QueryCtrl { } if (config.addNone) { - segments.unshift(this.uiSegmentSrv.newSegment({ type: 'template', value: 'None', expandable: true })); + segments.unshift(this.uiSegmentSrv.newSegment({ type: 'template', value: 'none', expandable: true })); } return segments; diff --git a/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts index ef8fdea951b..9559cd3c350 100644 --- a/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts +++ b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts @@ -19,8 +19,10 @@ describe('PostgresQuery', function() { describe('When generating time column SQL', function() { let query = new PostgresQuery({}, templateSrv); - expect(query.buildTimeColumn({ timeColumn: 'time' })).toBe('time AS "time"'); - expect(query.buildTimeColumn({ timeColumn: '"time"' })).toBe('"time" AS "time"'); + query.target.timeColumn = 'time'; + expect(query.buildTimeColumn()).toBe('time AS "time"'); + query.target.timeColumn = '"time"'; + expect(query.buildTimeColumn()).toBe('"time" AS "time"'); }); describe('When generating time column SQL with group by time', function() { @@ -28,65 +30,66 @@ describe('PostgresQuery', function() { { timeColumn: 'time', groupBy: [{ type: 'time', params: ['5m', 'none'] }] }, templateSrv ); - expect(query.buildTimeColumn(query.target)).toBe('$__timeGroup(time,5m)'); + expect(query.buildTimeColumn()).toBe('$__timeGroup(time,5m)'); query = new PostgresQuery({ timeColumn: 'time', groupBy: [{ type: 'time', params: ['5m', 'NULL'] }] }, templateSrv); - expect(query.buildTimeColumn(query.target)).toBe('$__timeGroup(time,5m,NULL)'); + expect(query.buildTimeColumn()).toBe('$__timeGroup(time,5m,NULL)'); }); describe('When generating metric column SQL', function() { let query = new PostgresQuery({}, templateSrv); - expect(query.buildMetricColumn({ metricColumn: 'host' })).toBe('host AS metric'); - expect(query.buildMetricColumn({ metricColumn: '"host"' })).toBe('"host" AS metric'); + query.target.metricColumn = 'host'; + expect(query.buildMetricColumn()).toBe('host AS metric'); + query.target.metricColumn = '"host"'; + expect(query.buildMetricColumn()).toBe('"host" AS metric'); }); describe('When generating value column SQL', function() { let query = new PostgresQuery({}, templateSrv); let column = [{ type: 'column', params: ['value'] }]; - expect(query.buildValueColumn(query.target, column)).toBe('value'); + expect(query.buildValueColumn(column)).toBe('value'); column = [{ type: 'column', params: ['value'] }, { type: 'alias', params: ['alias'] }]; - expect(query.buildValueColumn(query.target, column)).toBe('value AS "alias"'); + expect(query.buildValueColumn(column)).toBe('value AS "alias"'); column = [ { type: 'column', params: ['v'] }, { type: 'alias', params: ['a'] }, { type: 'aggregate', params: ['max'] }, ]; - expect(query.buildValueColumn(query.target, column)).toBe('max(v) AS "a"'); + expect(query.buildValueColumn(column)).toBe('max(v) AS "a"'); column = [ { type: 'column', params: ['v'] }, { type: 'alias', params: ['a'] }, { type: 'special', params: ['increase'] }, ]; - expect(query.buildValueColumn(query.target, column)).toBe('v - lag(v) OVER () AS "a"'); + expect(query.buildValueColumn(column)).toBe('v - lag(v) OVER () AS "a"'); }); describe('When generating WHERE clause', function() { let query = new PostgresQuery({ where: [] }, templateSrv); - let target; - expect(query.buildWhereClause(query.target)).toBe(''); - target = { where: [{ type: 'macro', name: '$__timeFilter' }], timeColumn: 't' }; - expect(query.buildWhereClause(target)).toBe('\nWHERE\n $__timeFilter(t)'); - target = { where: [{ type: 'expression', params: ['v', '=', '1'] }], timeColumn: 't' }; - expect(query.buildWhereClause(target)).toBe('\nWHERE\n v = 1'); - target = { - where: [{ type: 'macro', name: '$__timeFilter' }, { type: 'expression', params: ['v', '=', '1'] }], - timeColumn: 't', - }; - expect(query.buildWhereClause(target)).toBe('\nWHERE\n $__timeFilter(t) AND\n v = 1'); + expect(query.buildWhereClause()).toBe(''); + + query.target.timeColumn = 't'; + query.target.where = [{ type: 'macro', name: '$__timeFilter' }]; + expect(query.buildWhereClause()).toBe('\nWHERE\n $__timeFilter(t)'); + + query.target.where = [{ type: 'expression', params: ['v', '=', '1'] }]; + expect(query.buildWhereClause()).toBe('\nWHERE\n v = 1'); + + query.target.where = [{ type: 'macro', name: '$__timeFilter' }, { type: 'expression', params: ['v', '=', '1'] }]; + expect(query.buildWhereClause()).toBe('\nWHERE\n $__timeFilter(t) AND\n v = 1'); }); describe('When generating GROUP BY clause', function() { - let query = new PostgresQuery({ groupBy: [] }, templateSrv); - let target; + let query = new PostgresQuery({ groupBy: [], metricColumn: 'none' }, templateSrv); - expect(query.buildGroupByClause(query.target)).toBe(''); - target = { groupBy: [{ type: 'time', params: ['5m'] }], metricColumn: 'None' }; - expect(query.buildGroupByClause(target)).toBe('\nGROUP BY 1'); - target = { groupBy: [{ type: 'time', params: ['5m'] }], metricColumn: 'm' }; - expect(query.buildGroupByClause(target)).toBe('\nGROUP BY 1,2'); + expect(query.buildGroupByClause()).toBe(''); + query.target.groupBy = [{ type: 'time', params: ['5m'] }]; + expect(query.buildGroupByClause()).toBe('\nGROUP BY 1'); + query.target.metricColumn = 'm'; + expect(query.buildGroupByClause()).toBe('\nGROUP BY 1,2'); }); describe('When generating complete statement', function() { @@ -99,6 +102,6 @@ describe('PostgresQuery', function() { let result = 'SELECT\n t AS "time",\n value\nFROM public.table\nORDER BY 1'; let query = new PostgresQuery(target, templateSrv); - expect(query.buildQuery(query.target)).toBe(result); + expect(query.buildQuery()).toBe(result); }); }); From d9648f1fe764cbc898da95fec430cb1212bcf99f Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 15 Jul 2018 09:42:24 +0200 Subject: [PATCH 130/310] fix bug in query generation with metricColumn --- public/app/plugins/datasource/postgres/postgres_query.ts | 2 +- .../plugins/datasource/postgres/specs/postgres_query.jest.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 7826b455e99..5b0bd539abe 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -220,7 +220,7 @@ export default class PostgresQuery { query += '\n ' + this.buildTimeColumn(); if (this.hasMetricColumn()) { - query += '\n ' + this.buildMetricColumn(); + query += ',\n ' + this.buildMetricColumn(); } query += this.buildValueColumns(); diff --git a/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts index 9559cd3c350..42f1d5243d5 100644 --- a/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts +++ b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts @@ -103,5 +103,9 @@ describe('PostgresQuery', function() { let query = new PostgresQuery(target, templateSrv); expect(query.buildQuery()).toBe(result); + + query.target.metricColumn = 'm'; + result = 'SELECT\n t AS "time",\n m AS metric,\n value\nFROM public.table\nORDER BY 1'; + expect(query.buildQuery()).toBe(result); }); }); From f48060a1bb9eb8da82d14c872c2502d1f490eef2 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 15 Jul 2018 09:52:32 +0200 Subject: [PATCH 131/310] remove render code from sql_part --- .../app/core/components/sql_part/sql_part.ts | 38 ------------------- .../plugins/datasource/postgres/query_ctrl.ts | 9 +++-- .../plugins/datasource/postgres/sql_part.ts | 23 +---------- 3 files changed, 6 insertions(+), 64 deletions(-) diff --git a/public/app/core/components/sql_part/sql_part.ts b/public/app/core/components/sql_part/sql_part.ts index 5b049237881..1929cfdadfa 100644 --- a/public/app/core/components/sql_part/sql_part.ts +++ b/public/app/core/components/sql_part/sql_part.ts @@ -9,7 +9,6 @@ export class SqlPartDef { wrapOpen: string; wrapClose: string; separator: string; - renderer: any; category: any; addStrategy: any; @@ -32,7 +31,6 @@ export class SqlPartDef { } this.params = options.params; this.defaultParams = options.defaultParams; - this.renderer = options.renderer; this.category = options.category; this.addStrategy = options.addStrategy; } @@ -74,40 +72,4 @@ export class SqlPart { this.part.params = this.params; } - - render(innerExpr: string) { - return this.def.renderer(this, innerExpr); - } -} - -export function functionRenderer(part, innerExpr) { - var str = part.def.type + '('; - var parameters = _.map(part.params, (value, index) => { - var paramType = part.def.params[index]; - if (paramType.type === 'time') { - if (value === 'auto') { - value = '$__interval'; - } - } - if (paramType.quote === 'single') { - return "'" + value + "'"; - } else if (paramType.quote === 'double') { - return '"' + value + '"'; - } - - return value; - }); - - if (innerExpr) { - parameters.unshift(innerExpr); - } - return str + parameters.join(', ') + ')'; -} - -export function suffixRenderer(part, innerExpr) { - return innerExpr + ' ' + part.params[0]; -} - -export function identityRenderer(part, innerExpr) { - return part.params[0]; } diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 609342423fd..0b389ae1877 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -1,6 +1,7 @@ import _ from 'lodash'; import { PostgresQueryBuilder } from './query_builder'; import { QueryCtrl } from 'app/plugins/sdk'; +import { SqlPart } from 'app/core/components/sql_part/sql_part'; import PostgresQuery from './postgres_query'; import sqlPart from './sql_part'; @@ -32,10 +33,10 @@ export class PostgresQueryCtrl extends QueryCtrl { whereAdd: any; timeColumnSegment: any; metricColumnSegment: any; - selectMenu: any; - selectModels: any[]; - groupByParts: any[]; - whereParts: any[]; + selectMenu: any[]; + selectModels: SqlPart[][]; + groupByParts: SqlPart[][]; + whereParts: SqlPart[][]; groupByAdd: any; /** @ngInject **/ diff --git a/public/app/plugins/datasource/postgres/sql_part.ts b/public/app/plugins/datasource/postgres/sql_part.ts index 3e4664c0caf..b9ecc036cd9 100644 --- a/public/app/plugins/datasource/postgres/sql_part.ts +++ b/public/app/plugins/datasource/postgres/sql_part.ts @@ -1,5 +1,5 @@ import _ from 'lodash'; -import { SqlPartDef, SqlPart, functionRenderer, suffixRenderer } from 'app/core/components/sql_part/sql_part'; +import { SqlPartDef, SqlPart } from 'app/core/components/sql_part/sql_part'; var index = []; @@ -16,18 +16,6 @@ function register(options: any) { index[options.type] = new SqlPartDef(options); } -function aliasRenderer(part, innerExpr) { - return innerExpr + ' AS ' + '"' + part.params[0] + '"'; -} - -function aggregateRenderer(part, innerExpr) { - return part.params[0] + '(' + innerExpr + ')'; -} - -function columnRenderer(part, innerExpr) { - return part.params[0]; -} - function replaceAggregationAddStrategy(selectParts, partModel) { var hasAlias = false; @@ -133,7 +121,6 @@ register({ addStrategy: addColumnStrategy, params: [{ type: 'column', dynamicLookup: true }], defaultParams: ['value'], - renderer: columnRenderer, }); register({ @@ -147,7 +134,6 @@ register({ { name: 'right', type: 'string', dynamicLookup: true }, ], defaultParams: ['value', '=', 'value'], - renderer: columnRenderer, }); register({ @@ -157,7 +143,6 @@ register({ addStrategy: addExpressionStrategy, params: [], defaultParams: [], - renderer: columnRenderer, }); register({ @@ -166,7 +151,6 @@ register({ addStrategy: replaceAggregationAddStrategy, params: [{ name: 'name', type: 'string', dynamicLookup: true }], defaultParams: ['avg'], - renderer: aggregateRenderer, }); register({ @@ -175,7 +159,6 @@ register({ addStrategy: addMathStrategy, params: [{ name: 'expr', type: 'string' }], defaultParams: [' / 100'], - renderer: suffixRenderer, }); register({ @@ -184,8 +167,6 @@ register({ addStrategy: addAliasStrategy, params: [{ name: 'name', type: 'string', quote: 'double' }], defaultParams: ['alias'], - renderMode: 'suffix', - renderer: aliasRenderer, }); register({ @@ -205,7 +186,6 @@ register({ }, ], defaultParams: ['$__interval', 'none'], - renderer: functionRenderer, }); register({ @@ -220,7 +200,6 @@ register({ ], defaultParams: ['increase'], addStrategy: replaceSpecialAddStrategy, - renderer: aggregateRenderer, }); export default { From f85c9c012e214a8a35b2321f51486b76a1c60a28 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 15 Jul 2018 12:26:22 +0200 Subject: [PATCH 132/310] refactor adding sqlPart --- .../app/core/components/sql_part/sql_part.ts | 4 - .../postgres/partials/query.editor.html | 2 +- .../plugins/datasource/postgres/query_ctrl.ts | 41 ++++++- .../plugins/datasource/postgres/sql_part.ts | 113 ------------------ 4 files changed, 39 insertions(+), 121 deletions(-) diff --git a/public/app/core/components/sql_part/sql_part.ts b/public/app/core/components/sql_part/sql_part.ts index 1929cfdadfa..5fc6072fc8c 100644 --- a/public/app/core/components/sql_part/sql_part.ts +++ b/public/app/core/components/sql_part/sql_part.ts @@ -9,8 +9,6 @@ export class SqlPartDef { wrapOpen: string; wrapClose: string; separator: string; - category: any; - addStrategy: any; constructor(options: any) { this.type = options.type; @@ -31,8 +29,6 @@ export class SqlPartDef { } this.params = options.params; this.defaultParams = options.defaultParams; - this.category = options.category; - this.addStrategy = options.addStrategy; } } diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index f0e15fc1a50..29952c2106b 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -44,7 +44,7 @@
diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 0b389ae1877..fbd6907f767 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -222,9 +222,44 @@ export class PostgresQueryCtrl extends QueryCtrl { }; } - addSelectPart(selectParts, cat, subitem) { - let partModel = sqlPart.create({ type: cat.value }); - partModel.def.addStrategy(selectParts, partModel, this); + addSelectPart(selectParts, item) { + let partModel = sqlPart.create({ type: item.value }); + let addAlias = false; + + switch (item.value) { + case 'column': + let parts = _.map(selectParts, function(part: any) { + return sqlPart.create({ type: part.def.type, params: _.clone(part.params) }); + }); + this.selectModels.push(parts); + break; + case 'aggregate': + case 'special': + let index = _.findIndex(selectParts, (p: any) => p.def.type === item.value); + if (index !== -1) { + selectParts[index] = partModel; + } else { + selectParts.splice(1, 0, partModel); + } + if (!_.find(selectParts, (p: any) => p.def.type === 'alias')) { + addAlias = true; + } + break; + case 'alias': + addAlias = true; + break; + } + + if (addAlias) { + // set initial alias name to column name + partModel = sqlPart.create({ type: 'alias', params: [selectParts[0].params[0]] }); + if (selectParts[selectParts.length - 1].def.type === 'alias') { + selectParts[selectParts.length - 1] = partModel; + } else { + selectParts.push(partModel); + } + } + this.updatePersistedParts(); this.panelCtrl.refresh(); } diff --git a/public/app/plugins/datasource/postgres/sql_part.ts b/public/app/plugins/datasource/postgres/sql_part.ts index b9ecc036cd9..be1be82f6b7 100644 --- a/public/app/plugins/datasource/postgres/sql_part.ts +++ b/public/app/plugins/datasource/postgres/sql_part.ts @@ -16,109 +16,9 @@ function register(options: any) { index[options.type] = new SqlPartDef(options); } -function replaceAggregationAddStrategy(selectParts, partModel) { - var hasAlias = false; - - // look for existing aggregation - for (var i = 0; i < selectParts.length; i++) { - var part = selectParts[i]; - if (part.def.type === 'aggregate') { - selectParts[i] = partModel; - return; - } - if (part.def.type === 'alias') { - hasAlias = true; - } - } - - // add alias if none exists yet - if (!hasAlias) { - var aliasModel = createPart({ type: 'alias', params: [selectParts[0].params[0]] }); - selectParts.push(aliasModel); - } - - selectParts.splice(1, 0, partModel); -} - -function replaceSpecialAddStrategy(selectParts, partModel) { - var hasAlias = false; - - // look for existing aggregation - for (var i = 0; i < selectParts.length; i++) { - var part = selectParts[i]; - if (part.def.type === 'special') { - selectParts[i] = partModel; - return; - } - if (part.def.type === 'alias') { - hasAlias = true; - } - } - - // add alias if none exists yet - if (!hasAlias) { - var aliasModel = createPart({ type: 'alias', params: [selectParts[0].params[0]] }); - selectParts.push(aliasModel); - } - - selectParts.splice(1, 0, partModel); -} - -function addMathStrategy(selectParts, partModel) { - var partCount = selectParts.length; - if (partCount > 0) { - // if last is math, replace it - if (selectParts[partCount - 1].def.type === 'math') { - selectParts[partCount - 1] = partModel; - return; - } - // if next to last is math, replace it - if (partCount > 1 && selectParts[partCount - 2].def.type === 'math') { - selectParts[partCount - 2] = partModel; - return; - } else if (selectParts[partCount - 1].def.type === 'alias') { - // if last is alias add it before - selectParts.splice(partCount - 1, 0, partModel); - return; - } - } - selectParts.push(partModel); -} - -function addAliasStrategy(selectParts, partModel) { - var partCount = selectParts.length; - if (partCount > 0) { - // if last is alias, replace it - if (selectParts[partCount - 1].def.type === 'alias') { - selectParts[partCount - 1] = partModel; - return; - } - } - selectParts.push(partModel); -} - -function addColumnStrategy(selectParts, partModel, query) { - // copy all parts - var parts = _.map(selectParts, function(part: any) { - return createPart({ type: part.def.type, params: _.clone(part.params) }); - }); - - query.selectModels.push(parts); -} - -function addExpressionStrategy(selectParts, partModel, query) { - // copy all parts - var parts = _.map(selectParts, function(part: any) { - return createPart({ type: part.def.type, params: _.clone(part.params) }); - }); - - query.selectModels.push(parts); -} - register({ type: 'column', style: 'label', - addStrategy: addColumnStrategy, params: [{ type: 'column', dynamicLookup: true }], defaultParams: ['value'], }); @@ -127,7 +27,6 @@ register({ type: 'expression', style: 'expression', label: 'Expr:', - addStrategy: addExpressionStrategy, params: [ { name: 'left', type: 'string', dynamicLookup: true }, { name: 'op', type: 'string', dynamicLookup: true }, @@ -140,7 +39,6 @@ register({ type: 'macro', style: 'label', label: 'Macro:', - addStrategy: addExpressionStrategy, params: [], defaultParams: [], }); @@ -148,23 +46,13 @@ register({ register({ type: 'aggregate', style: 'label', - addStrategy: replaceAggregationAddStrategy, params: [{ name: 'name', type: 'string', dynamicLookup: true }], defaultParams: ['avg'], }); -register({ - type: 'math', - style: 'label', - addStrategy: addMathStrategy, - params: [{ name: 'expr', type: 'string' }], - defaultParams: [' / 100'], -}); - register({ type: 'alias', style: 'label', - addStrategy: addAliasStrategy, params: [{ name: 'name', type: 'string', quote: 'double' }], defaultParams: ['alias'], }); @@ -199,7 +87,6 @@ register({ }, ], defaultParams: ['increase'], - addStrategy: replaceSpecialAddStrategy, }); export default { From c3c20ef2e2308248147059cee9239cf4cac45c7b Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 15 Jul 2018 12:52:00 +0200 Subject: [PATCH 133/310] remove unused import --- public/app/plugins/datasource/postgres/sql_part.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/sql_part.ts b/public/app/plugins/datasource/postgres/sql_part.ts index be1be82f6b7..6413587f4d9 100644 --- a/public/app/plugins/datasource/postgres/sql_part.ts +++ b/public/app/plugins/datasource/postgres/sql_part.ts @@ -1,4 +1,3 @@ -import _ from 'lodash'; import { SqlPartDef, SqlPart } from 'app/core/components/sql_part/sql_part'; var index = []; From 6e824e81bf43b7315d62266ac2f896adf1dae442 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 15 Jul 2018 15:14:12 +0200 Subject: [PATCH 134/310] fix rate special function when using group by --- .../components/sql_part/sql_part_editor.ts | 40 +++++++++---------- .../plugins/datasource/postgres/datasource.ts | 6 +-- .../datasource/postgres/postgres_query.ts | 4 ++ .../plugins/datasource/postgres/sql_part.ts | 4 +- 4 files changed, 29 insertions(+), 25 deletions(-) diff --git a/public/app/core/components/sql_part/sql_part_editor.ts b/public/app/core/components/sql_part/sql_part_editor.ts index 1b0210af1e7..f272a6d7254 100644 --- a/public/app/core/components/sql_part/sql_part_editor.ts +++ b/public/app/core/components/sql_part/sql_part_editor.ts @@ -2,7 +2,7 @@ import _ from 'lodash'; import $ from 'jquery'; import coreModule from 'app/core/core_module'; -var template = ` +let template = `
{{part.label}} {{part.def.wrapOpen}}{{part.def.wrapClose}} @@ -15,7 +15,7 @@ var template = ` /** @ngInject */ export function sqlPartEditorDirective($compile, templateSrv) { - var paramTemplate = ''; + let paramTemplate = ''; return { restrict: 'E', @@ -26,18 +26,18 @@ export function sqlPartEditorDirective($compile, templateSrv) { debounce: '@', }, link: function postLink($scope, elem) { - var part = $scope.part; - var partDef = part.def; - var $paramsContainer = elem.find('.query-part-parameters'); - var debounceLookup = $scope.debounce; - var cancelBlur = null; + let part = $scope.part; + let partDef = part.def; + let $paramsContainer = elem.find('.query-part-parameters'); + let debounceLookup = $scope.debounce; + let cancelBlur = null; $scope.partActions = []; function clickFuncParam(paramIndex) { /*jshint validthis:true */ - var $link = $(this); - var $input = $link.next(); + let $link = $(this); + let $input = $link.next(); $input.val(part.params[paramIndex]); $input.css('width', $link.width() + 16 + 'px'); @@ -47,7 +47,7 @@ export function sqlPartEditorDirective($compile, templateSrv) { $input.focus(); $input.select(); - var typeahead = $input.data('typeahead'); + let typeahead = $input.data('typeahead'); if (typeahead) { $input.val(''); typeahead.lookup(); @@ -62,8 +62,8 @@ export function sqlPartEditorDirective($compile, templateSrv) { function switchToLink($input, paramIndex) { /*jshint validthis:true */ - var $link = $input.prev(); - var newValue = $input.val(); + let $link = $input.prev(); + let newValue = $input.val(); if (newValue !== '' || part.def.params[paramIndex].optional) { $link.html(templateSrv.highlightVariablesAsHtml(newValue)); @@ -95,9 +95,9 @@ export function sqlPartEditorDirective($compile, templateSrv) { return; } - var typeaheadSource = function(query, callback) { + let typeaheadSource = function(query, callback) { if (param.options) { - var options = param.options; + let options = param.options; if (param.type === 'int') { options = _.map(options, function(val) { return val.toString(); @@ -108,7 +108,7 @@ export function sqlPartEditorDirective($compile, templateSrv) { $scope.$apply(function() { $scope.handleEvent({ $event: { name: 'get-param-options', param: param } }).then(function(result) { - var dynamicOptions = _.map(result, function(op) { + let dynamicOptions = _.map(result, function(op) { return op.value; }); @@ -138,10 +138,10 @@ export function sqlPartEditorDirective($compile, templateSrv) { }, }); - var typeahead = $input.data('typeahead'); + let typeahead = $input.data('typeahead'); typeahead.lookup = function() { this.query = this.$element.val() || ''; - var items = this.source(this.query, $.proxy(this.process, this)); + let items = this.source(this.query, $.proxy(this.process, this)); return items ? this.process(items) : items; }; @@ -170,9 +170,9 @@ export function sqlPartEditorDirective($compile, templateSrv) { $('' + partDef.separator + '').appendTo($paramsContainer); } - var paramValue = templateSrv.highlightVariablesAsHtml(part.params[index]); - var $paramLink = $('' + paramValue + ''); - var $input = $(paramTemplate); + let paramValue = templateSrv.highlightVariablesAsHtml(part.params[index]); + let $paramLink = $('' + paramValue + ''); + let $input = $(paramTemplate); $paramLink.appendTo($paramsContainer); $input.appendTo($paramsContainer); diff --git a/public/app/plugins/datasource/postgres/datasource.ts b/public/app/plugins/datasource/postgres/datasource.ts index 1324dbadebf..01c6154865a 100644 --- a/public/app/plugins/datasource/postgres/datasource.ts +++ b/public/app/plugins/datasource/postgres/datasource.ts @@ -27,17 +27,17 @@ export class PostgresDatasource { return value; } - var quotedValues = _.map(value, function(val) { + let quotedValues = _.map(value, function(val) { return "'" + val + "'"; }); return quotedValues.join(','); } query(options) { - var queries = _.filter(options.targets, target => { + let queries = _.filter(options.targets, target => { return target.hide !== true; }).map(target => { - var queryModel = new PostgresQuery(target, this.templateSrv, options.scopedVars); + let queryModel = new PostgresQuery(target, this.templateSrv, options.scopedVars); return { refId: target.refId, diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 5b0bd539abe..54ce0a96e26 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -154,6 +154,10 @@ export default class PostgresQuery { break; case 'rate': let timeColumn = this.target.timeColumn; + if (aggregate) { + timeColumn = 'min(' + timeColumn + ')'; + } + let curr = query; let prev = 'lag(' + curr + ') OVER (' + over + ')'; query = '(CASE WHEN ' + curr + ' >= ' + prev + ' THEN ' + curr + ' - ' + prev + ' ELSE ' + curr + ' END)'; diff --git a/public/app/plugins/datasource/postgres/sql_part.ts b/public/app/plugins/datasource/postgres/sql_part.ts index 6413587f4d9..b0265a645ed 100644 --- a/public/app/plugins/datasource/postgres/sql_part.ts +++ b/public/app/plugins/datasource/postgres/sql_part.ts @@ -1,9 +1,9 @@ import { SqlPartDef, SqlPart } from 'app/core/components/sql_part/sql_part'; -var index = []; +let index = []; function createPart(part): any { - var def = index[part.type]; + let def = index[part.type]; if (!def) { return null; } From 2fcb09b23d262d7422600699554e3e027940fdb7 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 15 Jul 2018 16:22:23 +0200 Subject: [PATCH 135/310] filter datatype for groupby suggestions --- .../datasource/postgres/query_builder.ts | 17 +++++++++++------ .../plugins/datasource/postgres/query_ctrl.ts | 2 +- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/public/app/plugins/datasource/postgres/query_builder.ts b/public/app/plugins/datasource/postgres/query_builder.ts index ffa8f6ba270..c57fc288f56 100644 --- a/public/app/plugins/datasource/postgres/query_builder.ts +++ b/public/app/plugins/datasource/postgres/query_builder.ts @@ -1,6 +1,11 @@ export class PostgresQueryBuilder { constructor(private target, private queryModel) {} + // quote identifier as literal to use in metadata queries + quoteIdentAsLiteral(value) { + return this.queryModel.quoteLiteral(this.queryModel.unquoteIdentifier(value)); + } + buildSchemaQuery() { let query = 'SELECT quote_ident(schema_name) FROM information_schema.schemata WHERE'; query += " schema_name NOT LIKE 'pg_%' AND schema_name NOT LIKE '\\_%' AND schema_name <> 'information_schema';"; @@ -14,11 +19,6 @@ export class PostgresQueryBuilder { return query; } - // quote identifier as literal to use in metadata queries - quoteIdentAsLiteral(value) { - return this.queryModel.quoteLiteral(this.queryModel.unquoteIdentifier(value)); - } - buildColumnQuery(type?: string) { let query = 'SELECT quote_ident(column_name) FROM information_schema.columns WHERE '; query += 'table_schema = ' + this.quoteIdentAsLiteral(this.target.schema); @@ -31,11 +31,16 @@ export class PostgresQueryBuilder { break; } case 'metric': { - query += " AND data_type IN ('text','char','varchar','integer','bigint')"; + query += " AND data_type IN ('text','char','varchar')"; break; } case 'value': { query += " AND data_type IN ('bigint','integer','double precision','real')"; + query += ' AND column_name <> ' + this.quoteIdentAsLiteral(this.target.timeColumn); + break; + } + case 'groupby': { + query += " AND data_type IN ('text','char','varchar')"; break; } } diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index fbd6907f767..0aa0c77a63d 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -442,7 +442,7 @@ export class PostgresQueryCtrl extends QueryCtrl { getGroupByOptions() { return this.datasource - .metricFindQuery(this.queryBuilder.buildColumnQuery()) + .metricFindQuery(this.queryBuilder.buildColumnQuery('groupby')) .then(tags => { var options = []; if (!this.queryModel.hasGroupByTime()) { From 070b556fd39e5c4169f779aa30245ff789e17ffb Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 15 Jul 2018 17:02:26 +0200 Subject: [PATCH 136/310] dont expand variables in rawSql --- .../datasource/postgres/postgres_query.ts | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 54ce0a96e26..60838814a7f 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -77,22 +77,16 @@ export default class PostgresQuery { render(interpolate?) { let target = this.target; - let query; - if (target.rawQuery) { - if (interpolate) { - return this.templateSrv.replace(target.rawSql, this.scopedVars, this.interpolateQueryStr); - } else { - return target.rawSql; - } + if (!target.rawQuery) { + target.rawSql = this.buildQuery(); } - query = this.buildQuery(); if (interpolate) { - query = this.templateSrv.replace(query, this.scopedVars, this.interpolateQueryStr); + return this.templateSrv.replace(target.rawSql, this.scopedVars, this.interpolateQueryStr); + } else { + return target.rawSql; } - this.target.rawSql = query; - return query; } buildTimeColumn() { From b28bf1a4f9285145e7d1a129f463dda816f61214 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 15 Jul 2018 17:12:03 +0200 Subject: [PATCH 137/310] rename PostgresQueryBuilder to PostgresMetaQuery --- .../{query_builder.ts => meta_query.ts} | 2 +- .../datasource/postgres/postgres_query.ts | 1 - .../plugins/datasource/postgres/query_ctrl.ts | 28 +++++++++---------- 3 files changed, 15 insertions(+), 16 deletions(-) rename public/app/plugins/datasource/postgres/{query_builder.ts => meta_query.ts} (98%) diff --git a/public/app/plugins/datasource/postgres/query_builder.ts b/public/app/plugins/datasource/postgres/meta_query.ts similarity index 98% rename from public/app/plugins/datasource/postgres/query_builder.ts rename to public/app/plugins/datasource/postgres/meta_query.ts index c57fc288f56..bf3100f56f2 100644 --- a/public/app/plugins/datasource/postgres/query_builder.ts +++ b/public/app/plugins/datasource/postgres/meta_query.ts @@ -1,4 +1,4 @@ -export class PostgresQueryBuilder { +export class PostgresMetaQuery { constructor(private target, private queryModel) {} // quote identifier as literal to use in metadata queries diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 60838814a7f..0a172de850d 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -2,7 +2,6 @@ import _ from 'lodash'; export default class PostgresQuery { target: any; - queryBuilder: any; templateSrv: any; scopedVars: any; diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 0aa0c77a63d..a0523e10c7c 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -1,5 +1,5 @@ import _ from 'lodash'; -import { PostgresQueryBuilder } from './query_builder'; +import { PostgresMetaQuery } from './meta_query'; import { QueryCtrl } from 'app/plugins/sdk'; import { SqlPart } from 'app/core/components/sql_part/sql_part'; import PostgresQuery from './postgres_query'; @@ -24,7 +24,7 @@ export class PostgresQueryCtrl extends QueryCtrl { showLastQuerySQL: boolean; formats: any[]; queryModel: PostgresQuery; - queryBuilder: PostgresQueryBuilder; + metaBuilder: PostgresMetaQuery; lastQueryMeta: QueryMeta; lastQueryError: string; showHelp: boolean; @@ -44,7 +44,7 @@ export class PostgresQueryCtrl extends QueryCtrl { super($scope, $injector); this.target = this.target; this.queryModel = new PostgresQuery(this.target, templateSrv, this.panel.scopedVars); - this.queryBuilder = new PostgresQueryBuilder(this.target, this.queryModel); + this.metaBuilder = new PostgresMetaQuery(this.target, this.queryModel); this.updateProjection(); this.formats = [{ text: 'Time series', value: 'time_series' }, { text: 'Table', value: 'table' }]; @@ -121,7 +121,7 @@ export class PostgresQueryCtrl extends QueryCtrl { getSchemaSegments() { return this.datasource - .metricFindQuery(this.queryBuilder.buildSchemaQuery()) + .metricFindQuery(this.metaBuilder.buildSchemaQuery()) .then(this.transformToSegments({})) .catch(this.handleQueryError.bind(this)); } @@ -133,7 +133,7 @@ export class PostgresQueryCtrl extends QueryCtrl { getTableSegments() { return this.datasource - .metricFindQuery(this.queryBuilder.buildTableQuery()) + .metricFindQuery(this.metaBuilder.buildTableQuery()) .then(this.transformToSegments({})) .catch(this.handleQueryError.bind(this)); } @@ -145,7 +145,7 @@ export class PostgresQueryCtrl extends QueryCtrl { getTimeColumnSegments() { return this.datasource - .metricFindQuery(this.queryBuilder.buildColumnQuery('time')) + .metricFindQuery(this.metaBuilder.buildColumnQuery('time')) .then(this.transformToSegments({})) .catch(this.handleQueryError.bind(this)); } @@ -157,7 +157,7 @@ export class PostgresQueryCtrl extends QueryCtrl { getMetricColumnSegments() { return this.datasource - .metricFindQuery(this.queryBuilder.buildColumnQuery('metric')) + .metricFindQuery(this.metaBuilder.buildColumnQuery('metric')) .then(this.transformToSegments({ addNone: true })) .catch(this.handleQueryError.bind(this)); } @@ -285,12 +285,12 @@ export class PostgresQueryCtrl extends QueryCtrl { switch (part.def.type) { case 'aggregate': return this.datasource - .metricFindQuery(this.queryBuilder.buildAggregateQuery()) + .metricFindQuery(this.metaBuilder.buildAggregateQuery()) .then(this.transformToSegments({})) .catch(this.handleQueryError.bind(this)); case 'column': return this.datasource - .metricFindQuery(this.queryBuilder.buildColumnQuery('value')) + .metricFindQuery(this.metaBuilder.buildColumnQuery('value')) .then(this.transformToSegments({})) .catch(this.handleQueryError.bind(this)); } @@ -314,7 +314,7 @@ export class PostgresQueryCtrl extends QueryCtrl { switch (evt.name) { case 'get-param-options': { return this.datasource - .metricFindQuery(this.queryBuilder.buildColumnQuery()) + .metricFindQuery(this.metaBuilder.buildColumnQuery()) .then(this.transformToSegments({})) .catch(this.handleQueryError.bind(this)); } @@ -385,16 +385,16 @@ export class PostgresQueryCtrl extends QueryCtrl { switch (evt.param.name) { case 'left': return this.datasource - .metricFindQuery(this.queryBuilder.buildColumnQuery()) + .metricFindQuery(this.metaBuilder.buildColumnQuery()) .then(this.transformToSegments({})) .catch(this.handleQueryError.bind(this)); case 'right': return this.datasource - .metricFindQuery(this.queryBuilder.buildValueQuery(part.params[0])) + .metricFindQuery(this.metaBuilder.buildValueQuery(part.params[0])) .then(this.transformToSegments({ addTemplateVars: true, templateQuoter: this.queryModel.quoteLiteral })) .catch(this.handleQueryError.bind(this)); case 'op': - return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '<', '<=', '>', '>=', 'IN'])); + return this.$q.when(this.uiSegmentSrv.newOperators(['=', '!=', '<', '<=', '>', '>=', 'IN', 'NOT IN'])); default: return this.$q.when([]); } @@ -442,7 +442,7 @@ export class PostgresQueryCtrl extends QueryCtrl { getGroupByOptions() { return this.datasource - .metricFindQuery(this.queryBuilder.buildColumnQuery('groupby')) + .metricFindQuery(this.metaBuilder.buildColumnQuery('groupby')) .then(tags => { var options = []; if (!this.queryModel.hasGroupByTime()) { From b49d10104e785ee3c1195b696928727c9aa55ba3 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 15 Jul 2018 19:13:19 +0200 Subject: [PATCH 138/310] add popover for metric column --- .../plugins/datasource/postgres/partials/query.editor.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 29952c2106b..90dbdf2eee0 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -19,7 +19,10 @@ - +
From 43686616a06532f9aebd5b5c4acaebff17a4193c Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 15 Jul 2018 22:25:05 +0200 Subject: [PATCH 139/310] add query to find metric table --- .../plugins/datasource/postgres/meta_query.ts | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/public/app/plugins/datasource/postgres/meta_query.ts b/public/app/plugins/datasource/postgres/meta_query.ts index bf3100f56f2..df5a61ce3c4 100644 --- a/public/app/plugins/datasource/postgres/meta_query.ts +++ b/public/app/plugins/datasource/postgres/meta_query.ts @@ -6,6 +6,53 @@ export class PostgresMetaQuery { return this.queryModel.quoteLiteral(this.queryModel.unquoteIdentifier(value)); } + findMetricTable() { + // query that returns first table found that has a timestamptz column and a float column + let query = ` +SELECT + table_schema, + table_name, + ( SELECT + column_name + FROM information_schema.columns c + WHERE + c.table_schema = t.table_schema AND + c.table_name = t.table_name AND + udt_name IN ('timestamptz','timestamp') + ORDER BY ordinal_position LIMIT 1 + ) AS time_column, + ( SELECT + column_name + FROM information_schema.columns c + WHERE + c.table_schema = t.table_schema AND + c.table_name = t.table_name AND + udt_name='float8' + ORDER BY ordinal_position LIMIT 1 + ) AS value_column +FROM information_schema.tables t +WHERE + table_schema !~* '^_|^pg_|information_schema' AND + EXISTS + ( SELECT 1 + FROM information_schema.columns c + WHERE + c.table_schema = t.table_schema AND + c.table_name = t.table_name AND + udt_name IN ('timestamptz','timestamp') + ) + ( SELECT 1 + FROM information_schema.columns c + WHERE + c.table_schema = t.table_schema AND + c.table_name = t.table_name AND + udt_name='float8' + ) +LIMIT 1 +;`; + return query; + } + buildSchemaQuery() { let query = 'SELECT quote_ident(schema_name) FROM information_schema.schemata WHERE'; query += " schema_name NOT LIKE 'pg_%' AND schema_name NOT LIKE '\\_%' AND schema_name <> 'information_schema';"; From 7f348f38360963d0fdbb79e35783175a28276c0b Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 15 Jul 2018 22:26:52 +0200 Subject: [PATCH 140/310] dont run queries if target has no table set --- public/app/plugins/datasource/postgres/postgres_query.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 0a172de850d..6bec3f87a85 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -17,7 +17,7 @@ export default class PostgresQuery { target.metricColumn = target.metricColumn || 'none'; target.groupBy = target.groupBy || []; - target.where = target.where || []; + target.where = target.where || [{ type: 'macro', params: ['$__timeFilter'] }]; target.select = target.select || [[{ type: 'column', params: ['value'] }]]; // handle pre query gui panels gracefully @@ -77,6 +77,10 @@ export default class PostgresQuery { render(interpolate?) { let target = this.target; + if (!('table' in this.target)) { + return ''; + } + if (!target.rawQuery) { target.rawSql = this.buildQuery(); } From 0e608a08c25eaa5308021564852a8f3607a9f28e Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 15 Jul 2018 22:58:25 +0200 Subject: [PATCH 141/310] fix test for query generation --- public/app/plugins/datasource/postgres/postgres_query.ts | 2 +- .../plugins/datasource/postgres/specs/postgres_query.jest.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 6bec3f87a85..4e60eee9e11 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -17,7 +17,7 @@ export default class PostgresQuery { target.metricColumn = target.metricColumn || 'none'; target.groupBy = target.groupBy || []; - target.where = target.where || [{ type: 'macro', params: ['$__timeFilter'] }]; + target.where = target.where || [{ type: 'macro', name: '$__timeFilter', params: [] }]; target.select = target.select || [[{ type: 'column', params: ['value'] }]]; // handle pre query gui panels gracefully diff --git a/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts index 42f1d5243d5..d9b3f46b4bf 100644 --- a/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts +++ b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts @@ -98,6 +98,7 @@ describe('PostgresQuery', function() { schema: 'public', table: 'table', select: [[{ type: 'column', params: ['value'] }]], + where: [], }; let result = 'SELECT\n t AS "time",\n value\nFROM public.table\nORDER BY 1'; let query = new PostgresQuery(target, templateSrv); From 9f0b4e0aa779072d64da581976d74b825afc49fe Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 18 Jul 2018 13:29:47 +0200 Subject: [PATCH 142/310] add groupby when adding first aggregate --- public/app/plugins/datasource/postgres/query_ctrl.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index a0523e10c7c..68eac1cf34a 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -234,6 +234,10 @@ export class PostgresQueryCtrl extends QueryCtrl { this.selectModels.push(parts); break; case 'aggregate': + // add group by if no group by yet + if (this.target.groupBy.length === 0) { + this.addGroupBy('time', '1m'); + } case 'special': let index = _.findIndex(selectParts, (p: any) => p.def.type === item.value); if (index !== -1) { From 84d7743939c6dac456f25c1ad7103142b21e8136 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 21 Jul 2018 09:57:42 +0200 Subject: [PATCH 143/310] fix pre gui queries shortcircuit --- pkg/tsdb/postgres/postgres.go | 14 +++++++++++++- .../plugins/datasource/postgres/postgres_query.ts | 3 ++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/pkg/tsdb/postgres/postgres.go b/pkg/tsdb/postgres/postgres.go index fdf09216e51..12270da2a48 100644 --- a/pkg/tsdb/postgres/postgres.go +++ b/pkg/tsdb/postgres/postgres.go @@ -170,6 +170,8 @@ func (e PostgresQueryEndpoint) transformToTimeSeries(query *tsdb.Query, rows *co rowCount := 0 timeIndex := -1 metricIndex := -1 + metricPrefix := false + metricPrefixValue := "" // check columns of resultset: a column named time is mandatory // the first text column is treated as metric name unless a column named metric is present @@ -179,6 +181,10 @@ func (e PostgresQueryEndpoint) transformToTimeSeries(query *tsdb.Query, rows *co timeIndex = i case "metric": metricIndex = i + // use metric column as prefix with multiple value columns + if len(columnNames) > 3 { + metricPrefix = true + } default: if metricIndex == -1 { switch columnTypes[i].DatabaseTypeName() { @@ -234,7 +240,11 @@ func (e PostgresQueryEndpoint) transformToTimeSeries(query *tsdb.Query, rows *co if metricIndex >= 0 { if columnValue, ok := values[metricIndex].(string); ok { - metric = columnValue + if metricPrefix { + metricPrefixValue = columnValue + } else { + metric = columnValue + } } else { return fmt.Errorf("Column metric must be of type char,varchar or text, got: %T %v", values[metricIndex], values[metricIndex]) } @@ -251,6 +261,8 @@ func (e PostgresQueryEndpoint) transformToTimeSeries(query *tsdb.Query, rows *co if metricIndex == -1 { metric = col + } else if metricPrefix { + metric = metricPrefixValue + " " + col } series, exist := pointsBySeries[metric] diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 4e60eee9e11..2222e512e32 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -77,7 +77,8 @@ export default class PostgresQuery { render(interpolate?) { let target = this.target; - if (!('table' in this.target)) { + // new query with no table set yet + if (!this.target.rawQuery && !('table' in this.target)) { return ''; } From 7af9cd7dfc722eda93e6e2449ce29feca05b27f0 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 22 Jul 2018 15:06:59 +0200 Subject: [PATCH 144/310] set explicit order for rate and increase --- .../plugins/datasource/postgres/postgres_query.ts | 12 ++++++++++-- .../datasource/postgres/specs/postgres_query.jest.ts | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 2222e512e32..a16a9e5b458 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -136,16 +136,24 @@ export default class PostgresQuery { query = columnName.params[0]; let aggregate = _.find(column, (g: any) => g.type === 'aggregate'); + let special = _.find(column, (g: any) => g.type === 'special'); + if (aggregate) { - query = aggregate.params[0] + '(' + query + ')'; + if (special) { + query = aggregate.params[0] + '(' + query + ' ORDER BY ' + this.target.timeColumn + ')'; + } else { + query = aggregate.params[0] + '(' + query + ')'; + } } - let special = _.find(column, (g: any) => g.type === 'special'); if (special) { let over = ''; if (this.hasMetricColumn()) { over = 'PARTITION BY ' + this.target.metricColumn; } + if (!aggregate) { + over += 'ORDER BY ' + this.target.timeColumn; + } switch (special.params[0]) { case 'increase': query = query + ' - lag(' + query + ') OVER (' + over + ')'; diff --git a/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts index d9b3f46b4bf..659ca94496c 100644 --- a/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts +++ b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts @@ -63,7 +63,7 @@ describe('PostgresQuery', function() { { type: 'alias', params: ['a'] }, { type: 'special', params: ['increase'] }, ]; - expect(query.buildValueColumn(column)).toBe('v - lag(v) OVER () AS "a"'); + expect(query.buildValueColumn(column)).toBe('v - lag(v) OVER (ORDER BY time) AS "a"'); }); describe('When generating WHERE clause', function() { From e1a37cf27502e8a6aa9656ba13c5e1bf767bbf7b Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 22 Jul 2018 17:12:30 +0200 Subject: [PATCH 145/310] add order by to metadata queries --- public/app/plugins/datasource/postgres/meta_query.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/meta_query.ts b/public/app/plugins/datasource/postgres/meta_query.ts index df5a61ce3c4..66ff8867393 100644 --- a/public/app/plugins/datasource/postgres/meta_query.ts +++ b/public/app/plugins/datasource/postgres/meta_query.ts @@ -55,7 +55,7 @@ LIMIT 1 buildSchemaQuery() { let query = 'SELECT quote_ident(schema_name) FROM information_schema.schemata WHERE'; - query += " schema_name NOT LIKE 'pg_%' AND schema_name NOT LIKE '\\_%' AND schema_name <> 'information_schema';"; + query += " schema_name !~* '^pg_|^_|information_schema' ORDER BY schema_name"; return query; } @@ -63,6 +63,7 @@ LIMIT 1 buildTableQuery() { let query = 'SELECT quote_ident(table_name) FROM information_schema.tables WHERE '; query += 'table_schema = ' + this.quoteIdentAsLiteral(this.target.schema); + query += ' ORDER BY table_name'; return query; } @@ -92,6 +93,8 @@ LIMIT 1 } } + query += ' ORDER BY column_name'; + return query; } From b3ebc8609383b94584e3d1f083ae15940044c422 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Mon, 23 Jul 2018 07:52:42 +0200 Subject: [PATCH 146/310] fix window function query without group by --- public/app/plugins/datasource/postgres/postgres_query.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index a16a9e5b458..d0aa8a45841 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -152,7 +152,7 @@ export default class PostgresQuery { over = 'PARTITION BY ' + this.target.metricColumn; } if (!aggregate) { - over += 'ORDER BY ' + this.target.timeColumn; + over += ' ORDER BY ' + this.target.timeColumn; } switch (special.params[0]) { case 'increase': From 97f24733f5c2b3f1654663a11a8708f12f994820 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 27 Jul 2018 10:58:08 +0200 Subject: [PATCH 147/310] remove tableschema from query builder ui --- .../plugins/datasource/postgres/meta_query.ts | 51 ++++++++++++------- .../postgres/partials/query.editor.html | 1 - .../datasource/postgres/postgres_query.ts | 11 ++-- .../plugins/datasource/postgres/query_ctrl.ts | 15 ------ .../postgres/specs/postgres_query.jest.ts | 36 +++++++++++-- 5 files changed, 73 insertions(+), 41 deletions(-) diff --git a/public/app/plugins/datasource/postgres/meta_query.ts b/public/app/plugins/datasource/postgres/meta_query.ts index 66ff8867393..64271c022cc 100644 --- a/public/app/plugins/datasource/postgres/meta_query.ts +++ b/public/app/plugins/datasource/postgres/meta_query.ts @@ -10,7 +10,6 @@ export class PostgresMetaQuery { // query that returns first table found that has a timestamptz column and a float column let query = ` SELECT - table_schema, table_name, ( SELECT column_name @@ -32,7 +31,10 @@ SELECT ) AS value_column FROM information_schema.tables t WHERE - table_schema !~* '^_|^pg_|information_schema' AND + table_schema IN ( + SELECT CASE WHEN trim(unnest) = \'"$user"\' THEN user ELSE trim(unnest) END + FROM unnest(string_to_array(current_setting(\'search_path\'),\',\')) + ) AND EXISTS ( SELECT 1 FROM information_schema.columns c @@ -53,23 +55,30 @@ LIMIT 1 return query; } - buildSchemaQuery() { - let query = 'SELECT quote_ident(schema_name) FROM information_schema.schemata WHERE'; - query += " schema_name !~* '^pg_|^_|information_schema' ORDER BY schema_name"; - - return query; - } - buildTableQuery() { - let query = 'SELECT quote_ident(table_name) FROM information_schema.tables WHERE '; - query += 'table_schema = ' + this.quoteIdentAsLiteral(this.target.schema); - query += ' ORDER BY table_name'; + let query = ` +SELECT quote_ident(table_name) +FROM information_schema.tables +WHERE + table_schema IN ( + SELECT CASE WHEN trim(unnest) = \'"$user"\' THEN user ELSE trim(unnest) END + FROM unnest(string_to_array(current_setting(\'search_path\'),\',\')) + ) +ORDER BY table_name`; return query; } buildColumnQuery(type?: string) { - let query = 'SELECT quote_ident(column_name) FROM information_schema.columns WHERE '; - query += 'table_schema = ' + this.quoteIdentAsLiteral(this.target.schema); + let query = ` +SELECT quote_ident(column_name) +FROM information_schema.columns +WHERE + table_schema IN ( + SELECT CASE WHEN trim(unnest) = \'"$user"\' THEN user ELSE trim(unnest) END + FROM unnest(string_to_array(current_setting(\'search_path\'),\',\')) + LIMIT 1 + ) +`; query += ' AND table_name = ' + this.quoteIdentAsLiteral(this.target.table); switch (type) { @@ -100,15 +109,23 @@ LIMIT 1 buildValueQuery(column: string) { let query = 'SELECT DISTINCT quote_literal(' + column + ')'; - query += ' FROM ' + this.target.schema + '.' + this.target.table; + query += ' FROM ' + this.target.table; query += ' WHERE $__timeFilter(' + this.target.timeColumn + ')'; query += ' ORDER BY 1 LIMIT 100'; return query; } buildDatatypeQuery(column: string) { - let query = 'SELECT data_type FROM information_schema.columns WHERE '; - query += ' table_schema = ' + this.quoteIdentAsLiteral(this.target.schema); + let query = ` +SELECT data_type +FROM information_schema.columns +WHERE + table_schema IN ( + SELECT CASE WHEN trim(unnest) = \'"$user"\' THEN user ELSE trim(unnest) END + FROM unnest(string_to_array(current_setting(\'search_path\'),\',\')) + LIMIT 1 + ) +`; query += ' AND table_name = ' + this.quoteIdentAsLiteral(this.target.table); query += ' AND column_name = ' + this.quoteIdentAsLiteral(column); return query; diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 90dbdf2eee0..68711f3ea0b 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -13,7 +13,6 @@
- diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index d0aa8a45841..3c1b1b681b4 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -11,7 +11,6 @@ export default class PostgresQuery { this.templateSrv = templateSrv; this.scopedVars = scopedVars; - target.schema = target.schema || 'public'; target.format = target.format || 'time_series'; target.timeColumn = target.timeColumn || 'time'; target.metricColumn = target.metricColumn || 'none'; @@ -147,13 +146,15 @@ export default class PostgresQuery { } if (special) { - let over = ''; + let overParts = []; if (this.hasMetricColumn()) { - over = 'PARTITION BY ' + this.target.metricColumn; + overParts.push('PARTITION BY ' + this.target.metricColumn); } if (!aggregate) { - over += ' ORDER BY ' + this.target.timeColumn; + overParts.push('ORDER BY ' + this.target.timeColumn); } + + let over = overParts.join(' '); switch (special.params[0]) { case 'increase': query = query + ' - lag(' + query + ') OVER (' + over + ')'; @@ -234,7 +235,7 @@ export default class PostgresQuery { } query += this.buildValueColumns(); - query += '\nFROM ' + this.target.schema + '.' + this.target.table; + query += '\nFROM ' + this.target.table; query += this.buildWhereClause(); query += this.buildGroupByClause(); diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 68eac1cf34a..97db612e7db 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -28,7 +28,6 @@ export class PostgresQueryCtrl extends QueryCtrl { lastQueryMeta: QueryMeta; lastQueryError: string; showHelp: boolean; - schemaSegment: any; tableSegment: any; whereAdd: any; timeColumnSegment: any; @@ -59,8 +58,6 @@ export class PostgresQueryCtrl extends QueryCtrl { } } - this.schemaSegment = uiSegmentSrv.newSegment(this.target.schema); - if (!this.target.table) { this.tableSegment = uiSegmentSrv.newSegment({ value: 'select table', fake: true }); } else { @@ -119,18 +116,6 @@ export class PostgresQueryCtrl extends QueryCtrl { button.value = plusButton.value; } - getSchemaSegments() { - return this.datasource - .metricFindQuery(this.metaBuilder.buildSchemaQuery()) - .then(this.transformToSegments({})) - .catch(this.handleQueryError.bind(this)); - } - - schemaChanged() { - this.target.schema = this.schemaSegment.value; - this.panelCtrl.refresh(); - } - getTableSegments() { return this.datasource .metricFindQuery(this.metaBuilder.buildTableQuery()) diff --git a/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts index 659ca94496c..33d997d2d0a 100644 --- a/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts +++ b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts @@ -66,6 +66,37 @@ describe('PostgresQuery', function() { expect(query.buildValueColumn(column)).toBe('v - lag(v) OVER (ORDER BY time) AS "a"'); }); + describe('When generating value column SQL with metric column', function() { + let query = new PostgresQuery({}, templateSrv); + query.target.metricColumn = 'host'; + + let column = [{ type: 'column', params: ['value'] }]; + expect(query.buildValueColumn(column)).toBe('value'); + column = [{ type: 'column', params: ['value'] }, { type: 'alias', params: ['alias'] }]; + expect(query.buildValueColumn(column)).toBe('value AS "alias"'); + column = [ + { type: 'column', params: ['v'] }, + { type: 'alias', params: ['a'] }, + { type: 'aggregate', params: ['max'] }, + ]; + expect(query.buildValueColumn(column)).toBe('max(v) AS "a"'); + column = [ + { type: 'column', params: ['v'] }, + { type: 'alias', params: ['a'] }, + { type: 'special', params: ['increase'] }, + ]; + expect(query.buildValueColumn(column)).toBe('v - lag(v) OVER (PARTITION BY host ORDER BY time) AS "a"'); + column = [ + { type: 'column', params: ['v'] }, + { type: 'alias', params: ['a'] }, + { type: 'aggregate', params: ['max'] }, + { type: 'special', params: ['increase'] }, + ]; + expect(query.buildValueColumn(column)).toBe( + 'max(v ORDER BY time) - lag(max(v ORDER BY time)) OVER (PARTITION BY host) AS "a"' + ); + }); + describe('When generating WHERE clause', function() { let query = new PostgresQuery({ where: [] }, templateSrv); @@ -95,18 +126,17 @@ describe('PostgresQuery', function() { describe('When generating complete statement', function() { let target = { timeColumn: 't', - schema: 'public', table: 'table', select: [[{ type: 'column', params: ['value'] }]], where: [], }; - let result = 'SELECT\n t AS "time",\n value\nFROM public.table\nORDER BY 1'; + let result = 'SELECT\n t AS "time",\n value\nFROM table\nORDER BY 1'; let query = new PostgresQuery(target, templateSrv); expect(query.buildQuery()).toBe(result); query.target.metricColumn = 'm'; - result = 'SELECT\n t AS "time",\n m AS metric,\n value\nFROM public.table\nORDER BY 1'; + result = 'SELECT\n t AS "time",\n m AS metric,\n value\nFROM table\nORDER BY 1'; expect(query.buildQuery()).toBe(result); }); }); From ad26a319c50753053009fcb8b539cf118662e051 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Fri, 27 Jul 2018 14:02:12 +0200 Subject: [PATCH 148/310] refactor schema query generation --- .../plugins/datasource/postgres/meta_query.ts | 55 ++++++++++++------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/public/app/plugins/datasource/postgres/meta_query.ts b/public/app/plugins/datasource/postgres/meta_query.ts index 64271c022cc..c2fc8647137 100644 --- a/public/app/plugins/datasource/postgres/meta_query.ts +++ b/public/app/plugins/datasource/postgres/meta_query.ts @@ -55,31 +55,46 @@ LIMIT 1 return query; } - buildTableQuery() { + buildSchemaConstraint() { let query = ` -SELECT quote_ident(table_name) -FROM information_schema.tables -WHERE - table_schema IN ( - SELECT CASE WHEN trim(unnest) = \'"$user"\' THEN user ELSE trim(unnest) END - FROM unnest(string_to_array(current_setting(\'search_path\'),\',\')) - ) -ORDER BY table_name`; +table_schema IN ( + SELECT CASE WHEN trim(unnest) = \'"$user"\' THEN user ELSE trim(unnest) END + FROM unnest(string_to_array(current_setting(\'search_path\'),\',\')) +)`; + return query; + } + + buildTableConstraint(table: string) { + let query = ''; + + // check for schema qualified table + if (table.includes('.')) { + let parts = table.split('.'); + query = 'table_schema = ' + this.quoteIdentAsLiteral(parts[0]); + query += ' AND table_name = ' + this.quoteIdentAsLiteral(parts[1]); + return query; + } else { + query = ` +table_schema IN ( + SELECT CASE WHEN trim(unnest) = \'"$user"\' THEN user ELSE trim(unnest) END + FROM unnest(string_to_array(current_setting(\'search_path\'),\',\')) +)`; + query += ' AND table_name = ' + this.quoteIdentAsLiteral(table); + + return query; + } + } + + buildTableQuery() { + let query = 'SELECT quote_ident(table_name) FROM information_schema.tables WHERE '; + query += this.buildSchemaConstraint(); + query += ' ORDER BY table_name'; return query; } buildColumnQuery(type?: string) { - let query = ` -SELECT quote_ident(column_name) -FROM information_schema.columns -WHERE - table_schema IN ( - SELECT CASE WHEN trim(unnest) = \'"$user"\' THEN user ELSE trim(unnest) END - FROM unnest(string_to_array(current_setting(\'search_path\'),\',\')) - LIMIT 1 - ) -`; - query += ' AND table_name = ' + this.quoteIdentAsLiteral(this.target.table); + let query = 'SELECT quote_ident(column_name) FROM information_schema.columns WHERE '; + query += this.buildTableConstraint(this.target.table); switch (type) { case 'time': { From 6ca7a0397514596aa07baf462e4ad1e109f2a9b4 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 28 Jul 2018 12:53:36 +0200 Subject: [PATCH 149/310] consistent nameing fro group and select --- .../plugins/datasource/postgres/meta_query.ts | 2 +- .../postgres/partials/query.editor.html | 8 +-- .../datasource/postgres/postgres_query.ts | 28 ++++----- .../plugins/datasource/postgres/query_ctrl.ts | 62 +++++++++---------- .../postgres/specs/postgres_query.jest.ts | 14 ++--- 5 files changed, 57 insertions(+), 57 deletions(-) diff --git a/public/app/plugins/datasource/postgres/meta_query.ts b/public/app/plugins/datasource/postgres/meta_query.ts index c2fc8647137..fd29121313d 100644 --- a/public/app/plugins/datasource/postgres/meta_query.ts +++ b/public/app/plugins/datasource/postgres/meta_query.ts @@ -111,7 +111,7 @@ table_schema IN ( query += ' AND column_name <> ' + this.quoteIdentAsLiteral(this.target.timeColumn); break; } - case 'groupby': { + case 'group': { query += " AND data_type IN ('text','char','varchar')"; break; } diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 68711f3ea0b..3d3b7c43388 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -31,7 +31,7 @@
-
+
- + handle-event="ctrl.onGroupPartEvent(part, $index, $event)">
- +
diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 3c1b1b681b4..a6f45dbcee8 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -15,7 +15,7 @@ export default class PostgresQuery { target.timeColumn = target.timeColumn || 'time'; target.metricColumn = target.metricColumn || 'none'; - target.groupBy = target.groupBy || []; + target.group = target.group || []; target.where = target.where || [{ type: 'macro', name: '$__timeFilter', params: [] }]; target.select = target.select || [[{ type: 'column', params: ['value'] }]]; @@ -51,8 +51,8 @@ export default class PostgresQuery { return "'" + value.replace("'", "''") + "'"; } - hasGroupByTime() { - return _.find(this.target.groupBy, (g: any) => g.type === 'time'); + hasTimeGroup() { + return _.find(this.target.group, (g: any) => g.type === 'time'); } hasMetricColumn() { @@ -93,7 +93,7 @@ export default class PostgresQuery { } buildTimeColumn() { - let timeGroup = this.hasGroupByTime(); + let timeGroup = this.hasTimeGroup(); let query; if (timeGroup) { @@ -201,24 +201,24 @@ export default class PostgresQuery { return query; } - buildGroupByClause() { + buildGroupClause() { let query = ''; - let groupBySection = ''; + let groupSection = ''; - for (let i = 0; i < this.target.groupBy.length; i++) { - let part = this.target.groupBy[i]; + for (let i = 0; i < this.target.group.length; i++) { + let part = this.target.group[i]; if (i > 0) { - groupBySection += ', '; + groupSection += ', '; } if (part.type === 'time') { - groupBySection += '1'; + groupSection += '1'; } else { - groupBySection += part.params[0]; + groupSection += part.params[0]; } } - if (groupBySection.length) { - query = '\nGROUP BY ' + groupBySection; + if (groupSection.length) { + query = '\nGROUP BY ' + groupSection; if (this.hasMetricColumn()) { query += ',2'; } @@ -238,7 +238,7 @@ export default class PostgresQuery { query += '\nFROM ' + this.target.table; query += this.buildWhereClause(); - query += this.buildGroupByClause(); + query += this.buildGroupClause(); query += '\nORDER BY 1'; diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 97db612e7db..62bd1c63fab 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -33,10 +33,10 @@ export class PostgresQueryCtrl extends QueryCtrl { timeColumnSegment: any; metricColumnSegment: any; selectMenu: any[]; - selectModels: SqlPart[][]; - groupByParts: SqlPart[][]; - whereParts: SqlPart[][]; - groupByAdd: any; + selectParts: SqlPart[][]; + groupParts: SqlPart[]; + whereParts: SqlPart[]; + groupAdd: any; /** @ngInject **/ constructor($scope, $injector, private templateSrv, private $q, private uiSegmentSrv) { @@ -69,22 +69,22 @@ export class PostgresQueryCtrl extends QueryCtrl { this.buildSelectMenu(); this.whereAdd = this.uiSegmentSrv.newPlusButton(); - this.groupByAdd = this.uiSegmentSrv.newPlusButton(); + this.groupAdd = this.uiSegmentSrv.newPlusButton(); this.panelCtrl.events.on('data-received', this.onDataReceived.bind(this), $scope); this.panelCtrl.events.on('data-error', this.onDataError.bind(this), $scope); } updateProjection() { - this.selectModels = _.map(this.target.select, function(parts: any) { + this.selectParts = _.map(this.target.select, function(parts: any) { return _.map(parts, sqlPart.create).filter(n => n); }); this.whereParts = _.map(this.target.where, sqlPart.create).filter(n => n); - this.groupByParts = _.map(this.target.groupBy, sqlPart.create).filter(n => n); + this.groupParts = _.map(this.target.group, sqlPart.create).filter(n => n); } updatePersistedParts() { - this.target.select = _.map(this.selectModels, function(selectParts) { + this.target.select = _.map(this.selectParts, function(selectParts) { return _.map(selectParts, function(part: any) { return { type: part.def.type, params: part.params }; }); @@ -92,7 +92,7 @@ export class PostgresQueryCtrl extends QueryCtrl { this.target.where = _.map(this.whereParts, function(part: any) { return { type: part.def.type, name: part.name, params: part.params }; }); - this.target.groupBy = _.map(this.groupByParts, function(part: any) { + this.target.group = _.map(this.groupParts, function(part: any) { return { type: part.def.type, params: part.params }; }); } @@ -216,12 +216,12 @@ export class PostgresQueryCtrl extends QueryCtrl { let parts = _.map(selectParts, function(part: any) { return sqlPart.create({ type: part.def.type, params: _.clone(part.params) }); }); - this.selectModels.push(parts); + this.selectParts.push(parts); break; case 'aggregate': // add group by if no group by yet - if (this.target.groupBy.length === 0) { - this.addGroupBy('time', '1m'); + if (this.target.group.length === 0) { + this.addGroup('time', '1m'); } case 'special': let index = _.findIndex(selectParts, (p: any) => p.def.type === item.value); @@ -256,9 +256,9 @@ export class PostgresQueryCtrl extends QueryCtrl { removeSelectPart(selectParts, part) { if (part.def.type === 'column') { // remove all parts of column unless its last column - if (this.selectModels.length > 1) { - let modelsIndex = _.indexOf(this.selectModels, selectParts); - this.selectModels.splice(modelsIndex, 1); + if (this.selectParts.length > 1) { + let modelsIndex = _.indexOf(this.selectParts, selectParts); + this.selectParts.splice(modelsIndex, 1); } } else { let partIndex = _.indexOf(selectParts, part); @@ -299,7 +299,7 @@ export class PostgresQueryCtrl extends QueryCtrl { } } - handleGroupByPartEvent(part, index, evt) { + onGroupPartEvent(part, index, evt) { switch (evt.name) { case 'get-param-options': { return this.datasource @@ -312,7 +312,7 @@ export class PostgresQueryCtrl extends QueryCtrl { break; } case 'action': { - this.removeGroupBy(part, index); + this.removeGroup(part, index); this.panelCtrl.refresh(); break; } @@ -322,7 +322,7 @@ export class PostgresQueryCtrl extends QueryCtrl { } } - addGroupBy(partType, value) { + addGroup(partType, value) { let params = [value]; if (partType === 'time') { params = ['1m', 'none']; @@ -331,13 +331,13 @@ export class PostgresQueryCtrl extends QueryCtrl { if (partType === 'time') { // put timeGroup at start - this.groupByParts.splice(0, 0, partModel); + this.groupParts.splice(0, 0, partModel); } else { - this.groupByParts.push(partModel); + this.groupParts.push(partModel); } // add aggregates when adding group by - for (let selectParts of this.selectModels) { + for (let selectParts of this.selectParts) { if (!selectParts.some(part => part.def.type === 'aggregate')) { let aggregate = sqlPart.create({ type: 'aggregate', params: ['avg'] }); selectParts.splice(1, 0, aggregate); @@ -351,10 +351,10 @@ export class PostgresQueryCtrl extends QueryCtrl { this.updatePersistedParts(); } - removeGroupBy(part, index) { + removeGroup(part, index) { if (part.def.type === 'time') { // remove aggregations - this.selectModels = _.map(this.selectModels, (s: any) => { + this.selectParts = _.map(this.selectParts, (s: any) => { return _.filter(s, (part: any) => { if (part.def.type === 'aggregate') { return false; @@ -364,7 +364,7 @@ export class PostgresQueryCtrl extends QueryCtrl { }); } - this.groupByParts.splice(index, 1); + this.groupParts.splice(index, 1); this.updatePersistedParts(); } @@ -429,12 +429,12 @@ export class PostgresQueryCtrl extends QueryCtrl { this.panelCtrl.refresh(); } - getGroupByOptions() { + getGroupOptions() { return this.datasource - .metricFindQuery(this.metaBuilder.buildColumnQuery('groupby')) + .metricFindQuery(this.metaBuilder.buildColumnQuery('group')) .then(tags => { var options = []; - if (!this.queryModel.hasGroupByTime()) { + if (!this.queryModel.hasTimeGroup()) { options.push(this.uiSegmentSrv.newSegment({ type: 'time', value: 'time(1m,none)' })); } for (let tag of tags) { @@ -445,14 +445,14 @@ export class PostgresQueryCtrl extends QueryCtrl { .catch(this.handleQueryError.bind(this)); } - groupByAction() { - switch (this.groupByAdd.value) { + onGroupAction() { + switch (this.groupAdd.value) { default: { - this.addGroupBy(this.groupByAdd.type, this.groupByAdd.value); + this.addGroup(this.groupAdd.type, this.groupAdd.value); } } - this.resetPlusButton(this.groupByAdd); + this.resetPlusButton(this.groupAdd); this.panelCtrl.refresh(); } diff --git a/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts index 33d997d2d0a..00ac5ed0e56 100644 --- a/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts +++ b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts @@ -27,12 +27,12 @@ describe('PostgresQuery', function() { describe('When generating time column SQL with group by time', function() { let query = new PostgresQuery( - { timeColumn: 'time', groupBy: [{ type: 'time', params: ['5m', 'none'] }] }, + { timeColumn: 'time', group: [{ type: 'time', params: ['5m', 'none'] }] }, templateSrv ); expect(query.buildTimeColumn()).toBe('$__timeGroup(time,5m)'); - query = new PostgresQuery({ timeColumn: 'time', groupBy: [{ type: 'time', params: ['5m', 'NULL'] }] }, templateSrv); + query = new PostgresQuery({ timeColumn: 'time', group: [{ type: 'time', params: ['5m', 'NULL'] }] }, templateSrv); expect(query.buildTimeColumn()).toBe('$__timeGroup(time,5m,NULL)'); }); @@ -114,13 +114,13 @@ describe('PostgresQuery', function() { }); describe('When generating GROUP BY clause', function() { - let query = new PostgresQuery({ groupBy: [], metricColumn: 'none' }, templateSrv); + let query = new PostgresQuery({ group: [], metricColumn: 'none' }, templateSrv); - expect(query.buildGroupByClause()).toBe(''); - query.target.groupBy = [{ type: 'time', params: ['5m'] }]; - expect(query.buildGroupByClause()).toBe('\nGROUP BY 1'); + expect(query.buildGroupClause()).toBe(''); + query.target.group = [{ type: 'time', params: ['5m'] }]; + expect(query.buildGroupClause()).toBe('\nGROUP BY 1'); query.target.metricColumn = 'm'; - expect(query.buildGroupByClause()).toBe('\nGROUP BY 1,2'); + expect(query.buildGroupClause()).toBe('\nGROUP BY 1,2'); }); describe('When generating complete statement', function() { From 5327580939fbd242fc25da9e17e50f6ff8f02098 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sat, 28 Jul 2018 21:30:08 +0200 Subject: [PATCH 150/310] refactor column function handling --- .../postgres/partials/query.editor.html | 2 +- .../datasource/postgres/postgres_query.ts | 15 ++++++--- .../plugins/datasource/postgres/query_ctrl.ts | 31 ++++++++++++++++--- .../postgres/specs/postgres_query.jest.ts | 17 ++++++---- .../plugins/datasource/postgres/sql_part.ts | 4 +-- 5 files changed, 52 insertions(+), 17 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 3d3b7c43388..58d2415479a 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -46,7 +46,7 @@
diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index a6f45dbcee8..eb51449dd39 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -135,7 +135,7 @@ export default class PostgresQuery { query = columnName.params[0]; let aggregate = _.find(column, (g: any) => g.type === 'aggregate'); - let special = _.find(column, (g: any) => g.type === 'special'); + let special = _.find(column, (g: any) => g.type === 'window'); if (aggregate) { if (special) { @@ -155,9 +155,13 @@ export default class PostgresQuery { } let over = overParts.join(' '); + let curr: string; + let prev: string; switch (special.params[0]) { case 'increase': - query = query + ' - lag(' + query + ') OVER (' + over + ')'; + curr = query; + prev = 'lag(' + curr + ') OVER (' + over + ')'; + query = '(CASE WHEN ' + curr + ' >= ' + prev + ' THEN ' + curr + ' - ' + prev + ' ELSE ' + curr + ' END)'; break; case 'rate': let timeColumn = this.target.timeColumn; @@ -165,11 +169,14 @@ export default class PostgresQuery { timeColumn = 'min(' + timeColumn + ')'; } - let curr = query; - let prev = 'lag(' + curr + ') OVER (' + over + ')'; + curr = query; + prev = 'lag(' + curr + ') OVER (' + over + ')'; query = '(CASE WHEN ' + curr + ' >= ' + prev + ' THEN ' + curr + ' - ' + prev + ' ELSE ' + curr + ' END)'; query += '/extract(epoch from ' + timeColumn + ' - lag(' + timeColumn + ') OVER (' + over + '))'; break; + default: + query = special.params[0] + '(' + query + ') OVER (' + over + ')'; + break; } } diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 62bd1c63fab..88ef587037d 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -99,8 +99,28 @@ export class PostgresQueryCtrl extends QueryCtrl { buildSelectMenu() { this.selectMenu = [ - { text: 'Aggregate', value: 'aggregate' }, - { text: 'Special', value: 'special' }, + { + text: 'Aggregate Functions', + value: 'aggregate', + submenu: [ + { text: 'Average', value: 'avg' }, + { text: 'Count', value: 'count' }, + { text: 'Maximum', value: 'max' }, + { text: 'Minimum', value: 'min' }, + { text: 'Sum', value: 'sum' }, + { text: 'Standard deviation', value: 'stddev' }, + { text: 'Variance', value: 'variance' }, + ], + }, + { + text: 'Window Functions', + value: 'window', + submenu: [ + { text: 'Increase', value: 'increase' }, + { text: 'Rate', value: 'rate' }, + { text: 'Sum', value: 'sum' }, + ], + }, { text: 'Alias', value: 'alias' }, { text: 'Column', value: 'column' }, ]; @@ -207,8 +227,11 @@ export class PostgresQueryCtrl extends QueryCtrl { }; } - addSelectPart(selectParts, item) { + addSelectPart(selectParts, item, subItem) { let partModel = sqlPart.create({ type: item.value }); + if (subItem) { + partModel.params = [subItem.value]; + } let addAlias = false; switch (item.value) { @@ -223,7 +246,7 @@ export class PostgresQueryCtrl extends QueryCtrl { if (this.target.group.length === 0) { this.addGroup('time', '1m'); } - case 'special': + case 'window': let index = _.findIndex(selectParts, (p: any) => p.def.type === item.value); if (index !== -1) { selectParts[index] = partModel; diff --git a/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts index 00ac5ed0e56..c589eb3c43c 100644 --- a/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts +++ b/public/app/plugins/datasource/postgres/specs/postgres_query.jest.ts @@ -61,9 +61,11 @@ describe('PostgresQuery', function() { column = [ { type: 'column', params: ['v'] }, { type: 'alias', params: ['a'] }, - { type: 'special', params: ['increase'] }, + { type: 'window', params: ['increase'] }, ]; - expect(query.buildValueColumn(column)).toBe('v - lag(v) OVER (ORDER BY time) AS "a"'); + expect(query.buildValueColumn(column)).toBe( + '(CASE WHEN v >= lag(v) OVER (ORDER BY time) THEN v - lag(v) OVER (ORDER BY time) ELSE v END) AS "a"' + ); }); describe('When generating value column SQL with metric column', function() { @@ -83,17 +85,20 @@ describe('PostgresQuery', function() { column = [ { type: 'column', params: ['v'] }, { type: 'alias', params: ['a'] }, - { type: 'special', params: ['increase'] }, + { type: 'window', params: ['increase'] }, ]; - expect(query.buildValueColumn(column)).toBe('v - lag(v) OVER (PARTITION BY host ORDER BY time) AS "a"'); + expect(query.buildValueColumn(column)).toBe( + '(CASE WHEN v >= lag(v) OVER (PARTITION BY host ORDER BY time) THEN v - lag(v) OVER (PARTITION BY host ORDER BY time) ELSE v END) AS "a"' + ); column = [ { type: 'column', params: ['v'] }, { type: 'alias', params: ['a'] }, { type: 'aggregate', params: ['max'] }, - { type: 'special', params: ['increase'] }, + { type: 'window', params: ['increase'] }, ]; expect(query.buildValueColumn(column)).toBe( - 'max(v ORDER BY time) - lag(max(v ORDER BY time)) OVER (PARTITION BY host) AS "a"' + '(CASE WHEN max(v ORDER BY time) >= lag(max(v ORDER BY time)) OVER (PARTITION BY host) ' + + 'THEN max(v ORDER BY time) - lag(max(v ORDER BY time)) OVER (PARTITION BY host) ELSE max(v ORDER BY time) END) AS "a"' ); }); diff --git a/public/app/plugins/datasource/postgres/sql_part.ts b/public/app/plugins/datasource/postgres/sql_part.ts index b0265a645ed..487ddb50276 100644 --- a/public/app/plugins/datasource/postgres/sql_part.ts +++ b/public/app/plugins/datasource/postgres/sql_part.ts @@ -76,13 +76,13 @@ register({ }); register({ - type: 'special', + type: 'window', style: 'label', params: [ { name: 'function', type: 'string', - options: ['increase', 'rate'], + options: ['increase', 'rate', 'sum'], }, ], defaultParams: ['increase'], From 412bb6acab9bd594747e8483b801e1e5cf338b78 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 29 Jul 2018 13:30:21 +0200 Subject: [PATCH 151/310] refactor function handling in query builder --- .../postgres/partials/query.editor.html | 6 +-- .../datasource/postgres/postgres_query.ts | 17 +++++-- .../plugins/datasource/postgres/query_ctrl.ts | 50 +++++++++++++++---- .../plugins/datasource/postgres/sql_part.ts | 27 +++++++++- 4 files changed, 81 insertions(+), 19 deletions(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 58d2415479a..0c053388034 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -66,7 +66,7 @@
- +
@@ -83,12 +83,12 @@ + handle-event="ctrl.handleGroupPartEvent(part, $index, $event)">
- +
diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index eb51449dd39..4cd3ddefaa2 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -134,14 +134,21 @@ export default class PostgresQuery { let columnName = _.find(column, (g: any) => g.type === 'column'); query = columnName.params[0]; - let aggregate = _.find(column, (g: any) => g.type === 'aggregate'); + let aggregate = _.find(column, (g: any) => g.type === 'aggregate' || g.type === 'percentile'); let special = _.find(column, (g: any) => g.type === 'window'); if (aggregate) { - if (special) { - query = aggregate.params[0] + '(' + query + ' ORDER BY ' + this.target.timeColumn + ')'; - } else { - query = aggregate.params[0] + '(' + query + ')'; + switch (aggregate.type) { + case 'aggregate': + if (special) { + query = aggregate.params[0] + '(' + query + ' ORDER BY ' + this.target.timeColumn + ')'; + } else { + query = aggregate.params[0] + '(' + query + ')'; + } + break; + case 'percentile': + query = aggregate.params[0] + '(' + aggregate.params[1] + ') WITHIN GROUP (ORDER BY ' + query + ')'; + break; } } diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 88ef587037d..0191d46065f 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -112,6 +112,14 @@ export class PostgresQueryCtrl extends QueryCtrl { { text: 'Variance', value: 'variance' }, ], }, + { + text: 'Ordered-Set Aggregate Functions', + value: 'percentile', + submenu: [ + { text: 'Percentile (continuous)', value: 'percentile_cont' }, + { text: 'Percentile (discrete)', value: 'percentile_disc' }, + ], + }, { text: 'Window Functions', value: 'window', @@ -121,9 +129,9 @@ export class PostgresQueryCtrl extends QueryCtrl { { text: 'Sum', value: 'sum' }, ], }, - { text: 'Alias', value: 'alias' }, - { text: 'Column', value: 'column' }, ]; + this.selectMenu.push({ text: 'Alias', value: 'alias' }); + this.selectMenu.push({ text: 'Column', value: 'column' }); } toggleEditorMode() { @@ -241,15 +249,17 @@ export class PostgresQueryCtrl extends QueryCtrl { }); this.selectParts.push(parts); break; + case 'percentile': + partModel.params.push('0.95'); case 'aggregate': // add group by if no group by yet if (this.target.group.length === 0) { this.addGroup('time', '1m'); } - case 'window': - let index = _.findIndex(selectParts, (p: any) => p.def.type === item.value); - if (index !== -1) { - selectParts[index] = partModel; + let aggIndex = _.findIndex(selectParts, (p: any) => p.def.type === 'aggregate' || p.def.type === 'percentile'); + if (aggIndex !== -1) { + // replace current aggregation + selectParts[aggIndex] = partModel; } else { selectParts.splice(1, 0, partModel); } @@ -257,6 +267,26 @@ export class PostgresQueryCtrl extends QueryCtrl { addAlias = true; } break; + case 'window': + let windowIndex = _.findIndex(selectParts, (p: any) => p.def.type === 'window'); + if (windowIndex !== -1) { + // replace current window function + selectParts[windowIndex] = partModel; + } else { + let aggIndex = _.findIndex( + selectParts, + (p: any) => p.def.type === 'aggregate' || p.def.type === 'percentile' + ); + if (aggIndex !== -1) { + selectParts.splice(aggIndex + 1, 0, partModel); + } else { + selectParts.splice(1, 0, partModel); + } + } + if (!_.find(selectParts, (p: any) => p.def.type === 'alias')) { + addAlias = true; + } + break; case 'alias': addAlias = true; break; @@ -322,7 +352,7 @@ export class PostgresQueryCtrl extends QueryCtrl { } } - onGroupPartEvent(part, index, evt) { + handleGroupPartEvent(part, index, evt) { switch (evt.name) { case 'get-param-options': { return this.datasource @@ -379,7 +409,7 @@ export class PostgresQueryCtrl extends QueryCtrl { // remove aggregations this.selectParts = _.map(this.selectParts, (s: any) => { return _.filter(s, (part: any) => { - if (part.def.type === 'aggregate') { + if (part.def.type === 'aggregate' || part.def.type === 'percentile') { return false; } return true; @@ -436,7 +466,7 @@ export class PostgresQueryCtrl extends QueryCtrl { return this.$q.when(options); } - whereAddAction(part, index) { + addWhereAction(part, index) { switch (this.whereAdd.type) { case 'macro': { this.whereParts.push(sqlPart.create({ type: 'macro', name: this.whereAdd.value, params: [] })); @@ -468,7 +498,7 @@ export class PostgresQueryCtrl extends QueryCtrl { .catch(this.handleQueryError.bind(this)); } - onGroupAction() { + addGroupAction() { switch (this.groupAdd.value) { default: { this.addGroup(this.groupAdd.type, this.groupAdd.value); diff --git a/public/app/plugins/datasource/postgres/sql_part.ts b/public/app/plugins/datasource/postgres/sql_part.ts index 487ddb50276..9cf0bd8f425 100644 --- a/public/app/plugins/datasource/postgres/sql_part.ts +++ b/public/app/plugins/datasource/postgres/sql_part.ts @@ -45,10 +45,35 @@ register({ register({ type: 'aggregate', style: 'label', - params: [{ name: 'name', type: 'string', dynamicLookup: true }], + params: [ + { + name: 'name', + type: 'string', + options: ['avg', 'count', 'min', 'max', 'sum', 'stddev', 'variance'], + }, + ], defaultParams: ['avg'], }); +register({ + type: 'percentile', + label: 'Aggregate:', + style: 'label', + params: [ + { + name: 'name', + type: 'string', + options: ['percentile_cont', 'percentile_disc'], + }, + { + name: 'fraction', + type: 'number', + options: ['0.5', '0.75', '0.9', '0.95', '0.99'], + }, + ], + defaultParams: ['percentile_cont', '0.95'], +}); + register({ type: 'alias', style: 'label', From 26ea88252bee35e45d113ccc8333217a4df7e2dc Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 29 Jul 2018 15:00:13 +0200 Subject: [PATCH 152/310] add first and last support --- .../datasource/postgres/postgres_query.ts | 13 +++- .../plugins/datasource/postgres/query_ctrl.ts | 75 +++++++++++-------- 2 files changed, 52 insertions(+), 36 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 4cd3ddefaa2..4c9bd862658 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -138,16 +138,21 @@ export default class PostgresQuery { let special = _.find(column, (g: any) => g.type === 'window'); if (aggregate) { + let func = aggregate.params[0]; switch (aggregate.type) { case 'aggregate': - if (special) { - query = aggregate.params[0] + '(' + query + ' ORDER BY ' + this.target.timeColumn + ')'; + if (func === 'first' || func === 'last') { + query = func + '(' + query + ',' + this.target.timeColumn + ')'; } else { - query = aggregate.params[0] + '(' + query + ')'; + if (special) { + query = func + '(' + query + ' ORDER BY ' + this.target.timeColumn + ')'; + } else { + query = func + '(' + query + ')'; + } } break; case 'percentile': - query = aggregate.params[0] + '(' + aggregate.params[1] + ') WITHIN GROUP (ORDER BY ' + query + ')'; + query = func + '(' + aggregate.params[1] + ') WITHIN GROUP (ORDER BY ' + query + ')'; break; } } diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 0191d46065f..181a3a9234c 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -98,38 +98,49 @@ export class PostgresQueryCtrl extends QueryCtrl { } buildSelectMenu() { - this.selectMenu = [ - { - text: 'Aggregate Functions', - value: 'aggregate', - submenu: [ - { text: 'Average', value: 'avg' }, - { text: 'Count', value: 'count' }, - { text: 'Maximum', value: 'max' }, - { text: 'Minimum', value: 'min' }, - { text: 'Sum', value: 'sum' }, - { text: 'Standard deviation', value: 'stddev' }, - { text: 'Variance', value: 'variance' }, - ], - }, - { - text: 'Ordered-Set Aggregate Functions', - value: 'percentile', - submenu: [ - { text: 'Percentile (continuous)', value: 'percentile_cont' }, - { text: 'Percentile (discrete)', value: 'percentile_disc' }, - ], - }, - { - text: 'Window Functions', - value: 'window', - submenu: [ - { text: 'Increase', value: 'increase' }, - { text: 'Rate', value: 'rate' }, - { text: 'Sum', value: 'sum' }, - ], - }, - ]; + this.selectMenu = []; + let aggregates = { + text: 'Aggregate Functions', + value: 'aggregate', + submenu: [ + { text: 'Average', value: 'avg' }, + { text: 'Count', value: 'count' }, + { text: 'Maximum', value: 'max' }, + { text: 'Minimum', value: 'min' }, + { text: 'Sum', value: 'sum' }, + { text: 'Standard deviation', value: 'stddev' }, + { text: 'Variance', value: 'variance' }, + ], + }; + + // first and last are timescaledb specific + aggregates.submenu.push({ text: 'First', value: 'first' }); + aggregates.submenu.push({ text: 'Last', value: 'last' }); + + this.selectMenu.push(aggregates); + + // ordered set aggregates require postgres 9.4+ + let aggregates2 = { + text: 'Ordered-Set Aggregate Functions', + value: 'percentile', + submenu: [ + { text: 'Percentile (continuous)', value: 'percentile_cont' }, + { text: 'Percentile (discrete)', value: 'percentile_disc' }, + ], + }; + this.selectMenu.push(aggregates2); + + let windows = { + text: 'Window Functions', + value: 'window', + submenu: [ + { text: 'Increase', value: 'increase' }, + { text: 'Rate', value: 'rate' }, + { text: 'Sum', value: 'sum' }, + ], + }; + this.selectMenu.push(windows); + this.selectMenu.push({ text: 'Alias', value: 'alias' }); this.selectMenu.push({ text: 'Column', value: 'column' }); } From ace999b13fe82a750bc1fc722b558eb7471837de Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Sun, 29 Jul 2018 15:56:22 +0200 Subject: [PATCH 153/310] rename special to windows --- .../app/plugins/datasource/postgres/postgres_query.ts | 10 +++++----- public/app/plugins/datasource/postgres/query_ctrl.ts | 11 ++++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/public/app/plugins/datasource/postgres/postgres_query.ts b/public/app/plugins/datasource/postgres/postgres_query.ts index 4c9bd862658..9715665fd4b 100644 --- a/public/app/plugins/datasource/postgres/postgres_query.ts +++ b/public/app/plugins/datasource/postgres/postgres_query.ts @@ -135,7 +135,7 @@ export default class PostgresQuery { query = columnName.params[0]; let aggregate = _.find(column, (g: any) => g.type === 'aggregate' || g.type === 'percentile'); - let special = _.find(column, (g: any) => g.type === 'window'); + let windows = _.find(column, (g: any) => g.type === 'window'); if (aggregate) { let func = aggregate.params[0]; @@ -144,7 +144,7 @@ export default class PostgresQuery { if (func === 'first' || func === 'last') { query = func + '(' + query + ',' + this.target.timeColumn + ')'; } else { - if (special) { + if (windows) { query = func + '(' + query + ' ORDER BY ' + this.target.timeColumn + ')'; } else { query = func + '(' + query + ')'; @@ -157,7 +157,7 @@ export default class PostgresQuery { } } - if (special) { + if (windows) { let overParts = []; if (this.hasMetricColumn()) { overParts.push('PARTITION BY ' + this.target.metricColumn); @@ -169,7 +169,7 @@ export default class PostgresQuery { let over = overParts.join(' '); let curr: string; let prev: string; - switch (special.params[0]) { + switch (windows.params[0]) { case 'increase': curr = query; prev = 'lag(' + curr + ') OVER (' + over + ')'; @@ -187,7 +187,7 @@ export default class PostgresQuery { query += '/extract(epoch from ' + timeColumn + ' - lag(' + timeColumn + ') OVER (' + over + '))'; break; default: - query = special.params[0] + '(' + query + ') OVER (' + over + ')'; + query = windows.params[0] + '(' + query + ') OVER (' + over + ')'; break; } } diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index 181a3a9234c..b15a9773436 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -246,6 +246,10 @@ export class PostgresQueryCtrl extends QueryCtrl { }; } + findAggregateIndex(selectParts) { + return _.findIndex(selectParts, (p: any) => p.def.type === 'aggregate' || p.def.type === 'percentile'); + } + addSelectPart(selectParts, item, subItem) { let partModel = sqlPart.create({ type: item.value }); if (subItem) { @@ -267,7 +271,7 @@ export class PostgresQueryCtrl extends QueryCtrl { if (this.target.group.length === 0) { this.addGroup('time', '1m'); } - let aggIndex = _.findIndex(selectParts, (p: any) => p.def.type === 'aggregate' || p.def.type === 'percentile'); + let aggIndex = this.findAggregateIndex(selectParts); if (aggIndex !== -1) { // replace current aggregation selectParts[aggIndex] = partModel; @@ -284,10 +288,7 @@ export class PostgresQueryCtrl extends QueryCtrl { // replace current window function selectParts[windowIndex] = partModel; } else { - let aggIndex = _.findIndex( - selectParts, - (p: any) => p.def.type === 'aggregate' || p.def.type === 'percentile' - ); + let aggIndex = this.findAggregateIndex(selectParts); if (aggIndex !== -1) { selectParts.splice(aggIndex + 1, 0, partModel); } else { From a221d9ec84edad72f8fd1e1530a1b4efe2104bbe Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Thu, 2 Aug 2018 09:50:21 +0200 Subject: [PATCH 154/310] add more prominent button for switching edit mode --- .../plugins/datasource/postgres/partials/query.editor.html | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index d32e84f30db..218dd306985 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -105,6 +105,12 @@
+
+ +
+ +
User Permission
From 30fce34d9c74a8f49c8b0feda74dd6e51a5e6e9f Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 28 Aug 2018 12:10:55 +0200 Subject: [PATCH 203/310] make default mode for table panels raw editor --- public/app/plugins/datasource/postgres/query_ctrl.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/public/app/plugins/datasource/postgres/query_ctrl.ts b/public/app/plugins/datasource/postgres/query_ctrl.ts index b1da2b5517f..6f2c1ce5d57 100644 --- a/public/app/plugins/datasource/postgres/query_ctrl.ts +++ b/public/app/plugins/datasource/postgres/query_ctrl.ts @@ -54,6 +54,7 @@ export class PostgresQueryCtrl extends QueryCtrl { if (this.panelCtrl.panel.type === 'table') { this.target.format = 'table'; this.target.rawSql = 'SELECT 1'; + this.target.rawQuery = true; } else { this.target.rawSql = defaultQuery; this.datasource.metricFindQuery(this.metaBuilder.findMetricTable()).then(result => { From 8bdabad86e14a128907afecd66f77b0433aed2f1 Mon Sep 17 00:00:00 2001 From: Patrick O'Carroll Date: Tue, 28 Aug 2018 13:51:12 +0200 Subject: [PATCH 204/310] changed from rotating to bouncing, maybe to much squash and stretch --- public/views/index.template.html | 75 +++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 10 deletions(-) diff --git a/public/views/index.template.html b/public/views/index.template.html index 491846f8c0f..c88b39a7c94 100644 --- a/public/views/index.template.html +++ b/public/views/index.template.html @@ -21,20 +21,75 @@ - -
-
-
-
-
- Min time interval - - - A lower limit for the auto group by time interval. Recommended to be set to write frequency, - for example 1m if your data is written every minute. - -
-
-

PostgreSQL details

From a87b27c7d33b42fd5aee6c26ca87c1ea2b24887d Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 29 Aug 2018 16:50:59 +0200 Subject: [PATCH 215/310] check for correct quoting of multiple singlequotes --- public/app/plugins/datasource/postgres/specs/datasource.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/public/app/plugins/datasource/postgres/specs/datasource.test.ts b/public/app/plugins/datasource/postgres/specs/datasource.test.ts index 47493174989..cf5805bab05 100644 --- a/public/app/plugins/datasource/postgres/specs/datasource.test.ts +++ b/public/app/plugins/datasource/postgres/specs/datasource.test.ts @@ -230,6 +230,7 @@ describe('PostgreSQLDatasource', function() { it('should return a quoted value', () => { ctx.variable.multi = true; expect(ctx.ds.interpolateVariable("a'bc", ctx.variable)).toEqual("'a''bc'"); + expect(ctx.ds.interpolateVariable("a'b'c", ctx.variable)).toEqual("'a''b''c'"); }); }); From aadff18e9454ca75528b5f5de032ee6407001377 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Wed, 29 Aug 2018 18:38:18 +0200 Subject: [PATCH 216/310] hide Query Builder button for table panels --- .../app/plugins/datasource/postgres/partials/query.editor.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/plugins/datasource/postgres/partials/query.editor.html b/public/app/plugins/datasource/postgres/partials/query.editor.html index 19a77865a86..fd944f4266c 100644 --- a/public/app/plugins/datasource/postgres/partials/query.editor.html +++ b/public/app/plugins/datasource/postgres/partials/query.editor.html @@ -106,7 +106,7 @@
-