From 7aa753a25f4666e67a957c3db5979b00630d6c87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Thu, 28 Sep 2017 12:52:39 +0200 Subject: [PATCH] tech: migrating elasticsearch to typescript --- public/app/core/services/datasource_srv.js | 1 + .../dashboard/specs/history_ctrl_specs.ts | 2 +- .../features/dashboard/specs/history_mocks.ts | 193 ++++++ .../dashboard/specs/history_srv_specs.ts | 2 +- .../datasource/elasticsearch/datasource.d.ts | 3 - .../datasource/elasticsearch/datasource.js | 372 ----------- .../datasource/elasticsearch/datasource.ts | 376 +++++++++++ .../elasticsearch/elastic_response.ts | 619 +++++++++--------- .../elasticsearch/index_pattern.d.ts | 2 - .../datasource/elasticsearch/index_pattern.js | 48 -- .../datasource/elasticsearch/index_pattern.ts | 43 ++ .../elasticsearch/query_builder.d.ts | 2 - .../{query_builder.js => query_builder.ts} | 60 +- .../datasource/elasticsearch/query_def.d.ts | 2 - .../datasource/elasticsearch/query_def.js | 199 ------ .../datasource/elasticsearch/query_def.ts | 191 ++++++ .../specs/index_pattern_specs.ts | 4 +- .../specs/query_builder_specs.ts | 2 +- public/test/mocks/history-mocks.js | 197 ------ 19 files changed, 1148 insertions(+), 1170 deletions(-) create mode 100644 public/app/features/dashboard/specs/history_mocks.ts delete mode 100644 public/app/plugins/datasource/elasticsearch/datasource.d.ts delete mode 100644 public/app/plugins/datasource/elasticsearch/datasource.js create mode 100644 public/app/plugins/datasource/elasticsearch/datasource.ts delete mode 100644 public/app/plugins/datasource/elasticsearch/index_pattern.d.ts delete mode 100644 public/app/plugins/datasource/elasticsearch/index_pattern.js create mode 100644 public/app/plugins/datasource/elasticsearch/index_pattern.ts delete mode 100644 public/app/plugins/datasource/elasticsearch/query_builder.d.ts rename public/app/plugins/datasource/elasticsearch/{query_builder.js => query_builder.ts} (88%) delete mode 100644 public/app/plugins/datasource/elasticsearch/query_def.d.ts delete mode 100644 public/app/plugins/datasource/elasticsearch/query_def.js create mode 100644 public/app/plugins/datasource/elasticsearch/query_def.ts delete mode 100644 public/test/mocks/history-mocks.js diff --git a/public/app/core/services/datasource_srv.js b/public/app/core/services/datasource_srv.js index db8485bb2f1..2857aec5176 100644 --- a/public/app/core/services/datasource_srv.js +++ b/public/app/core/services/datasource_srv.js @@ -42,6 +42,7 @@ function (angular, _, coreModule, config) { var pluginDef = dsConfig.meta; System.import(pluginDef.module).then(function(plugin) { + console.log(plugin); // check if its in cache now if (self.datasources[name]) { deferred.resolve(self.datasources[name]); diff --git a/public/app/features/dashboard/specs/history_ctrl_specs.ts b/public/app/features/dashboard/specs/history_ctrl_specs.ts index d6365470887..42e34f97f79 100644 --- a/public/app/features/dashboard/specs/history_ctrl_specs.ts +++ b/public/app/features/dashboard/specs/history_ctrl_specs.ts @@ -2,7 +2,7 @@ import {describe, beforeEach, it, sinon, expect, angularMocks} from 'test/lib/co import _ from 'lodash'; import {HistoryListCtrl} from 'app/features/dashboard/history/history'; -import { versions, compare, restore } from 'test/mocks/history-mocks'; +import {versions, compare, restore} from './history_mocks'; describe('HistoryListCtrl', function() { var RESTORE_ID = 4; diff --git a/public/app/features/dashboard/specs/history_mocks.ts b/public/app/features/dashboard/specs/history_mocks.ts new file mode 100644 index 00000000000..9c89d71479b --- /dev/null +++ b/public/app/features/dashboard/specs/history_mocks.ts @@ -0,0 +1,193 @@ + +export function versions() { + return [{ + id: 4, + dashboardId: 1, + parentVersion: 3, + restoredFrom: 0, + version: 4, + created: '2017-02-22T17:43:01-08:00', + createdBy: 'admin', + message: '', + }, + { + id: 3, + dashboardId: 1, + parentVersion: 1, + restoredFrom: 1, + version: 3, + created: '2017-02-22T17:43:01-08:00', + createdBy: 'admin', + message: '', + }, + { + id: 2, + dashboardId: 1, + parentVersion: 0, + restoredFrom: -1, + version: 2, + created: '2017-02-22T17:29:52-08:00', + createdBy: 'admin', + message: '', + }, + { + id: 1, + dashboardId: 1, + parentVersion: 0, + restoredFrom: -1, + slug: 'history-dashboard', + version: 1, + created: '2017-02-22T17:06:37-08:00', + createdBy: 'admin', + message: '', + }]; +} + +export function compare(type) { + return type === 'basic' ? '
' : '
'; +} + +export function restore(version, restoredFrom?) { + return { + dashboard: { + meta: { + type: 'db', + canSave: true, + canEdit: true, + canStar: true, + slug: 'history-dashboard', + expires: '0001-01-01T00:00:00Z', + created: '2017-02-21T18:40:45-08:00', + updated: '2017-04-11T21:31:22.59219665-07:00', + updatedBy: 'admin', + createdBy: 'admin', + version: version, + }, + dashboard: { + annotations: { + list: [] + }, + description: 'A random dashboard for implementing the history list', + editable: true, + gnetId: null, + graphTooltip: 0, + hideControls: false, + id: 1, + links: [], + restoredFrom: restoredFrom, + rows: [{ + collapse: false, + height: '250px', + panels: [{ + aliasColors: {}, + bars: false, + datasource: null, + fill: 1, + id: 1, + legend: { + avg: false, + current: false, + max: false, + min: false, + show: true, + total: false, + values: false + }, + lines: true, + linewidth: 1, + nullPointMode: "null", + percentage: false, + pointradius: 5, + points: false, + renderer: 'flot', + seriesOverrides: [], + span: 12, + stack: false, + steppedLine: false, + targets: [{}], + thresholds: [], + timeFrom: null, + timeShift: null, + title: 'Panel Title', + tooltip: { + shared: true, + sort: 0, + value_type: 'individual' + }, + type: 'graph', + xaxis: { + mode: 'time', + name: null, + show: true, + values: [] + }, + yaxes: [{ + format: 'short', + label: null, + logBase: 1, + max: null, + min: null, + show: true + }, { + format: 'short', + label: null, + logBase: 1, + max: null, + min: null, + show: true + }] + }], + repeat: null, + repeatIteration: null, + repeatRowId: null, + showTitle: false, + title: 'Dashboard Row', + titleSize: 'h6' + } + ], + schemaVersion: 14, + style: 'dark', + tags: [ + 'development' + ], + templating: { + 'list': [] + }, + time: { + from: 'now-6h', + to: 'now' + }, + timepicker: { + refresh_intervals: [ + '5s', + '10s', + '30s', + '1m', + '5m', + '15m', + '30m', + '1h', + '2h', + '1d', + ], + time_options: [ + '5m', + '15m', + '1h', + '6h', + '12h', + '24h', + '2d', + '7d', + '30d' + ] + }, + timezone: 'utc', + title: 'History Dashboard', + version: version, + } + }, + message: 'Dashboard restored to version ' + version, + version: version + }; +} diff --git a/public/app/features/dashboard/specs/history_srv_specs.ts b/public/app/features/dashboard/specs/history_srv_specs.ts index e72a626467e..621e91c8f87 100644 --- a/public/app/features/dashboard/specs/history_srv_specs.ts +++ b/public/app/features/dashboard/specs/history_srv_specs.ts @@ -2,7 +2,7 @@ import {describe, beforeEach, it, expect, angularMocks} from 'test/lib/common'; import helpers from 'test/specs/helpers'; import '../history/history_srv'; -import {versions, restore} from 'test/mocks/history-mocks'; +import {versions, restore} from './history_mocks'; describe('historySrv', function() { var ctx = new helpers.ServiceTestContext(); diff --git a/public/app/plugins/datasource/elasticsearch/datasource.d.ts b/public/app/plugins/datasource/elasticsearch/datasource.d.ts deleted file mode 100644 index 3682abfb614..00000000000 --- a/public/app/plugins/datasource/elasticsearch/datasource.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -declare var ElasticDatasource: any; -export {ElasticDatasource}; - diff --git a/public/app/plugins/datasource/elasticsearch/datasource.js b/public/app/plugins/datasource/elasticsearch/datasource.js deleted file mode 100644 index d6c2568ae47..00000000000 --- a/public/app/plugins/datasource/elasticsearch/datasource.js +++ /dev/null @@ -1,372 +0,0 @@ -define([ - 'angular', - 'lodash', - 'moment', - 'app/core/utils/kbn', - './query_builder', - './index_pattern', - './elastic_response', - './query_ctrl', -], -function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticResponse) { - 'use strict'; - - ElasticResponse = ElasticResponse.ElasticResponse; - - /** @ngInject */ - function ElasticDatasource(instanceSettings, $q, backendSrv, templateSrv, timeSrv) { - this.basicAuth = instanceSettings.basicAuth; - this.withCredentials = instanceSettings.withCredentials; - this.url = instanceSettings.url; - this.name = instanceSettings.name; - this.index = instanceSettings.index; - this.timeField = instanceSettings.jsonData.timeField; - this.esVersion = instanceSettings.jsonData.esVersion; - this.indexPattern = new IndexPattern(instanceSettings.index, instanceSettings.jsonData.interval); - this.interval = instanceSettings.jsonData.timeInterval; - this.queryBuilder = new ElasticQueryBuilder({ - timeField: this.timeField, - esVersion: this.esVersion, - }); - - this._request = function(method, url, data) { - var options = { - url: this.url + "/" + url, - method: method, - data: data - }; - - if (this.basicAuth || this.withCredentials) { - options.withCredentials = true; - } - if (this.basicAuth) { - options.headers = { - "Authorization": this.basicAuth - }; - } - - return backendSrv.datasourceRequest(options); - }; - - this._get = function(url) { - var range = timeSrv.timeRange(); - var index_list = this.indexPattern.getIndexList(range.from.valueOf(), range.to.valueOf()); - if (_.isArray(index_list) && index_list.length) { - return this._request('GET', index_list[0] + url).then(function(results) { - results.data.$$config = results.config; - return results.data; - }); - } else { - return this._request('GET', this.indexPattern.getIndexForToday() + url).then(function(results) { - results.data.$$config = results.config; - return results.data; - }); - } - }; - - this._post = function(url, data) { - return this._request('POST', url, data).then(function(results) { - results.data.$$config = results.config; - return results.data; - }); - }; - - this.annotationQuery = function(options) { - var annotation = options.annotation; - var timeField = annotation.timeField || '@timestamp'; - var queryString = annotation.query || '*'; - var tagsField = annotation.tagsField || 'tags'; - var titleField = annotation.titleField || 'desc'; - var textField = annotation.textField || null; - - var range = {}; - range[timeField]= { - from: options.range.from.valueOf(), - to: options.range.to.valueOf(), - format: "epoch_millis", - }; - - var queryInterpolated = templateSrv.replace(queryString, {}, 'lucene'); - var query = { - "bool": { - "filter": [ - { "range": range }, - { - "query_string": { - "query": queryInterpolated - } - } - ] - } - }; - - var data = { - "query" : query, - "size": 10000 - }; - - // fields field not supported on ES 5.x - if (this.esVersion < 5) { - data["fields"] = [timeField, "_source"]; - } - - var header = {search_type: "query_then_fetch", "ignore_unavailable": true}; - - // old elastic annotations had index specified on them - if (annotation.index) { - header.index = annotation.index; - } else { - header.index = this.indexPattern.getIndexList(options.range.from, options.range.to); - } - - var payload = angular.toJson(header) + '\n' + angular.toJson(data) + '\n'; - - return this._post('_msearch', payload).then(function(res) { - var list = []; - var hits = res.responses[0].hits.hits; - - var getFieldFromSource = function(source, fieldName) { - if (!fieldName) { return; } - - var fieldNames = fieldName.split('.'); - var fieldValue = source; - - for (var i = 0; i < fieldNames.length; i++) { - fieldValue = fieldValue[fieldNames[i]]; - if (!fieldValue) { - console.log('could not find field in annotation: ', fieldName); - return ''; - } - } - - if (_.isArray(fieldValue)) { - fieldValue = fieldValue.join(', '); - } - return fieldValue; - }; - - for (var i = 0; i < hits.length; i++) { - var source = hits[i]._source; - var time = source[timeField]; - if (typeof hits[i].fields !== 'undefined') { - var fields = hits[i].fields; - if (_.isString(fields[timeField]) || _.isNumber(fields[timeField])) { - time = fields[timeField]; - } - } - - var event = { - annotation: annotation, - time: moment.utc(time).valueOf(), - title: getFieldFromSource(source, titleField), - tags: getFieldFromSource(source, tagsField), - text: getFieldFromSource(source, textField) - }; - - list.push(event); - } - return list; - }); - }; - - this.testDatasource = function() { - timeSrv.setTime({ from: 'now-1m', to: 'now' }, true); - // validate that the index exist and has date field - return this.getFields({type: 'date'}).then(function(dateFields) { - var timeField = _.find(dateFields, {text: this.timeField}); - if (!timeField) { - return { status: "error", message: "No date field named " + this.timeField + ' found' }; - } - return { status: "success", message: "Index OK. Time field name OK." }; - }.bind(this), function(err) { - console.log(err); - if (err.data && err.data.error) { - var message = angular.toJson(err.data.error); - if (err.data.error.reason) { - message = err.data.error.reason; - } - return { status: "error", message: message }; - } else { - return { status: "error", message: err.status }; - } - }); - }; - - this.getQueryHeader = function(searchType, timeFrom, timeTo) { - var header = {search_type: searchType, "ignore_unavailable": true}; - header.index = this.indexPattern.getIndexList(timeFrom, timeTo); - return angular.toJson(header); - }; - - this.query = function(options) { - var payload = ""; - var target; - var sentTargets = []; - - // add global adhoc filters to timeFilter - var adhocFilters = templateSrv.getAdhocFilters(this.name); - - for (var i = 0; i < options.targets.length; i++) { - target = options.targets[i]; - if (target.hide) {continue;} - - var queryString = templateSrv.replace(target.query || '*', options.scopedVars, 'lucene'); - var queryObj = this.queryBuilder.build(target, adhocFilters, queryString); - var esQuery = angular.toJson(queryObj); - - var searchType = (queryObj.size === 0 && this.esVersion < 5) ? 'count' : 'query_then_fetch'; - var header = this.getQueryHeader(searchType, options.range.from, options.range.to); - payload += header + '\n'; - - payload += esQuery + '\n'; - sentTargets.push(target); - } - - if (sentTargets.length === 0) { - return $q.when([]); - } - - payload = payload.replace(/\$timeFrom/g, options.range.from.valueOf()); - payload = payload.replace(/\$timeTo/g, options.range.to.valueOf()); - payload = templateSrv.replace(payload, options.scopedVars); - - return this._post('_msearch', payload).then(function(res) { - return new ElasticResponse(sentTargets, res).getTimeSeries(); - }); - }; - - this.getFields = function(query) { - return this._get('/_mapping').then(function(result) { - - var typeMap = { - 'float': 'number', - 'double': 'number', - 'integer': 'number', - 'long': 'number', - 'date': 'date', - 'string': 'string', - 'text': 'string', - 'scaled_float': 'number', - 'nested': 'nested' - }; - - function shouldAddField(obj, key, query) { - if (key[0] === '_') { - return false; - } - - if (!query.type) { - return true; - } - - // equal query type filter, or via typemap translation - return query.type === obj.type || query.type === typeMap[obj.type]; - } - - // Store subfield names: [system, process, cpu, total] -> system.process.cpu.total - var fieldNameParts = []; - var fields = {}; - - function getFieldsRecursively(obj) { - for (var key in obj) { - var subObj = obj[key]; - - // Check mapping field for nested fields - if (_.isObject(subObj.properties)) { - fieldNameParts.push(key); - getFieldsRecursively(subObj.properties); - } - - if (_.isObject(subObj.fields)) { - fieldNameParts.push(key); - getFieldsRecursively(subObj.fields); - } - - if (_.isString(subObj.type)) { - var fieldName = fieldNameParts.concat(key).join('.'); - - // Hide meta-fields and check field type - if (shouldAddField(subObj, key, query)) { - fields[fieldName] = { - text: fieldName, - type: subObj.type - }; - } - } - } - fieldNameParts.pop(); - } - - for (var indexName in result) { - var index = result[indexName]; - if (index && index.mappings) { - var mappings = index.mappings; - for (var typeName in mappings) { - var properties = mappings[typeName].properties; - getFieldsRecursively(properties); - } - } - } - - // transform to array - return _.map(fields, function(value) { - return value; - }); - }); - }; - - this.getTerms = function(queryDef) { - var range = timeSrv.timeRange(); - var searchType = this.esVersion >= 5 ? 'query_then_fetch' : 'count' ; - var header = this.getQueryHeader(searchType, range.from, range.to); - var esQuery = angular.toJson(this.queryBuilder.getTermsQuery(queryDef)); - - esQuery = esQuery.replace(/\$timeFrom/g, range.from.valueOf()); - esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf()); - esQuery = header + '\n' + esQuery + '\n'; - - return this._post('_msearch?search_type=' + searchType, esQuery).then(function(res) { - if (!res.responses[0].aggregations) { - return []; - } - - var buckets = res.responses[0].aggregations["1"].buckets; - return _.map(buckets, function(bucket) { - return { - text: bucket.key_as_string || bucket.key, - value: bucket.key - }; - }); - }); - }; - - this.metricFindQuery = function(query) { - query = angular.fromJson(query); - if (!query) { - return $q.when([]); - } - - if (query.find === 'fields') { - query.field = templateSrv.replace(query.field, {}, 'lucene'); - return this.getFields(query); - } - - if (query.find === 'terms') { - query.query = templateSrv.replace(query.query || '*', {}, 'lucene'); - return this.getTerms(query); - } - }; - - this.getTagKeys = function() { - return this.getFields({}); - }; - - this.getTagValues = function(options) { - return this.getTerms({field: options.key, query: '*'}); - }; - } - - return { - ElasticDatasource: ElasticDatasource - }; -}); diff --git a/public/app/plugins/datasource/elasticsearch/datasource.ts b/public/app/plugins/datasource/elasticsearch/datasource.ts new file mode 100644 index 00000000000..543e73d64e5 --- /dev/null +++ b/public/app/plugins/datasource/elasticsearch/datasource.ts @@ -0,0 +1,376 @@ +/// + +import angular from 'angular'; +import _ from 'lodash'; +import moment from 'moment'; +import {ElasticQueryBuilder} from './query_builder'; +import {IndexPattern} from './index_pattern'; +import {ElasticResponse} from './elastic_response'; + +export class ElasticDatasource { + basicAuth: string; + withCredentials: boolean; + url: string; + name: string; + index: string; + timeField: string; + esVersion: number; + interval: string; + queryBuilder: ElasticQueryBuilder; + indexPattern: IndexPattern; + + /** @ngInject */ + constructor(instanceSettings, private $q, private backendSrv, private templateSrv, private timeSrv) { + this.basicAuth = instanceSettings.basicAuth; + this.withCredentials = instanceSettings.withCredentials; + this.url = instanceSettings.url; + this.name = instanceSettings.name; + this.index = instanceSettings.index; + this.timeField = instanceSettings.jsonData.timeField; + this.esVersion = instanceSettings.jsonData.esVersion; + this.indexPattern = new IndexPattern(instanceSettings.index, instanceSettings.jsonData.interval); + this.interval = instanceSettings.jsonData.timeInterval; + this.queryBuilder = new ElasticQueryBuilder({ + timeField: this.timeField, + esVersion: this.esVersion, + }); + } + + private request(method, url, data?) { + var options: any = { + url: this.url + "/" + url, + method: method, + data: data + }; + + if (this.basicAuth || this.withCredentials) { + options.withCredentials = true; + } + if (this.basicAuth) { + options.headers = { + "Authorization": this.basicAuth + }; + } + + return this.backendSrv.datasourceRequest(options); + } + + private get(url) { + var range = this.timeSrv.timeRange(); + var index_list = this.indexPattern.getIndexList(range.from.valueOf(), range.to.valueOf()); + if (_.isArray(index_list) && index_list.length) { + return this.request('GET', index_list[0] + url).then(function(results) { + results.data.$$config = results.config; + return results.data; + }); + } else { + return this.request('GET', this.indexPattern.getIndexForToday() + url).then(function(results) { + results.data.$$config = results.config; + return results.data; + }); + } + } + + private post(url, data) { + return this.request('POST', url, data).then(function(results) { + results.data.$$config = results.config; + return results.data; + }); + } + + annotationQuery(options) { + var annotation = options.annotation; + var timeField = annotation.timeField || '@timestamp'; + var queryString = annotation.query || '*'; + var tagsField = annotation.tagsField || 'tags'; + var titleField = annotation.titleField || 'desc'; + var textField = annotation.textField || null; + + var range = {}; + range[timeField]= { + from: options.range.from.valueOf(), + to: options.range.to.valueOf(), + format: "epoch_millis", + }; + + var queryInterpolated = this.templateSrv.replace(queryString, {}, 'lucene'); + var query = { + "bool": { + "filter": [ + { "range": range }, + { + "query_string": { + "query": queryInterpolated + } + } + ] + } + }; + + var data = { + "query" : query, + "size": 10000 + }; + + // fields field not supported on ES 5.x + if (this.esVersion < 5) { + data["fields"] = [timeField, "_source"]; + } + + var header: any = {search_type: "query_then_fetch", "ignore_unavailable": true}; + + // old elastic annotations had index specified on them + if (annotation.index) { + header.index = annotation.index; + } else { + header.index = this.indexPattern.getIndexList(options.range.from, options.range.to); + } + + var payload = angular.toJson(header) + '\n' + angular.toJson(data) + '\n'; + + return this.post('_msearch', payload).then(res => { + var list = []; + var hits = res.responses[0].hits.hits; + + var getFieldFromSource = function(source, fieldName) { + if (!fieldName) { return; } + + var fieldNames = fieldName.split('.'); + var fieldValue = source; + + for (var i = 0; i < fieldNames.length; i++) { + fieldValue = fieldValue[fieldNames[i]]; + if (!fieldValue) { + console.log('could not find field in annotation: ', fieldName); + return ''; + } + } + + if (_.isArray(fieldValue)) { + fieldValue = fieldValue.join(', '); + } + return fieldValue; + }; + + for (var i = 0; i < hits.length; i++) { + var source = hits[i]._source; + var time = source[timeField]; + if (typeof hits[i].fields !== 'undefined') { + var fields = hits[i].fields; + if (_.isString(fields[timeField]) || _.isNumber(fields[timeField])) { + time = fields[timeField]; + } + } + + var event = { + annotation: annotation, + time: moment.utc(time).valueOf(), + title: getFieldFromSource(source, titleField), + tags: getFieldFromSource(source, tagsField), + text: getFieldFromSource(source, textField) + }; + + list.push(event); + } + return list; + }); + }; + + testDatasource() { + this.timeSrv.setTime({ from: 'now-1m', to: 'now' }, true); + // validate that the index exist and has date field + return this.getFields({type: 'date'}).then(function(dateFields) { + var timeField = _.find(dateFields, {text: this.timeField}); + if (!timeField) { + return { status: "error", message: "No date field named " + this.timeField + ' found' }; + } + return { status: "success", message: "Index OK. Time field name OK." }; + }.bind(this), function(err) { + console.log(err); + if (err.data && err.data.error) { + var message = angular.toJson(err.data.error); + if (err.data.error.reason) { + message = err.data.error.reason; + } + return { status: "error", message: message }; + } else { + return { status: "error", message: err.status }; + } + }); + } + + getQueryHeader(searchType, timeFrom, timeTo) { + return angular.toJson({ + search_type: searchType, + "ignore_unavailable": true, + index: this.indexPattern.getIndexList(timeFrom, timeTo), + }); + } + + query(options) { + var payload = ""; + var target; + var sentTargets = []; + + // add global adhoc filters to timeFilter + var adhocFilters = this.templateSrv.getAdhocFilters(this.name); + + for (var i = 0; i < options.targets.length; i++) { + target = options.targets[i]; + if (target.hide) {continue;} + + var queryString = this.templateSrv.replace(target.query || '*', options.scopedVars, 'lucene'); + var queryObj = this.queryBuilder.build(target, adhocFilters, queryString); + var esQuery = angular.toJson(queryObj); + + var searchType = (queryObj.size === 0 && this.esVersion < 5) ? 'count' : 'query_then_fetch'; + var header = this.getQueryHeader(searchType, options.range.from, options.range.to); + payload += header + '\n'; + + payload += esQuery + '\n'; + sentTargets.push(target); + } + + if (sentTargets.length === 0) { + return this.$q.when([]); + } + + payload = payload.replace(/\$timeFrom/g, options.range.from.valueOf()); + payload = payload.replace(/\$timeTo/g, options.range.to.valueOf()); + payload = this.templateSrv.replace(payload, options.scopedVars); + + return this.post('_msearch', payload).then(function(res) { + return new ElasticResponse(sentTargets, res).getTimeSeries(); + }); + }; + + getFields(query) { + return this.get('/_mapping').then(function(result) { + + var typeMap = { + 'float': 'number', + 'double': 'number', + 'integer': 'number', + 'long': 'number', + 'date': 'date', + 'string': 'string', + 'text': 'string', + 'scaled_float': 'number', + 'nested': 'nested' + }; + + function shouldAddField(obj, key, query) { + if (key[0] === '_') { + return false; + } + + if (!query.type) { + return true; + } + + // equal query type filter, or via typemap translation + return query.type === obj.type || query.type === typeMap[obj.type]; + } + + // Store subfield names: [system, process, cpu, total] -> system.process.cpu.total + var fieldNameParts = []; + var fields = {}; + + function getFieldsRecursively(obj) { + for (var key in obj) { + var subObj = obj[key]; + + // Check mapping field for nested fields + if (_.isObject(subObj.properties)) { + fieldNameParts.push(key); + getFieldsRecursively(subObj.properties); + } + + if (_.isObject(subObj.fields)) { + fieldNameParts.push(key); + getFieldsRecursively(subObj.fields); + } + + if (_.isString(subObj.type)) { + var fieldName = fieldNameParts.concat(key).join('.'); + + // Hide meta-fields and check field type + if (shouldAddField(subObj, key, query)) { + fields[fieldName] = { + text: fieldName, + type: subObj.type + }; + } + } + } + fieldNameParts.pop(); + } + + for (var indexName in result) { + var index = result[indexName]; + if (index && index.mappings) { + var mappings = index.mappings; + for (var typeName in mappings) { + var properties = mappings[typeName].properties; + getFieldsRecursively(properties); + } + } + } + + // transform to array + return _.map(fields, function(value) { + return value; + }); + }); + } + + getTerms(queryDef) { + var range = this.timeSrv.timeRange(); + var searchType = this.esVersion >= 5 ? 'query_then_fetch' : 'count' ; + var header = this.getQueryHeader(searchType, range.from, range.to); + var esQuery = angular.toJson(this.queryBuilder.getTermsQuery(queryDef)); + + esQuery = esQuery.replace(/\$timeFrom/g, range.from.valueOf()); + esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf()); + esQuery = header + '\n' + esQuery + '\n'; + + return this.post('_msearch?search_type=' + searchType, esQuery).then(function(res) { + if (!res.responses[0].aggregations) { + return []; + } + + var buckets = res.responses[0].aggregations["1"].buckets; + return _.map(buckets, function(bucket) { + return { + text: bucket.key_as_string || bucket.key, + value: bucket.key + }; + }); + }); + } + + metricFindQuery(query) { + query = angular.fromJson(query); + if (!query) { + return this.$q.when([]); + } + + if (query.find === 'fields') { + query.field = this.templateSrv.replace(query.field, {}, 'lucene'); + return this.getFields(query); + } + + if (query.find === 'terms') { + query.query = this.templateSrv.replace(query.query || '*', {}, 'lucene'); + return this.getTerms(query); + } + } + + getTagKeys() { + return this.getFields({}); + } + + getTagValues(options) { + return this.getTerms({field: options.key, query: '*'}); + } +} diff --git a/public/app/plugins/datasource/elasticsearch/elastic_response.ts b/public/app/plugins/datasource/elasticsearch/elastic_response.ts index 47fa14a99a6..84aa397cb1f 100644 --- a/public/app/plugins/datasource/elasticsearch/elastic_response.ts +++ b/public/app/plugins/datasource/elasticsearch/elastic_response.ts @@ -1,130 +1,55 @@ /// import _ from 'lodash'; -import queryDef from "./query_def"; +import * as queryDef from "./query_def"; import TableModel from 'app/core/table_model'; -export function ElasticResponse(targets, response) { - this.targets = targets; - this.response = response; -} +export class ElasticResponse { -ElasticResponse.prototype.processMetrics = function(esAgg, target, seriesList, props) { - var metric, y, i, newSeries, bucket, value; + constructor(private targets, private response) { + this.targets = targets; + this.response = response; + } - for (y = 0; y < target.metrics.length; y++) { - metric = target.metrics[y]; - if (metric.hide) { - continue; - } + processMetrics(esAgg, target, seriesList, props) { + var metric, y, i, newSeries, bucket, value; - switch (metric.type) { - case 'count': { - newSeries = { datapoints: [], metric: 'count', props: props}; - for (i = 0; i < esAgg.buckets.length; i++) { - bucket = esAgg.buckets[i]; - value = bucket.doc_count; - newSeries.datapoints.push([value, bucket.key]); - } - seriesList.push(newSeries); - break; + for (y = 0; y < target.metrics.length; y++) { + metric = target.metrics[y]; + if (metric.hide) { + continue; } - case 'percentiles': { - if (esAgg.buckets.length === 0) { + + switch (metric.type) { + case 'count': { + newSeries = { datapoints: [], metric: 'count', props: props}; + for (i = 0; i < esAgg.buckets.length; i++) { + bucket = esAgg.buckets[i]; + value = bucket.doc_count; + newSeries.datapoints.push([value, bucket.key]); + } + seriesList.push(newSeries); break; } - - var firstBucket = esAgg.buckets[0]; - var percentiles = firstBucket[metric.id].values; - - for (var percentileName in percentiles) { - newSeries = {datapoints: [], metric: 'p' + percentileName, props: props, field: metric.field}; - - for (i = 0; i < esAgg.buckets.length; i++) { - bucket = esAgg.buckets[i]; - var values = bucket[metric.id].values; - newSeries.datapoints.push([values[percentileName], bucket.key]); - } - seriesList.push(newSeries); - } - - break; - } - case 'extended_stats': { - for (var statName in metric.meta) { - if (!metric.meta[statName]) { - continue; + case 'percentiles': { + if (esAgg.buckets.length === 0) { + break; } - newSeries = {datapoints: [], metric: statName, props: props, field: metric.field}; + var firstBucket = esAgg.buckets[0]; + var percentiles = firstBucket[metric.id].values; - for (i = 0; i < esAgg.buckets.length; i++) { - bucket = esAgg.buckets[i]; - var stats = bucket[metric.id]; + for (var percentileName in percentiles) { + newSeries = {datapoints: [], metric: 'p' + percentileName, props: props, field: metric.field}; - // add stats that are in nested obj to top level obj - stats.std_deviation_bounds_upper = stats.std_deviation_bounds.upper; - stats.std_deviation_bounds_lower = stats.std_deviation_bounds.lower; - - newSeries.datapoints.push([stats[statName], bucket.key]); - } - - seriesList.push(newSeries); - } - - break; - } - default: { - newSeries = { datapoints: [], metric: metric.type, field: metric.field, props: props}; - for (i = 0; i < esAgg.buckets.length; i++) { - bucket = esAgg.buckets[i]; - - value = bucket[metric.id]; - if (value !== undefined) { - if (value.normalized_value) { - newSeries.datapoints.push([value.normalized_value, bucket.key]); - } else { - newSeries.datapoints.push([value.value, bucket.key]); + for (i = 0; i < esAgg.buckets.length; i++) { + bucket = esAgg.buckets[i]; + var values = bucket[metric.id].values; + newSeries.datapoints.push([values[percentileName], bucket.key]); } + seriesList.push(newSeries); } - } - seriesList.push(newSeries); - break; - } - } - } -}; - -ElasticResponse.prototype.processAggregationDocs = function(esAgg, aggDef, target, table, props) { - // add columns - if (table.columns.length === 0) { - for (let propKey of _.keys(props)) { - table.addColumn({text: propKey, filterable: true}); - } - table.addColumn({text: aggDef.field, filterable: true}); - } - - // helper func to add values to value array - let addMetricValue = (values, metricName, value) => { - table.addColumn({text: metricName}); - values.push(value); - }; - - for (let bucket of esAgg.buckets) { - let values = []; - - for (let propValues of _.values(props)) { - values.push(propValues); - } - - // add bucket key (value) - values.push(bucket.key); - - for (let metric of target.metrics) { - switch (metric.type) { - case "count": { - addMetricValue(values, this._getMetricName(metric.type), bucket.doc_count); break; } case 'extended_stats': { @@ -133,228 +58,304 @@ ElasticResponse.prototype.processAggregationDocs = function(esAgg, aggDef, targe continue; } - var stats = bucket[metric.id]; - // add stats that are in nested obj to top level obj - stats.std_deviation_bounds_upper = stats.std_deviation_bounds.upper; - stats.std_deviation_bounds_lower = stats.std_deviation_bounds.lower; + newSeries = {datapoints: [], metric: statName, props: props, field: metric.field}; - addMetricValue(values, this._getMetricName(statName), stats[statName]); + for (i = 0; i < esAgg.buckets.length; i++) { + bucket = esAgg.buckets[i]; + var stats = bucket[metric.id]; + + // add stats that are in nested obj to top level obj + stats.std_deviation_bounds_upper = stats.std_deviation_bounds.upper; + stats.std_deviation_bounds_lower = stats.std_deviation_bounds.lower; + + newSeries.datapoints.push([stats[statName], bucket.key]); + } + + seriesList.push(newSeries); } + break; } - default: { - let metricName = this._getMetricName(metric.type); - let otherMetrics = _.filter(target.metrics, {type: metric.type}); + default: { + newSeries = { datapoints: [], metric: metric.type, field: metric.field, props: props}; + for (i = 0; i < esAgg.buckets.length; i++) { + bucket = esAgg.buckets[i]; + + value = bucket[metric.id]; + if (value !== undefined) { + if (value.normalized_value) { + newSeries.datapoints.push([value.normalized_value, bucket.key]); + } else { + newSeries.datapoints.push([value.value, bucket.key]); + } + } - // if more of the same metric type include field field name in property - if (otherMetrics.length > 1) { - metricName += ' ' + metric.field; } - - addMetricValue(values, metricName, bucket[metric.id].value); + seriesList.push(newSeries); break; } } } - - table.rows.push(values); } -}; -// This is quite complex -// neeed to recurise down the nested buckets to build series -ElasticResponse.prototype.processBuckets = function(aggs, target, seriesList, table, props, depth) { - var bucket, aggDef, esAgg, aggId; - var maxDepth = target.bucketAggs.length-1; - - for (aggId in aggs) { - aggDef = _.find(target.bucketAggs, {id: aggId}); - esAgg = aggs[aggId]; - - if (!aggDef) { - continue; - } - - if (depth === maxDepth) { - if (aggDef.type === 'date_histogram') { - this.processMetrics(esAgg, target, seriesList, props); - } else { - this.processAggregationDocs(esAgg, aggDef, target, table, props); - } - } else { - for (var nameIndex in esAgg.buckets) { - bucket = esAgg.buckets[nameIndex]; - props = _.clone(props); - if (bucket.key !== void 0) { - props[aggDef.field] = bucket.key; - } else { - props["filter"] = nameIndex; - } - if (bucket.key_as_string) { - props[aggDef.field] = bucket.key_as_string; - } - this.processBuckets(bucket, target, seriesList, table, props, depth+1); + processAggregationDocs(esAgg, aggDef, target, table, props) { + // add columns + if (table.columns.length === 0) { + for (let propKey of _.keys(props)) { + table.addColumn({text: propKey, filterable: true}); } + table.addColumn({text: aggDef.field, filterable: true}); } - } -}; -ElasticResponse.prototype._getMetricName = function(metric) { - var metricDef = _.find(queryDef.metricAggTypes, {value: metric}); - if (!metricDef) { - metricDef = _.find(queryDef.extendedStats, {value: metric}); - } - - return metricDef ? metricDef.text : metric; -}; - -ElasticResponse.prototype._getSeriesName = function(series, target, metricTypeCount) { - var metricName = this._getMetricName(series.metric); - - if (target.alias) { - var regex = /\{\{([\s\S]+?)\}\}/g; - - return target.alias.replace(regex, function(match, g1, g2) { - var group = g1 || g2; - - if (group.indexOf('term ') === 0) { return series.props[group.substring(5)]; } - if (series.props[group] !== void 0) { return series.props[group]; } - if (group === 'metric') { return metricName; } - if (group === 'field') { return series.field; } - - return match; - }); - } - - if (series.field && queryDef.isPipelineAgg(series.metric)) { - var appliedAgg = _.find(target.metrics, { id: series.field }); - if (appliedAgg) { - metricName += ' ' + queryDef.describeMetric(appliedAgg); - } else { - metricName = 'Unset'; - } - } else if (series.field) { - metricName += ' ' + series.field; - } - - var propKeys = _.keys(series.props); - if (propKeys.length === 0) { - return metricName; - } - - var name = ''; - for (var propName in series.props) { - name += series.props[propName] + ' '; - } - - if (metricTypeCount === 1) { - return name.trim(); - } - - return name.trim() + ' ' + metricName; -}; - -ElasticResponse.prototype.nameSeries = function(seriesList, target) { - var metricTypeCount = _.uniq(_.map(seriesList, 'metric')).length; - var fieldNameCount = _.uniq(_.map(seriesList, 'field')).length; - - for (var i = 0; i < seriesList.length; i++) { - var series = seriesList[i]; - series.target = this._getSeriesName(series, target, metricTypeCount, fieldNameCount); - } -}; - -ElasticResponse.prototype.processHits = function(hits, seriesList) { - var series = {target: 'docs', type: 'docs', datapoints: [], total: hits.total, filterable: true}; - var propName, hit, doc, i; - - for (i = 0; i < hits.hits.length; i++) { - hit = hits.hits[i]; - doc = { - _id: hit._id, - _type: hit._type, - _index: hit._index + // helper func to add values to value array + let addMetricValue = (values, metricName, value) => { + table.addColumn({text: metricName}); + values.push(value); }; - if (hit._source) { - for (propName in hit._source) { - doc[propName] = hit._source[propName]; - } - } + for (let bucket of esAgg.buckets) { + let values = []; - for (propName in hit.fields) { - doc[propName] = hit.fields[propName]; - } - series.datapoints.push(doc); - } - - seriesList.push(series); -}; - -ElasticResponse.prototype.trimDatapoints = function(aggregations, target) { - var histogram = _.find(target.bucketAggs, { type: 'date_histogram'}); - - var shouldDropFirstAndLast = histogram && histogram.settings && histogram.settings.trimEdges; - if (shouldDropFirstAndLast) { - var trim = histogram.settings.trimEdges; - for (var prop in aggregations) { - var points = aggregations[prop]; - if (points.datapoints.length > trim * 2) { - points.datapoints = points.datapoints.slice(trim, points.datapoints.length - trim); - } - } - } -}; - -ElasticResponse.prototype.getErrorFromElasticResponse = function(response, err) { - var result: any = {}; - result.data = JSON.stringify(err, null, 4); - if (err.root_cause && err.root_cause.length > 0 && err.root_cause[0].reason) { - result.message = err.root_cause[0].reason; - } else { - result.message = err.reason || 'Unkown elatic error response'; - } - - if (response.$$config) { - result.config = response.$$config; - } - - return result; -}; - -ElasticResponse.prototype.getTimeSeries = function() { - var seriesList = []; - - for (var i = 0; i < this.response.responses.length; i++) { - var response = this.response.responses[i]; - if (response.error) { - throw this.getErrorFromElasticResponse(this.response, response.error); - } - - if (response.hits && response.hits.hits.length > 0) { - this.processHits(response.hits, seriesList); - } - - if (response.aggregations) { - var aggregations = response.aggregations; - var target = this.targets[i]; - var tmpSeriesList = []; - var table = new TableModel(); - - this.processBuckets(aggregations, target, tmpSeriesList, table, {}, 0); - this.trimDatapoints(tmpSeriesList, target); - this.nameSeries(tmpSeriesList, target); - - for (var y = 0; y < tmpSeriesList.length; y++) { - seriesList.push(tmpSeriesList[y]); + for (let propValues of _.values(props)) { + values.push(propValues); } - if (table.rows.length > 0) { - seriesList.push(table); + // add bucket key (value) + values.push(bucket.key); + + for (let metric of target.metrics) { + switch (metric.type) { + case "count": { + addMetricValue(values, this.getMetricName(metric.type), bucket.doc_count); + break; + } + case 'extended_stats': { + for (var statName in metric.meta) { + if (!metric.meta[statName]) { + continue; + } + + var stats = bucket[metric.id]; + // add stats that are in nested obj to top level obj + stats.std_deviation_bounds_upper = stats.std_deviation_bounds.upper; + stats.std_deviation_bounds_lower = stats.std_deviation_bounds.lower; + + addMetricValue(values, this.getMetricName(statName), stats[statName]); + } + break; + } + default: { + let metricName = this.getMetricName(metric.type); + let otherMetrics = _.filter(target.metrics, {type: metric.type}); + + // if more of the same metric type include field field name in property + if (otherMetrics.length > 1) { + metricName += ' ' + metric.field; + } + + addMetricValue(values, metricName, bucket[metric.id].value); + break; + } + } + } + + table.rows.push(values); + } + } + + // This is quite complex + // neeed to recurise down the nested buckets to build series + processBuckets(aggs, target, seriesList, table, props, depth) { + var bucket, aggDef, esAgg, aggId; + var maxDepth = target.bucketAggs.length-1; + + for (aggId in aggs) { + aggDef = _.find(target.bucketAggs, {id: aggId}); + esAgg = aggs[aggId]; + + if (!aggDef) { + continue; + } + + if (depth === maxDepth) { + if (aggDef.type === 'date_histogram') { + this.processMetrics(esAgg, target, seriesList, props); + } else { + this.processAggregationDocs(esAgg, aggDef, target, table, props); + } + } else { + for (var nameIndex in esAgg.buckets) { + bucket = esAgg.buckets[nameIndex]; + props = _.clone(props); + if (bucket.key !== void 0) { + props[aggDef.field] = bucket.key; + } else { + props["filter"] = nameIndex; + } + if (bucket.key_as_string) { + props[aggDef.field] = bucket.key_as_string; + } + this.processBuckets(bucket, target, seriesList, table, props, depth+1); + } } } } - return { data: seriesList }; -}; + private getMetricName(metric) { + var metricDef = _.find(queryDef.metricAggTypes, {value: metric}); + if (!metricDef) { + metricDef = _.find(queryDef.extendedStats, {value: metric}); + } + return metricDef ? metricDef.text : metric; + } + + private getSeriesName(series, target, metricTypeCount) { + var metricName = this.getMetricName(series.metric); + + if (target.alias) { + var regex = /\{\{([\s\S]+?)\}\}/g; + + return target.alias.replace(regex, function(match, g1, g2) { + var group = g1 || g2; + + if (group.indexOf('term ') === 0) { return series.props[group.substring(5)]; } + if (series.props[group] !== void 0) { return series.props[group]; } + if (group === 'metric') { return metricName; } + if (group === 'field') { return series.field; } + + return match; + }); + } + + if (series.field && queryDef.isPipelineAgg(series.metric)) { + var appliedAgg = _.find(target.metrics, { id: series.field }); + if (appliedAgg) { + metricName += ' ' + queryDef.describeMetric(appliedAgg); + } else { + metricName = 'Unset'; + } + } else if (series.field) { + metricName += ' ' + series.field; + } + + var propKeys = _.keys(series.props); + if (propKeys.length === 0) { + return metricName; + } + + var name = ''; + for (var propName in series.props) { + name += series.props[propName] + ' '; + } + + if (metricTypeCount === 1) { + return name.trim(); + } + + return name.trim() + ' ' + metricName; + } + + nameSeries(seriesList, target) { + var metricTypeCount = _.uniq(_.map(seriesList, 'metric')).length; + + for (var i = 0; i < seriesList.length; i++) { + var series = seriesList[i]; + series.target = this.getSeriesName(series, target, metricTypeCount); + } + } + + processHits(hits, seriesList) { + var series = {target: 'docs', type: 'docs', datapoints: [], total: hits.total, filterable: true}; + var propName, hit, doc, i; + + for (i = 0; i < hits.hits.length; i++) { + hit = hits.hits[i]; + doc = { + _id: hit._id, + _type: hit._type, + _index: hit._index + }; + + if (hit._source) { + for (propName in hit._source) { + doc[propName] = hit._source[propName]; + } + } + + for (propName in hit.fields) { + doc[propName] = hit.fields[propName]; + } + series.datapoints.push(doc); + } + + seriesList.push(series); + } + + trimDatapoints(aggregations, target) { + var histogram = _.find(target.bucketAggs, { type: 'date_histogram'}); + + var shouldDropFirstAndLast = histogram && histogram.settings && histogram.settings.trimEdges; + if (shouldDropFirstAndLast) { + var trim = histogram.settings.trimEdges; + for (var prop in aggregations) { + var points = aggregations[prop]; + if (points.datapoints.length > trim * 2) { + points.datapoints = points.datapoints.slice(trim, points.datapoints.length - trim); + } + } + } + } + + getErrorFromElasticResponse(response, err) { + var result: any = {}; + result.data = JSON.stringify(err, null, 4); + if (err.root_cause && err.root_cause.length > 0 && err.root_cause[0].reason) { + result.message = err.root_cause[0].reason; + } else { + result.message = err.reason || 'Unkown elatic error response'; + } + + if (response.$$config) { + result.config = response.$$config; + } + + return result; + } + + getTimeSeries() { + var seriesList = []; + + for (var i = 0; i < this.response.responses.length; i++) { + var response = this.response.responses[i]; + if (response.error) { + throw this.getErrorFromElasticResponse(this.response, response.error); + } + + if (response.hits && response.hits.hits.length > 0) { + this.processHits(response.hits, seriesList); + } + + if (response.aggregations) { + var aggregations = response.aggregations; + var target = this.targets[i]; + var tmpSeriesList = []; + var table = new TableModel(); + + this.processBuckets(aggregations, target, tmpSeriesList, table, {}, 0); + this.trimDatapoints(tmpSeriesList, target); + this.nameSeries(tmpSeriesList, target); + + for (var y = 0; y < tmpSeriesList.length; y++) { + seriesList.push(tmpSeriesList[y]); + } + + if (table.rows.length > 0) { + seriesList.push(table); + } + } + } + + return { data: seriesList }; + } +} diff --git a/public/app/plugins/datasource/elasticsearch/index_pattern.d.ts b/public/app/plugins/datasource/elasticsearch/index_pattern.d.ts deleted file mode 100644 index c3318b8e133..00000000000 --- a/public/app/plugins/datasource/elasticsearch/index_pattern.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -declare var test: any; -export default test; diff --git a/public/app/plugins/datasource/elasticsearch/index_pattern.js b/public/app/plugins/datasource/elasticsearch/index_pattern.js deleted file mode 100644 index 6b26e41e1c5..00000000000 --- a/public/app/plugins/datasource/elasticsearch/index_pattern.js +++ /dev/null @@ -1,48 +0,0 @@ -define([ - 'lodash', - 'moment', -], -function (_, moment) { - 'use strict'; - - function IndexPattern(pattern, interval) { - this.pattern = pattern; - this.interval = interval; - } - - IndexPattern.intervalMap = { - "Hourly": { startOf: 'hour', amount: 'hours'}, - "Daily": { startOf: 'day', amount: 'days'}, - "Weekly": { startOf: 'isoWeek', amount: 'weeks'}, - "Monthly": { startOf: 'month', amount: 'months'}, - "Yearly": { startOf: 'year', amount: 'years'}, - }; - - IndexPattern.prototype.getIndexForToday = function() { - if (this.interval) { - return moment.utc().format(this.pattern); - } else { - return this.pattern; - } - }; - - IndexPattern.prototype.getIndexList = function(from, to) { - if (!this.interval) { - return this.pattern; - } - - var intervalInfo = IndexPattern.intervalMap[this.interval]; - var start = moment(from).utc().startOf(intervalInfo.startOf); - var end = moment(to).utc().startOf(intervalInfo.startOf).valueOf(); - var indexList = []; - - while (start <= end) { - indexList.push(start.format(this.pattern)); - start.add(1, intervalInfo.amount); - } - - return indexList; - }; - - return IndexPattern; -}); diff --git a/public/app/plugins/datasource/elasticsearch/index_pattern.ts b/public/app/plugins/datasource/elasticsearch/index_pattern.ts new file mode 100644 index 00000000000..4778dafa48f --- /dev/null +++ b/public/app/plugins/datasource/elasticsearch/index_pattern.ts @@ -0,0 +1,43 @@ +/// + +import moment from 'moment'; + +const intervalMap = { + "Hourly": { startOf: 'hour', amount: 'hours'}, + "Daily": { startOf: 'day', amount: 'days'}, + "Weekly": { startOf: 'isoWeek', amount: 'weeks'}, + "Monthly": { startOf: 'month', amount: 'months'}, + "Yearly": { startOf: 'year', amount: 'years'}, +}; + +export class IndexPattern { + + constructor(private pattern, private interval: string | null) { } + + getIndexForToday() { + if (this.interval) { + return moment.utc().format(this.pattern); + } else { + return this.pattern; + } + }; + + getIndexList(from, to) { + if (!this.interval) { + return this.pattern; + } + + var intervalInfo = intervalMap[this.interval]; + var start = moment(from).utc().startOf(intervalInfo.startOf); + var end = moment(to).utc().startOf(intervalInfo.startOf).valueOf(); + var indexList = []; + + while (start <= end) { + indexList.push(start.format(this.pattern)); + start.add(1, intervalInfo.amount); + } + + return indexList; + } +} + diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.d.ts b/public/app/plugins/datasource/elasticsearch/query_builder.d.ts deleted file mode 100644 index c3318b8e133..00000000000 --- a/public/app/plugins/datasource/elasticsearch/query_builder.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -declare var test: any; -export default test; diff --git a/public/app/plugins/datasource/elasticsearch/query_builder.js b/public/app/plugins/datasource/elasticsearch/query_builder.ts similarity index 88% rename from public/app/plugins/datasource/elasticsearch/query_builder.js rename to public/app/plugins/datasource/elasticsearch/query_builder.ts index 274a0e8f72f..ba4e1e436d4 100644 --- a/public/app/plugins/datasource/elasticsearch/query_builder.js +++ b/public/app/plugins/datasource/elasticsearch/query_builder.ts @@ -1,15 +1,15 @@ -define([ - './query_def', -], -function (queryDef) { - 'use strict'; +import * as queryDef from './query_def'; - function ElasticQueryBuilder(options) { +export class ElasticQueryBuilder { + timeField: string; + esVersion: number; + + constructor(options) { this.timeField = options.timeField; this.esVersion = options.esVersion; } - ElasticQueryBuilder.prototype.getRangeFilter = function() { + getRangeFilter() { var filter = {}; filter[this.timeField] = { gte: "$timeFrom", @@ -18,9 +18,9 @@ function (queryDef) { }; return filter; - }; + } - ElasticQueryBuilder.prototype.buildTermsAgg = function(aggDef, queryNode, target) { + buildTermsAgg(aggDef, queryNode, target) { var metricRef, metric, y; queryNode.terms = { "field": aggDef.field }; @@ -57,10 +57,10 @@ function (queryDef) { } return queryNode; - }; + } - ElasticQueryBuilder.prototype.getDateHistogramAgg = function(aggDef) { - var esAgg = {}; + getDateHistogramAgg(aggDef) { + var esAgg: any = {}; var settings = aggDef.settings || {}; esAgg.interval = settings.interval; esAgg.field = this.timeField; @@ -77,10 +77,10 @@ function (queryDef) { } return esAgg; - }; + } - ElasticQueryBuilder.prototype.getHistogramAgg = function(aggDef) { - var esAgg = {}; + getHistogramAgg(aggDef) { + var esAgg: any = {}; var settings = aggDef.settings || {}; esAgg.interval = settings.interval; esAgg.field = aggDef.field; @@ -90,9 +90,9 @@ function (queryDef) { esAgg.missing = settings.missing; } return esAgg; - }; + } - ElasticQueryBuilder.prototype.getFiltersAgg = function(aggDef) { + getFiltersAgg(aggDef) { var filterObj = {}; for (var i = 0; i < aggDef.settings.filters.length; i++) { var query = aggDef.settings.filters[i].query; @@ -107,9 +107,9 @@ function (queryDef) { } return filterObj; - }; + } - ElasticQueryBuilder.prototype.documentQuery = function(query, size) { + documentQuery(query, size) { query.size = size; query.sort = {}; query.sort[this.timeField] = {order: 'desc', unmapped_type: 'boolean'}; @@ -126,9 +126,9 @@ function (queryDef) { query.docvalue_fields = [this.timeField]; } return query; - }; + } - ElasticQueryBuilder.prototype.addAdhocFilters = function(query, adhocFilters) { + addAdhocFilters(query, adhocFilters) { if (!adhocFilters) { return; } @@ -142,7 +142,7 @@ function (queryDef) { queryCondition = {}; queryCondition[filter.key] = {query: filter.value}; - switch(filter.operator){ + switch (filter.operator){ case "=": if (!query.query.bool.must) { query.query.bool.must = []; } query.query.bool.must.push({match_phrase: queryCondition}); @@ -169,7 +169,7 @@ function (queryDef) { } }; - ElasticQueryBuilder.prototype.build = function(target, adhocFilters, queryString) { + build(target, adhocFilters?, queryString?) { // make sure query has defaults; target.metrics = target.metrics || [{ type: 'count', id: '1' }]; target.dsType = 'elasticsearch'; @@ -213,7 +213,7 @@ function (queryDef) { var aggDef = target.bucketAggs[i]; var esAgg = {}; - switch(aggDef.type) { + switch (aggDef.type) { case 'date_histogram': { esAgg["date_histogram"] = this.getDateHistogramAgg(aggDef); break; @@ -273,10 +273,10 @@ function (queryDef) { } return query; - }; + } - ElasticQueryBuilder.prototype.getTermsQuery = function(queryDef) { - var query = { + getTermsQuery(queryDef) { + var query: any = { "size": 0, "query": { "bool": { @@ -311,7 +311,5 @@ function (queryDef) { } }; return query; - }; - - return ElasticQueryBuilder; -}); + } +} diff --git a/public/app/plugins/datasource/elasticsearch/query_def.d.ts b/public/app/plugins/datasource/elasticsearch/query_def.d.ts deleted file mode 100644 index c3318b8e133..00000000000 --- a/public/app/plugins/datasource/elasticsearch/query_def.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -declare var test: any; -export default test; diff --git a/public/app/plugins/datasource/elasticsearch/query_def.js b/public/app/plugins/datasource/elasticsearch/query_def.js deleted file mode 100644 index a55e2dfd9e5..00000000000 --- a/public/app/plugins/datasource/elasticsearch/query_def.js +++ /dev/null @@ -1,199 +0,0 @@ -define([ - 'lodash' -], -function (_) { - 'use strict'; - - return { - metricAggTypes: [ - {text: "Count", value: 'count', requiresField: false}, - {text: "Average", value: 'avg', requiresField: true, supportsInlineScript: true, supportsMissing: true}, - {text: "Sum", value: 'sum', requiresField: true, supportsInlineScript: true, supportsMissing: true}, - {text: "Max", value: 'max', requiresField: true, supportsInlineScript: true, supportsMissing: true}, - {text: "Min", value: 'min', requiresField: true, supportsInlineScript: true, supportsMissing: true}, - {text: "Extended Stats", value: 'extended_stats', requiresField: true, supportsMissing: true, supportsInlineScript: true}, - {text: "Percentiles", value: 'percentiles', requiresField: true, supportsMissing: true, supportsInlineScript: true}, - {text: "Unique Count", value: "cardinality", requiresField: true, supportsMissing: true}, - {text: "Moving Average", value: 'moving_avg', requiresField: false, isPipelineAgg: true, minVersion: 2}, - {text: "Derivative", value: 'derivative', requiresField: false, isPipelineAgg: true, minVersion: 2 }, - {text: "Raw Document", value: "raw_document", requiresField: false} - ], - - bucketAggTypes: [ - {text: "Terms", value: 'terms', requiresField: true}, - {text: "Filters", value: 'filters' }, - {text: "Geo Hash Grid", value: 'geohash_grid', requiresField: true}, - {text: "Date Histogram", value: 'date_histogram', requiresField: true}, - {text: "Histogram", value: 'histogram', requiresField: true}, - ], - - orderByOptions: [ - {text: "Doc Count", value: '_count' }, - {text: "Term value", value: '_term' }, - ], - - orderOptions: [ - {text: "Top", value: 'desc' }, - {text: "Bottom", value: 'asc' }, - ], - - sizeOptions: [ - {text: "No limit", value: '0' }, - {text: "1", value: '1' }, - {text: "2", value: '2' }, - {text: "3", value: '3' }, - {text: "5", value: '5' }, - {text: "10", value: '10' }, - {text: "15", value: '15' }, - {text: "20", value: '20' }, - ], - - extendedStats: [ - {text: 'Avg', value: 'avg'}, - {text: 'Min', value: 'min'}, - {text: 'Max', value: 'max'}, - {text: 'Sum', value: 'sum'}, - {text: 'Count', value: 'count'}, - {text: 'Std Dev', value: 'std_deviation'}, - {text: 'Std Dev Upper', value: 'std_deviation_bounds_upper'}, - {text: 'Std Dev Lower', value: 'std_deviation_bounds_lower'}, - ], - - intervalOptions: [ - {text: 'auto', value: 'auto'}, - {text: '10s', value: '10s'}, - {text: '1m', value: '1m'}, - {text: '5m', value: '5m'}, - {text: '10m', value: '10m'}, - {text: '20m', value: '20m'}, - {text: '1h', value: '1h'}, - {text: '1d', value: '1d'}, - ], - - movingAvgModelOptions: [ - {text: 'Simple', value: 'simple'}, - {text: 'Linear', value: 'linear'}, - {text: 'Exponentially Weighted', value: 'ewma'}, - {text: 'Holt Linear', value: 'holt'}, - {text: 'Holt Winters', value: 'holt_winters'}, - ], - - pipelineOptions: { - 'moving_avg' : [ - {text: 'window', default: 5}, - {text: 'model', default: 'simple'}, - {text: 'predict', default: undefined}, - {text: 'minimize', default: false}, - ], - 'derivative': [ - {text: 'unit', default: undefined}, - ] - }, - - movingAvgModelSettings: { - 'simple' : [], - 'linear' : [], - 'ewma' : [ - {text: "Alpha", value: "alpha", default: undefined}], - 'holt' : [ - {text: "Alpha", value: "alpha", default: undefined}, - {text: "Beta", value: "beta", default: undefined}, - ], - 'holt_winters' : [ - {text: "Alpha", value: "alpha", default: undefined}, - {text: "Beta", value: "beta", default: undefined}, - {text: "Gamma", value: "gamma", default: undefined}, - {text: "Period", value: "period", default: undefined}, - {text: "Pad", value: "pad", default: undefined, isCheckbox: true}, - ], - }, - - getMetricAggTypes: function(esVersion) { - return _.filter(this.metricAggTypes, function(f) { - if (f.minVersion) { - return f.minVersion <= esVersion; - } else { - return true; - } - }); - }, - - getPipelineOptions: function(metric) { - if (!this.isPipelineAgg(metric.type)) { - return []; - } - - return this.pipelineOptions[metric.type]; - }, - - isPipelineAgg: function(metricType) { - if (metricType) { - var po = this.pipelineOptions[metricType]; - return po !== null && po !== undefined; - } - - return false; - }, - - getPipelineAggOptions: function(targets) { - var self = this; - var result = []; - _.each(targets.metrics, function(metric) { - if (!self.isPipelineAgg(metric.type)) { - result.push({text: self.describeMetric(metric), value: metric.id }); - } - }); - - return result; - }, - - getMovingAvgSettings: function(model, filtered) { - var filteredResult = []; - if (filtered) { - _.each(this.movingAvgModelSettings[model], function(setting) { - if (!(setting.isCheckbox)) { - filteredResult.push(setting); - } - }); - return filteredResult; - } - return this.movingAvgModelSettings[model]; - }, - - getOrderByOptions: function(target) { - var self = this; - var metricRefs = []; - _.each(target.metrics, function(metric) { - if (metric.type !== 'count') { - metricRefs.push({text: self.describeMetric(metric), value: metric.id}); - } - }); - - return this.orderByOptions.concat(metricRefs); - }, - - describeOrder: function(order) { - var def = _.find(this.orderOptions, {value: order}); - return def.text; - }, - - describeMetric: function(metric) { - var def = _.find(this.metricAggTypes, {value: metric.type}); - return def.text + ' ' + metric.field; - }, - - describeOrderBy: function(orderBy, target) { - var def = _.find(this.orderByOptions, {value: orderBy}); - if (def) { - return def.text; - } - var metric = _.find(target.metrics, {id: orderBy}); - if (metric) { - return this.describeMetric(metric); - } else { - return "metric not found"; - } - }, - }; - -}); diff --git a/public/app/plugins/datasource/elasticsearch/query_def.ts b/public/app/plugins/datasource/elasticsearch/query_def.ts new file mode 100644 index 00000000000..dac48acae48 --- /dev/null +++ b/public/app/plugins/datasource/elasticsearch/query_def.ts @@ -0,0 +1,191 @@ +/// + +import _ from 'lodash'; + +export const metricAggTypes = [ + {text: "Count", value: 'count', requiresField: false}, + {text: "Average", value: 'avg', requiresField: true, supportsInlineScript: true, supportsMissing: true}, + {text: "Sum", value: 'sum', requiresField: true, supportsInlineScript: true, supportsMissing: true}, + {text: "Max", value: 'max', requiresField: true, supportsInlineScript: true, supportsMissing: true}, + {text: "Min", value: 'min', requiresField: true, supportsInlineScript: true, supportsMissing: true}, + {text: "Extended Stats", value: 'extended_stats', requiresField: true, supportsMissing: true, supportsInlineScript: true}, + {text: "Percentiles", value: 'percentiles', requiresField: true, supportsMissing: true, supportsInlineScript: true}, + {text: "Unique Count", value: "cardinality", requiresField: true, supportsMissing: true}, + {text: "Moving Average", value: 'moving_avg', requiresField: false, isPipelineAgg: true, minVersion: 2}, + {text: "Derivative", value: 'derivative', requiresField: false, isPipelineAgg: true, minVersion: 2 }, + {text: "Raw Document", value: "raw_document", requiresField: false} +]; + +export const bucketAggTypes = [ + {text: "Terms", value: 'terms', requiresField: true}, + {text: "Filters", value: 'filters' }, + {text: "Geo Hash Grid", value: 'geohash_grid', requiresField: true}, + {text: "Date Histogram", value: 'date_histogram', requiresField: true}, + {text: "Histogram", value: 'histogram', requiresField: true}, +]; + +export const orderByOptions = [ + {text: "Doc Count", value: '_count' }, + {text: "Term value", value: '_term' }, +]; + +export const orderOptions = [ + {text: "Top", value: 'desc' }, + {text: "Bottom", value: 'asc' }, +]; + +export const sizeOptions = [ + {text: "No limit", value: '0' }, + {text: "1", value: '1' }, + {text: "2", value: '2' }, + {text: "3", value: '3' }, + {text: "5", value: '5' }, + {text: "10", value: '10' }, + {text: "15", value: '15' }, + {text: "20", value: '20' }, +]; + +export const extendedStats = [ + {text: 'Avg', value: 'avg'}, + {text: 'Min', value: 'min'}, + {text: 'Max', value: 'max'}, + {text: 'Sum', value: 'sum'}, + {text: 'Count', value: 'count'}, + {text: 'Std Dev', value: 'std_deviation'}, + {text: 'Std Dev Upper', value: 'std_deviation_bounds_upper'}, + {text: 'Std Dev Lower', value: 'std_deviation_bounds_lower'}, +]; + +export const intervalOptions = [ + {text: 'auto', value: 'auto'}, + {text: '10s', value: '10s'}, + {text: '1m', value: '1m'}, + {text: '5m', value: '5m'}, + {text: '10m', value: '10m'}, + {text: '20m', value: '20m'}, + {text: '1h', value: '1h'}, + {text: '1d', value: '1d'}, +]; + +export const movingAvgModelOptions = [ + {text: 'Simple', value: 'simple'}, + {text: 'Linear', value: 'linear'}, + {text: 'Exponentially Weighted', value: 'ewma'}, + {text: 'Holt Linear', value: 'holt'}, + {text: 'Holt Winters', value: 'holt_winters'}, +]; + +export const pipelineOptions = { + 'moving_avg' : [ + {text: 'window', default: 5}, + {text: 'model', default: 'simple'}, + {text: 'predict', default: undefined}, + {text: 'minimize', default: false}, + ], + 'derivative': [ + {text: 'unit', default: undefined}, + ] +}; + +export const movingAvgModelSettings = { + 'simple' : [], + 'linear' : [], + 'ewma' : [ + {text: "Alpha", value: "alpha", default: undefined}], + 'holt' : [ + {text: "Alpha", value: "alpha", default: undefined}, + {text: "Beta", value: "beta", default: undefined}, + ], + 'holt_winters' : [ + {text: "Alpha", value: "alpha", default: undefined}, + {text: "Beta", value: "beta", default: undefined}, + {text: "Gamma", value: "gamma", default: undefined}, + {text: "Period", value: "period", default: undefined}, + {text: "Pad", value: "pad", default: undefined, isCheckbox: true}, + ], +}; + +export function getMetricAggTypes(esVersion) { + return _.filter(metricAggTypes, function(f) { + if (f.minVersion) { + return f.minVersion <= esVersion; + } else { + return true; + } + }); +} + +export function getPipelineOptions(metric) { + if (!isPipelineAgg(metric.type)) { + return []; + } + + return pipelineOptions[metric.type]; +} + +export function isPipelineAgg(metricType) { + if (metricType) { + var po = pipelineOptions[metricType]; + return po !== null && po !== undefined; + } + + return false; +} + +export function getPipelineAggOptions(targets) { + var result = []; + _.each(targets.metrics, function(metric) { + if (!isPipelineAgg(metric.type)) { + result.push({text: describeMetric(metric), value: metric.id }); + } + }); + + return result; +} + +export function getMovingAvgSettings(model, filtered) { + var filteredResult = []; + if (filtered) { + _.each(movingAvgModelSettings[model], function(setting) { + if (!(setting.isCheckbox)) { + filteredResult.push(setting); + } + }); + return filteredResult; + } + return movingAvgModelSettings[model]; +} + +export function getOrderByOptions(target) { + var metricRefs = []; + _.each(target.metrics, function(metric) { + if (metric.type !== 'count') { + metricRefs.push({text: describeMetric(metric), value: metric.id}); + } + }); + + return orderByOptions.concat(metricRefs); +} + +export function describeOrder(order) { + var def = _.find(orderOptions, {value: order}); + return def.text; +} + +export function describeMetric(metric) { + var def = _.find(metricAggTypes, {value: metric.type}); + return def.text + ' ' + metric.field; +} + +export function describeOrderBy(orderBy, target) { + var def = _.find(orderByOptions, {value: orderBy}); + if (def) { + return def.text; + } + var metric = _.find(target.metrics, {id: orderBy}); + if (metric) { + return describeMetric(metric); + } else { + return "metric not found"; + } +}; diff --git a/public/app/plugins/datasource/elasticsearch/specs/index_pattern_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/index_pattern_specs.ts index 3fee81083b7..16ec678f1c0 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/index_pattern_specs.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/index_pattern_specs.ts @@ -2,7 +2,7 @@ import {describe, it, expect} from 'test/lib/common'; import moment from 'moment'; -import IndexPattern from '../index_pattern'; +import {IndexPattern} from '../index_pattern'; describe('IndexPattern', function() { @@ -19,7 +19,7 @@ describe('IndexPattern', function() { describe('no interval', function() { it('should return correct index', function() { - var pattern = new IndexPattern('my-metrics'); + var pattern = new IndexPattern('my-metrics', null); var from = new Date(2015, 4, 30, 1, 2, 3); var to = new Date(2015, 5, 1, 12, 5 , 6); expect(pattern.getIndexList(from, to)).to.eql('my-metrics'); diff --git a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts index 3440309897b..29876d1ea42 100644 --- a/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts +++ b/public/app/plugins/datasource/elasticsearch/specs/query_builder_specs.ts @@ -1,6 +1,6 @@ import {describe, beforeEach, it, expect} from 'test/lib/common'; -import ElasticQueryBuilder from '../query_builder'; +import {ElasticQueryBuilder} from '../query_builder'; describe('ElasticQueryBuilder', function() { var builder; diff --git a/public/test/mocks/history-mocks.js b/public/test/mocks/history-mocks.js deleted file mode 100644 index 6bbed54920d..00000000000 --- a/public/test/mocks/history-mocks.js +++ /dev/null @@ -1,197 +0,0 @@ -define([], - function() { - 'use strict'; - - return { - versions: function() { - return [{ - id: 4, - dashboardId: 1, - parentVersion: 3, - restoredFrom: 0, - version: 4, - created: '2017-02-22T17:43:01-08:00', - createdBy: 'admin', - message: '', - }, - { - id: 3, - dashboardId: 1, - parentVersion: 1, - restoredFrom: 1, - version: 3, - created: '2017-02-22T17:43:01-08:00', - createdBy: 'admin', - message: '', - }, - { - id: 2, - dashboardId: 1, - parentVersion: 0, - restoredFrom: -1, - version: 2, - created: '2017-02-22T17:29:52-08:00', - createdBy: 'admin', - message: '', - }, - { - id: 1, - dashboardId: 1, - parentVersion: 0, - restoredFrom: -1, - slug: 'history-dashboard', - version: 1, - created: '2017-02-22T17:06:37-08:00', - createdBy: 'admin', - message: '', - }]; - }, - compare: function(type) { - return type === 'basic' ? '
' : '
'; - }, - restore: function(version, restoredFrom) { - return { - dashboard: { - meta: { - type: 'db', - canSave: true, - canEdit: true, - canStar: true, - slug: 'history-dashboard', - expires: '0001-01-01T00:00:00Z', - created: '2017-02-21T18:40:45-08:00', - updated: '2017-04-11T21:31:22.59219665-07:00', - updatedBy: 'admin', - createdBy: 'admin', - version: version, - }, - dashboard: { - annotations: { - list: [] - }, - description: 'A random dashboard for implementing the history list', - editable: true, - gnetId: null, - graphTooltip: 0, - hideControls: false, - id: 1, - links: [], - restoredFrom: restoredFrom, - rows: [{ - collapse: false, - height: '250px', - panels: [{ - aliasColors: {}, - bars: false, - datasource: null, - fill: 1, - id: 1, - legend: { - avg: false, - current: false, - max: false, - min: false, - show: true, - total: false, - values: false - }, - lines: true, - linewidth: 1, - nullPointMode: "null", - percentage: false, - pointradius: 5, - points: false, - renderer: 'flot', - seriesOverrides: [], - span: 12, - stack: false, - steppedLine: false, - targets: [{}], - thresholds: [], - timeFrom: null, - timeShift: null, - title: 'Panel Title', - tooltip: { - shared: true, - sort: 0, - value_type: 'individual' - }, - type: 'graph', - xaxis: { - mode: 'time', - name: null, - show: true, - values: [] - }, - yaxes: [{ - format: 'short', - label: null, - logBase: 1, - max: null, - min: null, - show: true - }, { - format: 'short', - label: null, - logBase: 1, - max: null, - min: null, - show: true - }] - }], - repeat: null, - repeatIteration: null, - repeatRowId: null, - showTitle: false, - title: 'Dashboard Row', - titleSize: 'h6' - } - ], - schemaVersion: 14, - style: 'dark', - tags: [ - 'development' - ], - templating: { - 'list': [] - }, - time: { - from: 'now-6h', - to: 'now' - }, - timepicker: { - refresh_intervals: [ - '5s', - '10s', - '30s', - '1m', - '5m', - '15m', - '30m', - '1h', - '2h', - '1d', - ], - time_options: [ - '5m', - '15m', - '1h', - '6h', - '12h', - '24h', - '2d', - '7d', - '30d' - ] - }, - timezone: 'utc', - title: 'History Dashboard', - version: version, - } - }, - message: 'Dashboard restored to version ' + version, - version: version - }; - }, - }; -});