From 411719bc70da9a8e1233299f204f91b7b0fc8934 Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Tue, 13 Nov 2018 15:35:20 +0000 Subject: [PATCH] Explore: POC for datasource query importers Explore is about keeping context between datasources if possible. When changing from metrics to logging, some of the filtering can be kept to narrow down logging streams relevant to the metrics. - adds `importQueries` function in language providers - query import dependent on origin datasource - implemented prometheus-to-logging import: keeping label selectors that are common to both datasources - added types --- public/app/features/explore/Explore.tsx | 38 ++++++++-- public/app/features/plugins/datasource_srv.ts | 4 +- .../plugins/datasource/logging/datasource.ts | 7 +- .../logging/language_provider.test.ts | 74 +++++++++++++++++++ .../datasource/logging/language_provider.ts | 54 +++++++++++++- .../datasource/prometheus/language_utils.ts | 4 +- public/app/types/datasources.ts | 2 - public/app/types/series.ts | 11 +++ 8 files changed, 178 insertions(+), 16 deletions(-) create mode 100644 public/app/plugins/datasource/logging/language_provider.test.ts diff --git a/public/app/features/explore/Explore.tsx b/public/app/features/explore/Explore.tsx index 753f158fd9f..37b0036d3f2 100644 --- a/public/app/features/explore/Explore.tsx +++ b/public/app/features/explore/Explore.tsx @@ -3,8 +3,9 @@ import { hot } from 'react-hot-loader'; import Select from 'react-select'; import _ from 'lodash'; +import { DataSource } from 'app/types/datasources'; import { ExploreState, ExploreUrlState, HistoryItem, Query, QueryTransaction, ResultType } from 'app/types/explore'; -import { RawTimeRange } from 'app/types/series'; +import { RawTimeRange, DataQuery } from 'app/types/series'; import kbn from 'app/core/utils/kbn'; import colors from 'app/core/utils/colors'; import store from 'app/core/store'; @@ -16,6 +17,7 @@ import PickerOption from 'app/core/components/Picker/PickerOption'; import IndicatorsContainer from 'app/core/components/Picker/IndicatorsContainer'; import NoOptionsMessage from 'app/core/components/Picker/NoOptionsMessage'; import TableModel, { mergeTablesIntoModel } from 'app/core/table_model'; +import { DatasourceSrv } from 'app/features/plugins/datasource_srv'; import QueryRows from './QueryRows'; import Graph from './Graph'; @@ -24,7 +26,6 @@ import Table from './Table'; import ErrorBoundary from './ErrorBoundary'; import TimePicker from './TimePicker'; import { ensureQueries, generateQueryKey, hasQuery } from './utils/query'; -import { DataSource } from 'app/types/datasources'; const MAX_HISTORY_ITEMS = 100; @@ -77,7 +78,7 @@ function updateHistory(history: HistoryItem[], datasourceId: string, queries: st } interface ExploreProps { - datasourceSrv: any; + datasourceSrv: DatasourceSrv; onChangeSplit: (split: boolean, state?: ExploreState) => void; onSaveState: (key: string, state: ExploreState) => void; position: string; @@ -92,6 +93,7 @@ export class Explore extends React.PureComponent { /** * Current query expressions of the rows including their modifications, used for running queries. * Not kept in component state to prevent edit-render roundtrips. + * TODO: make this generic (other datasources might not have string representations of current query state) */ queryExpressions: string[]; @@ -160,7 +162,7 @@ export class Explore extends React.PureComponent { } } - async setDatasource(datasource: DataSource) { + async setDatasource(datasource: any, origin?: DataSource) { const supportsGraph = datasource.meta.metrics; const supportsLogs = datasource.meta.logs; const supportsTable = datasource.meta.metrics; @@ -181,12 +183,33 @@ export class Explore extends React.PureComponent { datasource.init(); } - // Keep queries but reset edit state + // Check if queries can be imported from previously selected datasource + let queryExpressions = this.queryExpressions; + if (origin) { + if (origin.meta.id === datasource.meta.id) { + // Keep same queries if same type of datasource + queryExpressions = [...this.queryExpressions]; + } else if (datasource.importQueries) { + // Datasource-specific importers, wrapping to satisfy interface + const wrappedQueries: DataQuery[] = this.queryExpressions.map((query, index) => ({ + refId: String(index), + expr: query, + })); + const modifiedQueries: DataQuery[] = await datasource.importQueries(wrappedQueries, origin.meta); + queryExpressions = modifiedQueries.map(({ expr }) => expr); + } else { + // Default is blank queries + queryExpressions = this.queryExpressions.map(() => ''); + } + } + + // Reset edit state with new queries const nextQueries = this.state.queries.map((q, i) => ({ ...q, key: generateQueryKey(i), - query: this.queryExpressions[i], + query: queryExpressions[i], })); + this.queryExpressions = queryExpressions; // Custom components const StartPage = datasource.pluginExports.ExploreStartPage; @@ -246,6 +269,7 @@ export class Explore extends React.PureComponent { }; onChangeDatasource = async option => { + const origin = this.state.datasource; this.setState({ datasource: null, datasourceError: null, @@ -254,7 +278,7 @@ export class Explore extends React.PureComponent { }); const datasourceName = option.value; const datasource = await this.props.datasourceSrv.get(datasourceName); - this.setDatasource(datasource); + this.setDatasource(datasource as any, origin); }; onChangeQuery = (value: string, index: number, override?: boolean) => { diff --git a/public/app/features/plugins/datasource_srv.ts b/public/app/features/plugins/datasource_srv.ts index fed455472c9..f7bd6b4d1ee 100644 --- a/public/app/features/plugins/datasource_srv.ts +++ b/public/app/features/plugins/datasource_srv.ts @@ -22,7 +22,7 @@ export class DatasourceSrv { this.datasources = {}; } - get(name?): Promise { + get(name?: string): Promise { if (!name) { return this.get(config.defaultDatasource); } @@ -40,7 +40,7 @@ export class DatasourceSrv { return this.loadDatasource(name); } - loadDatasource(name) { + loadDatasource(name: string): Promise { const dsConfig = config.datasources[name]; if (!dsConfig) { return this.$q.reject({ message: 'Datasource named ' + name + ' was not found' }); diff --git a/public/app/plugins/datasource/logging/datasource.ts b/public/app/plugins/datasource/logging/datasource.ts index fcf3028c025..494dcd78d6c 100644 --- a/public/app/plugins/datasource/logging/datasource.ts +++ b/public/app/plugins/datasource/logging/datasource.ts @@ -1,10 +1,11 @@ import _ from 'lodash'; import * as dateMath from 'app/core/utils/datemath'; +import { LogsStream, LogsModel, makeSeriesForLogs } from 'app/core/logs_model'; +import { PluginMeta, DataQuery } from 'app/types'; import LanguageProvider from './language_provider'; import { mergeStreamsToLogs } from './result_transformer'; -import { LogsStream, LogsModel, makeSeriesForLogs } from 'app/core/logs_model'; export const DEFAULT_LIMIT = 1000; @@ -111,6 +112,10 @@ export default class LoggingDatasource { }); } + async importQueries(queries: DataQuery[], originMeta: PluginMeta): Promise { + return this.languageProvider.importQueries(queries, originMeta.id); + } + metadataRequest(url) { // HACK to get label values for {job=|}, will be replaced when implementing LoggingQueryField const apiUrl = url.replace('v1', 'prom'); diff --git a/public/app/plugins/datasource/logging/language_provider.test.ts b/public/app/plugins/datasource/logging/language_provider.test.ts new file mode 100644 index 00000000000..e0844cf0c7a --- /dev/null +++ b/public/app/plugins/datasource/logging/language_provider.test.ts @@ -0,0 +1,74 @@ +import Plain from 'slate-plain-serializer'; + +import LanguageProvider from './language_provider'; + +describe('Language completion provider', () => { + const datasource = { + metadataRequest: () => ({ data: { data: [] } }), + }; + + it('returns default suggestions on emtpty context', () => { + const instance = new LanguageProvider(datasource); + const result = instance.provideCompletionItems({ text: '', prefix: '', wrapperClasses: [] }); + expect(result.context).toBeUndefined(); + expect(result.refresher).toBeUndefined(); + expect(result.suggestions.length).toEqual(0); + }); + + describe('label suggestions', () => { + it('returns default label suggestions on label context', () => { + const instance = new LanguageProvider(datasource); + const value = Plain.deserialize('{}'); + const range = value.selection.merge({ + anchorOffset: 1, + }); + const valueWithSelection = value.change().select(range).value; + const result = instance.provideCompletionItems({ + text: '', + prefix: '', + wrapperClasses: ['context-labels'], + value: valueWithSelection, + }); + expect(result.context).toBe('context-labels'); + expect(result.suggestions).toEqual([{ items: [{ label: 'job' }, { label: 'namespace' }], label: 'Labels' }]); + }); + }); +}); + +describe('Query imports', () => { + const datasource = { + metadataRequest: () => ({ data: { data: [] } }), + }; + + it('returns empty queries for unknown origin datasource', async () => { + const instance = new LanguageProvider(datasource); + const result = await instance.importQueries([{ refId: 'bar', expr: 'foo' }], 'unknown'); + expect(result).toEqual([{ refId: 'bar', expr: '' }]); + }); + + describe('prometheus query imports', () => { + it('returns empty query from metric-only query', async () => { + const instance = new LanguageProvider(datasource); + const result = await instance.importPrometheusQuery('foo'); + expect(result).toEqual(''); + }); + + it('returns empty query from selector query if label is not available', async () => { + const datasourceWithLabels = { + metadataRequest: url => (url === '/api/prom/label' ? { data: { data: ['other'] } } : { data: { data: [] } }), + }; + const instance = new LanguageProvider(datasourceWithLabels); + const result = await instance.importPrometheusQuery('{foo="bar"}'); + expect(result).toEqual('{}'); + }); + + it('returns selector query from selector query with common labels', async () => { + const datasourceWithLabels = { + metadataRequest: url => (url === '/api/prom/label' ? { data: { data: ['foo'] } } : { data: { data: [] } }), + }; + const instance = new LanguageProvider(datasourceWithLabels); + const result = await instance.importPrometheusQuery('metric{foo="bar",baz="42"}'); + expect(result).toEqual('{foo="bar"}'); + }); + }); +}); diff --git a/public/app/plugins/datasource/logging/language_provider.ts b/public/app/plugins/datasource/logging/language_provider.ts index 0896168ca56..00745d2eee8 100644 --- a/public/app/plugins/datasource/logging/language_provider.ts +++ b/public/app/plugins/datasource/logging/language_provider.ts @@ -8,9 +8,9 @@ import { TypeaheadInput, TypeaheadOutput, } from 'app/types/explore'; - -import { parseSelector } from 'app/plugins/datasource/prometheus/language_utils'; +import { parseSelector, labelRegexp, selectorRegexp } from 'app/plugins/datasource/prometheus/language_utils'; import PromqlSyntax from 'app/plugins/datasource/prometheus/promql'; +import { DataQuery } from 'app/types'; const DEFAULT_KEYS = ['job', 'namespace']; const EMPTY_SELECTOR = '{}'; @@ -158,6 +158,56 @@ export default class LoggingLanguageProvider extends LanguageProvider { return { context, refresher, suggestions }; } + async importQueries(queries: DataQuery[], datasourceType: string): Promise { + if (datasourceType === 'prometheus') { + return Promise.all( + queries.map(async query => { + const expr = await this.importPrometheusQuery(query.expr); + return { + ...query, + expr, + }; + }) + ); + } + return queries.map(query => ({ + ...query, + expr: '', + })); + } + + async importPrometheusQuery(query: string): Promise { + // Consider only first selector in query + const selectorMatch = query.match(selectorRegexp); + if (selectorMatch) { + const selector = selectorMatch[0]; + const labels = {}; + selector.replace(labelRegexp, (_, key, operator, value) => { + labels[key] = { value, operator }; + return ''; + }); + + // Keep only labels that exist on origin and target datasource + await this.start(); // fetches all existing label keys + const commonLabels = {}; + for (const key in labels) { + const existingKeys = this.labelKeys[EMPTY_SELECTOR]; + if (existingKeys.indexOf(key) > -1) { + // Should we check for label value equality here? + commonLabels[key] = labels[key]; + } + } + const labelKeys = Object.keys(commonLabels).sort(); + const cleanSelector = labelKeys + .map(key => `${key}${commonLabels[key].operator}${commonLabels[key].value}`) + .join(','); + + return ['{', cleanSelector, '}'].join(''); + } + + return ''; + } + async fetchLogLabels() { const url = '/api/prom/label'; try { diff --git a/public/app/plugins/datasource/prometheus/language_utils.ts b/public/app/plugins/datasource/prometheus/language_utils.ts index 5995c427cd1..0f01dc3f767 100644 --- a/public/app/plugins/datasource/prometheus/language_utils.ts +++ b/public/app/plugins/datasource/prometheus/language_utils.ts @@ -24,8 +24,8 @@ export function processLabels(labels, withName = false) { } // const cleanSelectorRegexp = /\{(\w+="[^"\n]*?")(,\w+="[^"\n]*?")*\}/; -const selectorRegexp = /\{[^}]*?\}/; -const labelRegexp = /\b(\w+)(!?=~?)("[^"\n]*?")/g; +export const selectorRegexp = /\{[^}]*?\}/; +export const labelRegexp = /\b(\w+)(!?=~?)("[^"\n]*?")/g; export function parseSelector(query: string, cursorOffset = 1): { labelKeys: any[]; selector: string } { if (!query.match(selectorRegexp)) { // Special matcher for metrics diff --git a/public/app/types/datasources.ts b/public/app/types/datasources.ts index 5522f3a11ce..705d1f36a54 100644 --- a/public/app/types/datasources.ts +++ b/public/app/types/datasources.ts @@ -18,8 +18,6 @@ export interface DataSource { readOnly: boolean; meta?: PluginMeta; pluginExports?: PluginExports; - init?: () => void; - testDatasource?: () => Promise; } export interface DataSourcesState { diff --git a/public/app/types/series.ts b/public/app/types/series.ts index 5396880611b..18ebbc5f648 100644 --- a/public/app/types/series.ts +++ b/public/app/types/series.ts @@ -1,4 +1,5 @@ import { Moment } from 'moment'; +import { PluginMeta } from './plugins'; export enum LoadingState { NotStarted = 'NotStarted', @@ -70,6 +71,7 @@ export interface DataQueryResponse { export interface DataQuery { refId: string; + [key: string]: any; } export interface DataQueryOptions { @@ -87,5 +89,14 @@ export interface DataQueryOptions { } export interface DataSourceApi { + /** + * Imports queries from a different datasource + */ + importQueries?(queries: DataQuery[], originMeta: PluginMeta): Promise; + /** + * Initializes a datasource after instantiation + */ + init?: () => void; query(options: DataQueryOptions): Promise; + testDatasource?: () => Promise; }