From a9c94ec93bc772c086e8d7eab29b37b6bc82c1ee Mon Sep 17 00:00:00 2001 From: Dominik Prokop Date: Wed, 22 May 2019 23:10:05 +0200 Subject: [PATCH] Explore: Update the way Loki retrieve log context (#17204) * Move log's typings into grafana/ui * Update the way context is retrieved for Loki Major changes: 1. getLogRowContext expects row to be of LogRowModel type 2. getLogRowContext accepts generic options object, specific to a datasource of interest. limit option has been removed, and now it's a part of Loki's context query options (see below) 3. LogRowContextProvider performs two queries now. Before, it was Loki ds that performed queries in both directions when getLogRowContext. 4. Loki's getLogRowContext accepts options object of a type: interface LokiContextQueryOptions { direction?: 'BACKWARD' | 'FORWARD'; limit?: number; } This will enable querying in either direction independently and also slightly simplifies the way query errors are handled. LogRowContextProvider maps the results to a Loki specific context types, basically string[][], as raw log lines are displayed in first version. --- package.json | 1 + packages/grafana-ui/src/types/datasource.ts | 10 +- packages/grafana-ui/src/types/logs.ts | 84 +++++++++++++++ public/app/core/logs_model.ts | 90 ++-------------- public/app/core/specs/logs_model.test.ts | 7 +- public/app/core/utils/explore.test.ts | 3 +- public/app/core/utils/explore.ts | 5 +- public/app/features/explore/LiveLogs.tsx | 11 +- public/app/features/explore/LogLabel.tsx | 3 +- public/app/features/explore/LogLabelStats.tsx | 2 +- public/app/features/explore/LogLabels.tsx | 3 +- public/app/features/explore/LogRow.tsx | 14 ++- public/app/features/explore/LogRowContext.tsx | 2 +- .../explore/LogRowContextProvider.tsx | 69 +++++++----- public/app/features/explore/Logs.tsx | 18 +++- public/app/features/explore/LogsContainer.tsx | 9 +- .../features/explore/state/actions.test.ts | 3 +- public/app/features/explore/state/actions.ts | 4 +- .../features/explore/state/reducers.test.ts | 3 +- public/app/features/explore/state/reducers.ts | 4 +- .../features/explore/state/selectors.test.ts | 2 +- .../app/plugins/datasource/loki/datasource.ts | 100 ++++++++---------- public/app/types/explore.ts | 3 +- 23 files changed, 245 insertions(+), 205 deletions(-) diff --git a/package.json b/package.json index 30721ddaf6a..e2153dcf341 100644 --- a/package.json +++ b/package.json @@ -155,6 +155,7 @@ "storybook:build": "cd packages/grafana-ui && yarn storybook:build", "themes:generate": "ts-node --project ./scripts/cli/tsconfig.json ./scripts/cli/generateSassVariableFiles.ts", "prettier:check": "prettier --list-different \"**/*.{ts,tsx,scss}\"", + "prettier:write": "prettier --list-different \"**/*.{ts,tsx,scss}\" --write", "gui:tslint": "tslint -c ./packages/grafana-ui/tslint.json --project ./packages/grafana-ui/tsconfig.json", "gui:build": "ts-node --project ./scripts/cli/tsconfig.json ./scripts/cli/index.ts gui:build", "gui:releasePrepare": "ts-node --project ./scripts/cli/tsconfig.json ./scripts/cli/index.ts gui:release", diff --git a/packages/grafana-ui/src/types/datasource.ts b/packages/grafana-ui/src/types/datasource.ts index 596e5851cf0..dd364a7d9c9 100644 --- a/packages/grafana-ui/src/types/datasource.ts +++ b/packages/grafana-ui/src/types/datasource.ts @@ -3,6 +3,7 @@ import { TimeRange } from './time'; import { PluginMeta, GrafanaPlugin } from './plugin'; import { TableData, TimeSeries, SeriesData, LoadingState } from './data'; import { PanelData } from './panel'; +import { LogRowModel } from './logs'; // NOTE: this seems more general than just DataSource export interface DataSourcePluginOptionsEditorProps { @@ -187,7 +188,10 @@ export abstract class DataSourceApi< /** * Retrieve context for a given log row */ - getLogRowContext?(row: any, limit?: number): Promise; + getLogRowContext?: ( + row: LogRowModel, + options?: TContextQueryOptions + ) => Promise; /** * Set after constructor call, as the data source instance is the most common thing to pass around @@ -299,10 +303,6 @@ export interface DataQueryResponse { data: DataQueryResponseData[]; } -export interface LogRowContextQueryResponse { - data: Array>; -} - export interface DataQuery { /** * A - Z diff --git a/packages/grafana-ui/src/types/logs.ts b/packages/grafana-ui/src/types/logs.ts index 63f264fff4f..3d8cc0951b8 100644 --- a/packages/grafana-ui/src/types/logs.ts +++ b/packages/grafana-ui/src/types/logs.ts @@ -1,3 +1,5 @@ +import { Labels, TimeSeries } from './data'; + /** * Mapping of log level abbreviation to canonical log level. * Supported levels are reduce to limit color variation. @@ -19,3 +21,85 @@ export enum LogLevel { trace = 'trace', unknown = 'unknown', } + +export enum LogsMetaKind { + Number, + String, + LabelsMap, +} + +export interface LogsMetaItem { + label: string; + value: string | number | Labels; + kind: LogsMetaKind; +} + +export interface LogRowModel { + duplicates?: number; + entry: string; + hasAnsi: boolean; + labels: Labels; + logLevel: LogLevel; + raw: string; + searchWords?: string[]; + timestamp: string; // ISO with nanosec precision + timeFromNow: string; + timeEpochMs: number; + timeLocal: string; + uniqueLabels?: Labels; +} + +export interface LogsModel { + hasUniqueLabels: boolean; + meta?: LogsMetaItem[]; + rows: LogRowModel[]; + series?: TimeSeries[]; +} + +export interface LogSearchMatch { + start: number; + length: number; + text: string; +} + +export interface LogLabelStatsModel { + active?: boolean; + count: number; + proportion: number; + value: string; +} + +export enum LogsDedupStrategy { + none = 'none', + exact = 'exact', + numbers = 'numbers', + signature = 'signature', +} + +export interface LogsParser { + /** + * Value-agnostic matcher for a field label. + * Used to filter rows, and first capture group contains the value. + */ + buildMatcher: (label: string) => RegExp; + + /** + * Returns all parsable substrings from a line, used for highlighting + */ + getFields: (line: string) => string[]; + + /** + * Gets the label name from a parsable substring of a line + */ + getLabelFromField: (field: string) => string; + + /** + * Gets the label value from a parsable substring of a line + */ + getValueFromField: (field: string) => string; + /** + * Function to verify if this is a valid parser for the given line. + * The parser accepts the line unless it returns undefined. + */ + test: (line: string) => any; +} diff --git a/public/app/core/logs_model.ts b/public/app/core/logs_model.ts index fd7e5c638a9..d2a4780b62a 100644 --- a/public/app/core/logs_model.ts +++ b/public/app/core/logs_model.ts @@ -13,6 +13,13 @@ import { toLegacyResponseData, FieldCache, FieldType, + LogRowModel, + LogsModel, + LogsMetaItem, + LogsMetaKind, + LogsParser, + LogLabelStatsModel, + LogsDedupStrategy, } from '@grafana/ui'; import { getThemeColor } from 'app/core/utils/colors'; import { hasAnsiCodes } from 'app/core/utils/text'; @@ -28,95 +35,12 @@ export const LogLevelColor = { [LogLevel.unknown]: getThemeColor('#8e8e8e', '#dde4ed'), }; -export interface LogSearchMatch { - start: number; - length: number; - text: string; -} - -export interface LogRowModel { - duplicates?: number; - entry: string; - hasAnsi: boolean; - labels: Labels; - logLevel: LogLevel; - raw: string; - searchWords?: string[]; - timestamp: string; // ISO with nanosec precision - timeFromNow: string; - timeEpochMs: number; - timeLocal: string; - uniqueLabels?: Labels; -} - -export interface LogLabelStatsModel { - active?: boolean; - count: number; - proportion: number; - value: string; -} - -export enum LogsMetaKind { - Number, - String, - LabelsMap, -} - -export interface LogsMetaItem { - label: string; - value: string | number | Labels; - kind: LogsMetaKind; -} - -export interface LogsModel { - hasUniqueLabels: boolean; - meta?: LogsMetaItem[]; - rows: LogRowModel[]; - series?: TimeSeries[]; -} - export enum LogsDedupDescription { none = 'No de-duplication', exact = 'De-duplication of successive lines that are identical, ignoring ISO datetimes.', numbers = 'De-duplication of successive lines that are identical when ignoring numbers, e.g., IP addresses, latencies.', signature = 'De-duplication of successive lines that have identical punctuation and whitespace.', } - -export enum LogsDedupStrategy { - none = 'none', - exact = 'exact', - numbers = 'numbers', - signature = 'signature', -} - -export interface LogsParser { - /** - * Value-agnostic matcher for a field label. - * Used to filter rows, and first capture group contains the value. - */ - buildMatcher: (label: string) => RegExp; - - /** - * Returns all parsable substrings from a line, used for highlighting - */ - getFields: (line: string) => string[]; - - /** - * Gets the label name from a parsable substring of a line - */ - getLabelFromField: (field: string) => string; - - /** - * Gets the label value from a parsable substring of a line - */ - getValueFromField: (field: string) => string; - /** - * Function to verify if this is a valid parser for the given line. - * The parser accepts the line unless it returns undefined. - */ - test: (line: string) => any; -} - const LOGFMT_REGEXP = /(?:^|\s)(\w+)=("[^"]*"|\S+)/; export const LogsParsers: { [name: string]: LogsParser } = { diff --git a/public/app/core/specs/logs_model.test.ts b/public/app/core/specs/logs_model.test.ts index c30a3ebfa39..a2d47412bd0 100644 --- a/public/app/core/specs/logs_model.test.ts +++ b/public/app/core/specs/logs_model.test.ts @@ -1,15 +1,12 @@ +import { SeriesData, FieldType, LogsModel, LogsMetaKind, LogsDedupStrategy } from '@grafana/ui'; import { + dedupLogRows, calculateFieldStats, calculateLogsLabelStats, - dedupLogRows, getParser, - LogsDedupStrategy, - LogsModel, LogsParsers, seriesDataToLogsModel, - LogsMetaKind, } from '../logs_model'; -import { SeriesData, FieldType } from '@grafana/ui'; describe('dedupLogRows()', () => { test('should return rows as is when dedup is set to none', () => { diff --git a/public/app/core/utils/explore.test.ts b/public/app/core/utils/explore.test.ts index 3a0752d2a5e..9e11fddd629 100644 --- a/public/app/core/utils/explore.test.ts +++ b/public/app/core/utils/explore.test.ts @@ -12,8 +12,7 @@ import { } from './explore'; import { ExploreUrlState } from 'app/types/explore'; import store from 'app/core/store'; -import { LogsDedupStrategy } from 'app/core/logs_model'; -import { DataQueryError } from '@grafana/ui'; +import { DataQueryError, LogsDedupStrategy } from '@grafana/ui'; const DEFAULT_EXPLORE_STATE: ExploreUrlState = { datasource: null, diff --git a/public/app/core/utils/explore.ts b/public/app/core/utils/explore.ts index e82ba9c3409..99e168b8590 100644 --- a/public/app/core/utils/explore.ts +++ b/public/app/core/utils/explore.ts @@ -22,6 +22,9 @@ import { guessFieldTypes, TimeFragment, DataQueryError, + LogRowModel, + LogsModel, + LogsDedupStrategy, } from '@grafana/ui'; import TimeSeries from 'app/core/time_series2'; import { @@ -33,7 +36,7 @@ import { QueryOptions, ResultGetter, } from 'app/types/explore'; -import { LogsDedupStrategy, seriesDataToLogsModel, LogsModel, LogRowModel } from 'app/core/logs_model'; +import { seriesDataToLogsModel } from 'app/core/logs_model'; import { toUtc } from '@grafana/ui/src/utils/moment_wrapper'; import { isLive } from '@grafana/ui/src/components/RefreshPicker/RefreshPicker'; diff --git a/public/app/features/explore/LiveLogs.tsx b/public/app/features/explore/LiveLogs.tsx index 120b136b127..040beae2aec 100644 --- a/public/app/features/explore/LiveLogs.tsx +++ b/public/app/features/explore/LiveLogs.tsx @@ -1,8 +1,15 @@ import React, { PureComponent } from 'react'; import { css, cx } from 'emotion'; -import { Themeable, withTheme, GrafanaTheme, selectThemeVariant, LinkButton } from '@grafana/ui'; +import { + Themeable, + withTheme, + GrafanaTheme, + selectThemeVariant, + LinkButton, + LogsModel, + LogRowModel, +} from '@grafana/ui'; -import { LogsModel, LogRowModel } from 'app/core/logs_model'; import ElapsedTime from './ElapsedTime'; import { ButtonSize, ButtonVariant } from '@grafana/ui/src/components/Button/AbstractButton'; diff --git a/public/app/features/explore/LogLabel.tsx b/public/app/features/explore/LogLabel.tsx index b4570f10c82..1794d60b689 100644 --- a/public/app/features/explore/LogLabel.tsx +++ b/public/app/features/explore/LogLabel.tsx @@ -1,7 +1,8 @@ import React, { PureComponent } from 'react'; -import { calculateLogsLabelStats, LogLabelStatsModel, LogRowModel } from 'app/core/logs_model'; import { LogLabelStats } from './LogLabelStats'; +import { LogRowModel, LogLabelStatsModel } from '@grafana/ui'; +import { calculateLogsLabelStats } from 'app/core/logs_model'; interface Props { getRows?: () => LogRowModel[]; diff --git a/public/app/features/explore/LogLabelStats.tsx b/public/app/features/explore/LogLabelStats.tsx index 466cc050e43..07f33d8aff7 100644 --- a/public/app/features/explore/LogLabelStats.tsx +++ b/public/app/features/explore/LogLabelStats.tsx @@ -1,6 +1,6 @@ import React, { PureComponent } from 'react'; import classnames from 'classnames'; -import { LogLabelStatsModel } from 'app/core/logs_model'; +import { LogLabelStatsModel } from '@grafana/ui'; function LogLabelStatsRow(logLabelStatsModel: LogLabelStatsModel) { const { active, count, proportion, value } = logLabelStatsModel; diff --git a/public/app/features/explore/LogLabels.tsx b/public/app/features/explore/LogLabels.tsx index f89836055c5..9dffca38a66 100644 --- a/public/app/features/explore/LogLabels.tsx +++ b/public/app/features/explore/LogLabels.tsx @@ -1,8 +1,7 @@ import React, { PureComponent } from 'react'; -import { LogRowModel } from 'app/core/logs_model'; import { LogLabel } from './LogLabel'; -import { Labels } from '@grafana/ui'; +import { Labels, LogRowModel } from '@grafana/ui'; interface Props { getRows?: () => LogRowModel[]; diff --git a/public/app/features/explore/LogRow.tsx b/public/app/features/explore/LogRow.tsx index 719c4eba816..ad2aced592f 100644 --- a/public/app/features/explore/LogRow.tsx +++ b/public/app/features/explore/LogRow.tsx @@ -3,7 +3,7 @@ import _ from 'lodash'; import Highlighter from 'react-highlight-words'; import classnames from 'classnames'; -import { LogRowModel, LogLabelStatsModel, LogsParser, calculateFieldStats, getParser } from 'app/core/logs_model'; +import { calculateFieldStats, getParser } from 'app/core/logs_model'; import { LogLabels } from './LogLabels'; import { findHighlightChunksInText } from 'app/core/utils/text'; import { LogLabelStats } from './LogLabelStats'; @@ -15,7 +15,15 @@ import { HasMoreContextRows, LogRowContextQueryErrors, } from './LogRowContextProvider'; -import { ThemeContext, selectThemeVariant, GrafanaTheme, DataQueryResponse } from '@grafana/ui'; +import { + ThemeContext, + selectThemeVariant, + GrafanaTheme, + DataQueryResponse, + LogRowModel, + LogLabelStatsModel, + LogsParser, +} from '@grafana/ui'; import { LogRowContext } from './LogRowContext'; import tinycolor from 'tinycolor2'; @@ -29,7 +37,7 @@ interface Props { getRows: () => LogRowModel[]; onClickLabel?: (label: string, value: string) => void; onContextClick?: () => void; - getRowContext?: (row: LogRowModel, limit: number) => Promise; + getRowContext?: (row: LogRowModel, options?: any) => Promise; className?: string; } diff --git a/public/app/features/explore/LogRowContext.tsx b/public/app/features/explore/LogRowContext.tsx index f873b50958b..da9c3ec4812 100644 --- a/public/app/features/explore/LogRowContext.tsx +++ b/public/app/features/explore/LogRowContext.tsx @@ -7,10 +7,10 @@ import { ClickOutsideWrapper, CustomScrollbar, DataQueryError, + LogRowModel, } from '@grafana/ui'; import { css, cx } from 'emotion'; import { LogRowContextRows, HasMoreContextRows, LogRowContextQueryErrors } from './LogRowContextProvider'; -import { LogRowModel } from 'app/core/logs_model'; import { Alert } from './Error'; interface LogRowContextProps { diff --git a/public/app/features/explore/LogRowContextProvider.tsx b/public/app/features/explore/LogRowContextProvider.tsx index d3fa0bfdcc8..a43d982e2b4 100644 --- a/public/app/features/explore/LogRowContextProvider.tsx +++ b/public/app/features/explore/LogRowContextProvider.tsx @@ -1,11 +1,11 @@ -import { LogRowModel } from 'app/core/logs_model'; -import { LogRowContextQueryResponse, SeriesData, DataQueryResponse, DataQueryError } from '@grafana/ui'; +import { DataQueryResponse, DataQueryError, LogRowModel } from '@grafana/ui'; import { useState, useEffect } from 'react'; +import flatten from 'lodash/flatten'; import useAsync from 'react-use/lib/useAsync'; export interface LogRowContextRows { - before?: Array; - after?: Array; + before?: string[]; + after?: string[]; } export interface LogRowContextQueryErrors { before?: string; @@ -19,7 +19,7 @@ export interface HasMoreContextRows { interface LogRowContextProviderProps { row: LogRowModel; - getRowContext: (row: LogRowModel, limit: number) => Promise; + getRowContext: (row: LogRowModel, options?: any) => Promise; children: (props: { result: LogRowContextRows; errors: LogRowContextQueryErrors; @@ -34,23 +34,44 @@ export const LogRowContextProvider: React.FunctionComponent { const [limit, setLimit] = useState(10); - const [result, setResult] = useState(null); - const [errors, setErrors] = useState(null); + const [result, setResult] = useState<{ + data: string[][]; + errors: string[]; + }>(null); const [hasMoreContextRows, setHasMoreContextRows] = useState({ before: true, after: true, }); const { value } = useAsync(async () => { - const context = await getRowContext(row, limit); + const promises = [ + getRowContext(row, { + limit, + }), + getRowContext(row, { + limit, + direction: 'FORWARD', + }), + ]; + + const results: Array = await Promise.all(promises.map(p => p.catch(e => e))); + return { - data: context.data.map(series => { - if ((series as SeriesData).rows) { - return (series as SeriesData).rows.map(row => row[1]); + data: results.map(result => { + if ((result as DataQueryResponse).data) { + return (result as DataQueryResponse).data.map(series => { + return series.rows.map(row => row[1]); + }); } else { - return [series]; + return []; + } + }), + errors: results.map(result => { + if ((result as DataQueryError).message) { + return (result as DataQueryError).message; + } else { + return null; } - return []; }), }; }, [limit]); @@ -60,7 +81,6 @@ export const LogRowContextProvider: React.FunctionComponent { let hasMoreLogsBefore = true, hasMoreLogsAfter = true; - let beforeContextError, afterContextError; if (currentResult && currentResult.data[0].length === value.data[0].length) { hasMoreLogsBefore = false; @@ -70,23 +90,11 @@ export const LogRowContextProvider: React.FunctionComponent 0 && value.data[0][0].message) { - beforeContextError = value.data[0][0].message; - } - if (value.data[1] && value.data[1].length > 0 && value.data[1][0].message) { - afterContextError = value.data[1][0].message; - } - setHasMoreContextRows({ before: hasMoreLogsBefore, after: hasMoreLogsAfter, }); - setErrors({ - before: beforeContextError, - after: afterContextError, - }); - return value; }); } @@ -94,10 +102,13 @@ export const LogRowContextProvider: React.FunctionComponent setLimit(limit + 10), }); diff --git a/public/app/features/explore/Logs.tsx b/public/app/features/explore/Logs.tsx index f12830b63b7..b6ee5a7f731 100644 --- a/public/app/features/explore/Logs.tsx +++ b/public/app/features/explore/Logs.tsx @@ -2,16 +2,26 @@ import _ from 'lodash'; import React, { PureComponent } from 'react'; import * as rangeUtil from '@grafana/ui/src/utils/rangeutil'; -import { RawTimeRange, Switch, LogLevel, TimeZone, TimeRange, AbsoluteTimeRange } from '@grafana/ui'; +import { + RawTimeRange, + Switch, + LogLevel, + TimeZone, + TimeRange, + AbsoluteTimeRange, + LogsMetaKind, + LogsModel, + LogsDedupStrategy, + LogRowModel, +} from '@grafana/ui'; import TimeSeries from 'app/core/time_series2'; -import { LogsDedupDescription, LogsDedupStrategy, LogsModel, LogsMetaKind, LogRowModel } from 'app/core/logs_model'; - import ToggleButtonGroup, { ToggleButton } from 'app/core/components/ToggleButtonGroup/ToggleButtonGroup'; import Graph from './Graph'; import { LogLabels } from './LogLabels'; import { LogRow } from './LogRow'; +import { LogsDedupDescription } from 'app/core/logs_model'; const PREVIEW_LIMIT = 100; @@ -60,7 +70,7 @@ interface Props { onStopScanning?: () => void; onDedupStrategyChange: (dedupStrategy: LogsDedupStrategy) => void; onToggleLogLevel: (hiddenLogLevels: Set) => void; - getRowContext?: (row: LogRowModel, limit: number) => Promise; + getRowContext?: (row: LogRowModel, options?: any) => Promise; } interface State { diff --git a/public/app/features/explore/LogsContainer.tsx b/public/app/features/explore/LogsContainer.tsx index 484dd993cec..d8d85efcc13 100644 --- a/public/app/features/explore/LogsContainer.tsx +++ b/public/app/features/explore/LogsContainer.tsx @@ -10,10 +10,12 @@ import { toUtc, dateTime, DataSourceApi, + LogsModel, + LogRowModel, + LogsDedupStrategy, } from '@grafana/ui'; import { ExploreId, ExploreItemState } from 'app/types/explore'; -import { LogsModel, LogsDedupStrategy, LogRowModel } from 'app/core/logs_model'; import { StoreState } from 'app/types'; import { changeDedupStrategy, changeTime } from './state/actions'; @@ -29,6 +31,7 @@ interface LogsContainerProps { datasourceInstance: DataSourceApi | null; exploreId: ExploreId; loading: boolean; + logsHighlighterExpressions?: string[]; logsResult?: LogsModel; dedupedResult?: LogsModel; @@ -77,11 +80,11 @@ export class LogsContainer extends PureComponent { }); }; - getLogRowContext = async (row: LogRowModel, limit: number) => { + getLogRowContext = async (row: LogRowModel, options?: any) => { const { datasourceInstance } = this.props; if (datasourceInstance) { - return datasourceInstance.getLogRowContext(row, limit); + return datasourceInstance.getLogRowContext(row, options); } return []; diff --git a/public/app/features/explore/state/actions.test.ts b/public/app/features/explore/state/actions.test.ts index 0bae4dc1d95..ba096602c05 100644 --- a/public/app/features/explore/state/actions.test.ts +++ b/public/app/features/explore/state/actions.test.ts @@ -1,7 +1,6 @@ import { refreshExplore, testDatasource, loadDatasource } from './actions'; import { ExploreId, ExploreUrlState, ExploreUpdateState } from 'app/types'; import { thunkTester } from 'test/core/thunk/thunkTester'; -import { LogsDedupStrategy } from 'app/core/logs_model'; import { initializeExploreAction, InitializeExplorePayload, @@ -18,7 +17,7 @@ import { Emitter } from 'app/core/core'; import { ActionOf } from 'app/core/redux/actionCreatorFactory'; import { makeInitialUpdateState } from './reducers'; import { DataQuery } from '@grafana/ui/src/types/datasource'; -import { DefaultTimeZone, RawTimeRange } from '@grafana/ui'; +import { DefaultTimeZone, RawTimeRange, LogsDedupStrategy } from '@grafana/ui'; import { toUtc } from '@grafana/ui/src/utils/moment_wrapper'; jest.mock('app/features/plugins/datasource_srv', () => ({ diff --git a/public/app/features/explore/state/actions.ts b/public/app/features/explore/state/actions.ts index e0621898826..09f950905ae 100644 --- a/public/app/features/explore/state/actions.ts +++ b/public/app/features/explore/state/actions.ts @@ -35,7 +35,8 @@ import { DataSourceSelectItem, QueryFixAction, TimeRange, -} from '@grafana/ui/src/types'; + LogsDedupStrategy, +} from '@grafana/ui'; import { ExploreId, ExploreUrlState, @@ -87,7 +88,6 @@ import { changeModeAction, } from './actionTypes'; import { ActionOf, ActionCreator } from 'app/core/redux/actionCreatorFactory'; -import { LogsDedupStrategy } from 'app/core/logs_model'; import { getTimeZone } from 'app/features/profile/state/selectors'; import { isDateTime } from '@grafana/ui/src/utils/moment_wrapper'; import { toDataQueryError } from 'app/features/dashboard/state/PanelQueryState'; diff --git a/public/app/features/explore/state/reducers.test.ts b/public/app/features/explore/state/reducers.test.ts index 5af71c29f8d..da9bdfabe26 100644 --- a/public/app/features/explore/state/reducers.test.ts +++ b/public/app/features/explore/state/reducers.test.ts @@ -29,10 +29,9 @@ import { import { Reducer } from 'redux'; import { ActionOf } from 'app/core/redux/actionCreatorFactory'; import { updateLocation } from 'app/core/actions/location'; -import { LogsDedupStrategy, LogsModel } from 'app/core/logs_model'; import { serializeStateToUrlParam } from 'app/core/utils/explore'; import TableModel from 'app/core/table_model'; -import { DataSourceApi, DataQuery } from '@grafana/ui'; +import { DataSourceApi, DataQuery, LogsModel, LogsDedupStrategy } from '@grafana/ui'; describe('Explore item reducer', () => { describe('scanning', () => { diff --git a/public/app/features/explore/state/reducers.ts b/public/app/features/explore/state/reducers.ts index 7e8fbb9d6ef..1291f3d749b 100644 --- a/public/app/features/explore/state/reducers.ts +++ b/public/app/features/explore/state/reducers.ts @@ -10,7 +10,7 @@ import { sortLogsResult, } from 'app/core/utils/explore'; import { ExploreItemState, ExploreState, ExploreId, ExploreUpdateState, ExploreMode } from 'app/types/explore'; -import { DataQuery } from '@grafana/ui/src/types'; +import { DataQuery, LogsModel } from '@grafana/ui'; import { HigherOrderAction, ActionTypes, @@ -58,7 +58,7 @@ import { LocationUpdate } from 'app/types'; import TableModel from 'app/core/table_model'; import { isLive } from '@grafana/ui/src/components/RefreshPicker/RefreshPicker'; import { subscriptionDataReceivedAction, startSubscriptionAction } from './epics'; -import { LogsModel, seriesDataToLogsModel } from 'app/core/logs_model'; +import { seriesDataToLogsModel } from 'app/core/logs_model'; export const DEFAULT_RANGE = { from: 'now-6h', diff --git a/public/app/features/explore/state/selectors.test.ts b/public/app/features/explore/state/selectors.test.ts index 3a1fa5102ae..52f8d27811d 100644 --- a/public/app/features/explore/state/selectors.test.ts +++ b/public/app/features/explore/state/selectors.test.ts @@ -1,5 +1,5 @@ import { deduplicatedLogsSelector } from './selectors'; -import { LogsDedupStrategy } from 'app/core/logs_model'; +import { LogsDedupStrategy } from '@grafana/ui'; import { ExploreItemState } from 'app/types'; const state = { diff --git a/public/app/plugins/datasource/loki/datasource.ts b/public/app/plugins/datasource/loki/datasource.ts index 932c9c0a6e2..d86e5fe1922 100644 --- a/public/app/plugins/datasource/loki/datasource.ts +++ b/public/app/plugins/datasource/loki/datasource.ts @@ -16,12 +16,12 @@ import { DataSourceApi, DataSourceInstanceSettings, DataQueryError, -} from '@grafana/ui/src/types'; + LogRowModel, +} from '@grafana/ui'; import { LokiQuery, LokiOptions } from './types'; import { BackendSrv } from 'app/core/services/backend_srv'; import { TemplateSrv } from 'app/features/templating/template_srv'; import { safeStringifyValue } from 'app/core/utils/explore'; -import { LogRowModel } from 'app/core/logs_model'; export const DEFAULT_MAX_LINES = 1000; @@ -41,6 +41,11 @@ function serializeParams(data: any) { .join('&'); } +interface LokiContextQueryOptions { + direction?: 'BACKWARD' | 'FORWARD'; + limit?: number; +} + export class LokiDatasource extends DataSourceApi { languageProvider: LanguageProvider; maxLines: number; @@ -224,7 +229,7 @@ export class LokiDatasource extends DataSourceApi { return Math.ceil(date.valueOf() * 1e6); } - prepareLogRowContextQueryTargets = (row: LogRowModel, limit: number) => { + prepareLogRowContextQueryTarget = (row: LogRowModel, limit: number, direction: 'BACKWARD' | 'FORWARD') => { const query = Object.keys(row.labels) .map(label => { return `${label}="${row.labels[label]}"`; @@ -236,69 +241,58 @@ export class LokiDatasource extends DataSourceApi { const commontTargetOptons = { limit, query: `{${query}}`, + direction, }; - return [ - // Target for "before" context - { + + if (direction === 'BACKWARD') { + return { ...commontTargetOptons, start: timeEpochNs - contextTimeBuffer, end: timeEpochNs, - direction: 'BACKWARD', - }, - // Target for "after" context - { + direction, + }; + } else { + return { ...commontTargetOptons, start: timeEpochNs, // TODO: We should add 1ns here for the original row not no be included in the result end: timeEpochNs + contextTimeBuffer, - direction: 'FORWARD', - }, - ]; + }; + } }; - getLogRowContext = (row: LogRowModel, limit?: number) => { - // Preparing two targets, for preceeding and following log queries - const targets = this.prepareLogRowContextQueryTargets(row, limit || 10); + getLogRowContext = async (row: LogRowModel, options?: LokiContextQueryOptions) => { + const target = this.prepareLogRowContextQueryTarget( + row, + (options && options.limit) || 10, + (options && options.direction) || 'BACKWARD' + ); + const series: SeriesData[] = []; - return Promise.all( - targets.map(target => { - return this._request('/api/prom/query', target).catch(e => { - const error: DataQueryError = { - message: 'Error during context query. Please check JS console logs.', - status: e.status, - statusText: e.statusText, - }; - return error; - }); - }) - ).then((results: any[]) => { - const series: Array> = []; - const emptySeries = { - fields: [], - rows: [], - } as SeriesData; - - for (let i = 0; i < results.length; i++) { - const result = results[i]; - series[i] = []; - if (result.data) { - for (const stream of result.data.streams || []) { - const seriesData = logStreamToSeriesData(stream); - series[i].push(seriesData); - } - } else { - series[i].push(result); + try { + const result = await this._request('/api/prom/query', target); + if (result.data) { + for (const stream of result.data.streams || []) { + const seriesData = logStreamToSeriesData(stream); + series.push(seriesData); + } + } + if (options && options.direction === 'FORWARD') { + if (series[0] && series[0].rows) { + series[0].rows.reverse(); } } - // Following context logs are requested in "forward" direction. - // This means, that we need to reverse those to make them sorted - // in descending order (by timestamp) - if (series[1][0] && (series[1][0] as SeriesData).rows) { - (series[1][0] as SeriesData).rows.reverse(); - } - - return { data: [series[0][0] || emptySeries, series[1][0] || emptySeries] }; - }); + return { + data: series, + }; + } catch (e) { + const error: DataQueryError = { + message: 'Error during context query. Please check JS console logs.', + status: e.status, + statusText: e.statusText, + }; + throw error; + } }; testDatasource() { diff --git a/public/app/types/explore.ts b/public/app/types/explore.ts index 7bb8695e1da..c852d92ee29 100644 --- a/public/app/types/explore.ts +++ b/public/app/types/explore.ts @@ -11,10 +11,11 @@ import { LogLevel, TimeRange, DataQueryError, + LogsModel, + LogsDedupStrategy, } from '@grafana/ui'; import { Emitter, TimeSeries } from 'app/core/core'; -import { LogsModel, LogsDedupStrategy } from 'app/core/logs_model'; import TableModel from 'app/core/table_model'; export enum ExploreMode {