Explore: Support wide data frames (#28393)

* Change how isTimeSeries work

* Simplify the decorators and update tests

public/app/core/reducers/appNotification.ts

@@ -5,6 +5,12 @@ export const initialState: AppNotificationsState = {
   appNotifications: [] as AppNotification[],
 };
 
+/**
+ * Reducer and action to show toast notifications of various types (success, warnings, errors etc). Use to show
+ * transient info to user, like errors that cannot be otherwise handled or success after an action.
+ *
+ * Use factory functions in core/copy/appNotifications to create the payload.
+ */
 const appNotificationsSlice = createSlice({
   name: 'appNotifications',
   initialState,

public/app/features/explore/state/actions.ts

@@ -1,5 +1,5 @@
 // Libraries
-import { map, throttleTime } from 'rxjs/operators';
+import { map, mergeMap, throttleTime } from 'rxjs/operators';
 import { identity } from 'rxjs';
 import { PayloadAction } from '@reduxjs/toolkit';
 import { DataSourceSrv } from '@grafana/runtime';
@@ -84,7 +84,7 @@ import {
 } from './actionTypes';
 import { getTimeZone } from 'app/features/profile/state/selectors';
 import { getShiftedTimeRange } from 'app/core/utils/timePicker';
-import { updateLocation } from '../../../core/actions';
+import { notifyApp, updateLocation } from '../../../core/actions';
 import { getTimeSrv, TimeSrv } from '../../dashboard/services/TimeSrv';
 import { preProcessPanelData, runRequest } from '../../dashboard/state/runRequest';
 import { DashboardModel, PanelModel } from 'app/features/dashboard/state';
@@ -96,6 +96,7 @@ import {
   decorateWithLogsResult,
   decorateWithTableResult,
 } from '../utils/decorators';
+import { createErrorNotification } from '../../../core/copy/appNotification';
 
 /**
  * Adds a query row after the row with the given index.
@@ -427,6 +428,8 @@ export const runQueries = (exploreId: ExploreId): ThunkResult<void> => {
       queryResponse,
       querySubscription,
       history,
+      refreshInterval,
+      absoluteRange,
     } = exploreItemState;
 
     if (!hasNonEmptyQuery(queries)) {
@@ -473,12 +476,13 @@ export const runQueries = (exploreId: ExploreId): ThunkResult<void> => {
         // actually can see what is happening.
         live ? throttleTime(500) : identity,
         map((data: PanelData) => preProcessPanelData(data, queryResponse)),
-        decorateWithGraphLogsTraceAndTable(getState().explore[exploreId].datasourceInstance),
+        map(decorateWithGraphLogsTraceAndTable),
-        decorateWithGraphResult(),
+        map(decorateWithGraphResult),
-        decorateWithTableResult(),
+        map(decorateWithLogsResult({ absoluteRange, refreshInterval })),
-        decorateWithLogsResult(getState().explore[exploreId])
+        mergeMap(decorateWithTableResult)
       )
-      .subscribe(data => {
+      .subscribe(
+        data => {
           if (!data.error && firstResponse) {
             // Side-effect: Saving history in localstorage
             const nextHistory = updateHistory(history, datasourceId, queries);
@@ -513,7 +517,13 @@ export const runQueries = (exploreId: ExploreId): ThunkResult<void> => {
               dispatch(scanStopAction({ exploreId }));
             }
           }
-      });
+        },
+        error => {
+          dispatch(notifyApp(createErrorNotification('Query processing error', error)));
+          dispatch(changeLoadingStateAction({ exploreId, loadingState: LoadingState.Error }));
+          console.error(error);
+        }
+      );
 
     dispatch(queryStoreSubscriptionAction({ exploreId, querySubscription: newQuerySub }));
   };

public/app/features/explore/utils/decorators.test.ts

@@ -3,15 +3,12 @@ jest.mock('@grafana/data/src/datetime/formatter', () => ({
   dateTimeFormatTimeAgo: (ts: any) => 'fromNow() jest mocked',
 }));
 
-import { of } from 'rxjs';
 import {
   ArrayVector,
   DataFrame,
   DataQueryRequest,
-  DataSourceApi,
   FieldType,
   LoadingState,
-  observableTester,
   PanelData,
   TimeRange,
   toDataFrame,
@@ -24,7 +21,7 @@ import {
   decorateWithTableResult,
 } from './decorators';
 import { describe } from '../../../../test/lib/common';
-import { ExploreItemState, ExplorePanelData } from 'app/types';
+import { ExplorePanelData } from 'app/types';
 import TableModel from 'app/core/table_model';
 
 const getTestContext = () => {
@@ -37,6 +34,7 @@ const getTestContext = () => {
     fields: [
       { name: 'time', type: FieldType.time, values: [100, 200, 300] },
       { name: 'A-series', type: FieldType.number, values: [4, 5, 6] },
+      { name: 'B-series', type: FieldType.number, values: [7, 8, 9] },
     ],
   });
 
@@ -86,10 +84,8 @@ const createExplorePanelData = (args: Partial<ExplorePanelData>): ExplorePanelDa
 };
 
 describe('decorateWithGraphLogsTraceAndTable', () => {
-  describe('when used without error', () => {
+  it('should correctly classify the dataFrames', () => {
-    it('then the result should be correct', done => {
     const { table, logs, timeSeries, emptyTable } = getTestContext();
-      const datasourceInstance = ({ meta: { id: 'prometheus' } } as unknown) as DataSourceApi;
     const series = [table, logs, timeSeries, emptyTable];
     const panelData: PanelData = {
       series,
@@ -97,10 +93,7 @@ describe('decorateWithGraphLogsTraceAndTable', () => {
       timeRange: ({} as unknown) as TimeRange,
     };
 
-      observableTester().subscribeAndExpectOnNext({
+    expect(decorateWithGraphLogsTraceAndTable(panelData)).toEqual({
-        observable: of(panelData).pipe(decorateWithGraphLogsTraceAndTable(datasourceInstance)),
-        expect: value => {
-          expect(value).toEqual({
       series,
       state: LoadingState.Done,
       timeRange: {},
@@ -112,15 +105,9 @@ describe('decorateWithGraphLogsTraceAndTable', () => {
       tableResult: null,
       logsResult: null,
     });
-        },
-        done,
-      });
-    });
   });
 
-  describe('when used without frames', () => {
+  it('should handle empty array', () => {
-    it('then the result should be correct', done => {
-      const datasourceInstance = ({ meta: { id: 'prometheus' } } as unknown) as DataSourceApi;
     const series: DataFrame[] = [];
     const panelData: PanelData = {
       series,
@@ -128,10 +115,7 @@ describe('decorateWithGraphLogsTraceAndTable', () => {
       timeRange: ({} as unknown) as TimeRange,
     };
 
-      observableTester().subscribeAndExpectOnNext({
+    expect(decorateWithGraphLogsTraceAndTable(panelData)).toEqual({
-        observable: of(panelData).pipe(decorateWithGraphLogsTraceAndTable(datasourceInstance)),
-        expect: value => {
-          expect(value).toEqual({
       series: [],
       state: LoadingState.Done,
       timeRange: {},
@@ -143,16 +127,10 @@ describe('decorateWithGraphLogsTraceAndTable', () => {
       tableResult: null,
       logsResult: null,
     });
-        },
-        done,
-      });
-    });
   });
 
-  describe('when used with an error', () => {
+  it('should handle query error', () => {
-    it('then the result should be correct', done => {
     const { timeSeries, logs, table } = getTestContext();
-      const datasourceInstance = ({ meta: { id: 'prometheus' } } as unknown) as DataSourceApi;
     const series: DataFrame[] = [timeSeries, logs, table];
     const panelData: PanelData = {
       series,
@@ -161,10 +139,7 @@ describe('decorateWithGraphLogsTraceAndTable', () => {
       timeRange: ({} as unknown) as TimeRange,
     };
 
-      observableTester().subscribeAndExpectOnNext({
+    expect(decorateWithGraphLogsTraceAndTable(panelData)).toEqual({
-        observable: of(panelData).pipe(decorateWithGraphLogsTraceAndTable(datasourceInstance)),
-        expect: value => {
-          expect(value).toEqual({
       series: [timeSeries, logs, table],
       error: {},
       state: LoadingState.Error,
@@ -177,88 +152,65 @@ describe('decorateWithGraphLogsTraceAndTable', () => {
       tableResult: null,
       logsResult: null,
     });
-        },
-        done,
-      });
-    });
   });
 });
 
 describe('decorateWithGraphResult', () => {
-  describe('when used without error', () => {
+  it('should process the graph dataFrames', () => {
-    it('then the graphResult should be correct', done => {
     const { timeSeries } = getTestContext();
-      const timeField = timeSeries.fields[0];
-      const valueField = timeSeries.fields[1];
     const panelData = createExplorePanelData({ graphFrames: [timeSeries] });
-
+    console.log(decorateWithGraphResult(panelData).graphResult);
-      observableTester().subscribeAndExpectOnNext({
+    expect(decorateWithGraphResult(panelData).graphResult).toMatchObject([
-        observable: of(panelData).pipe(decorateWithGraphResult()),
+      {
-        expect: panelData => {
-          expect(panelData.graphResult![0]).toEqual({
         label: 'A-series',
-            color: '#7EB26D',
         data: [
           [100, 4],
           [200, 5],
           [300, 6],
         ],
-            info: [],
         isVisible: true,
         yAxis: {
           index: 1,
         },
         seriesIndex: 0,
-            timeField,
-            valueField,
         timeStep: 100,
-          });
       },
-        done,
+      {
-      });
+        label: 'B-series',
-    });
+        data: [
+          [100, 7],
+          [200, 8],
+          [300, 9],
+        ],
+        isVisible: true,
+        yAxis: {
+          index: 1,
+        },
+        seriesIndex: 1,
+        timeStep: 100,
+      },
+    ]);
   });
 
-  describe('when used without error but graph frames are empty', () => {
+  it('returns null if it gets empty array', () => {
-    it('then the graphResult should be null', done => {
     const panelData = createExplorePanelData({ graphFrames: [] });
-
+    expect(decorateWithGraphResult(panelData).graphResult).toBeNull();
-      observableTester().subscribeAndExpectOnNext({
-        observable: of(panelData).pipe(decorateWithGraphResult()),
-        expect: panelData => {
-          expect(panelData.graphResult).toBeNull();
-        },
-        done,
-      });
-    });
   });
 
-  describe('when used with error', () => {
+  it('returns null if panelData has error', () => {
-    it('then the graphResult should be null', done => {
     const { timeSeries } = getTestContext();
     const panelData = createExplorePanelData({ error: {}, graphFrames: [timeSeries] });
-
+    expect(decorateWithGraphResult(panelData).graphResult).toBeNull();
-      observableTester().subscribeAndExpectOnNext({
-        observable: of(panelData).pipe(decorateWithGraphResult()),
-        expect: panelData => {
-          expect(panelData.graphResult).toBeNull();
-        },
-        done,
-      });
-    });
   });
 });
 
 describe('decorateWithTableResult', () => {
-  describe('when used without error', () => {
+  it('should process table type dataFrame', async () => {
-    it('then the tableResult should be correct', done => {
     const { table, emptyTable } = getTestContext();
     const panelData = createExplorePanelData({ tableFrames: [table, emptyTable] });
+    const panelResult = await decorateWithTableResult(panelData).toPromise();
 
-      observableTester().subscribeAndExpectOnNext({
+    let theResult = panelResult.tableResult;
-        observable: of(panelData).pipe(decorateWithTableResult()),
-        expect: panelData => {
-          let theResult = panelData.tableResult;
 
     expect(theResult?.fields[0].name).toEqual('value');
     expect(theResult?.fields[1].name).toEqual('time');
@@ -291,12 +243,9 @@ describe('decorateWithTableResult', () => {
     expect(theResult.fields[3].name).toEqual('message');
     expect(theResult.fields[1].display).not.toBeNull();
     expect(theResult.length).toBe(3);
-        },
-        done,
-      });
   });
 
-    it('should do join transform if all series are timeseries', done => {
+  it('should do join transform if all series are timeseries', async () => {
     const tableFrames = [
       toDataFrame({
         name: 'A-series',
@@ -316,11 +265,8 @@ describe('decorateWithTableResult', () => {
       }),
     ];
     const panelData = createExplorePanelData({ tableFrames });
-
+    const panelResult = await decorateWithTableResult(panelData).toPromise();
-      observableTester().subscribeAndExpectOnNext({
+    const result = panelResult.tableResult;
-        observable: of(panelData).pipe(decorateWithTableResult()),
-        expect: panelData => {
-          const result = panelData.tableResult;
 
     expect(result?.fields[0].name).toBe('Time');
     expect(result?.fields[1].name).toBe('A-series');
@@ -328,12 +274,9 @@ describe('decorateWithTableResult', () => {
     expect(result?.fields[0].values.toArray()).toEqual([100, 200, 300]);
     expect(result?.fields[1].values.toArray()).toEqual([4, 5, 6]);
     expect(result?.fields[2].values.toArray()).toEqual([4, 5, 6]);
-        },
-        done,
-      });
   });
 
-    it('should not override fields display property when filled', done => {
+  it('should not override fields display property when filled', async () => {
     const tableFrames = [
       toDataFrame({
         name: 'A-series',
@@ -345,64 +288,30 @@ describe('decorateWithTableResult', () => {
     tableFrames[0].fields[0].display = displayFunctionMock;
 
     const panelData = createExplorePanelData({ tableFrames });
-
+    const panelResult = await decorateWithTableResult(panelData).toPromise();
-      observableTester().subscribeAndExpectOnNext({
+    expect(panelResult.tableResult?.fields[0].display).toBe(displayFunctionMock);
-        observable: of(panelData).pipe(decorateWithTableResult()),
-        expect: panelData => {
-          const data = panelData.tableResult;
-          expect(data?.fields[0].display).toBe(displayFunctionMock);
-        },
-        done,
-      });
-    });
   });
 
-  describe('when used without error but table frames are empty', () => {
+  it('should return null when passed empty array', async () => {
-    it('then the tableResult should be null', done => {
     const panelData = createExplorePanelData({ tableFrames: [] });
-
+    const panelResult = await decorateWithTableResult(panelData).toPromise();
-      observableTester().subscribeAndExpectOnNext({
+    expect(panelResult.tableResult).toBeNull();
-        observable: of(panelData).pipe(decorateWithTableResult()),
-        expect: panelData => {
-          expect(panelData.tableResult).toBeNull();
-        },
-        done,
-      });
-    });
   });
 
-  describe('when used with error', () => {
+  it('returns null if panelData has error', async () => {
-    it('then the tableResult should be null', done => {
     const { table, emptyTable } = getTestContext();
     const panelData = createExplorePanelData({ error: {}, tableFrames: [table, emptyTable] });
-
+    const panelResult = await decorateWithTableResult(panelData).toPromise();
-      observableTester().subscribeAndExpectOnNext({
+    expect(panelResult.tableResult).toBeNull();
-        observable: of(panelData).pipe(decorateWithTableResult()),
-        expect: panelData => {
-          expect(panelData.tableResult).toBeNull();
-        },
-        done,
-      });
-    });
   });
 });
 
 describe('decorateWithLogsResult', () => {
-  describe('when used without error', () => {
+  it('should correctly transform logs dataFrames', () => {
-    it('then the logsResult should be correct', done => {
     const { logs } = getTestContext();
-      const state = ({
-        queryIntervals: { intervalMs: 10 },
-      } as unknown) as ExploreItemState;
     const request = ({ timezone: 'utc', intervalMs: 60000 } as unknown) as DataQueryRequest;
     const panelData = createExplorePanelData({ logsFrames: [logs], request });
-
+    expect(decorateWithLogsResult()(panelData).logsResult).toEqual({
-      observableTester().subscribeAndExpectOnNext({
-        observable: of(panelData).pipe(decorateWithLogsResult(state)),
-        expect: panelData => {
-          const theResult = panelData.logsResult;
-
-          expect(theResult).toEqual({
       hasUniqueLabels: false,
       meta: [],
       rows: [
@@ -496,40 +405,16 @@ describe('decorateWithLogsResult', () => {
       ],
       visibleRange: undefined,
     });
-        },
-        done,
-      });
-    });
   });
 
-  describe('when used without error but logs frames are empty', () => {
+  it('returns null if passed empty array', () => {
-    it('then the graphResult should be null', done => {
     const panelData = createExplorePanelData({ logsFrames: [] });
-      const state = ({} as unknown) as ExploreItemState;
+    expect(decorateWithLogsResult()(panelData).logsResult).toBeNull();
-
-      observableTester().subscribeAndExpectOnNext({
-        observable: of(panelData).pipe(decorateWithLogsResult(state)),
-        expect: panelData => {
-          expect(panelData.logsResult).toBeNull();
-        },
-        done,
-      });
-    });
   });
 
-  describe('when used with error', () => {
+  it('returns null if panelData has error', () => {
-    it('then the graphResult should be null', done => {
     const { logs } = getTestContext();
     const panelData = createExplorePanelData({ error: {}, logsFrames: [logs] });
-      const state = ({} as unknown) as ExploreItemState;
+    expect(decorateWithLogsResult()(panelData).logsResult).toBeNull();
-
-      observableTester().subscribeAndExpectOnNext({
-        observable: of(panelData).pipe(decorateWithLogsResult(state)),
-        expect: panelData => {
-          expect(panelData.logsResult).toBeNull();
-        },
-        done,
-      });
-    });
   });
 });

public/app/features/explore/utils/decorators.ts

@@ -1,27 +1,28 @@
-import { MonoTypeOperatorFunction, of, OperatorFunction } from 'rxjs';
+import { Observable, of } from 'rxjs';
-import { map, mergeMap } from 'rxjs/operators';
+import { map } from 'rxjs/operators';
 import {
+  AbsoluteTimeRange,
   DataFrame,
-  DataSourceApi,
   FieldType,
   getDisplayProcessor,
   PanelData,
-  PreferredVisualisationType,
   sortLogsResult,
   standardTransformers,
 } from '@grafana/data';
 import { config } from '@grafana/runtime';
+import { groupBy } from 'lodash';
 
-import { ExploreItemState, ExplorePanelData } from '../../../types';
+import { ExplorePanelData } from '../../../types';
 import { getGraphSeriesModel } from '../../../plugins/panel/graph2/getGraphSeriesModel';
 import { dataFrameToLogsModel } from '../../../core/logs_model';
 import { refreshIntervalToSortOrder } from '../../../core/utils/explore';
 
-export const decorateWithGraphLogsTraceAndTable = (
+/**
-  datasourceInstance?: DataSourceApi | null
+ * When processing response first we try to determine what kind of dataframes we got as one query can return multiple
-): OperatorFunction<PanelData, ExplorePanelData> => inputStream =>
+ * dataFrames with different type of data. This is later used for type specific processing. As we use this in
-  inputStream.pipe(
+ * Observable pipeline, it decorates the existing panelData to pass the results to later processing stages.
-    map(data => {
+ */
+export const decorateWithGraphLogsTraceAndTable = (data: PanelData): ExplorePanelData => {
   if (data.error) {
     return {
       ...data,
@@ -41,26 +42,29 @@ export const decorateWithGraphLogsTraceAndTable = (
   const traceFrames: DataFrame[] = [];
 
   for (const frame of data.series) {
-        if (shouldShowInVisualisationTypeStrict(frame, 'logs')) {
+    switch (frame.meta?.preferredVisualisationType) {
+      case 'logs':
         logsFrames.push(frame);
-        } else if (shouldShowInVisualisationTypeStrict(frame, 'graph')) {
+        break;
+      case 'graph':
         graphFrames.push(frame);
-        } else if (shouldShowInVisualisationTypeStrict(frame, 'trace')) {
+        break;
+      case 'trace':
         traceFrames.push(frame);
-        } else if (shouldShowInVisualisationTypeStrict(frame, 'table')) {
+        break;
+      case 'table':
         tableFrames.push(frame);
-        } else if (isTimeSeries(frame, datasourceInstance?.meta.id)) {
+        break;
-          if (shouldShowInVisualisationType(frame, 'graph')) {
+      default:
+        if (isTimeSeries(frame)) {
           graphFrames.push(frame);
-          }
-          if (shouldShowInVisualisationType(frame, 'table')) {
           tableFrames.push(frame);
-          }
         } else {
           // We fallback to table if we do not have any better meta info about the dataframe.
           tableFrames.push(frame);
         }
     }
+  }
 
   return {
     ...data,
@@ -72,12 +76,9 @@ export const decorateWithGraphLogsTraceAndTable = (
     tableResult: null,
     logsResult: null,
   };
-    })
+};
-  );
 
-export const decorateWithGraphResult = (): MonoTypeOperatorFunction<ExplorePanelData> => inputStream =>
+export const decorateWithGraphResult = (data: ExplorePanelData): ExplorePanelData => {
-  inputStream.pipe(
-    map(data => {
   if (data.error) {
     return { ...data, graphResult: null };
   }
@@ -94,12 +95,14 @@ export const decorateWithGraphResult = (): MonoTypeOperatorFunction<ExplorePanel
         );
 
   return { ...data, graphResult };
-    })
+};
-  );
 
-export const decorateWithTableResult = (): MonoTypeOperatorFunction<ExplorePanelData> => inputStream =>
+/**
-  inputStream.pipe(
+ * This processing returns Observable because it uses Transformer internally which result type is also Observable.
-    mergeMap(data => {
+ * In this case the transformer should return single result but it is possible that in the future it could return
+ * multiple results and so this should be used with mergeMap or similar to unbox the internal observable.
+ */
+export const decorateWithTableResult = (data: ExplorePanelData): Observable<ExplorePanelData> => {
   if (data.error) {
     return of({ ...data, tableResult: null });
   }
@@ -148,67 +151,37 @@ export const decorateWithTableResult = (): MonoTypeOperatorFunction<ExplorePanel
       return { ...data, tableResult: frame };
     })
   );
-    })
+};
-  );
 
 export const decorateWithLogsResult = (
-  state: ExploreItemState
+  options: { absoluteRange?: AbsoluteTimeRange; refreshInterval?: string } = {}
-): MonoTypeOperatorFunction<ExplorePanelData> => inputStream =>
+) => (data: ExplorePanelData): ExplorePanelData => {
-  inputStream.pipe(
-    map(data => {
   if (data.error) {
     return { ...data, logsResult: null };
   }
 
-      const { absoluteRange, refreshInterval } = state;
   if (data.logsFrames.length === 0) {
     return { ...data, logsResult: null };
   }
 
   const timeZone = data.request?.timezone ?? 'browser';
   const intervalMs = data.request?.intervalMs;
-      const newResults = dataFrameToLogsModel(data.logsFrames, intervalMs, timeZone, absoluteRange);
+  const newResults = dataFrameToLogsModel(data.logsFrames, intervalMs, timeZone, options.absoluteRange);
-      const sortOrder = refreshIntervalToSortOrder(refreshInterval);
+  const sortOrder = refreshIntervalToSortOrder(options.refreshInterval);
   const sortedNewResults = sortLogsResult(newResults, sortOrder);
   const rows = sortedNewResults.rows;
   const series = sortedNewResults.series;
   const logsResult = { ...sortedNewResults, rows, series };
 
   return { ...data, logsResult };
-    })
+};
-  );
 
-function isTimeSeries(frame: DataFrame, datasource?: string): boolean {
+/**
-  // TEMP: Temporary hack. Remove when logs/metrics unification is done
+ * Check if frame contains time series, which for our purpose means 1 time column and 1 or more numeric columns.
-  if (datasource && datasource === 'cloudwatch') {
+ */
-    return isTimeSeriesCloudWatch(frame);
+function isTimeSeries(frame: DataFrame): boolean {
-  }
+  const grouped = groupBy(frame.fields, field => field.type);
-
+  return Boolean(
-  if (frame.fields.length === 2) {
+    Object.keys(grouped).length === 2 && grouped[FieldType.time]?.length === 1 && grouped[FieldType.number]
-    if (frame.fields[0].type === FieldType.time) {
-      return true;
-    }
-  }
-
-  return false;
-}
-
-function shouldShowInVisualisationType(frame: DataFrame, visualisation: PreferredVisualisationType) {
-  if (frame.meta?.preferredVisualisationType && frame.meta?.preferredVisualisationType !== visualisation) {
-    return false;
-  }
-
-  return true;
-}
-
-function shouldShowInVisualisationTypeStrict(frame: DataFrame, visualisation: PreferredVisualisationType) {
-  return frame.meta?.preferredVisualisationType === visualisation;
-}
-
-// TEMP: Temporary hack. Remove when logs/metrics unification is done
-function isTimeSeriesCloudWatch(frame: DataFrame): boolean {
-  return (
-    frame.fields.some(field => field.type === FieldType.time) &&
-    frame.fields.some(field => field.type === FieldType.number)
   );
 }