Influx: Make max series limit configurable and show the limiting message if applied (#31025)

* Add configuration in ConfigEditor and default to 1000

* Show data in explore if any even if there is an error

* Update pkg/tsdb/influxdb/flux/executor.go

* Better handling of defaults

* Add test for runQuery to show data even with error

* Update public/app/store/configureStore.ts

Co-authored-by: Giordano Ricci <gio.ricci@grafana.com>

* Update public/app/plugins/datasource/influxdb/components/ConfigEditor.tsx

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

* Update tooltip

* Update input

* Lint fixes

* Update snapshots

* Update decorator tests

Co-authored-by: Giordano Ricci <gio.ricci@grafana.com>
Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

pkg/tsdb/influxdb/flux/executor.go

@@ -85,7 +85,9 @@ func readDataFrames(result *api.QueryTableResult, maxPoints int, maxSeries int)
 		}
 	}
 
-	// Attach any errors (may be null)
+	// result.Err() is probably more important then the other errors
-	dr.Error = result.Err()
+	if result.Err() != nil {
+		dr.Error = result.Err()
+	}
 	return dr
 }

pkg/tsdb/influxdb/flux/flux.go

@@ -39,7 +39,9 @@ func Query(ctx context.Context, dsInfo *models.DataSource, tsdbQuery *tsdb.TsdbQ
 			continue
 		}
 
-		res := executeQuery(ctx, *qm, r, 50)
+		// If the default changes also update labels/placeholder in config page.
+		maxSeries := dsInfo.JsonData.Get("maxSeries").MustInt(1000)
+		res := executeQuery(ctx, *qm, r, maxSeries)
 
 		tRes.Results[query.RefId] = backendDataResponseToTSDBResponse(&res, query.RefId)
 	}

public/app/features/explore/state/query.test.ts

@@ -4,15 +4,19 @@ import {
   cancelQueriesAction,
   queryReducer,
   removeQueryRowAction,
+  runQueries,
   scanStartAction,
   scanStopAction,
 } from './query';
 import { ExploreId, ExploreItemState } from 'app/types';
-import { interval } from 'rxjs';
+import { interval, of } from 'rxjs';
-import { RawTimeRange, toUtc } from '@grafana/data';
+import { ArrayVector, DataQueryResponse, DefaultTimeZone, MutableDataFrame, RawTimeRange, toUtc } from '@grafana/data';
 import { thunkTester } from 'test/core/thunk/thunkTester';
 import { makeExplorePaneState } from './utils';
 import { reducerTester } from '../../../../test/core/redux/reducerTester';
+import { configureStore } from '../../../store/configureStore';
+import { setTimeSrv } from '../../dashboard/services/TimeSrv';
+import Mock = jest.Mock;
 
 const QUERY_KEY_REGEX = /Q-(?:[a-z0-9]+-){5}(?:[0-9]+)/;
 const t = toUtc();
@@ -24,6 +28,58 @@ const testRange = {
     to: t,
   },
 };
+const defaultInitialState = {
+  user: {
+    orgId: '1',
+    timeZone: DefaultTimeZone,
+  },
+  explore: {
+    [ExploreId.left]: {
+      datasourceInstance: {
+        query: jest.fn(),
+        meta: {
+          id: 'something',
+        },
+      },
+      initialized: true,
+      containerWidth: 1920,
+      eventBridge: { emit: () => {} } as any,
+      queries: [{ expr: 'test' }] as any[],
+      range: testRange,
+      refreshInterval: {
+        label: 'Off',
+        value: 0,
+      },
+    },
+  },
+};
+
+describe('runQueries', () => {
+  it('should pass dataFrames to state even if there is error in response', async () => {
+    setTimeSrv({
+      init() {},
+    } as any);
+    const store = configureStore({
+      ...(defaultInitialState as any),
+    });
+    (store.getState().explore[ExploreId.left].datasourceInstance?.query as Mock).mockReturnValueOnce(
+      of({
+        error: { message: 'test error' },
+        data: [
+          new MutableDataFrame({
+            fields: [{ name: 'test', values: new ArrayVector() }],
+            meta: {
+              preferredVisualisationType: 'graph',
+            },
+          }),
+        ],
+      } as DataQueryResponse)
+    );
+    await store.dispatch(runQueries(ExploreId.left));
+    expect(store.getState().explore[ExploreId.left].showMetrics).toBeTruthy();
+    expect(store.getState().explore[ExploreId.left].graphResult).toBeDefined();
+  });
+});
 
 describe('running queries', () => {
   it('should cancel running query when cancelQueries is dispatched', async () => {

public/app/features/explore/state/query.ts

@@ -664,16 +664,6 @@ export const processQueryResponse = (
 
     // For Angular editors
     state.eventBridge.emit(PanelEvents.dataError, error);
-
-    return {
-      ...state,
-      loading: loadingState === LoadingState.Loading || loadingState === LoadingState.Streaming,
-      queryResponse: response,
-      graphResult: null,
-      tableResult: null,
-      logsResult: null,
-      update: makeInitialUpdateState(),
-    };
   }
 
   if (!request) {

public/app/features/explore/utils/decorators.test.ts

@@ -132,7 +132,7 @@ describe('decorateWithGraphLogsTraceAndTable', () => {
     });
   });
 
-  it('should handle query error', () => {
+  it('should return frames even if there is an error', () => {
     const { timeSeries, logs, table } = getTestContext();
     const series: DataFrame[] = [timeSeries, logs, table];
     const panelData: PanelData = {
@@ -147,9 +147,9 @@ describe('decorateWithGraphLogsTraceAndTable', () => {
       error: {},
       state: LoadingState.Error,
       timeRange: {},
-      graphFrames: [],
+      graphFrames: [timeSeries],
-      tableFrames: [],
+      tableFrames: [table],
-      logsFrames: [],
+      logsFrames: [logs],
       traceFrames: [],
       nodeGraphFrames: [],
       graphResult: null,
@@ -171,10 +171,10 @@ describe('decorateWithGraphResult', () => {
     expect(decorateWithGraphResult(panelData).graphResult).toBeNull();
   });
 
-  it('returns null if panelData has error', () => {
+  it('returns data if panelData has error', () => {
     const { timeSeries } = getTestContext();
     const panelData = createExplorePanelData({ error: {}, graphFrames: [timeSeries] });
-    expect(decorateWithGraphResult(panelData).graphResult).toBeNull();
+    expect(decorateWithGraphResult(panelData).graphResult).toMatchObject([timeSeries]);
   });
 });
 
@@ -272,11 +272,11 @@ describe('decorateWithTableResult', () => {
     expect(panelResult.tableResult).toBeNull();
   });
 
-  it('returns null if panelData has error', async () => {
+  it('returns data if panelData has error', async () => {
     const { table, emptyTable } = getTestContext();
     const panelData = createExplorePanelData({ error: {}, tableFrames: [table, emptyTable] });
     const panelResult = await decorateWithTableResult(panelData).toPromise();
-    expect(panelResult.tableResult).toBeNull();
+    expect(panelResult.tableResult).not.toBeNull();
   });
 });
 
@@ -386,9 +386,9 @@ describe('decorateWithLogsResult', () => {
     expect(decorateWithLogsResult()(panelData).logsResult).toBeNull();
   });
 
-  it('returns null if panelData has error', () => {
+  it('returns data if panelData has error', () => {
     const { logs } = getTestContext();
     const panelData = createExplorePanelData({ error: {}, logsFrames: [logs] });
-    expect(decorateWithLogsResult()(panelData).logsResult).toBeNull();
+    expect(decorateWithLogsResult()(panelData).logsResult).not.toBeNull();
   });
 });

public/app/features/explore/utils/decorators.ts

@@ -21,20 +21,6 @@ import { ExplorePanelData } from '../../../types';
  * Observable pipeline, it decorates the existing panelData to pass the results to later processing stages.
  */
 export const decorateWithFrameTypeMetadata = (data: PanelData): ExplorePanelData => {
-  if (data.error) {
-    return {
-      ...data,
-      graphFrames: [],
-      tableFrames: [],
-      logsFrames: [],
-      traceFrames: [],
-      nodeGraphFrames: [],
-      graphResult: null,
-      tableResult: null,
-      logsResult: null,
-    };
-  }
-
   const graphFrames: DataFrame[] = [];
   const tableFrames: DataFrame[] = [];
   const logsFrames: DataFrame[] = [];
@@ -83,7 +69,7 @@ export const decorateWithFrameTypeMetadata = (data: PanelData): ExplorePanelData
 };
 
 export const decorateWithGraphResult = (data: ExplorePanelData): ExplorePanelData => {
-  if (data.error || !data.graphFrames.length) {
+  if (!data.graphFrames.length) {
     return { ...data, graphResult: null };
   }
 
@@ -96,10 +82,6 @@ export const decorateWithGraphResult = (data: ExplorePanelData): ExplorePanelDat
  * multiple results and so this should be used with mergeMap or similar to unbox the internal observable.
  */
 export const decorateWithTableResult = (data: ExplorePanelData): Observable<ExplorePanelData> => {
-  if (data.error) {
-    return of({ ...data, tableResult: null });
-  }
-
   if (data.tableFrames.length === 0) {
     return of({ ...data, tableResult: null });
   }
@@ -149,10 +131,6 @@ export const decorateWithTableResult = (data: ExplorePanelData): Observable<Expl
 export const decorateWithLogsResult = (
   options: { absoluteRange?: AbsoluteTimeRange; refreshInterval?: string } = {}
 ) => (data: ExplorePanelData): ExplorePanelData => {
-  if (data.error) {
-    return { ...data, logsResult: null };
-  }
-
   if (data.logsFrames.length === 0) {
     return { ...data, logsResult: null };
   }

public/app/plugins/datasource/influxdb/components/ConfigEditor.tsx

@@ -7,8 +7,9 @@ import {
   onUpdateDatasourceJsonDataOption,
   onUpdateDatasourceJsonDataOptionSelect,
   onUpdateDatasourceSecureJsonDataOption,
+  updateDatasourcePluginJsonDataOption,
 } from '@grafana/data';
-import { DataSourceHttpSettings, InfoBox, InlineFormLabel, LegacyForms } from '@grafana/ui';
+import { DataSourceHttpSettings, InfoBox, InlineField, InlineFormLabel, LegacyForms } from '@grafana/ui';
 const { Select, Input, SecretFormField } = LegacyForms;
 import { InfluxOptions, InfluxSecureJsonData, InfluxVersion } from '../types';
 
@@ -31,8 +32,20 @@ const versions = [
 ] as Array<SelectableValue<InfluxVersion>>;
 
 export type Props = DataSourcePluginOptionsEditorProps<InfluxOptions>;
+type State = {
+  maxSeries: string | undefined;
+};
+
+export class ConfigEditor extends PureComponent<Props, State> {
+  state = {
+    maxSeries: '',
+  };
+
+  constructor(props: Props) {
+    super(props);
+    this.state.maxSeries = props.options.jsonData.maxSeries?.toString() || '';
+  }
 
-export class ConfigEditor extends PureComponent<Props> {
   // 1x
   onResetPassword = () => {
     updateDatasourcePluginResetOption(this.props, 'password');
@@ -67,33 +80,12 @@ export class ConfigEditor extends PureComponent<Props> {
   };
 
   renderInflux2x() {
-    const { options, onOptionsChange } = this.props;
+    const { options } = this.props;
     const { secureJsonFields } = options;
     const secureJsonData = (options.secureJsonData || {}) as InfluxSecureJsonData;
 
     return (
-      <div>
+      <>
-        <div className="gf-form-group">
-          <InfoBox>
-            <h5>Support for flux in Grafana is currently in beta</h5>
-            <p>
-              Please report any issues to: <br />
-              <a href="https://github.com/grafana/grafana/issues/new/choose">
-                https://github.com/grafana/grafana/issues
-              </a>
-            </p>
-          </InfoBox>
-        </div>
-        <br />
-
-        <DataSourceHttpSettings
-          showAccessOptions={false}
-          dataSourceConfig={options}
-          defaultUrl="http://localhost:8086"
-          onChange={onOptionsChange}
-        />
-
-        <h3 className="page-heading">InfluxDB Details</h3>
         <div className="gf-form-inline">
           <div className="gf-form">
             <InlineFormLabel className="width-10">Organization</InlineFormLabel>
@@ -152,125 +144,111 @@ export class ConfigEditor extends PureComponent<Props> {
             </div>
           </div>
         </div>
-      </div>
+      </>
     );
   }
 
   renderInflux1x() {
-    const { options, onOptionsChange } = this.props;
+    const { options } = this.props;
     const { secureJsonFields } = options;
     const secureJsonData = (options.secureJsonData || {}) as InfluxSecureJsonData;
 
     return (
-      <div>
+      <>
-        <DataSourceHttpSettings
+        <InfoBox>
-          showAccessOptions={true}
+          <h5>Database Access</h5>
-          dataSourceConfig={options}
+          <p>
-          defaultUrl="http://localhost:8086"
+            Setting the database for this datasource does not deny access to other databases. The InfluxDB query syntax
-          onChange={onOptionsChange}
+            allows switching the database in the query. For example:
-        />
+            <code>SHOW MEASUREMENTS ON _internal</code> or
-
+            <code>SELECT * FROM &quot;_internal&quot;..&quot;database&quot; LIMIT 10</code>
-        <h3 className="page-heading">InfluxDB Details</h3>
+            <br />
-        <div className="gf-form-group">
+            <br />
-          <div className="gf-form-inline">
+            To support data isolation and security, make sure appropriate permissions are configured in InfluxDB.
-            <div className="gf-form">
+          </p>
-              <InlineFormLabel className="width-10">Database</InlineFormLabel>
+        </InfoBox>
-              <div className="width-20">
+        <div className="gf-form-inline">
-                <Input
+          <div className="gf-form">
-                  className="width-20"
+            <InlineFormLabel className="width-10">Database</InlineFormLabel>
-                  value={options.database || ''}
+            <div className="width-20">
-                  onChange={onUpdateDatasourceOption(this.props, 'database')}
+              <Input
-                />
+                className="width-20"
-              </div>
+                value={options.database || ''}
-            </div>
+                onChange={onUpdateDatasourceOption(this.props, 'database')}
-          </div>
-          <div className="gf-form-inline">
-            <div className="gf-form">
-              <InlineFormLabel className="width-10">User</InlineFormLabel>
-              <div className="width-10">
-                <Input
-                  className="width-20"
-                  value={options.user || ''}
-                  onChange={onUpdateDatasourceOption(this.props, 'user')}
-                />
-              </div>
-            </div>
-          </div>
-          <div className="gf-form-inline">
-            <div className="gf-form">
-              <SecretFormField
-                isConfigured={(secureJsonFields && secureJsonFields.password) as boolean}
-                value={secureJsonData.password || ''}
-                label="Password"
-                labelWidth={10}
-                inputWidth={20}
-                onReset={this.onResetPassword}
-                onChange={onUpdateDatasourceSecureJsonDataOption(this.props, 'password')}
               />
             </div>
           </div>
-          <div className="gf-form-inline">
+        </div>
-            <div className="gf-form">
+        <div className="gf-form-inline">
-              <InlineFormLabel
+          <div className="gf-form">
-                className="width-10"
+            <InlineFormLabel className="width-10">User</InlineFormLabel>
-                tooltip="You can use either GET or POST HTTP method to query your InfluxDB database. The POST
+            <div className="width-10">
+              <Input
+                className="width-20"
+                value={options.user || ''}
+                onChange={onUpdateDatasourceOption(this.props, 'user')}
+              />
+            </div>
+          </div>
+        </div>
+        <div className="gf-form-inline">
+          <div className="gf-form">
+            <SecretFormField
+              isConfigured={(secureJsonFields && secureJsonFields.password) as boolean}
+              value={secureJsonData.password || ''}
+              label="Password"
+              labelWidth={10}
+              inputWidth={20}
+              onReset={this.onResetPassword}
+              onChange={onUpdateDatasourceSecureJsonDataOption(this.props, 'password')}
+            />
+          </div>
+        </div>
+        <div className="gf-form-inline">
+          <div className="gf-form">
+            <InlineFormLabel
+              className="width-10"
+              tooltip="You can use either GET or POST HTTP method to query your InfluxDB database. The POST
           method allows you to perform heavy requests (with a lots of WHERE clause) while the GET method
           will restrict you and return an error if the query is too large."
-              >
+            >
-                HTTP Method
+              HTTP Method
-              </InlineFormLabel>
+            </InlineFormLabel>
-              <Select
+            <Select
+              className="width-10"
+              value={httpModes.find((httpMode) => httpMode.value === options.jsonData.httpMode)}
+              options={httpModes}
+              defaultValue={options.jsonData.httpMode}
+              onChange={onUpdateDatasourceJsonDataOptionSelect(this.props, 'httpMode')}
+            />
+          </div>
+        </div>
+
+        <div className="gf-form-inline">
+          <div className="gf-form">
+            <InlineFormLabel
+              className="width-10"
+              tooltip="A lower limit for the auto group by time interval. Recommended to be set to write frequency,
+				for example 1m if your data is written every minute."
+            >
+              Min time interval
+            </InlineFormLabel>
+            <div className="width-10">
+              <Input
                 className="width-10"
-                value={httpModes.find((httpMode) => httpMode.value === options.jsonData.httpMode)}
+                placeholder="10s"
-                options={httpModes}
+                value={options.jsonData.timeInterval || ''}
-                defaultValue={options.jsonData.httpMode}
+                onChange={onUpdateDatasourceJsonDataOption(this.props, 'timeInterval')}
-                onChange={onUpdateDatasourceJsonDataOptionSelect(this.props, 'httpMode')}
               />
             </div>
           </div>
         </div>
-
+      </>
-        <div className="gf-form-group">
-          <InfoBox>
-            <h5>Database Access</h5>
-            <p>
-              Setting the database for this datasource does not deny access to other databases. The InfluxDB query
-              syntax allows switching the database in the query. For example:
-              <code>SHOW MEASUREMENTS ON _internal</code> or
-              <code>SELECT * FROM &quot;_internal&quot;..&quot;database&quot; LIMIT 10</code>
-              <br />
-              <br />
-              To support data isolation and security, make sure appropriate permissions are configured in InfluxDB.
-            </p>
-          </InfoBox>
-        </div>
-        <div className="gf-form-group">
-          <div className="gf-form-inline">
-            <div className="gf-form">
-              <InlineFormLabel
-                className="width-10"
-                tooltip="A lower limit for the auto group by time interval. Recommended to be set to write frequency,
-				for example 1m if your data is written every minute."
-              >
-                Min time interval
-              </InlineFormLabel>
-              <div className="width-10">
-                <Input
-                  className="width-10"
-                  placeholder="10s"
-                  value={options.jsonData.timeInterval || ''}
-                  onChange={onUpdateDatasourceJsonDataOption(this.props, 'timeInterval')}
-                />
-              </div>
-            </div>
-          </div>
-        </div>
-      </div>
     );
   }
 
   render() {
-    const { options } = this.props;
+    const { options, onOptionsChange } = this.props;
 
     return (
       <>
@@ -289,7 +267,52 @@ export class ConfigEditor extends PureComponent<Props> {
           </div>
         </div>
 
-        {options.jsonData.version === InfluxVersion.Flux ? this.renderInflux2x() : this.renderInflux1x()}
+        {options.jsonData.version === InfluxVersion.Flux && (
+          <InfoBox>
+            <h5>Support for Flux in Grafana is currently in beta</h5>
+            <p>
+              Please report any issues to: <br />
+              <a href="https://github.com/grafana/grafana/issues/new/choose">
+                https://github.com/grafana/grafana/issues
+              </a>
+            </p>
+          </InfoBox>
+        )}
+
+        <DataSourceHttpSettings
+          showAccessOptions={true}
+          dataSourceConfig={options}
+          defaultUrl="http://localhost:8086"
+          onChange={onOptionsChange}
+        />
+
+        <div className="gf-form-group">
+          <div>
+            <h3 className="page-heading">InfluxDB Details</h3>
+          </div>
+          {options.jsonData.version === InfluxVersion.Flux ? this.renderInflux2x() : this.renderInflux1x()}
+          <div className="gf-form-inline">
+            <InlineField
+              labelWidth={20}
+              label="Max series"
+              tooltip="Limit the number of series/tables that Grafana will process. Lower this number to prevent abuse, and increase it if you have lots of small time series and not all are shown. Defaults to 1000."
+            >
+              <Input
+                placeholder="1000"
+                type="number"
+                className="width-10"
+                value={this.state.maxSeries}
+                onChange={(event) => {
+                  // We duplicate this state so that we allow to write freely inside the input. We don't have
+                  // any influence over saving so this seems to be only way to do this.
+                  this.setState({ maxSeries: event.currentTarget.value });
+                  const val = parseInt(event.currentTarget.value, 10);
+                  updateDatasourcePluginJsonDataOption(this.props, 'maxSeries', Number.isFinite(val) ? val : undefined);
+                }}
+              />
+            </InlineField>
+          </div>
+        </div>
       </>
     );
   }

public/app/store/configureStore.ts

@@ -14,7 +14,7 @@ export function addRootReducer(reducers: any) {
   addReducer(reducers);
 }
 
-export function configureStore() {
+export function configureStore(initialState?: Partial<StoreState>) {
   const logger = createLogger({
     predicate: (getState) => {
       return getState().application.logActions;
@@ -35,6 +35,7 @@ export function configureStore() {
     devTools: process.env.NODE_ENV !== 'production',
     preloadedState: {
       navIndex: buildInitialState(),
+      ...initialState,
     },
   });
 
@@ -42,7 +43,7 @@ export function configureStore() {
   return store;
 }
 
-/* 
+/*
 function getActionsToIgnoreSerializableCheckOn() {
   return [
     'dashboard/setPanelAngularComponent',
@@ -58,7 +59,7 @@ function getActionsToIgnoreSerializableCheckOn() {
 }
 
 function getPathsToIgnoreMutationAndSerializableCheckOn() {
-  return [    
+  return [
     'plugins.panels',
     'dashboard.panels',
     'dashboard.getModel',
@@ -75,7 +76,7 @@ function getPathsToIgnoreMutationAndSerializableCheckOn() {
     'explore.right.eventBridge',
     'explore.right.range',
     'explore.left.querySubscription',
-    'explore.right.querySubscription',    
+    'explore.right.querySubscription',
   ];
 }
 */