IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

InfluxDB: Fix retention policy handling for InfluxDB 3.0 engine by bringing back the hardcoded default (#72467)

Browse Source 
* Revert retention policy handling

* Remove comment

* Send retention policy with request when it's possible
This commit is contained in:
ismail simsek 2023-08-04 16:57:41 +03:00 committed by GitHub
parent 0d6e911fff
commit 3395ad03a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 29 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
public/app/plugins/datasource/influxdb/components/editor/query/influxql/hooks/useRetentionPolicies.test.ts
View File

@ -1,55 +0,0 @@
import { renderHook } from '@testing-library/react-hooks';
import { FetchResponse } from '@grafana/runtime/src';
import config from 'app/core/config';
import { getMockDS, getMockDSInstanceSettings, mockBackendService } from '../../../../../specs/mocks';
import { useRetentionPolicies } from './useRetentionPolicies';
jest.mock('@grafana/runtime', () => ({
...jest.requireActual('@grafana/runtime'),
}));
describe('useRetentionPolicies', () => {
it('should return all policies when influxdbBackendMigration feature toggle enabled', async () => {
const instanceSettings = getMockDSInstanceSettings();
const datasource = getMockDS(instanceSettings);
mockBackendService(response);
config.featureToggles.influxdbBackendMigration = true;
const { result, waitForNextUpdate } = renderHook(() => useRetentionPolicies(datasource));
await waitForNextUpdate();
expect(result.current.retentionPolicies.length).toEqual(4);
expect(result.current.retentionPolicies[0]).toEqual('autogen');
});
});
const response: FetchResponse = {
config: {
url: 'mock-response-url',
},
headers: new Headers(),
ok: false,
redirected: false,
status: 0,
statusText: '',
type: 'basic',
url: '',
data: {
results: {
metadataQuery: {
status: 200,
frames: [
{
schema: {
refId: 'metadataQuery',
fields: [{ name: 'value', type: 'string', typeInfo: { frame: 'string' } }],
},
data: { values: [['autogen', 'bar', '5m_avg', '1m_avg']] },
},
],
},
},
},
};

15
public/app/plugins/datasource/influxdb/components/editor/query/influxql/hooks/useRetentionPolicies.ts
View File

@ -1,15 +0,0 @@
import { useEffect, useState } from 'react';
import InfluxDatasource from '../../../../../datasource';
import { getAllPolicies } from '../../../../../influxql_metadata_query';
export const useRetentionPolicies = (datasource: InfluxDatasource) => {
const [retentionPolicies, setRetentionPolicies] = useState<string[]>([]);
useEffect(() => {
getAllPolicies(datasource).then((data) => {
setRetentionPolicies(data);
});
}, [datasource]);
return { retentionPolicies };
};

10
public/app/plugins/datasource/influxdb/components/editor/query/influxql/visual/FromSection.tsx
View File

@ -1,5 +1,6 @@
import React from 'react';
import { DEFAULT_POLICY } from '../../../../../types';
import { toSelectableValue } from '../utils/toSelectableValue';
import { Seg } from './Seg';
@ -32,7 +33,12 @@ export const FromSection = ({
}: Props): JSX.Element => {
const handlePolicyLoadOptions = async () => {
const allPolicies = await getPolicyOptions();
return allPolicies.map(toSelectableValue);
// if `default` does not exist in the list of policies, we add it
const allPoliciesWithDefault = allPolicies.some((p) => p === 'default')
? allPolicies
: [DEFAULT_POLICY, ...allPolicies];
return allPoliciesWithDefault.map(toSelectableValue);
};
const handleMeasurementLoadOptions = async (filter: string) => {
@ -44,7 +50,7 @@ export const FromSection = ({
<>
<Seg
allowCustomValue
value={policy ?? ''}
value={policy ?? 'using default policy'}
loadOptions={handlePolicyLoadOptions}
onChange={(v) => {
onChange(v.value, measurement);

14
public/app/plugins/datasource/influxdb/components/editor/query/influxql/visual/VisualInfluxQLEditor.tsx
View File

@ -1,5 +1,5 @@
import { css } from '@emotion/css';
import React, { useEffect, useId, useMemo } from 'react';
import React, { useId, useMemo } from 'react';
import { GrafanaTheme2 } from '@grafana/data';
import { InlineLabel, SegmentSection, useStyles2 } from '@grafana/ui';
@ -23,7 +23,6 @@ import {
} from '../../../../../queryUtils';
import { InfluxQuery, InfluxQueryTag } from '../../../../../types';
import { DEFAULT_RESULT_FORMAT } from '../../../constants';
import { useRetentionPolicies } from '../hooks/useRetentionPolicies';
import { filterTags } from '../utils/filterTags';
import { getNewGroupByPartOptions, getNewSelectPartOptions, makePartList } from '../utils/partListUtils';
import { withTemplateVariableOptions } from '../utils/withTemplateVariableOptions';
@ -52,17 +51,6 @@ export const VisualInfluxQLEditor = (props: Props): JSX.Element => {
const query = normalizeQuery(props.query);
const { datasource } = props;
const { measurement, policy } = query;
const { retentionPolicies } = useRetentionPolicies(datasource);
useEffect(() => {
if (!policy && retentionPolicies.length > 0) {
props.onChange({
...query,
policy: retentionPolicies[0],
});
props.onRunQuery();
}
}, [policy, props, query, retentionPolicies]);
const allTagKeys = useMemo(async () => {
const tagKeys = (await getTagKeysForMeasurementAndTags(datasource, [], measurement, policy)).map(

48
public/app/plugins/datasource/influxdb/datasource.ts
View File

@ -1,5 +1,5 @@
import { cloneDeep, extend, groupBy, has, isString, map as _map, omit, pick, reduce } from 'lodash';
import { defer, lastValueFrom, merge, mergeMap, Observable, of, throwError } from 'rxjs';
import { lastValueFrom, merge, Observable, of, throwError } from 'rxjs';
import { catchError, map } from 'rxjs/operators';
import {
@ -36,12 +36,11 @@ import { FluxQueryEditor } from './components/editor/query/flux/FluxQueryEditor'
import { BROWSER_MODE_DISABLED_MESSAGE } from './constants';
import InfluxQueryModel from './influx_query_model';
import InfluxSeries from './influx_series';
import { getAllPolicies } from './influxql_metadata_query';
import { buildMetadataQuery } from './influxql_query_builder';
import { prepareAnnotation } from './migrations';
import { buildRawQuery, replaceHardCodedRetentionPolicy } from './queryUtils';
import { buildRawQuery } from './queryUtils';
import ResponseParser from './response_parser';
import { InfluxOptions, InfluxQuery, InfluxVersion } from './types';
import { DEFAULT_POLICY, InfluxOptions, InfluxQuery, InfluxVersion } from './types';
export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery, InfluxOptions> {
type: string;
@ -57,7 +56,6 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
httpMode: string;
version?: InfluxVersion;
isProxyAccess: boolean;
retentionPolicies: string[];
constructor(
instanceSettings: DataSourceInstanceSettings<InfluxOptions>,
@ -83,7 +81,6 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
this.responseParser = new ResponseParser();
this.version = settingsData.version ?? InfluxVersion.InfluxQL;
this.isProxyAccess = instanceSettings.access === 'proxy';
this.retentionPolicies = [];
if (this.version === InfluxVersion.Flux) {
// When flux, use an annotation processor rather than the `annotationQuery` lifecycle
@ -98,48 +95,13 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
}
}
async getRetentionPolicies(): Promise<string[]> {
// Only For InfluxQL Mode
if (this.version === InfluxVersion.InfluxQL && !this.retentionPolicies.length) {
return getAllPolicies(this).catch((err) => {
console.error(
'Unable to fetch retention policies. Queries will be run without specifying retention policy.',
err
);
return Promise.resolve(this.retentionPolicies);
});
}
return Promise.resolve(this.retentionPolicies);
}
query(request: DataQueryRequest<InfluxQuery>): Observable<DataQueryResponse> {
if (!this.isProxyAccess) {
const error = new Error(BROWSER_MODE_DISABLED_MESSAGE);
return throwError(() => error);
}
// When the dashboard first load or on dashboard panel edit mode
// PanelQueryRunner runs the queries to have a visualization on the panel.
// At that point datasource doesn't have the retention policies fetched.
// So hardcoded policy is being sent. Which causes problems.
// To overcome this we check/load policies first and then do the query.
return defer(() => this.getRetentionPolicies()).pipe(
mergeMap((allPolicies) => {
this.retentionPolicies = allPolicies;
const policyFixedRequests = {
...request,
targets: request.targets.map((t) => {
t.policy = t.policy ? this.templateSrv.replace(t.policy, {}, 'regex') : '';
return {
...t,
policy: replaceHardCodedRetentionPolicy(t.policy, this.retentionPolicies),
};
}),
};
return this._query(policyFixedRequests);
})
);
return this._query(request);
}
_query(request: DataQueryRequest<InfluxQuery>): Observable<DataQueryResponse> {
@ -463,7 +425,7 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
params.db = this.database;
}
if (options?.policy) {
if (options?.policy && options.policy !== DEFAULT_POLICY) {
params.rp = options.policy;
}

9
public/app/plugins/datasource/influxdb/influx_query_model.ts
View File

@ -20,6 +20,7 @@ export default class InfluxQueryModel {
this.templateSrv = templateSrv;
this.scopedVars = scopedVars;
target.policy = target.policy || 'default';
target.resultFormat = target.resultFormat || 'time_series';
target.orderByTime = target.orderByTime || 'ASC';
target.tags = target.tags || [];
@ -192,7 +193,13 @@ export default class InfluxQueryModel {
measurement = this.templateSrv.replace(measurement, this.scopedVars, 'regex');
}
return `"${policy}".${measurement}`;
if (policy !== 'default') {
policy = '"' + this.target.policy + '".';
} else {
policy = '';
}
return policy + measurement;
}
interpolateQueryStr(value: any[], variable: { multi: any; includeAll: any }, defaultFormatFn: any) {

3
public/app/plugins/datasource/influxdb/influxql_metadata_query.ts
View File

@ -3,7 +3,6 @@ import config from 'app/core/config';
import InfluxDatasource from './datasource';
import { buildMetadataQuery } from './influxql_query_builder';
import { replaceHardCodedRetentionPolicy } from './queryUtils';
import { InfluxQuery, InfluxQueryTag, MetadataQueryType } from './types';
type MetadataQueryOptions = {
@ -33,9 +32,9 @@ const runExploreQuery = async (options: MetadataQueryOptions): Promise<Array<{ t
const policy = retentionPolicy ? datasource.templateSrv.replace(retentionPolicy, {}, 'regex') : '';
const target: InfluxQuery = {
query,
policy,
rawQuery: true,
refId: 'metadataQuery',
policy: replaceHardCodedRetentionPolicy(policy, datasource.retentionPolicies),
};
if (config.featureToggles.influxdbBackendMigration) {
return datasource.runMetadataQuery(target);

24
public/app/plugins/datasource/influxdb/queryUtils.test.ts
View File

@ -1,12 +1,6 @@
import { cloneDeep } from 'lodash';
import {
buildRawQuery,
changeGroupByPart,
changeSelectPart,
normalizeQuery,
replaceHardCodedRetentionPolicy,
} from './queryUtils';
import { buildRawQuery, changeGroupByPart, changeSelectPart, normalizeQuery } from './queryUtils';
import { InfluxQuery } from './types';
describe('InfluxDB query utils', () => {
@ -43,9 +37,7 @@ describe('InfluxDB query utils', () => {
],
],
})
).toBe(
'SELECT mean("value") FROM "default"."measurement" WHERE $timeFilter GROUP BY time($__interval)' + ' fill(null)'
);
).toBe('SELECT mean("value") FROM "measurement" WHERE $timeFilter GROUP BY time($__interval)' + ' fill(null)');
});
it('should handle small query', () => {
expect(
@ -245,7 +237,7 @@ describe('InfluxDB query utils', () => {
})
).toBe(
`SELECT holt_winters_with_fit(mean("usage_idle"), 30, 5), median("usage_guest") ` +
`FROM "default"."cpu" ` +
`FROM "cpu" ` +
`WHERE ("cpu" = 'cpu2' OR "cpu" = 'cpu3' AND "cpu" = 'cpu1') ` +
`AND $timeFilter ` +
`GROUP BY time($__interval), "cpu", "host" fill(none) ` +
@ -441,14 +433,4 @@ describe('InfluxDB query utils', () => {
});
});
});
describe('replaceHardCodedRetentionPolicy', () => {
it('should replace non-existing hardcoded retention policy', () => {
const hardCodedRetentionPolicy = 'default';
const fetchedRetentionPolicies = ['foo', 'fighters', 'nirvana'];
expect(replaceHardCodedRetentionPolicy(hardCodedRetentionPolicy, fetchedRetentionPolicies)).toBe('foo');
expect(replaceHardCodedRetentionPolicy('foo', fetchedRetentionPolicies)).toBe('foo');
expect(replaceHardCodedRetentionPolicy(undefined, fetchedRetentionPolicies)).toBe('foo');
expect(replaceHardCodedRetentionPolicy(hardCodedRetentionPolicy, [])).toBe('');
});
});
});

16
public/app/plugins/datasource/influxdb/queryUtils.ts
View File

@ -90,19 +90,3 @@ export function changeGroupByPart(query: InfluxQuery, partIndex: number, newPara
};
return { ...query, groupBy: newGroupBy };
}
// Retention policy was hardcoded as `default` in
// public/app/plugins/datasource/influxdb/components/VisualInfluxQLEditor/FromSection.tsx
// We opted out hardcoded the policy in public/app/plugins/datasource/influxdb/influx_query_model.ts
// Which means if a user has a default retention policy named `default` they cannot use it.
// In https://github.com/grafana/grafana/pull/63820 we introduced a feature to use actual retention policies.
// But this did not consider that some users have hardcoded `default` retention policy in their dashboards.
// This function checks whether the given target has hardcoded retention policy not.
// If it is hardcoded it returns the actual default policy.
export function replaceHardCodedRetentionPolicy(policy: string | undefined, retentionPolicies: string[]): string {
if (!policy || !retentionPolicies.includes(policy)) {
return retentionPolicies[0] ?? '';
}
return policy;
}

15
public/app/plugins/datasource/influxdb/response_parser.ts
View File

@ -38,20 +38,7 @@ export default class ResponseParser {
// (while the newer versions—first).
if (isValueFirst) {
// We want to know whether the given retention policy is the default one or not.
// If it is default policy then we should add it to the beginning.
// The index 4 gives us if that policy is default or not.
// https://docs.influxdata.com/influxdb/v1.8/query_language/explore-schema/#show-retention-policies
// Only difference is v0.9. In that version we don't receive shardGroupDuration value.
// https://archive.docs.influxdata.com/influxdb/v0.9/query_language/schema_exploration/#show-retention-policies
// Since it is always the last value we will check that last value always.
if (isRetentionPolicyQuery && value[value.length - 1] === true) {
const newSetValues = [value[0].toString(), ...Array.from(res)];
res.clear();
newSetValues.forEach((sv) => res.add(sv));
} else {
res.add(value[0].toString());
}
res.add(value[0].toString());
} else if (value[1] !== undefined) {
res.add(value[1].toString());
} else {

2
public/app/plugins/datasource/influxdb/types.ts
View File

@ -1,5 +1,7 @@
import { AdHocVariableFilter, DataQuery, DataSourceJsonData } from '@grafana/data';
export const DEFAULT_POLICY = 'default';
export enum InfluxVersion {
InfluxQL = 'InfluxQL',
Flux = 'Flux',