IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Prometheus: Browser resource caching (#60711)

Browse Source 
Add cache control headers and range snapping to Prometheus resource API calls.
This commit is contained in:
Galen Kistler 2023-04-03 09:07:17 -05:00 committed by GitHub
parent 008bf143ac
commit 96e9e80739
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 1033 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
docs/sources/administration/provisioning/index.md
View File

@ -169,56 +169,57 @@ Common settings in the [built-in core data sources]({{< relref "../../datasource
> **Note:** Data sources tagged with _HTTP\*_ communicate using the HTTP protocol, which includes all core data source plugins except MySQL, PostgreSQL, and MSSQL.
| Name | Type | Data source | Description |
| -------------------------- | ------- | ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- |
| tlsAuth | boolean | _HTTP\*_, MySQL | Enable TLS authentication using client cert configured in secure json data |
| tlsAuthWithCACert | boolean | _HTTP\*_, MySQL, PostgreSQL | Enable TLS authentication using CA cert |
| tlsSkipVerify | boolean | _HTTP\*_, MySQL, PostgreSQL, MSSQL | Controls whether a client verifies the server's certificate chain and host name. |
| serverName | string | _HTTP\*_, MSSQL | Optional. Controls the server name used for certificate common name/subject alternative name verification. Defaults to using the data source URL. |
| timeout | string | _HTTP\*_ | Request timeout in seconds. Overrides dataproxy.timeout option |
| graphiteVersion | string | Graphite | Graphite version |
| timeInterval | string | Prometheus, Elasticsearch, InfluxDB, MySQL, PostgreSQL and MSSQL | Lowest interval/step value that should be used for this data source. |
| httpMode | string | Influxdb | HTTP Method. 'GET', 'POST', defaults to GET |
| maxSeries | number | Influxdb | Max number of series/tables that Grafana processes |
| httpMethod | string | Prometheus | HTTP Method. 'GET', 'POST', defaults to POST |
| customQueryParameters | string | Prometheus | Query parameters to add, as a URL-encoded string. |
| manageAlerts | boolean | Prometheus and Loki | Manage alerts via Alerting UI |
| alertmanagerUid | string | Prometheus and Loki | UID of Alert Manager that manages Alert for this data source. |
| timeField | string | Elasticsearch | Which field that should be used as timestamp |
| interval | string | Elasticsearch | Index date time format. nil(No Pattern), 'Hourly', 'Daily', 'Weekly', 'Monthly' or 'Yearly' |
| logMessageField | string | Elasticsearch | Which field should be used as the log message |
| logLevelField | string | Elasticsearch | Which field should be used to indicate the priority of the log message |
| maxConcurrentShardRequests | number | Elasticsearch | Maximum number of concurrent shard requests that each sub-search request executes per node |
| sigV4Auth | boolean | Elasticsearch and Prometheus | Enable usage of SigV4 |
| sigV4AuthType | string | Elasticsearch and Prometheus | SigV4 auth provider. default/credentials/keys |
| sigV4ExternalId | string | Elasticsearch and Prometheus | Optional SigV4 External ID |
| sigV4AssumeRoleArn | string | Elasticsearch and Prometheus | Optional SigV4 ARN role to assume |
| sigV4Region | string | Elasticsearch and Prometheus | SigV4 AWS region |
| sigV4Profile | string | Elasticsearch and Prometheus | Optional SigV4 credentials profile |
| authType | string | Cloudwatch | Auth provider. default/credentials/keys |
| externalId | string | Cloudwatch | Optional External ID |
| assumeRoleArn | string | Cloudwatch | Optional ARN role to assume |
| defaultRegion | string | Cloudwatch | Optional default AWS region |
| customMetricsNamespaces | string | Cloudwatch | Namespaces of Custom Metrics |
| profile | string | Cloudwatch | Optional credentials profile |
| tsdbVersion | string | OpenTSDB | Version |
| tsdbResolution | string | OpenTSDB | Resolution |
| sslmode | string | PostgreSQL | SSLmode. 'disable', 'require', 'verify-ca' or 'verify-full' |
| tlsConfigurationMethod | string | PostgreSQL | SSL Certificate configuration, either by 'file-path' or 'file-content' |
| sslRootCertFile | string | PostgreSQL, MSSQL | SSL server root certificate file, must be readable by the Grafana user |
| sslCertFile | string | PostgreSQL | SSL client certificate file, must be readable by the Grafana user |
| sslKeyFile | string | PostgreSQL | SSL client key file, must be readable by _only_ the Grafana user |
| encrypt | string | MSSQL | Connection SSL encryption handling. 'disable', 'false' or 'true' |
| postgresVersion | number | PostgreSQL | Postgres version as a number (903/904/905/906/1000) meaning v9.3, v9.4, ..., v10 |
| timescaledb | boolean | PostgreSQL | Enable usage of TimescaleDB extension |
| maxOpenConns | number | MySQL, PostgreSQL and MSSQL | Maximum number of open connections to the database (Grafana v5.4+) |
| maxIdleConns | number | MySQL, PostgreSQL and MSSQL | Maximum number of connections in the idle connection pool (Grafana v5.4+) |
| connMaxLifetime | number | MySQL, PostgreSQL and MSSQL | Maximum amount of time in seconds a connection may be reused (Grafana v5.4+) |
| keepCookies | array | _HTTP\*_ | Cookies that needs to be passed along while communicating with data sources |
| prometheusVersion | string | Prometheus | The version of the Prometheus data source, such as `2.37.0`, `2.24.0` |
| prometheusType | string | Prometheus | The type of the Prometheus data sources. such as `Prometheus`, `Cortex`, `Thanos`, `Mimir` |
| implementation | string | AlertManager | The implementation of the AlertManager data source, such as `prometheus`, `cortex` or `mimir` |
| handleGrafanaManagedAlerts | boolean | AlertManager | When enabled, Grafana-managed alerts are sent to this Alertmanager |
| Name | Type | Data source | Description |
| -------------------------- | ------- | ---------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| tlsAuth | boolean | _HTTP\*_, MySQL | Enable TLS authentication using client cert configured in secure json data |
| tlsAuthWithCACert | boolean | _HTTP\*_, MySQL, PostgreSQL | Enable TLS authentication using CA cert |
| tlsSkipVerify | boolean | _HTTP\*_, MySQL, PostgreSQL, MSSQL | Controls whether a client verifies the server's certificate chain and host name. |
| serverName | string | _HTTP\*_, MSSQL | Optional. Controls the server name used for certificate common name/subject alternative name verification. Defaults to using the data source URL. |
| timeout | string | _HTTP\*_ | Request timeout in seconds. Overrides dataproxy.timeout option |
| graphiteVersion | string | Graphite | Graphite version |
| timeInterval | string | Prometheus, Elasticsearch, InfluxDB, MySQL, PostgreSQL and MSSQL | Lowest interval/step value that should be used for this data source. |
| httpMode | string | Influxdb | HTTP Method. 'GET', 'POST', defaults to GET |
| maxSeries | number | Influxdb | Max number of series/tables that Grafana processes |
| httpMethod | string | Prometheus | HTTP Method. 'GET', 'POST', defaults to POST |
| customQueryParameters | string | Prometheus | Query parameters to add, as a URL-encoded string. |
| manageAlerts | boolean | Prometheus and Loki | Manage alerts via Alerting UI |
| alertmanagerUid | string | Prometheus and Loki | UID of Alert Manager that manages Alert for this data source. |
| timeField | string | Elasticsearch | Which field that should be used as timestamp |
| interval | string | Elasticsearch | Index date time format. nil(No Pattern), 'Hourly', 'Daily', 'Weekly', 'Monthly' or 'Yearly' |
| logMessageField | string | Elasticsearch | Which field should be used as the log message |
| logLevelField | string | Elasticsearch | Which field should be used to indicate the priority of the log message |
| maxConcurrentShardRequests | number | Elasticsearch | Maximum number of concurrent shard requests that each sub-search request executes per node |
| sigV4Auth | boolean | Elasticsearch and Prometheus | Enable usage of SigV4 |
| sigV4AuthType | string | Elasticsearch and Prometheus | SigV4 auth provider. default/credentials/keys |
| sigV4ExternalId | string | Elasticsearch and Prometheus | Optional SigV4 External ID |
| sigV4AssumeRoleArn | string | Elasticsearch and Prometheus | Optional SigV4 ARN role to assume |
| sigV4Region | string | Elasticsearch and Prometheus | SigV4 AWS region |
| sigV4Profile | string | Elasticsearch and Prometheus | Optional SigV4 credentials profile |
| authType | string | Cloudwatch | Auth provider. default/credentials/keys |
| externalId | string | Cloudwatch | Optional External ID |
| assumeRoleArn | string | Cloudwatch | Optional ARN role to assume |
| defaultRegion | string | Cloudwatch | Optional default AWS region |
| customMetricsNamespaces | string | Cloudwatch | Namespaces of Custom Metrics |
| profile | string | Cloudwatch | Optional credentials profile |
| tsdbVersion | string | OpenTSDB | Version |
| tsdbResolution | string | OpenTSDB | Resolution |
| sslmode | string | PostgreSQL | SSLmode. 'disable', 'require', 'verify-ca' or 'verify-full' |
| tlsConfigurationMethod | string | PostgreSQL | SSL Certificate configuration, either by 'file-path' or 'file-content' |
| sslRootCertFile | string | PostgreSQL, MSSQL | SSL server root certificate file, must be readable by the Grafana user |
| sslCertFile | string | PostgreSQL | SSL client certificate file, must be readable by the Grafana user |
| sslKeyFile | string | PostgreSQL | SSL client key file, must be readable by _only_ the Grafana user |
| encrypt | string | MSSQL | Connection SSL encryption handling. 'disable', 'false' or 'true' |
| postgresVersion | number | PostgreSQL | Postgres version as a number (903/904/905/906/1000) meaning v9.3, v9.4, ..., v10 |
| timescaledb | boolean | PostgreSQL | Enable usage of TimescaleDB extension |
| maxOpenConns | number | MySQL, PostgreSQL and MSSQL | Maximum number of open connections to the database (Grafana v5.4+) |
| maxIdleConns | number | MySQL, PostgreSQL and MSSQL | Maximum number of connections in the idle connection pool (Grafana v5.4+) |
| connMaxLifetime | number | MySQL, PostgreSQL and MSSQL | Maximum amount of time in seconds a connection may be reused (Grafana v5.4+) |
| keepCookies | array | _HTTP\*_ | Cookies that needs to be passed along while communicating with data sources |
| prometheusVersion | string | Prometheus | The version of the Prometheus data source, such as `2.37.0`, `2.24.0` |
| prometheusType | string | Prometheus | The type of the Prometheus data sources. such as `Prometheus`, `Cortex`, `Thanos`, `Mimir` |
| cacheLevel | string | Prometheus | This determines the duration of the browser cache. Valid values include: `Low`, `Medium`, `High`, and `None`. This field is configurable when you enable the `prometheusResourceBrowserCache` feature flag. |
| implementation | string | AlertManager | The implementation of the AlertManager data source, such as `prometheus`, `cortex` or `mimir` |
| handleGrafanaManagedAlerts | boolean | AlertManager | When enabled, Grafana-managed alerts are sent to this Alertmanager |
For examples of specific data sources' JSON data, refer to that [data source's documentation]({{< relref "../../datasources" >}}).

1
docs/sources/datasources/prometheus/_index.md
View File

@ -96,6 +96,7 @@ datasources:
manageAlerts: true
prometheusType: Prometheus
prometheusVersion: 2.37.0
cacheLevel: 'High'
exemplarTraceIdDestinations:
# Field with internal link pointing to data source in Grafana.
# datasourceUid value can be anything, but it should be unique across all defined data source uids.

1
docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md
View File

@ -96,6 +96,7 @@ Alpha features might be changed or removed without prior notice.
| `traceqlSearch` | Enables the 'TraceQL Search' tab for the Tempo datasource which provides a UI to generate TraceQL queries |
| `prometheusMetricEncyclopedia` | Replaces the Prometheus query builder metric select option with a paginated and filterable component |
| `timeSeriesTable` | Enable time series table transformer & sparkline cell type |
| `prometheusResourceBrowserCache` | Displays browser caching options in Prometheus data source configuration |
| `influxdbBackendMigration` | Query InfluxDB InfluxQL without the proxy |
| `clientTokenRotation` | Replaces the current in-request token rotation so that the client initiates the rotation |
| `prometheusDataplane` | Changes responses to from Prometheus to be compliant with the dataplane specification. In particular it sets the numeric Field.Name from 'Value' to the value of the `__name__` label when present. |

1
packages/grafana-data/src/types/featureToggles.gen.ts
View File

@ -83,6 +83,7 @@ export interface FeatureToggles {
traceqlSearch?: boolean;
prometheusMetricEncyclopedia?: boolean;
timeSeriesTable?: boolean;
prometheusResourceBrowserCache?: boolean;
influxdbBackendMigration?: boolean;
clientTokenRotation?: boolean;
disableElasticsearchBackendExploreQuery?: boolean;

7
pkg/services/featuremgmt/registry.go
View File

@ -438,6 +438,13 @@ var (
FrontendOnly: true,
Owner: appO11ySquad,
},
{
Name: "prometheusResourceBrowserCache",
Description: "Displays browser caching options in Prometheus data source configuration",
State: FeatureStateAlpha,
FrontendOnly: true,
Owner: grafanaObservabilityMetricsSquad,
},
{
Name: "influxdbBackendMigration",
Description: "Query InfluxDB InfluxQL without the proxy",

1
pkg/services/featuremgmt/toggles_gen.csv
View File

@ -64,6 +64,7 @@ drawerDataSourcePicker,alpha,@grafana/grafana-bi-squad,false,false,false,true
traceqlSearch,alpha,@grafana/observability-traces-and-profiling,false,false,false,true
prometheusMetricEncyclopedia,alpha,@grafana/observability-metrics,false,false,false,true
timeSeriesTable,alpha,@grafana/app-o11y,false,false,false,true
prometheusResourceBrowserCache,alpha,@grafana/observability-metrics,false,false,false,true
influxdbBackendMigration,alpha,@grafana/observability-metrics,false,false,false,true
clientTokenRotation,alpha,@grafana/grafana-authnz-team,false,false,false,false
disableElasticsearchBackendExploreQuery,beta,@grafana/observability-logs,false,false,false,false

1 Name State Owner requiresDevMode RequiresLicense RequiresRestart FrontendOnly
64 traceqlSearch alpha @grafana/observability-traces-and-profiling false false false true
65 prometheusMetricEncyclopedia alpha @grafana/observability-metrics false false false true
66 timeSeriesTable alpha @grafana/app-o11y false false false true
67 prometheusResourceBrowserCache alpha @grafana/observability-metrics false false false true
68 influxdbBackendMigration alpha @grafana/observability-metrics false false false true
69 clientTokenRotation alpha @grafana/grafana-authnz-team false false false false
70 disableElasticsearchBackendExploreQuery beta @grafana/observability-logs false false false false

4
pkg/services/featuremgmt/toggles_gen.go
View File

@ -267,6 +267,10 @@ const (
// Enable time series table transformer &amp; sparkline cell type
FlagTimeSeriesTable = "timeSeriesTable"
// FlagPrometheusResourceBrowserCache
// Displays browser caching options in Prometheus data source configuration
FlagPrometheusResourceBrowserCache = "prometheusResourceBrowserCache"
// FlagInfluxdbBackendMigration
// Query InfluxDB InfluxQL without the proxy
FlagInfluxdbBackendMigration = "influxdbBackendMigration"

7
pkg/tsdb/prometheus/resource/resource.go
View File

@ -7,7 +7,6 @@ import (
"net/http"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/tsdb/prometheus/client"
"github.com/grafana/grafana/pkg/tsdb/prometheus/utils"
@ -47,6 +46,12 @@ func (r *Resource) Execute(ctx context.Context, req *backend.CallResourceRequest
return nil, fmt.Errorf("error querying resource: %v", err)
}
// frontend sets the X-Grafana-Cache with the desired response cache control value
if len(req.GetHTTPHeaders().Get("X-Grafana-Cache")) > 0 {
resp.Header.Set("X-Grafana-Cache", "y")
resp.Header.Set("Cache-Control", req.GetHTTPHeaders().Get("X-Grafana-Cache"))
}
defer func() {
tmpErr := resp.Body.Close()
if tmpErr != nil && err == nil {

27
public/app/plugins/datasource/prometheus/components/PromLink.test.tsx
View File

@ -1,7 +1,7 @@
import { render, screen } from '@testing-library/react';
import React from 'react';
import { PanelData } from '@grafana/data';
import { dateTime, PanelData, TimeRange } from '@grafana/data';
import { PrometheusDatasource } from '../datasource';
import { PromQuery } from '../types';
@ -15,6 +15,10 @@ jest.mock('@grafana/data', () => ({
},
}));
const now = dateTime().valueOf();
const intervalInSeconds = 60 * 5;
const endInput = encodeURIComponent(dateTime(now).add(5, 'hours').format('Y-MM-DD HH:mm'));
const getPanelData = (panelDataOverrides?: Partial<PanelData>) => {
const panelData = {
request: {
@ -24,12 +28,10 @@ const getPanelData = (panelDataOverrides?: Partial<PanelData>) => {
{ refId: 'B', datasource: 'prom2' },
],
range: {
to: {
utc: () => ({
format: jest.fn(),
}),
},
},
raw: {},
to: dateTime(now), // "now"
from: dateTime(now - 1000 * intervalInSeconds), // 5 minutes ago from "now"
} as TimeRange,
},
};
@ -38,7 +40,6 @@ const getPanelData = (panelDataOverrides?: Partial<PanelData>) => {
const getDataSource = (datasourceOverrides?: Partial<PrometheusDatasource>) => {
const datasource = {
getPrometheusTime: () => 123,
createQuery: () => ({ expr: 'up', step: 15 }),
directUrl: 'prom1',
getRateIntervalScopedVariable: jest.fn(() => ({ __rate_interval: { text: '60s', value: '60s' } })),
@ -49,7 +50,7 @@ const getDataSource = (datasourceOverrides?: Partial<PrometheusDatasource>) => {
const getDataSourceWithCustomQueryParameters = (datasourceOverrides?: Partial<PrometheusDatasource>) => {
const datasource = {
getPrometheusTime: () => 124,
getPrometheusTime: () => 1677870470,
createQuery: () => ({ expr: 'up', step: 20 }),
directUrl: 'prom3',
getRateIntervalScopedVariable: jest.fn(() => ({ __rate_interval: { text: '60s', value: '60s' } })),
@ -68,7 +69,7 @@ describe('PromLink', () => {
);
expect(screen.getByText('Prometheus')).toHaveAttribute(
'href',
'prom1/graph?g0.expr=up&g0.range_input=0s&g0.end_input=undefined&g0.step_input=15&g0.tab=0'
`prom1/graph?g0.expr=up&g0.range_input=${intervalInSeconds}s&g0.end_input=${endInput}&g0.step_input=15&g0.tab=0`
);
});
it('should show different link when there are 2 components with the same panel data', () => {
@ -85,11 +86,11 @@ describe('PromLink', () => {
const promLinkButtons = screen.getAllByText('Prometheus');
expect(promLinkButtons[0]).toHaveAttribute(
'href',
'prom1/graph?g0.expr=up&g0.range_input=0s&g0.end_input=undefined&g0.step_input=15&g0.tab=0'
`prom1/graph?g0.expr=up&g0.range_input=${intervalInSeconds}s&g0.end_input=${endInput}&g0.step_input=15&g0.tab=0`
);
expect(promLinkButtons[1]).toHaveAttribute(
'href',
'prom2/graph?g0.expr=up&g0.range_input=0s&g0.end_input=undefined&g0.step_input=15&g0.tab=0'
`prom2/graph?g0.expr=up&g0.range_input=${intervalInSeconds}s&g0.end_input=${endInput}&g0.step_input=15&g0.tab=0`
);
});
it('should create sanitized link', async () => {
@ -116,7 +117,7 @@ describe('PromLink', () => {
);
expect(screen.getByText('Prometheus')).toHaveAttribute(
'href',
'prom3/graph?g0.foo=1&g0.expr=up&g0.range_input=0s&g0.end_input=undefined&g0.step_input=20&g0.tab=0'
`prom3/graph?g0.foo=1&g0.expr=up&g0.range_input=${intervalInSeconds}s&g0.end_input=${endInput}&g0.step_input=20&g0.tab=0`
);
});
});

5
public/app/plugins/datasource/prometheus/components/PromLink.tsx
View File

@ -4,6 +4,7 @@ import React, { useEffect, useState, memo } from 'react';
import { DataQueryRequest, PanelData, ScopedVars, textUtil, rangeUtil } from '@grafana/data';
import { PrometheusDatasource } from '../datasource';
import { getPrometheusTime } from '../language_utils';
import { PromQuery } from '../types';
interface Props {
@ -26,8 +27,8 @@ const PromLink = ({ panelData, query, datasource }: Props) => {
request: { range, interval, scopedVars },
} = panelData;
const start = datasource.getPrometheusTime(range.from, false);
const end = datasource.getPrometheusTime(range.to, true);
const start = getPrometheusTime(range.from, false);
const end = getPrometheusTime(range.to, true);
const rangeDiff = Math.ceil(end - start);
const endTime = range.to.utc().format('YYYY-MM-DD HH:mm');

1
public/app/plugins/datasource/prometheus/components/PromQueryEditorByApp.test.tsx
View File

@ -28,6 +28,7 @@ function setup(app: CoreApp): { onRunQuery: jest.Mock } {
getInitHints: () => [],
getPrometheusTime: jest.fn((date, roundup) => 123),
getQueryHints: jest.fn(() => []),
getDebounceTimeInMilliseconds: jest.fn(() => 300),
languageProvider: {
start: () => Promise.resolve([]),
syntax: () => {},

3
public/app/plugins/datasource/prometheus/components/monaco-query-field/MonacoQueryField.tsx
View File

@ -165,6 +165,7 @@ const MonacoQueryField = (props: Props) => {
const getSeriesValues = lpRef.current.getSeriesValues;
const getSeriesLabels = lpRef.current.getSeriesLabels;
const dataProvider = {
getHistory,
getAllMetricNames,
@ -230,7 +231,7 @@ const MonacoQueryField = (props: Props) => {
const updateCurrentEditorValue = debounce(() => {
const editorValue = editor.getValue();
onChangeRef.current(editorValue);
}, 300);
}, lpRef.current.datasource.getDebounceTimeInMilliseconds());
editor.getModel()?.onDidChangeContent(() => {
updateCurrentEditorValue();

31
public/app/plugins/datasource/prometheus/configuration/PromSettings.tsx
View File

@ -19,10 +19,11 @@ import {
Select,
} from '@grafana/ui';
import config from '../../../../core/config';
import { useUpdateDatasource } from '../../../../features/datasources/state';
import { PromApplication, PromBuildInfoResponse } from '../../../../types/unified-alerting-dto';
import { QueryEditorMode } from '../querybuilder/shared/types';
import { PromOptions } from '../types';
import { PrometheusCacheLevel, PromOptions } from '../types';
import { ExemplarsSettings } from './ExemplarsSettings';
import { PromFlavorVersions } from './PromFlavorVersions';
@ -39,6 +40,13 @@ const editorOptions = [
{ value: QueryEditorMode.Code, label: 'Code' },
];
const cacheValueOptions = [
{ value: PrometheusCacheLevel.Low, label: 'Low' },
{ value: PrometheusCacheLevel.Medium, label: 'Medium' },
{ value: PrometheusCacheLevel.High, label: 'High' },
{ value: PrometheusCacheLevel.None, label: 'None' },
];
type PrometheusSelectItemsType = Array<{ value: PromApplication; label: PromApplication }>;
const prometheusFlavorSelectItems: PrometheusSelectItemsType = [
@ -301,7 +309,7 @@ export const PromSettings = (props: Props) => {
</div>
<div className="gf-form">
<FormField
label="Default Editor"
label="Default editor"
labelWidth={14}
inputEl={
<Select
@ -335,6 +343,25 @@ export const PromSettings = (props: Props) => {
/>
</div>
</div>
{config.featureToggles.prometheusResourceBrowserCache && (
<div className="gf-form-inline">
<div className="gf-form max-width-30">
<FormField
label="Cache level"
labelWidth={14}
tooltip="Sets the browser caching level for editor queries. Higher cache settings are recommended for high cardinality data sources."
inputEl={
<Select
className={`width-25`}
onChange={onChangeHandler('cacheLevel', options, onOptionsChange)}
options={cacheValueOptions}
value={cacheValueOptions.find((o) => o.value === options.jsonData.cacheLevel)}
/>
}
/>
</div>
</div>
)}
</div>
<ExemplarsSettings
options={options.jsonData.exemplarTraceIdDestinations}

141
public/app/plugins/datasource/prometheus/datasource.test.ts
View File

@ -15,6 +15,7 @@ import {
LoadingState,
toDataFrame,
} from '@grafana/data';
import { config } from '@grafana/runtime';
import { TimeSrv } from 'app/features/dashboard/services/TimeSrv';
import { TemplateSrv } from 'app/features/templating/template_srv';
import { QueryOptions } from 'app/types';
@ -29,7 +30,7 @@ import {
prometheusSpecialRegexEscape,
} from './datasource';
import PromQlLanguageProvider from './language_provider';
import { PromOptions, PromQuery, PromQueryRequest } from './types';
import { PrometheusCacheLevel, PromOptions, PromQuery, PromQueryRequest } from './types';
const fetchMock = jest.fn().mockReturnValue(of(createDefaultPromResponse()));
@ -49,14 +50,26 @@ const templateSrvStub = {
replace: replaceMock,
} as unknown as TemplateSrv;
const timeSrvStub = {
const fromSeconds = 1674500289215;
const toSeconds = 1674500349215;
const timeSrvStubOld = {
timeRange() {
return {
from: dateTime(1531468681),
to: dateTime(1531489712),
};
},
} as unknown as TimeSrv;
} as TimeSrv;
const timeSrvStub: TimeSrv = {
timeRange() {
return {
from: dateTime(fromSeconds),
to: dateTime(toSeconds),
};
},
} as TimeSrv;
beforeEach(() => {
jest.clearAllMocks();
@ -73,7 +86,8 @@ describe('PrometheusDatasource', () => {
password: 'mupp',
jsonData: {
customQueryParameters: '',
},
cacheLevel: PrometheusCacheLevel.Low,
} as Partial<PromOptions>,
} as unknown as DataSourceInstanceSettings<PromOptions>;
beforeEach(() => {
@ -421,6 +435,106 @@ describe('PrometheusDatasource', () => {
});
});
// Remove when prometheusResourceBrowserCache is removed
describe('When prometheusResourceBrowserCache feature flag is off, there should be no change to the query intervals ', () => {
beforeEach(() => {
config.featureToggles.prometheusResourceBrowserCache = false;
});
it('test default 1 minute quantization', () => {
const dataSource = new PrometheusDatasource(
{
...instanceSettings,
jsonData: { ...instanceSettings.jsonData, cacheLevel: PrometheusCacheLevel.Low },
},
templateSrvStub as unknown as TemplateSrv,
timeSrvStub as unknown as TimeSrv
);
const quantizedRange = dataSource.getAdjustedInterval();
const oldRange = dataSource.getTimeRangeParams();
// For "1 minute" the window is unchanged
expect(parseInt(quantizedRange.end, 10) - parseInt(quantizedRange.start, 10)).toBe(60);
expect(parseInt(oldRange.end, 10) - parseInt(oldRange.start, 10)).toBe(60);
});
it('test 10 minute quantization', () => {
const dataSource = new PrometheusDatasource(
{
...instanceSettings,
jsonData: { ...instanceSettings.jsonData, cacheLevel: PrometheusCacheLevel.Medium },
},
templateSrvStub as unknown as TemplateSrv,
timeSrvStub as unknown as TimeSrv
);
const quantizedRange = dataSource.getAdjustedInterval();
const oldRange = dataSource.getTimeRangeParams();
expect(parseInt(quantizedRange.end, 10) - parseInt(quantizedRange.start, 10)).toBe(60);
expect(parseInt(oldRange.end, 10) - parseInt(oldRange.start, 10)).toBe(60);
});
});
describe('Test query range snapping', () => {
beforeEach(() => {
config.featureToggles.prometheusResourceBrowserCache = true;
});
it('test default 1 minute quantization', () => {
const dataSource = new PrometheusDatasource(
{
...instanceSettings,
jsonData: { ...instanceSettings.jsonData, cacheLevel: PrometheusCacheLevel.Low },
},
templateSrvStub as unknown as TemplateSrv,
timeSrvStub as unknown as TimeSrv
);
const quantizedRange = dataSource.getAdjustedInterval();
// For "1 minute" the window contains all the minutes, so a query from 1:11:09 - 1:12:09 becomes 1:11 - 1:13
expect(parseInt(quantizedRange.end, 10) - parseInt(quantizedRange.start, 10)).toBe(120);
});
it('test 10 minute quantization', () => {
const dataSource = new PrometheusDatasource(
{
...instanceSettings,
jsonData: { ...instanceSettings.jsonData, cacheLevel: PrometheusCacheLevel.Medium },
},
templateSrvStub as unknown as TemplateSrv,
timeSrvStub as unknown as TimeSrv
);
const quantizedRange = dataSource.getAdjustedInterval();
expect(parseInt(quantizedRange.end, 10) - parseInt(quantizedRange.start, 10)).toBe(600);
});
it('test 60 minute quantization', () => {
const dataSource = new PrometheusDatasource(
{
...instanceSettings,
jsonData: { ...instanceSettings.jsonData, cacheLevel: PrometheusCacheLevel.High },
},
templateSrvStub as unknown as TemplateSrv,
timeSrvStub as unknown as TimeSrv
);
const quantizedRange = dataSource.getAdjustedInterval();
expect(parseInt(quantizedRange.end, 10) - parseInt(quantizedRange.start, 10)).toBe(3600);
});
it('test quantization turned off', () => {
const dataSource = new PrometheusDatasource(
{
...instanceSettings,
jsonData: { ...instanceSettings.jsonData, cacheLevel: PrometheusCacheLevel.None },
},
templateSrvStub as unknown as TemplateSrv,
timeSrvStub as unknown as TimeSrv
);
const quantizedRange = dataSource.getAdjustedInterval();
expect(parseInt(quantizedRange.end, 10) - parseInt(quantizedRange.start, 10)).toBe(
(toSeconds - fromSeconds) / 1000
);
});
});
describe('alignRange', () => {
it('does not modify already aligned intervals with perfect step', () => {
const range = alignRange(0, 3, 3, 0);
@ -716,8 +830,13 @@ describe('PrometheusDatasource', () => {
describe('metricFindQuery', () => {
beforeEach(() => {
const prometheusDatasource = new PrometheusDatasource(
{ ...instanceSettings, jsonData: { ...instanceSettings.jsonData, cacheLevel: PrometheusCacheLevel.None } },
templateSrvStub,
timeSrvStubOld
);
const query = 'query_result(topk(5,rate(http_request_duration_microseconds_count[$__interval])))';
ds.metricFindQuery(query);
prometheusDatasource.metricFindQuery(query);
});
it('should call templateSrv.replace with scopedVars', () => {
@ -756,7 +875,7 @@ describe('PrometheusDatasource2', () => {
directUrl: 'direct',
user: 'test',
password: 'mupp',
jsonData: { httpMethod: 'GET' },
jsonData: { httpMethod: 'GET', cacheLevel: PrometheusCacheLevel.None },
} as unknown as DataSourceInstanceSettings<PromOptions>;
let ds: PrometheusDatasource;
@ -1398,7 +1517,13 @@ describe('PrometheusDatasource2', () => {
const step = 55;
const adjusted = alignRange(start, end, step, timeSrvStub.timeRange().to.utcOffset() * 60);
const urlExpected =
'proxied/api/v1/query_range?query=test' + '&start=' + adjusted.start + '&end=' + adjusted.end + '&step=' + step;
'proxied/api/v1/query_range?query=test' +
'&start=' +
adjusted.start +
'&end=' +
(adjusted.end + step) +
'&step=' +
step;
fetchMock.mockImplementation(() => of(response));
ds.query(query);
const res = fetchMock.mock.calls[0][0];
@ -1651,7 +1776,7 @@ describe('PrometheusDatasource2', () => {
'&start=' +
adjusted.start +
'&end=' +
adjusted.end +
(adjusted.end + step) +
'&step=' +
step;
fetchMock.mockImplementation(() => of(response));

75
public/app/plugins/datasource/prometheus/datasource.tsx
View File

@ -17,8 +17,6 @@ import {
DataSourceInstanceSettings,
DataSourceWithQueryExportSupport,
DataSourceWithQueryImportSupport,
dateMath,
DateTime,
dateTime,
LoadingState,
QueryFixAction,
@ -43,10 +41,17 @@ import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv';
import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv';
import { PromApiFeatures, PromApplication } from 'app/types/unified-alerting-dto';
import config from '../../../core/config';
import { addLabelToQuery } from './add_label_to_query';
import { AnnotationQueryEditor } from './components/AnnotationQueryEditor';
import PrometheusLanguageProvider from './language_provider';
import { expandRecordingRules } from './language_utils';
import {
expandRecordingRules,
getClientCacheDurationInMinutes,
getPrometheusTime,
getRangeSnapInterval,
} from './language_utils';
import { renderLegendFormat } from './legend';
import PrometheusMetricFindQuery from './metric_find_query';
import { getInitHints, getQueryHints } from './query_hints';
@ -57,6 +62,7 @@ import {
ExemplarTraceIdDestination,
PromDataErrorResponse,
PromDataSuccessResponse,
PrometheusCacheLevel,
PromExemplarData,
PromMatrixData,
PromOptions,
@ -98,6 +104,7 @@ export class PrometheusDatasource
exemplarsAvailable: boolean;
subType: PromApplication;
rulerEnabled: boolean;
cacheLevel: PrometheusCacheLevel;
constructor(
instanceSettings: DataSourceInstanceSettings<PromOptions>,
@ -131,6 +138,7 @@ export class PrometheusDatasource
this.defaultEditor = instanceSettings.jsonData.defaultEditor;
this.variables = new PrometheusVariableSupport(this, this.templateSrv, this.timeSrv);
this.exemplarsAvailable = true;
this.cacheLevel = instanceSettings.jsonData.cacheLevel ?? PrometheusCacheLevel.Low;
// This needs to be here and cannot be static because of how annotations typing affects casting of data source
// objects to DataSourceApi types.
@ -451,8 +459,8 @@ export class PrometheusDatasource
);
// Run queries trough browser/proxy
} else {
const start = this.getPrometheusTime(request.range.from, false);
const end = this.getPrometheusTime(request.range.to, true);
const start = getPrometheusTime(request.range.from, false);
const end = getPrometheusTime(request.range.to, true);
const { queries, activeTargets } = this.prepareTargets(request, start, end);
// No valid targets, return the empty result to save a round trip.
@ -794,8 +802,8 @@ export class PrometheusDatasource
method: 'POST',
headers: this.getRequestHeaders(),
data: {
from: (this.getPrometheusTime(options.range.from, false) * 1000).toString(),
to: (this.getPrometheusTime(options.range.to, true) * 1000).toString(),
from: (getPrometheusTime(options.range.from, false) * 1000).toString(),
to: (getPrometheusTime(options.range.to, true) * 1000).toString(),
queries: [this.applyTemplateVariables(queryModel, {})],
},
requestId: `prom-query-${annotation.name}`,
@ -1164,19 +1172,32 @@ export class PrometheusDatasource
return { ...query, expr: expression };
}
getPrometheusTime(date: string | DateTime, roundUp: boolean) {
if (typeof date === 'string') {
date = dateMath.parse(date, roundUp)!;
/**
* Returns the adjusted "snapped" interval parameters
*/
getAdjustedInterval(): { start: string; end: string } {
if (!config.featureToggles.prometheusResourceBrowserCache) {
return this.getTimeRangeParams();
}
return Math.ceil(date.valueOf() / 1000);
const range = this.timeSrv.timeRange();
return getRangeSnapInterval(this.cacheLevel, range);
}
/**
* This will return a time range that always includes the users current time range,
* and then a little extra padding to round up/down to the nearest nth minute,
* defined by the result of the getCacheDurationInMinutes.
*
* For longer cache durations, and shorter query durations, the window we're calculating might be much bigger then the user's current window,
* resulting in us returning labels/values that might not be applicable for the given window, this is a necessary trade off if we want to cache larger durations
*
*/
getTimeRangeParams(): { start: string; end: string } {
const range = this.timeSrv.timeRange();
return {
start: this.getPrometheusTime(range.from, false).toString(),
end: this.getPrometheusTime(range.to, true).toString(),
start: getPrometheusTime(range.from, false).toString(),
end: getPrometheusTime(range.to, true).toString(),
};
}
@ -1232,6 +1253,32 @@ export class PrometheusDatasource
interpolateString(string: string) {
return this.templateSrv.replace(string, undefined, this.interpolateQueryExpr);
}
getDebounceTimeInMilliseconds(): number {
switch (this.cacheLevel) {
case PrometheusCacheLevel.Medium:
return 600;
case PrometheusCacheLevel.High:
return 1200;
default:
return 350;
}
}
getDaysToCacheMetadata(): number {
switch (this.cacheLevel) {
case PrometheusCacheLevel.Medium:
return 7;
case PrometheusCacheLevel.High:
return 30;
default:
return 1;
}
}
getCacheDurationInMinutes(): number {
return getClientCacheDurationInMinutes(this.cacheLevel);
}
}
/**

329
public/app/plugins/datasource/prometheus/language_provider.test.ts
View File

@ -1,24 +1,54 @@
import { Editor as SlateEditor } from 'slate';
import Plain from 'slate-plain-serializer';
import { AbstractLabelOperator, HistoryItem } from '@grafana/data';
import { AbstractLabelOperator, dateTime, HistoryItem, TimeRange } from '@grafana/data';
import { config } from '@grafana/runtime';
import { SearchFunctionType } from '@grafana/ui';
import { Label } from './components/monaco-query-field/monaco-completion-provider/situation';
import { PrometheusDatasource } from './datasource';
import LanguageProvider from './language_provider';
import { PromQuery } from './types';
import { getClientCacheDurationInMinutes, getPrometheusTime, getRangeSnapInterval } from './language_utils';
import { PrometheusCacheLevel, PromQuery } from './types';
const now = new Date().getTime();
const timeRangeDurationSeconds = 1;
const toPrometheusTime = getPrometheusTime(dateTime(now), false);
const fromPrometheusTime = getPrometheusTime(dateTime(now - timeRangeDurationSeconds * 1000), false);
const toPrometheusTimeString = toPrometheusTime.toString(10);
const fromPrometheusTimeString = fromPrometheusTime.toString(10);
const getTimeRangeParams = (override?: Partial<{ start: string; end: string }>): { start: string; end: string } => ({
start: fromPrometheusTimeString,
end: toPrometheusTimeString,
...override,
});
const getMockQuantizedTimeRangeParams = (override?: Partial<TimeRange>): TimeRange => ({
from: dateTime(fromPrometheusTime * 1000),
to: dateTime(toPrometheusTime * 1000),
raw: {
from: `now-${timeRangeDurationSeconds}s`,
to: 'now',
},
...override,
});
describe('Language completion provider', () => {
const datasource: PrometheusDatasource = {
const defaultDatasource: PrometheusDatasource = {
metadataRequest: () => ({ data: { data: [] } }),
getTimeRangeParams: () => ({ start: '0', end: '1' }),
getTimeRangeParams: getTimeRangeParams,
interpolateString: (string: string) => string,
hasLabelsMatchAPISupport: () => false,
getQuantizedTimeRangeParams: () =>
getRangeSnapInterval(PrometheusCacheLevel.None, getMockQuantizedTimeRangeParams()),
getDaysToCacheMetadata: () => 1,
getAdjustedInterval: () => getRangeSnapInterval(PrometheusCacheLevel.None, getMockQuantizedTimeRangeParams()),
cacheLevel: PrometheusCacheLevel.None,
} as unknown as PrometheusDatasource;
describe('cleanText', () => {
const cleanText = new LanguageProvider(datasource).cleanText;
const cleanText = new LanguageProvider(defaultDatasource).cleanText;
it('does not remove metric or label keys', () => {
expect(cleanText('foo')).toBe('foo');
expect(cleanText('foo_bar')).toBe('foo_bar');
@ -68,26 +98,14 @@ describe('Language completion provider', () => {
});
});
describe('getSeriesLabels', () => {
it('should call series endpoint', () => {
const languageProvider = new LanguageProvider({ ...datasource } as PrometheusDatasource);
const getSeriesLabels = languageProvider.getSeriesLabels;
const requestSpy = jest.spyOn(languageProvider, 'request');
const labelName = 'job';
const labelValue = 'grafana';
getSeriesLabels(`{${labelName}="${labelValue}"}`, [{ name: labelName, value: labelValue, op: '=' }] as Label[]);
expect(requestSpy).toHaveBeenCalled();
expect(requestSpy).toHaveBeenCalledWith('/api/v1/series', [], {
end: '1',
'match[]': '{job="grafana"}',
start: '0',
});
// @todo clean up prometheusResourceBrowserCache feature flag
describe('getSeriesLabelsDeprecatedLRU', () => {
beforeEach(() => {
config.featureToggles.prometheusResourceBrowserCache = false;
});
it('should call labels endpoint', () => {
const languageProvider = new LanguageProvider({
...datasource,
...defaultDatasource,
hasLabelsMatchAPISupport: () => true,
} as PrometheusDatasource);
const getSeriesLabels = languageProvider.getSeriesLabels;
@ -98,30 +116,145 @@ describe('Language completion provider', () => {
getSeriesLabels(`{${labelName}="${labelValue}"}`, [{ name: labelName, value: labelValue, op: '=' }] as Label[]);
expect(requestSpy).toHaveBeenCalled();
expect(requestSpy).toHaveBeenCalledWith(`/api/v1/labels`, [], {
end: '1',
end: toPrometheusTimeString,
'match[]': '{job="grafana"}',
start: '0',
start: fromPrometheusTimeString,
});
});
it('should call series endpoint', () => {
const languageProvider = new LanguageProvider({
...defaultDatasource,
getAdjustedInterval: () => getRangeSnapInterval(PrometheusCacheLevel.None, getMockQuantizedTimeRangeParams()),
} as PrometheusDatasource);
const getSeriesLabels = languageProvider.getSeriesLabels;
const requestSpy = jest.spyOn(languageProvider, 'request');
const labelName = 'job';
const labelValue = 'grafana';
getSeriesLabels(`{${labelName}="${labelValue}"}`, [{ name: labelName, value: labelValue, op: '=' }] as Label[]);
expect(requestSpy).toHaveBeenCalled();
expect(requestSpy).toHaveBeenCalledWith('/api/v1/series', [], {
end: toPrometheusTimeString,
'match[]': '{job="grafana"}',
start: fromPrometheusTimeString,
});
});
});
describe('getSeriesLabels', () => {
beforeEach(() => {
config.featureToggles.prometheusResourceBrowserCache = true;
});
it('should call labels endpoint', () => {
const languageProvider = new LanguageProvider({
...defaultDatasource,
hasLabelsMatchAPISupport: () => true,
} as PrometheusDatasource);
const getSeriesLabels = languageProvider.getSeriesLabels;
const requestSpy = jest.spyOn(languageProvider, 'request');
const labelName = 'job';
const labelValue = 'grafana';
getSeriesLabels(`{${labelName}="${labelValue}"}`, [{ name: labelName, value: labelValue, op: '=' }] as Label[]);
expect(requestSpy).toHaveBeenCalled();
expect(requestSpy).toHaveBeenCalledWith(
`/api/v1/labels`,
[],
{
end: toPrometheusTimeString,
'match[]': '{job="grafana"}',
start: fromPrometheusTimeString,
},
undefined
);
});
it('should call series endpoint', () => {
const languageProvider = new LanguageProvider({
...defaultDatasource,
getAdjustedInterval: () => getRangeSnapInterval(PrometheusCacheLevel.None, getMockQuantizedTimeRangeParams()),
} as PrometheusDatasource);
const getSeriesLabels = languageProvider.getSeriesLabels;
const requestSpy = jest.spyOn(languageProvider, 'request');
const labelName = 'job';
const labelValue = 'grafana';
getSeriesLabels(`{${labelName}="${labelValue}"}`, [{ name: labelName, value: labelValue, op: '=' }] as Label[]);
expect(requestSpy).toHaveBeenCalled();
expect(requestSpy).toHaveBeenCalledWith(
'/api/v1/series',
[],
{
end: toPrometheusTimeString,
'match[]': '{job="grafana"}',
start: fromPrometheusTimeString,
},
undefined
);
});
it('should call labels endpoint with quantized start', () => {
const timeSnapMinutes = getClientCacheDurationInMinutes(PrometheusCacheLevel.Low);
const languageProvider = new LanguageProvider({
...defaultDatasource,
hasLabelsMatchAPISupport: () => true,
cacheLevel: PrometheusCacheLevel.Low,
getAdjustedInterval: () => getRangeSnapInterval(PrometheusCacheLevel.Low, getMockQuantizedTimeRangeParams()),
getCacheDurationInMinutes: () => timeSnapMinutes,
} as PrometheusDatasource);
const getSeriesLabels = languageProvider.getSeriesLabels;
const requestSpy = jest.spyOn(languageProvider, 'request');
const labelName = 'job';
const labelValue = 'grafana';
getSeriesLabels(`{${labelName}="${labelValue}"}`, [{ name: labelName, value: labelValue, op: '=' }] as Label[]);
expect(requestSpy).toHaveBeenCalled();
expect(requestSpy).toHaveBeenCalledWith(
`/api/v1/labels`,
[],
{
end: (
dateTime(fromPrometheusTime * 1000)
.add(timeSnapMinutes, 'minute')
.startOf('minute')
.valueOf() / 1000
).toString(),
'match[]': '{job="grafana"}',
start: (
dateTime(toPrometheusTime * 1000)
.startOf('minute')
.valueOf() / 1000
).toString(),
},
{ headers: { 'X-Grafana-Cache': `private, max-age=${timeSnapMinutes * 60}` } }
);
});
});
describe('getSeriesValues', () => {
it('should call old series endpoint and should use match[] parameter', () => {
const languageProvider = new LanguageProvider(datasource);
const languageProvider = new LanguageProvider({
...defaultDatasource,
} as PrometheusDatasource);
const getSeriesValues = languageProvider.getSeriesValues;
const requestSpy = jest.spyOn(languageProvider, 'request');
getSeriesValues('job', '{job="grafana"}');
expect(requestSpy).toHaveBeenCalled();
expect(requestSpy).toHaveBeenCalledWith('/api/v1/series', [], {
end: '1',
'match[]': '{job="grafana"}',
start: '0',
});
expect(requestSpy).toHaveBeenCalledWith(
'/api/v1/series',
[],
{
end: toPrometheusTimeString,
'match[]': '{job="grafana"}',
start: fromPrometheusTimeString,
},
undefined
);
});
it('should call new series endpoint and should use match[] parameter', () => {
const languageProvider = new LanguageProvider({
...datasource,
...defaultDatasource,
hasLabelsMatchAPISupport: () => true,
} as PrometheusDatasource);
const getSeriesValues = languageProvider.getSeriesValues;
@ -130,17 +263,22 @@ describe('Language completion provider', () => {
const labelValue = 'grafana';
getSeriesValues(labelName, `{${labelName}="${labelValue}"}`);
expect(requestSpy).toHaveBeenCalled();
expect(requestSpy).toHaveBeenCalledWith(`/api/v1/label/${labelName}/values`, [], {
end: '1',
'match[]': `{${labelName}="${labelValue}"}`,
start: '0',
});
expect(requestSpy).toHaveBeenCalledWith(
`/api/v1/label/${labelName}/values`,
[],
{
end: toPrometheusTimeString,
'match[]': `{${labelName}="${labelValue}"}`,
start: fromPrometheusTimeString,
},
undefined
);
});
});
describe('fetchSeries', () => {
it('should use match[] parameter', () => {
const languageProvider = new LanguageProvider(datasource);
const languageProvider = new LanguageProvider(defaultDatasource);
const fetchSeries = languageProvider.fetchSeries;
const requestSpy = jest.spyOn(languageProvider, 'request');
fetchSeries('{job="grafana"}');
@ -148,7 +286,8 @@ describe('Language completion provider', () => {
expect(requestSpy).toHaveBeenCalledWith(
'/api/v1/series',
{},
{ end: '1', 'match[]': '{job="grafana"}', start: '0' }
{ end: toPrometheusTimeString, 'match[]': '{job="grafana"}', start: fromPrometheusTimeString },
undefined
);
});
});
@ -156,41 +295,51 @@ describe('Language completion provider', () => {
describe('fetchSeriesLabels', () => {
it('should interpolate variable in series', () => {
const languageProvider = new LanguageProvider({
...datasource,
...defaultDatasource,
interpolateString: (string: string) => string.replace(/\$/, 'interpolated-'),
} as PrometheusDatasource);
const fetchSeriesLabels = languageProvider.fetchSeriesLabels;
const requestSpy = jest.spyOn(languageProvider, 'request');
fetchSeriesLabels('$metric');
expect(requestSpy).toHaveBeenCalled();
expect(requestSpy).toHaveBeenCalledWith('/api/v1/series', [], {
end: '1',
'match[]': 'interpolated-metric',
start: '0',
});
expect(requestSpy).toHaveBeenCalledWith(
'/api/v1/series',
[],
{
end: toPrometheusTimeString,
'match[]': 'interpolated-metric',
start: fromPrometheusTimeString,
},
undefined
);
});
});
describe('fetchLabelValues', () => {
it('should interpolate variable in series', () => {
const languageProvider = new LanguageProvider({
...datasource,
...defaultDatasource,
interpolateString: (string: string) => string.replace(/\$/, 'interpolated-'),
} as PrometheusDatasource);
const fetchLabelValues = languageProvider.fetchLabelValues;
const requestSpy = jest.spyOn(languageProvider, 'request');
fetchLabelValues('$job');
expect(requestSpy).toHaveBeenCalled();
expect(requestSpy).toHaveBeenCalledWith('/api/v1/label/interpolated-job/values', [], {
end: '1',
start: '0',
});
expect(requestSpy).toHaveBeenCalledWith(
'/api/v1/label/interpolated-job/values',
[],
{
end: toPrometheusTimeString,
start: fromPrometheusTimeString,
},
undefined
);
});
});
describe('empty query suggestions', () => {
it('returns no suggestions on empty context', async () => {
const instance = new LanguageProvider(datasource);
const instance = new LanguageProvider(defaultDatasource);
const value = Plain.deserialize('');
const result = await instance.provideCompletionItems({ text: '', prefix: '', value, wrapperClasses: [] });
expect(result.context).toBeUndefined();
@ -198,7 +347,7 @@ describe('Language completion provider', () => {
});
it('returns no suggestions with metrics on empty context even when metrics were provided', async () => {
const instance = new LanguageProvider(datasource);
const instance = new LanguageProvider(defaultDatasource);
instance.metrics = ['foo', 'bar'];
const value = Plain.deserialize('');
const result = await instance.provideCompletionItems({ text: '', prefix: '', value, wrapperClasses: [] });
@ -207,7 +356,7 @@ describe('Language completion provider', () => {
});
it('returns history on empty context when history was provided', async () => {
const instance = new LanguageProvider(datasource);
const instance = new LanguageProvider(defaultDatasource);
const value = Plain.deserialize('');
const history: Array<HistoryItem<PromQuery>> = [
{
@ -236,7 +385,7 @@ describe('Language completion provider', () => {
describe('range suggestions', () => {
it('returns range suggestions in range context', async () => {
const instance = new LanguageProvider(datasource);
const instance = new LanguageProvider(defaultDatasource);
const value = Plain.deserialize('1');
const result = await instance.provideCompletionItems({
text: '1',
@ -266,7 +415,7 @@ describe('Language completion provider', () => {
describe('metric suggestions', () => {
it('returns history, metrics and function suggestions in an uknown context ', async () => {
const instance = new LanguageProvider(datasource);
const instance = new LanguageProvider(defaultDatasource);
instance.metrics = ['foo', 'bar'];
const history: Array<HistoryItem<PromQuery>> = [
{
@ -301,7 +450,7 @@ describe('Language completion provider', () => {
});
it('returns no suggestions directly after a binary operator', async () => {
const instance = new LanguageProvider(datasource);
const instance = new LanguageProvider(defaultDatasource);
instance.metrics = ['foo', 'bar'];
const value = Plain.deserialize('*');
const result = await instance.provideCompletionItems({ text: '*', prefix: '', value, wrapperClasses: [] });
@ -310,7 +459,7 @@ describe('Language completion provider', () => {
});
it('returns metric suggestions with prefix after a binary operator', async () => {
const instance = new LanguageProvider(datasource);
const instance = new LanguageProvider(defaultDatasource);
instance.metrics = ['foo', 'bar'];
const value = Plain.deserialize('foo + b');
const ed = new SlateEditor({ value });
@ -333,7 +482,7 @@ describe('Language completion provider', () => {
});
it('returns no suggestions at the beginning of a non-empty function', async () => {
const instance = new LanguageProvider(datasource);
const instance = new LanguageProvider(defaultDatasource);
const value = Plain.deserialize('sum(up)');
const ed = new SlateEditor({ value });
@ -351,7 +500,7 @@ describe('Language completion provider', () => {
describe('label suggestions', () => {
it('returns default label suggestions on label context and no metric', async () => {
const instance = new LanguageProvider(datasource);
const instance = new LanguageProvider(defaultDatasource);
const value = Plain.deserialize('{}');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(1).value;
@ -373,9 +522,8 @@ describe('Language completion provider', () => {
it('returns label suggestions on label context and metric', async () => {
const datasources: PrometheusDatasource = {
...defaultDatasource,
metadataRequest: () => ({ data: { data: [{ __name__: 'metric', bar: 'bazinga' }] } }),
getTimeRangeParams: () => ({ start: '0', end: '1' }),
interpolateString: (string: string) => string,
} as unknown as PrometheusDatasource;
const instance = new LanguageProvider(datasources);
const value = Plain.deserialize('metric{}');
@ -394,7 +542,8 @@ describe('Language completion provider', () => {
});
it('returns label suggestions on label context but leaves out labels that already exist', async () => {
const datasource: PrometheusDatasource = {
const testDatasource: PrometheusDatasource = {
...defaultDatasource,
metadataRequest: () => ({
data: {
data: [
@ -408,10 +557,8 @@ describe('Language completion provider', () => {
],
},
}),
getTimeRangeParams: () => ({ start: '0', end: '1' }),
interpolateString: (string: string) => string,
} as unknown as PrometheusDatasource;
const instance = new LanguageProvider(datasource);
const instance = new LanguageProvider(testDatasource);
const value = Plain.deserialize('{job1="foo",job2!="foo",job3=~"foo",__name__="metric",}');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(54).value;
@ -429,7 +576,7 @@ describe('Language completion provider', () => {
it('returns label value suggestions inside a label value context after a negated matching operator', async () => {
const instance = new LanguageProvider({
...datasource,
...defaultDatasource,
metadataRequest: () => {
return { data: { data: ['value1', 'value2'] } };
},
@ -456,7 +603,7 @@ describe('Language completion provider', () => {
it('returns a refresher on label context and unavailable metric', async () => {
jest.spyOn(console, 'warn').mockImplementation(() => {});
const instance = new LanguageProvider(datasource);
const instance = new LanguageProvider(defaultDatasource);
const value = Plain.deserialize('metric{}');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(7).value;
@ -473,7 +620,7 @@ describe('Language completion provider', () => {
it('returns label values on label context when given a metric and a label key', async () => {
const instance = new LanguageProvider({
...datasource,
...defaultDatasource,
metadataRequest: () => simpleMetricLabelsResponse,
} as unknown as PrometheusDatasource);
const value = Plain.deserialize('metric{bar=ba}');
@ -494,7 +641,7 @@ describe('Language completion provider', () => {
it('returns label suggestions on aggregation context and metric w/ selector', async () => {
const instance = new LanguageProvider({
...datasource,
...defaultDatasource,
metadataRequest: () => simpleMetricLabelsResponse,
} as unknown as PrometheusDatasource);
const value = Plain.deserialize('sum(metric{foo="xx"}) by ()');
@ -514,7 +661,7 @@ describe('Language completion provider', () => {
it('returns label suggestions on aggregation context and metric w/o selector', async () => {
const instance = new LanguageProvider({
...datasource,
...defaultDatasource,
metadataRequest: () => simpleMetricLabelsResponse,
} as unknown as PrometheusDatasource);
const value = Plain.deserialize('sum(metric) by ()');
@ -534,7 +681,7 @@ describe('Language completion provider', () => {
it('returns label suggestions inside a multi-line aggregation context', async () => {
const instance = new LanguageProvider({
...datasource,
...defaultDatasource,
metadataRequest: () => simpleMetricLabelsResponse,
} as unknown as PrometheusDatasource);
const value = Plain.deserialize('sum(\nmetric\n)\nby ()');
@ -560,7 +707,7 @@ describe('Language completion provider', () => {
it('returns label suggestions inside an aggregation context with a range vector', async () => {
const instance = new LanguageProvider({
...datasource,
...defaultDatasource,
metadataRequest: () => simpleMetricLabelsResponse,
} as unknown as PrometheusDatasource);
const value = Plain.deserialize('sum(rate(metric[1h])) by ()');
@ -584,7 +731,7 @@ describe('Language completion provider', () => {
it('returns label suggestions inside an aggregation context with a range vector and label', async () => {
const instance = new LanguageProvider({
...datasource,
...defaultDatasource,
metadataRequest: () => simpleMetricLabelsResponse,
} as unknown as PrometheusDatasource);
const value = Plain.deserialize('sum(rate(metric{label1="value"}[1h])) by ()');
@ -607,7 +754,7 @@ describe('Language completion provider', () => {
});
it('returns no suggestions inside an unclear aggregation context using alternate syntax', async () => {
const instance = new LanguageProvider(datasource);
const instance = new LanguageProvider(defaultDatasource);
const value = Plain.deserialize('sum by ()');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(8).value;
@ -623,7 +770,7 @@ describe('Language completion provider', () => {
it('returns label suggestions inside an aggregation context using alternate syntax', async () => {
const instance = new LanguageProvider({
...datasource,
...defaultDatasource,
metadataRequest: () => simpleMetricLabelsResponse,
} as unknown as PrometheusDatasource);
const value = Plain.deserialize('sum by () (metric)');
@ -646,15 +793,16 @@ describe('Language completion provider', () => {
});
it('does not re-fetch default labels', async () => {
const datasource: PrometheusDatasource = {
const testDatasource: PrometheusDatasource = {
...defaultDatasource,
metadataRequest: jest.fn(() => ({ data: { data: [] } })),
getTimeRangeParams: jest.fn(() => ({ start: '0', end: '1' })),
interpolateString: (string: string) => string,
getQuantizedTimeRangeParams: getMockQuantizedTimeRangeParams,
} as unknown as PrometheusDatasource;
const mockedMetadataRequest = jest.mocked(datasource.metadataRequest);
const mockedMetadataRequest = jest.mocked(testDatasource.metadataRequest);
const instance = new LanguageProvider(datasource);
const instance = new LanguageProvider(testDatasource);
const value = Plain.deserialize('{}');
const ed = new SlateEditor({ value });
const valueWithSelection = ed.moveForward(1).value;
@ -678,7 +826,6 @@ describe('Language completion provider', () => {
jest.spyOn(console, 'warn').mockImplementation(() => {});
const datasource: PrometheusDatasource = {
metadataRequest: jest.fn(() => ({ data: { data: ['foo', 'bar'] as string[] } })),
getTimeRangeParams: jest.fn(() => ({ start: '0', end: '1' })),
lookupsDisabled: true,
} as unknown as PrometheusDatasource;
const mockedMetadataRequest = jest.mocked(datasource.metadataRequest);
@ -702,10 +849,9 @@ describe('Language completion provider', () => {
});
it('issues metadata requests when lookup is not disabled', async () => {
const datasource: PrometheusDatasource = {
...defaultDatasource,
metadataRequest: jest.fn(() => ({ data: { data: ['foo', 'bar'] as string[] } })),
getTimeRangeParams: jest.fn(() => ({ start: '0', end: '1' })),
lookupsDisabled: false,
interpolateString: (string: string) => string,
} as unknown as PrometheusDatasource;
const mockedMetadataRequest = jest.mocked(datasource.metadataRequest);
const instance = new LanguageProvider(datasource);
@ -714,18 +860,35 @@ describe('Language completion provider', () => {
await instance.start();
expect(mockedMetadataRequest.mock.calls.length).toBeGreaterThan(0);
});
it('doesnt blow up if metadata or fetchLabels rejects', async () => {
jest.spyOn(console, 'error').mockImplementation();
const datasource: PrometheusDatasource = {
...defaultDatasource,
metadataRequest: jest.fn(() => Promise.reject('rejected')),
lookupsDisabled: false,
} as unknown as PrometheusDatasource;
const mockedMetadataRequest = jest.mocked(datasource.metadataRequest);
const instance = new LanguageProvider(datasource);
expect(mockedMetadataRequest.mock.calls.length).toBe(0);
const result = await instance.start();
expect(result[0]).toBeUndefined();
expect(result[1]).toEqual([]);
expect(mockedMetadataRequest.mock.calls.length).toBe(3);
});
});
describe('Query imports', () => {
it('returns empty queries', async () => {
const instance = new LanguageProvider(datasource);
const instance = new LanguageProvider(defaultDatasource);
const result = await instance.importFromAbstractQuery({ refId: 'bar', labelMatchers: [] });
expect(result).toEqual({ refId: 'bar', expr: '', range: true });
});
describe('exporting to abstract query', () => {
it('exports labels with metric name', async () => {
const instance = new LanguageProvider(datasource);
const instance = new LanguageProvider(defaultDatasource);
const abstractQuery = instance.exportToAbstractQuery({
refId: 'bar',
expr: 'metric_name{label1="value1", label2!="value2", label3=~"value3", label4!~"value4"}',

152
public/app/plugins/datasource/prometheus/language_provider.ts
View File

@ -11,7 +11,7 @@ import {
HistoryItem,
LanguageProvider,
} from '@grafana/data';
import { BackendSrvRequest } from '@grafana/runtime';
import { BackendSrvRequest, config } from '@grafana/runtime';
import { CompletionItem, CompletionItemGroup, SearchFunctionType, TypeaheadInput, TypeaheadOutput } from '@grafana/ui';
import { Label } from './components/monaco-query-field/monaco-completion-provider/situation';
@ -27,7 +27,7 @@ import {
toPromLikeQuery,
} from './language_utils';
import PromqlSyntax, { FUNCTIONS, RATE_RANGES } from './promql';
import { PromMetricsMetadata, PromQuery } from './types';
import { PrometheusCacheLevel, PromMetricsMetadata, PromQuery } from './types';
const DEFAULT_KEYS = ['job', 'instance'];
const EMPTY_SELECTOR = '{}';
@ -43,6 +43,14 @@ const setFunctionKind = (suggestion: CompletionItem): CompletionItem => {
return suggestion;
};
const buildCacheHeaders = (durationInSeconds: number) => {
return {
headers: {
'X-Grafana-Cache': `private, max-age=${durationInSeconds}`,
},
};
};
export function addHistoryMetadata(item: CompletionItem, history: any[]): CompletionItem {
const cutoffTs = Date.now() - HISTORY_COUNT_CUTOFF;
const historyForItem = history.filter((h) => h.ts > cutoffTs && h.query === item.label);
@ -97,6 +105,8 @@ const PREFIX_DELIMITER_REGEX =
interface AutocompleteContext {
history?: Array<HistoryItem<PromQuery>>;
}
const secondsInDay = 86400;
export default class PromQlLanguageProvider extends LanguageProvider {
histogramMetrics: string[];
timeRange?: { start: number; end: number };
@ -106,7 +116,6 @@ export default class PromQlLanguageProvider extends LanguageProvider {
datasource: PrometheusDatasource;
labelKeys: string[] = [];
declare labelFetchTs: number;
/**
* Cache for labels of series. This is bit simplistic in the sense that it just counts responses each as a 1 and does
* not account for different size of a response. If that is needed a `length` function can be added in the options.
@ -114,7 +123,6 @@ export default class PromQlLanguageProvider extends LanguageProvider {
*/
private labelsCache = new LRU<string, Record<string, string[]>>({ max: 10 });
private labelValuesCache = new LRU<string, string[]>({ max: 10 });
constructor(datasource: PrometheusDatasource, initialValues?: Partial<PromQlLanguageProvider>) {
super();
@ -126,6 +134,16 @@ export default class PromQlLanguageProvider extends LanguageProvider {
Object.assign(this, initialValues);
}
getDefaultCacheHeaders() {
// @todo clean up prometheusResourceBrowserCache feature flag
if (config.featureToggles.prometheusResourceBrowserCache) {
if (this.datasource.cacheLevel !== PrometheusCacheLevel.None) {
return buildCacheHeaders(this.datasource.getCacheDurationInMinutes() * 60);
}
}
return;
}
// Strip syntax chars so that typeahead suggestions can work on clean inputs
cleanText(s: string) {
const parts = s.split(PREFIX_DELIMITER_REGEX);
@ -153,17 +171,26 @@ export default class PromQlLanguageProvider extends LanguageProvider {
return [];
}
// TODO #33976: make those requests parallel
await this.fetchLabels();
this.metrics = (await this.fetchLabelValues('__name__')) || [];
await this.loadMetricsMetadata();
this.histogramMetrics = processHistogramMetrics(this.metrics).sort();
return [];
return Promise.all([this.loadMetricsMetadata(), this.fetchLabels()]);
};
async loadMetricsMetadata() {
// @todo clean up prometheusResourceBrowserCache feature flag
const headers = config.featureToggles.prometheusResourceBrowserCache
? buildCacheHeaders(this.datasource.getDaysToCacheMetadata() * secondsInDay)
: {};
this.metricsMetadata = fixSummariesMetadata(
await this.request('/api/v1/metadata', {}, {}, { showErrorAlert: false })
await this.request(
'/api/v1/metadata',
{},
{},
{
showErrorAlert: false,
...headers,
}
)
);
}
@ -488,13 +515,14 @@ export default class PromQlLanguageProvider extends LanguageProvider {
}
/**
* @todo cache
* @param key
*/
fetchLabelValues = async (key: string): Promise<string[]> => {
const params = this.datasource.getTimeRangeParams();
const url = `/api/v1/label/${this.datasource.interpolateString(key)}/values`;
return await this.request(url, [], params);
const params = this.datasource.getAdjustedInterval();
const interpolatedName = this.datasource.interpolateString(key);
const url = `/api/v1/label/${interpolatedName}/values`;
const value = await this.request(url, [], params, this.getDefaultCacheHeaders());
return value ?? [];
};
async getLabelValues(key: string): Promise<string[]> {
@ -506,10 +534,10 @@ export default class PromQlLanguageProvider extends LanguageProvider {
*/
async fetchLabels(): Promise<string[]> {
const url = '/api/v1/labels';
const params = this.datasource.getTimeRangeParams();
const params = this.datasource.getAdjustedInterval();
this.labelFetchTs = Date.now().valueOf();
const res = await this.request(url, [], params);
const res = await this.request(url, [], params, this.getDefaultCacheHeaders());
if (Array.isArray(res)) {
this.labelKeys = res.slice().sort();
}
@ -539,12 +567,41 @@ export default class PromQlLanguageProvider extends LanguageProvider {
*/
fetchSeriesValuesWithMatch = async (name: string, match?: string): Promise<string[]> => {
const interpolatedName = name ? this.datasource.interpolateString(name) : null;
const range = this.datasource.getTimeRangeParams();
const range = this.datasource.getAdjustedInterval();
const urlParams = {
...range,
...(match && { 'match[]': match }),
};
// @todo clean up prometheusResourceBrowserCache feature flag
if (!config.featureToggles.prometheusResourceBrowserCache) {
return await this.fetchSeriesValuesLRUCache(interpolatedName, range, name, urlParams);
}
const value = await this.request(
`/api/v1/label/${interpolatedName}/values`,
[],
urlParams,
this.getDefaultCacheHeaders()
);
return value ?? [];
};
/**
* @deprecated
* @todo clean up prometheusResourceBrowserCache feature flag
* @param interpolatedName
* @param range
* @param name
* @param urlParams
* @private
*/
private async fetchSeriesValuesLRUCache(
interpolatedName: string | null,
range: { start: string; end: string },
name: string,
urlParams: { start: string; end: string }
) {
const cacheParams = new URLSearchParams({
'match[]': interpolatedName ?? '',
start: roundSecToMin(parseInt(range.start, 10)).toString(),
@ -561,7 +618,7 @@ export default class PromQlLanguageProvider extends LanguageProvider {
}
}
return value ?? [];
};
}
/**
* Gets series labels
@ -597,12 +654,38 @@ export default class PromQlLanguageProvider extends LanguageProvider {
*/
fetchSeriesLabels = async (name: string, withName?: boolean): Promise<Record<string, string[]>> => {
const interpolatedName = this.datasource.interpolateString(name);
const range = this.datasource.getTimeRangeParams();
const range = this.datasource.getAdjustedInterval();
const urlParams = {
...range,
'match[]': interpolatedName,
};
const url = `/api/v1/series`;
if (!config.featureToggles.prometheusResourceBrowserCache) {
return await this.fetchSeriesLabelsLRUCache(interpolatedName, range, withName, url, urlParams);
}
const data = await this.request(url, [], urlParams, this.getDefaultCacheHeaders());
const { values } = processLabels(data, withName);
return values;
};
/**
* @deprecated
* @param interpolatedName
* @param range
* @param withName
* @param url
* @param urlParams
* @private
*/
private async fetchSeriesLabelsLRUCache(
interpolatedName: string,
range: { start: string; end: string },
withName: boolean | undefined,
url: string,
urlParams: { start: string; 'match[]': string; end: string }
) {
// Cache key is a bit different here. We add the `withName` param and also round up to a minute the intervals.
// The rounding may seem strange but makes relative intervals like now-1h less prone to need separate request every
// millisecond while still actually getting all the keys for the correct interval. This still can create problems
@ -623,7 +706,7 @@ export default class PromQlLanguageProvider extends LanguageProvider {
this.labelsCache.set(cacheKey, value);
}
return value;
};
}
/**
* Fetch labels for a series using /labels endpoint. This is cached by its args but also by the global timeRange currently selected as
@ -633,12 +716,37 @@ export default class PromQlLanguageProvider extends LanguageProvider {
*/
fetchSeriesLabelsMatch = async (name: string, withName?: boolean): Promise<Record<string, string[]>> => {
const interpolatedName = this.datasource.interpolateString(name);
const range = this.datasource.getTimeRangeParams();
const range = this.datasource.getAdjustedInterval();
const urlParams = {
...range,
'match[]': interpolatedName,
};
const url = `/api/v1/labels`;
if (!config.featureToggles.prometheusResourceBrowserCache) {
return await this.fetchSeriesLabelMatchLRUCache(interpolatedName, range, withName, url, urlParams);
}
const data: string[] = await this.request(url, [], urlParams, this.getDefaultCacheHeaders());
// Convert string array to Record<string , []>
return data.reduce((ac, a) => ({ ...ac, [a]: '' }), {});
};
/**
* @deprecated
* @param interpolatedName
* @param range
* @param withName
* @param url
* @param urlParams
* @private
*/
private async fetchSeriesLabelMatchLRUCache(
interpolatedName: string,
range: { start: string; end: string },
withName: boolean | undefined,
url: string,
urlParams: { start: string; 'match[]': string; end: string }
) {
// Cache key is a bit different here. We add the `withName` param and also round up to a minute the intervals.
// The rounding may seem strange but makes relative intervals like now-1h less prone to need separate request every
// millisecond while still actually getting all the keys for the correct interval. This still can create problems
@ -659,7 +767,7 @@ export default class PromQlLanguageProvider extends LanguageProvider {
this.labelsCache.set(cacheKey, value);
}
return value;
};
}
/**
* Fetch series for a selector. Use this for raw results. Use fetchSeriesLabels() to get labels.
@ -669,7 +777,7 @@ export default class PromQlLanguageProvider extends LanguageProvider {
const url = '/api/v1/series';
const range = this.datasource.getTimeRangeParams();
const params = { ...range, 'match[]': match };
return await this.request(url, {}, params);
return await this.request(url, {}, params, this.getDefaultCacheHeaders());
};
/**

235
public/app/plugins/datasource/prometheus/language_utils.test.ts
View File

@ -1,13 +1,18 @@
import { AbstractLabelOperator, AbstractQuery } from '@grafana/data';
import { Moment } from 'moment';
import { AbstractLabelOperator, AbstractQuery, DateTime, dateTime, TimeRange } from '@grafana/data';
import {
escapeLabelValueInExactSelector,
escapeLabelValueInRegexSelector,
expandRecordingRules,
fixSummariesMetadata,
getPrometheusTime,
getRangeSnapInterval,
parseSelector,
toPromLikeQuery,
} from './language_utils';
import { PrometheusCacheLevel } from './types';
describe('parseSelector()', () => {
let parsed;
@ -222,6 +227,234 @@ describe('escapeLabelValueInRegexSelector()', () => {
});
});
describe('getRangeSnapInterval', () => {
it('will not change input if set to no cache', () => {
const intervalSeconds = 10 * 60; // 10 minutes
const now = new Date().valueOf();
const expectedFrom = dateTime(now - intervalSeconds * 1000);
const expectedTo = dateTime(now);
const range: TimeRange = {
from: expectedFrom,
to: expectedTo,
} as TimeRange;
expect(getRangeSnapInterval(PrometheusCacheLevel.None, range)).toEqual({
start: getPrometheusTime(expectedFrom, false).toString(),
end: getPrometheusTime(expectedTo, true).toString(),
});
});
it('will snap range to closest minute', () => {
const queryDurationMinutes = 10;
const intervalSeconds = queryDurationMinutes * 60; // 10 minutes
const now = new Date().valueOf();
const nowPlusOneMinute = now + 1000 * 60;
const nowPlusTwoMinute = now + 1000 * 60 * 2;
const nowTime = dateTime(now) as Moment;
const expectedFrom = nowTime.clone().startOf('minute').subtract(queryDurationMinutes, 'minute');
const expectedTo = nowTime.clone().startOf('minute').add(1, 'minute');
const range: TimeRange = {
from: dateTime(now - intervalSeconds * 1000),
to: dateTime(now),
} as TimeRange;
const range2: TimeRange = {
from: dateTime(nowPlusOneMinute - intervalSeconds * 1000),
to: dateTime(nowPlusOneMinute),
raw: {
from: dateTime(nowPlusOneMinute - intervalSeconds * 1000),
to: dateTime(nowPlusOneMinute),
},
};
const range3: TimeRange = {
from: dateTime(nowPlusTwoMinute - intervalSeconds * 1000),
to: dateTime(nowPlusTwoMinute),
raw: {
from: dateTime(nowPlusTwoMinute - intervalSeconds * 1000),
to: dateTime(nowPlusTwoMinute),
},
};
const first = getRangeSnapInterval(PrometheusCacheLevel.Low, range);
const second = getRangeSnapInterval(PrometheusCacheLevel.Low, range2);
const third = getRangeSnapInterval(PrometheusCacheLevel.Low, range3);
expect(first).toEqual({
start: getPrometheusTime(expectedFrom as DateTime, false).toString(10),
end: getPrometheusTime(expectedTo as DateTime, false).toString(10),
});
expect(second).toEqual({
start: getPrometheusTime(expectedFrom.clone().add(1, 'minute') as DateTime, false).toString(10),
end: getPrometheusTime(expectedTo.clone().add(1, 'minute') as DateTime, false).toString(10),
});
expect(third).toEqual({
start: getPrometheusTime(expectedFrom.clone().add(2, 'minute') as DateTime, false).toString(10),
end: getPrometheusTime(expectedTo.clone().add(2, 'minute') as DateTime, false).toString(10),
});
});
it('will snap range to closest 10 minute', () => {
const queryDurationMinutes = 60;
const intervalSeconds = queryDurationMinutes * 60; // 10 minutes
const now = new Date().valueOf();
const nowPlusOneMinute = now + 1000 * 60;
const nowPlusTwoMinute = now + 1000 * 60 * 2;
const nowTime = dateTime(now) as Moment;
const nowTimePlusOne = dateTime(nowPlusOneMinute) as Moment;
const nowTimePlusTwo = dateTime(nowPlusTwoMinute) as Moment;
const calculateClosest10 = (date: Moment): Moment => {
const numberOfMinutes = Math.floor(date.minutes() / 10) * 10;
const numberOfHours = numberOfMinutes < 60 ? date.hours() : date.hours() + 1;
return date
.clone()
.minutes(numberOfMinutes % 60)
.hours(numberOfHours);
};
const expectedFromFirst = calculateClosest10(
nowTime.clone().startOf('minute').subtract(queryDurationMinutes, 'minute')
);
const expectedToFirst = calculateClosest10(nowTime.clone().startOf('minute').add(1, 'minute'));
const expectedFromSecond = calculateClosest10(
nowTimePlusOne.clone().startOf('minute').subtract(queryDurationMinutes, 'minute')
);
const expectedToSecond = calculateClosest10(nowTimePlusOne.clone().startOf('minute').add(1, 'minute'));
const expectedFromThird = calculateClosest10(
nowTimePlusTwo.clone().startOf('minute').subtract(queryDurationMinutes, 'minute')
);
const expectedToThird = calculateClosest10(nowTimePlusTwo.clone().startOf('minute').add(1, 'minute'));
const range: TimeRange = {
from: dateTime(now - intervalSeconds * 1000),
to: dateTime(now),
} as TimeRange;
const range2: TimeRange = {
from: dateTime(nowPlusOneMinute - intervalSeconds * 1000),
to: dateTime(nowPlusOneMinute),
raw: {
from: dateTime(nowPlusOneMinute - intervalSeconds * 1000),
to: dateTime(nowPlusOneMinute),
},
};
const range3: TimeRange = {
from: dateTime(nowPlusTwoMinute - intervalSeconds * 1000),
to: dateTime(nowPlusTwoMinute),
raw: {
from: dateTime(nowPlusTwoMinute - intervalSeconds * 1000),
to: dateTime(nowPlusTwoMinute),
},
};
const first = getRangeSnapInterval(PrometheusCacheLevel.Medium, range);
const second = getRangeSnapInterval(PrometheusCacheLevel.Medium, range2);
const third = getRangeSnapInterval(PrometheusCacheLevel.Medium, range3);
expect(first).toEqual({
start: getPrometheusTime(expectedFromFirst as DateTime, false).toString(10),
end: getPrometheusTime(expectedToFirst as DateTime, false).toString(10),
});
expect(second).toEqual({
start: getPrometheusTime(expectedFromSecond.clone() as DateTime, false).toString(10),
end: getPrometheusTime(expectedToSecond.clone() as DateTime, false).toString(10),
});
expect(third).toEqual({
start: getPrometheusTime(expectedFromThird.clone() as DateTime, false).toString(10),
end: getPrometheusTime(expectedToThird.clone() as DateTime, false).toString(10),
});
});
it('will snap range to closest 60 minute', () => {
const queryDurationMinutes = 120;
const intervalSeconds = queryDurationMinutes * 60;
const now = new Date().valueOf();
const nowPlusOneMinute = now + 1000 * 60;
const nowPlusTwoMinute = now + 1000 * 60 * 2;
const nowTime = dateTime(now) as Moment;
const nowTimePlusOne = dateTime(nowPlusOneMinute) as Moment;
const nowTimePlusTwo = dateTime(nowPlusTwoMinute) as Moment;
const calculateClosest60 = (date: Moment): Moment => {
const numberOfMinutes = Math.floor(date.minutes() / 60) * 60;
const numberOfHours = numberOfMinutes < 60 ? date.hours() : date.hours() + 1;
return date
.clone()
.minutes(numberOfMinutes % 60)
.hours(numberOfHours);
};
const expectedFromFirst = calculateClosest60(
nowTime.clone().startOf('minute').subtract(queryDurationMinutes, 'minute')
);
const expectedToFirst = calculateClosest60(nowTime.clone().startOf('minute').add(1, 'minute'));
const expectedFromSecond = calculateClosest60(
nowTimePlusOne.clone().startOf('minute').subtract(queryDurationMinutes, 'minute')
);
const expectedToSecond = calculateClosest60(nowTimePlusOne.clone().startOf('minute').add(1, 'minute'));
const expectedFromThird = calculateClosest60(
nowTimePlusTwo.clone().startOf('minute').subtract(queryDurationMinutes, 'minute')
);
const expectedToThird = calculateClosest60(nowTimePlusTwo.clone().startOf('minute').add(1, 'minute'));
const range: TimeRange = {
from: dateTime(now - intervalSeconds * 1000),
to: dateTime(now),
} as TimeRange;
const range2: TimeRange = {
from: dateTime(nowPlusOneMinute - intervalSeconds * 1000),
to: dateTime(nowPlusOneMinute),
raw: {
from: dateTime(nowPlusOneMinute - intervalSeconds * 1000),
to: dateTime(nowPlusOneMinute),
},
};
const range3: TimeRange = {
from: dateTime(nowPlusTwoMinute - intervalSeconds * 1000),
to: dateTime(nowPlusTwoMinute),
raw: {
from: dateTime(nowPlusTwoMinute - intervalSeconds * 1000),
to: dateTime(nowPlusTwoMinute),
},
};
const first = getRangeSnapInterval(PrometheusCacheLevel.High, range);
const second = getRangeSnapInterval(PrometheusCacheLevel.High, range2);
const third = getRangeSnapInterval(PrometheusCacheLevel.High, range3);
expect(first).toEqual({
start: getPrometheusTime(expectedFromFirst as DateTime, false).toString(10),
end: getPrometheusTime(expectedToFirst as DateTime, false).toString(10),
});
expect(second).toEqual({
start: getPrometheusTime(expectedFromSecond.clone() as DateTime, false).toString(10),
end: getPrometheusTime(expectedToSecond.clone() as DateTime, false).toString(10),
});
expect(third).toEqual({
start: getPrometheusTime(expectedFromThird.clone() as DateTime, false).toString(10),
end: getPrometheusTime(expectedToThird.clone() as DateTime, false).toString(10),
});
});
});
describe('toPromLikeQuery', () => {
it('export abstract query to PromQL-like query', () => {
const abstractQuery: AbstractQuery = {

76
public/app/plugins/datasource/prometheus/language_utils.ts
View File

@ -1,11 +1,20 @@
import { invert } from 'lodash';
import { Token } from 'prismjs';
import { DataQuery, AbstractQuery, AbstractLabelOperator, AbstractLabelMatcher } from '@grafana/data';
import {
AbstractLabelMatcher,
AbstractLabelOperator,
AbstractQuery,
DataQuery,
dateMath,
DateTime,
incrRoundDn,
TimeRange,
} from '@grafana/data';
import { addLabelToQuery } from './add_label_to_query';
import { SUGGESTIONS_LIMIT } from './language_provider';
import { PromMetricsMetadata, PromMetricsMetadataItem } from './types';
import { PrometheusCacheLevel, PromMetricsMetadata, PromMetricsMetadataItem } from './types';
export const processHistogramMetrics = (metrics: string[]) => {
const resultSet: Set<string> = new Set();
@ -55,6 +64,7 @@ export function processLabels(labels: Array<{ [key: string]: string }>, withName
// const cleanSelectorRegexp = /\{(\w+="[^"\n]*?")(,\w+="[^"\n]*?")*\}/;
export const selectorRegexp = /\{[^}]*?(\}|$)/;
export const labelRegexp = /\b(\w+)(!?=~?)("[^"\n]*?")/g;
export function parseSelector(query: string, cursorOffset = 1): { labelKeys: any[]; selector: string } {
if (!query.match(selectorRegexp)) {
// Special matcher for metrics
@ -231,6 +241,11 @@ export function roundSecToMin(seconds: number): number {
return Math.floor(seconds / 60);
}
// Returns number of minutes rounded up to the nearest nth minute
export function roundSecToNextMin(seconds: number, secondsToRound = 1): number {
return Math.ceil(seconds / 60) - (Math.ceil(seconds / 60) % secondsToRound);
}
export function limitSuggestions(items: string[]) {
return items.slice(0, SUGGESTIONS_LIMIT);
}
@ -248,6 +263,7 @@ export function addLimitInfo(items: any[] | undefined): string {
// the list of metacharacters is: *+?()|\.[]{}^$
// we make a javascript regular expression that matches those characters:
const RE2_METACHARACTERS = /[*+?()|\\.\[\]{}^$]/g;
function escapePrometheusRegexp(value: string): string {
return value.replace(RE2_METACHARACTERS, '\\$&');
}
@ -344,3 +360,59 @@ export function extractLabelMatchers(tokens: Array<string | Token>): AbstractLab
return labelMatchers;
}
/**
* Calculates new interval "snapped" to the closest Nth minute, depending on cache level datasource setting
* @param cacheLevel
* @param range
*/
export function getRangeSnapInterval(
cacheLevel: PrometheusCacheLevel,
range: TimeRange
): { start: string; end: string } {
// Don't round the range if we're not caching
if (cacheLevel === PrometheusCacheLevel.None) {
return {
start: getPrometheusTime(range.from, false).toString(),
end: getPrometheusTime(range.to, true).toString(),
};
}
// Otherwise round down to the nearest nth minute for the start time
const startTime = getPrometheusTime(range.from, false);
// const startTimeQuantizedSeconds = roundSecToLastMin(startTime, getClientCacheDurationInMinutes(cacheLevel)) * 60;
const startTimeQuantizedSeconds = incrRoundDn(startTime, getClientCacheDurationInMinutes(cacheLevel) * 60);
// And round up to the nearest nth minute for the end time
const endTime = getPrometheusTime(range.to, true);
const endTimeQuantizedSeconds = roundSecToNextMin(endTime, getClientCacheDurationInMinutes(cacheLevel)) * 60;
// If the interval was too short, we could have rounded both start and end to the same time, if so let's add one step to the end
if (startTimeQuantizedSeconds === endTimeQuantizedSeconds) {
const endTimePlusOneStep = endTimeQuantizedSeconds + getClientCacheDurationInMinutes(cacheLevel) * 60;
return { start: startTimeQuantizedSeconds.toString(), end: endTimePlusOneStep.toString() };
}
const start = startTimeQuantizedSeconds.toString();
const end = endTimeQuantizedSeconds.toString();
return { start, end };
}
export function getClientCacheDurationInMinutes(cacheLevel: PrometheusCacheLevel) {
switch (cacheLevel) {
case PrometheusCacheLevel.Medium:
return 10;
case PrometheusCacheLevel.High:
return 60;
default:
return 1;
}
}
export function getPrometheusTime(date: string | DateTime, roundUp: boolean) {
if (typeof date === 'string') {
date = dateMath.parse(date, roundUp)!;
}
return Math.ceil(date.valueOf() / 1000);
}

15
public/app/plugins/datasource/prometheus/metric_find_query.ts
View File

@ -6,6 +6,7 @@ import { MetricFindValue, TimeRange } from '@grafana/data';
import { getTimeSrv } from 'app/features/dashboard/services/TimeSrv';
import { PrometheusDatasource } from './datasource';
import { getPrometheusTime } from './language_utils';
import { PromQueryRequest } from './types';
export default class PrometheusMetricFindQuery {
@ -56,8 +57,8 @@ export default class PrometheusMetricFindQuery {
}
labelValuesQuery(label: string, metric?: string) {
const start = this.datasource.getPrometheusTime(this.range.from, false);
const end = this.datasource.getPrometheusTime(this.range.to, true);
const start = getPrometheusTime(this.range.from, false);
const end = getPrometheusTime(this.range.to, true);
const params = { ...(metric && { 'match[]': metric }), start: start.toString(), end: end.toString() };
if (!metric || this.datasource.hasLabelsMatchAPISupport()) {
@ -89,8 +90,8 @@ export default class PrometheusMetricFindQuery {
}
metricNameQuery(metricFilterPattern: string) {
const start = this.datasource.getPrometheusTime(this.range.from, false);
const end = this.datasource.getPrometheusTime(this.range.to, true);
const start = getPrometheusTime(this.range.from, false);
const end = getPrometheusTime(this.range.to, true);
const params = {
start: start.toString(),
end: end.toString(),
@ -114,7 +115,7 @@ export default class PrometheusMetricFindQuery {
}
queryResultQuery(query: string) {
const end = this.datasource.getPrometheusTime(this.range.to, true);
const end = getPrometheusTime(this.range.to, true);
const instantQuery: PromQueryRequest = { expr: query } as PromQueryRequest;
return this.datasource.performInstantQuery(instantQuery, end).pipe(
map((result) => {
@ -152,8 +153,8 @@ export default class PrometheusMetricFindQuery {
}
metricNameAndLabelsQuery(query: string): Promise<MetricFindValue[]> {
const start = this.datasource.getPrometheusTime(this.range.from, false);
const end = this.datasource.getPrometheusTime(this.range.to, true);
const start = getPrometheusTime(this.range.from, false);
const end = getPrometheusTime(this.range.to, true);
const params = {
'match[]': query,
start: start.toString(),

7
public/app/plugins/datasource/prometheus/querybuilder/components/LabelFilterItem.tsx
View File

@ -20,6 +20,7 @@ export interface Props {
invalidLabel?: boolean;
invalidValue?: boolean;
getLabelValuesAutofillSuggestions: (query: string, labelName?: string) => Promise<SelectableValue[]>;
debounceDuration: number;
}
export function LabelFilterItem({
@ -32,6 +33,7 @@ export function LabelFilterItem({
invalidLabel,
invalidValue,
getLabelValuesAutofillSuggestions,
debounceDuration,
}: Props) {
const [state, setState] = useState<{
labelNames?: SelectableValue[];
@ -54,7 +56,10 @@ export function LabelFilterItem({
return [];
};
const labelValueSearch = debounce((query: string) => getLabelValuesAutofillSuggestions(query, item.label), 350);
const labelValueSearch = debounce(
(query: string) => getLabelValuesAutofillSuggestions(query, item.label),
debounceDuration
);
return (
<div data-testid="prometheus-dimensions-filter-item">

7
public/app/plugins/datasource/prometheus/querybuilder/components/LabelFilters.test.tsx
View File

@ -5,7 +5,7 @@ import { selectOptionInTest } from 'test/helpers/selectOptionInTest';
import { getLabelSelects } from '../testUtils';
import { LabelFilters, MISSING_LABEL_FILTER_ERROR_MESSAGE } from './LabelFilters';
import { LabelFilters, MISSING_LABEL_FILTER_ERROR_MESSAGE, Props } from './LabelFilters';
describe('LabelFilters', () => {
it('renders empty input without labels', async () => {
@ -81,6 +81,7 @@ describe('LabelFilters', () => {
getLabelValuesAutofillSuggestions={jest.fn()}
onGetLabelValues={jest.fn()}
labelsFilters={[]}
debounceDuration={300}
/>
);
expect(screen.getAllByText('Select label')).toHaveLength(1);
@ -101,9 +102,8 @@ describe('LabelFilters', () => {
});
function setup(propOverrides?: Partial<ComponentProps<typeof LabelFilters>>) {
const defaultProps = {
const defaultProps: Props = {
onChange: jest.fn(),
getLabelValues: jest.fn(),
getLabelValuesAutofillSuggestions: async (query: string, labelName?: string) => [
{ label: 'bar', value: 'bar' },
{ label: 'qux', value: 'qux' },
@ -119,6 +119,7 @@ function setup(propOverrides?: Partial<ComponentProps<typeof LabelFilters>>) {
{ label: 'qux', value: 'qux' },
{ label: 'quux', value: 'quux' },
],
debounceDuration: 300,
labelsFilters: [],
};

3
public/app/plugins/datasource/prometheus/querybuilder/components/LabelFilters.tsx
View File

@ -18,6 +18,7 @@ export interface Props {
/** If set to true, component will show error message until at least 1 filter is selected */
labelFilterRequired?: boolean;
getLabelValuesAutofillSuggestions: (query: string, labelName?: string) => Promise<SelectableValue[]>;
debounceDuration: number;
}
export function LabelFilters({
@ -27,6 +28,7 @@ export function LabelFilters({
onGetLabelValues,
labelFilterRequired,
getLabelValuesAutofillSuggestions,
debounceDuration,
}: Props) {
const defaultOp = '=';
const [items, setItems] = useState<Array<Partial<QueryBuilderLabelFilter>>>([{ op: defaultOp }]);
@ -63,6 +65,7 @@ export function LabelFilters({
onChange={onLabelsChange}
renderItem={(item: Partial<QueryBuilderLabelFilter>, onChangeItem, onDelete) => (
<LabelFilterItem
debounceDuration={debounceDuration}
item={item}
defaultOp={defaultOp}
onChange={onChangeItem}

2
public/app/plugins/datasource/prometheus/querybuilder/components/MetricEncyclopediaModal.tsx
View File

@ -327,7 +327,7 @@ export const MetricEncyclopediaModal = (props: Props) => {
setMetrics(metrics);
setFilteredMetricCount(metrics.length);
setIsLoading(false);
}, 300),
}, datasource.getDebounceTimeInMilliseconds()),
[datasource, query.labels]
);

5
public/app/plugins/datasource/prometheus/querybuilder/components/MetricSelect.tsx
View File

@ -114,7 +114,10 @@ export function MetricSelect({ datasource, query, onChange, onGetMetrics, labels
});
};
const debouncedSearch = debounce((query: string) => getMetricLabels(query), 300);
const debouncedSearch = debounce(
(query: string) => getMetricLabels(query),
datasource.getDebounceTimeInMilliseconds()
);
return (
<EditorFieldGroup>

1
public/app/plugins/datasource/prometheus/querybuilder/components/PromQueryBuilder.tsx
View File

@ -255,6 +255,7 @@ export const PromQueryBuilder = React.memo<Props>((props) => {
/>
)}
<LabelFilters
debounceDuration={datasource.getDebounceTimeInMilliseconds()}
getLabelValuesAutofillSuggestions={getLabelValuesAutocompleteSuggestions}
labelsFilters={query.labels}
// eslint-ignore

7
public/app/plugins/datasource/prometheus/types.ts
View File

@ -20,6 +20,12 @@ export interface PromQuery extends GenPromQuery, DataQuery {
intervalFactor?: number;
}
export enum PrometheusCacheLevel {
Low = 'Low',
Medium = 'Medium',
High = 'High',
None = 'None',
}
export interface PromOptions extends DataSourceJsonData {
timeInterval?: string;
queryTimeout?: string;
@ -30,6 +36,7 @@ export interface PromOptions extends DataSourceJsonData {
exemplarTraceIdDestinations?: ExemplarTraceIdDestination[];
prometheusType?: PromApplication;
prometheusVersion?: string;
cacheLevel?: PrometheusCacheLevel;
defaultEditor?: QueryEditorMode;
}