Loki: Introduce $__auto range variable for metric queries (#72690)

* Loki: Add  interpolation to backend

* Loki: Replace default variable  with  in frontend

* Loki: Update docs in query builder fro __auto

* Loki: Update test for change default __auto

* Loki: Remove  and  from suggestions as  should be used

* Update docs

* Update pkg/tsdb/loki/parse_query.go

* Fix backend lint

* Fix lint and test

* Update

* Update docs/sources/datasources/loki/template-variables/index.md

Co-authored-by: lwandz13 <126723338+lwandz13@users.noreply.github.com>

* Update public/app/plugins/datasource/loki/querybuilder/operationUtils.ts

Co-authored-by: Matias Chomicki <matyax@gmail.com>

---------

Co-authored-by: lwandz13 <126723338+lwandz13@users.noreply.github.com>
Co-authored-by: Matias Chomicki <matyax@gmail.com>
This commit is contained in:
Ivana Huckova 2023-08-03 16:27:23 +02:00 committed by GitHub
parent b1fd399c10
commit 7bb0ff7055
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 103 additions and 61 deletions

View File

@ -47,11 +47,11 @@ You can use this variable type to specify any number of key/value filters, and G
For more information, refer to [Add ad hoc filters][add-template-variables-add-ad-hoc-filters].
## Use interval and range variables
## Use $\_\_auto variable for Loki metric queries
You can use some global built-in variables in query variables: `$__interval`, `$__interval_ms`, `$__range`, `$__range_s`, and `$__range_ms`.
Consider using the `$__auto` variable in your Loki metric queries, which will automatically be substituted with the [step value](https://grafana.com/docs/grafana/next/datasources/loki/query-editor/#options) for range queries, and with the selected time range's value (computed from the starting and ending times) for instant queries.
For more information, refer to [Global built-in variables][add-template-variables-global-variables].
For more information about variables, refer to [Global built-in variables][add-template-variables-global-variables].
## Label extraction and indexing in Loki

View File

@ -14,7 +14,7 @@ const addDataSource = () => {
});
};
const finalQuery = 'rate({instance=~"instance1|instance2"} | logfmt | __error__=`` [$__interval]';
const finalQuery = 'rate({instance=~"instance1|instance2"} | logfmt | __error__=`` [$__auto]';
describe('Loki query builder', () => {
beforeEach(() => {
@ -64,7 +64,7 @@ describe('Loki query builder', () => {
e2e().contains('Operations').should('be.visible').click();
e2e().contains('Range functions').should('be.visible').click();
e2e().contains('Rate').should('be.visible').click();
e2e().contains('rate({} | logfmt | __error__=`` [$__interval]').should('be.visible');
e2e().contains('rate({} | logfmt | __error__=`` [$__auto]').should('be.visible');
// Check for expected error
e2e().contains(MISSING_LABEL_FILTER_ERROR_MESSAGE).should('be.visible');
@ -90,7 +90,7 @@ describe('Loki query builder', () => {
e2e().contains('instance1|instance2').should('be.visible');
e2e().contains('logfmt').should('be.visible');
e2e().contains('__error__').should('be.visible');
e2e().contains('$__interval').should('be.visible');
e2e().contains('$__auto').should('be.visible');
// Checks the explain mode toggle
e2e().contains('label', 'Explain').click();

View File

@ -10,6 +10,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/tsdb/intervalv2"
"github.com/grafana/grafana/pkg/tsdb/loki/kinds/dataquery"
)
const (
@ -18,6 +19,7 @@ const (
varRange = "$__range"
varRangeS = "$__range_s"
varRangeMs = "$__range_ms"
varAuto = "$__auto"
)
const (
@ -26,10 +28,12 @@ const (
varRangeAlt = "${__range}"
varRangeSAlt = "${__range_s}"
varRangeMsAlt = "${__range_ms}"
// $__auto is a new variable and we don't want to support this templating format
)
func interpolateVariables(expr string, interval time.Duration, timeRange time.Duration) string {
func interpolateVariables(expr string, interval time.Duration, timeRange time.Duration, queryType dataquery.LokiQueryType, step time.Duration) string {
intervalText := intervalv2.FormatDuration(interval)
stepText := intervalv2.FormatDuration(step)
intervalMsText := strconv.FormatInt(int64(interval/time.Millisecond), 10)
rangeMs := timeRange.Milliseconds()
@ -42,6 +46,13 @@ func interpolateVariables(expr string, interval time.Duration, timeRange time.Du
expr = strings.ReplaceAll(expr, varRangeMs, rangeMsText)
expr = strings.ReplaceAll(expr, varRangeS, rangeSText)
expr = strings.ReplaceAll(expr, varRange, rangeSText+"s")
if queryType == dataquery.LokiQueryTypeInstant {
expr = strings.ReplaceAll(expr, varAuto, rangeSText+"s")
}
if queryType == dataquery.LokiQueryTypeRange {
expr = strings.ReplaceAll(expr, varAuto, stepText)
}
// this is duplicated code, hopefully this can be handled in a nicer way when
// https://github.com/grafana/grafana/issues/42928 is done.
@ -131,13 +142,13 @@ func parseQuery(queryContext *backend.QueryDataRequest) ([]*lokiQuery, error) {
return nil, err
}
expr := interpolateVariables(model.Expr, interval, timeRange)
queryType, err := parseQueryType(model.QueryType)
if err != nil {
return nil, err
}
expr := interpolateVariables(model.Expr, interval, timeRange, queryType, step)
direction, err := parseDirection(model.Direction)
if err != nil {
return nil, err

View File

@ -5,6 +5,7 @@ import (
"time"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/tsdb/loki/kinds/dataquery"
"github.com/stretchr/testify/require"
)
@ -36,26 +37,59 @@ func TestParseQuery(t *testing.T) {
})
t.Run("interpolate variables, range between 1s and 0.5s", func(t *testing.T) {
expr := "go_goroutines $__interval $__interval_ms $__range $__range_s $__range_ms"
queryType := dataquery.LokiQueryTypeRange
interval := time.Millisecond * 50
step := time.Millisecond * 100
timeRange := time.Millisecond * 750
require.Equal(t, "go_goroutines 50ms 50 1s 1 750", interpolateVariables(expr, interval, timeRange))
require.Equal(t, "go_goroutines 50ms 50 1s 1 750", interpolateVariables(expr, interval, timeRange, queryType, step))
})
t.Run("parsing query model, range below 0.5s", func(t *testing.T) {
expr := "go_goroutines $__interval $__interval_ms $__range $__range_s $__range_ms"
queryType := dataquery.LokiQueryTypeRange
interval := time.Millisecond * 50
step := time.Millisecond * 100
timeRange := time.Millisecond * 250
require.Equal(t, "go_goroutines 50ms 50 0s 0 250", interpolateVariables(expr, interval, timeRange))
require.Equal(t, "go_goroutines 50ms 50 0s 0 250", interpolateVariables(expr, interval, timeRange, queryType, step))
})
t.Run("interpolate variables, curly-braces syntax", func(t *testing.T) {
expr := "go_goroutines ${__interval} ${__interval_ms} ${__range} ${__range_s} ${__range_ms}"
queryType := dataquery.LokiQueryTypeRange
interval := time.Second * 2
step := time.Millisecond * 100
timeRange := time.Second * 50
require.Equal(t, "go_goroutines 2s 2000 50s 50 50000", interpolateVariables(expr, interval, timeRange))
require.Equal(t, "go_goroutines 2s 2000 50s 50 50000", interpolateVariables(expr, interval, timeRange, queryType, step))
})
t.Run("interpolate variables should work with $__auto and instant query type", func(t *testing.T) {
expr := "rate({compose_project=\"docker-compose\"}[$__auto])"
queryType := dataquery.LokiQueryTypeInstant
interval := time.Second * 2
step := time.Millisecond * 100
timeRange := time.Second * 50
require.Equal(t, "rate({compose_project=\"docker-compose\"}[50s])", interpolateVariables(expr, interval, timeRange, queryType, step))
})
t.Run("interpolate variables should work with $__auto and range query type", func(t *testing.T) {
expr := "rate({compose_project=\"docker-compose\"}[$__auto])"
queryType := dataquery.LokiQueryTypeRange
interval := time.Second * 2
step := time.Millisecond * 100
timeRange := time.Second * 50
require.Equal(t, "rate({compose_project=\"docker-compose\"}[100ms])", interpolateVariables(expr, interval, timeRange, queryType, step))
})
t.Run("interpolate variables should return original query if no variables", func(t *testing.T) {
expr := "rate({compose_project=\"docker-compose\"}[10s])"
queryType := dataquery.LokiQueryTypeRange
interval := time.Second * 2
step := time.Millisecond * 100
timeRange := time.Second * 50
require.Equal(t, "rate({compose_project=\"docker-compose\"}[10s])", interpolateVariables(expr, interval, timeRange, queryType, step))
})
}

View File

@ -29,8 +29,7 @@ const NS_IN_MS = 1000000;
// When changing RATE_RANGES, check if Prometheus/PromQL ranges should be changed too
// @see public/app/plugins/datasource/prometheus/promql.ts
const RATE_RANGES: CompletionItem[] = [
{ label: '$__interval', sortValue: '$__interval' },
{ label: '$__range', sortValue: '$__range' },
{ label: '$__auto', sortValue: '$__auto' },
{ label: '1m', sortValue: '00:01:00' },
{ label: '5m', sortValue: '00:05:00' },
{ label: '10m', sortValue: '00:10:00' },

View File

@ -201,7 +201,7 @@ export class UnthemedLokiLabelBrowser extends React.Component<BrowserProps, Brow
closeType: 'showLogsRateButton',
});
const selector = buildSelector(this.state.labels);
const query = `rate(${selector}[$__interval])`;
const query = `rate(${selector}[$__auto])`;
this.props.onChange(query);
};

View File

@ -212,8 +212,7 @@ describe('getCompletions', () => {
const completions = await getCompletions(situation, completionProvider);
expect(completions).toEqual([
{ insertText: '$__interval', label: '$__interval', type: 'DURATION' },
{ insertText: '$__range', label: '$__range', type: 'DURATION' },
{ insertText: '$__auto', label: '$__auto', type: 'DURATION' },
{ insertText: '1m', label: '1m', type: 'DURATION' },
{ insertText: '5m', label: '5m', type: 'DURATION' },
{ insertText: '10m', label: '10m', type: 'DURATION' },

View File

@ -54,7 +54,7 @@ const AGGREGATION_COMPLETIONS: Completion[] = AGGREGATION_OPERATORS.map((f) => (
const FUNCTION_COMPLETIONS: Completion[] = RANGE_VEC_FUNCTIONS.map((f) => ({
type: 'FUNCTION',
label: f.label,
insertText: `${f.insertText ?? ''}({$0}[\\$__interval])`, // i don't know what to do when this is nullish. it should not be.
insertText: `${f.insertText ?? ''}({$0}[\\$__auto])`, // i don't know what to do when this is nullish. it should not be.
isSnippet: true,
triggerOnInsert: true,
detail: f.detail,
@ -71,13 +71,11 @@ const BUILT_IN_FUNCTIONS_COMPLETIONS: Completion[] = BUILT_IN_FUNCTIONS.map((f)
documentation: f.documentation,
}));
const DURATION_COMPLETIONS: Completion[] = ['$__interval', '$__range', '1m', '5m', '10m', '30m', '1h', '1d'].map(
(text) => ({
const DURATION_COMPLETIONS: Completion[] = ['$__auto', '1m', '5m', '10m', '30m', '1h', '1d'].map((text) => ({
type: 'DURATION',
label: text,
insertText: text,
})
);
}));
const UNWRAP_FUNCTION_COMPLETIONS: Completion[] = [
{

View File

@ -115,6 +115,7 @@ function isErrorBoundary(boundary: ParserErrorBoundary | null): boundary is Pars
export const placeHolderScopedVars = {
__interval: { text: '1s', value: '1s' },
__auto: { text: '1s', value: '1s' },
__interval_ms: { text: '1000', value: 1000 },
__range_ms: { text: '1000', value: 1000 },
__range_s: { text: '1', value: 1 },

View File

@ -1104,7 +1104,7 @@ describe('LokiDatasource', () => {
}
)
).toEqual({
expr: 'sum by (level) (count_over_time({label=value}[$__interval]))',
expr: 'sum by (level) (count_over_time({label=value}[$__auto]))',
queryType: LokiQueryType.Range,
refId: 'log-volume-A',
supportingQueryType: SupportingQueryType.LogsVolume,
@ -1122,7 +1122,7 @@ describe('LokiDatasource', () => {
}
)
).toEqual({
expr: 'sum by (level) (count_over_time({label=value}[$__interval]))',
expr: 'sum by (level) (count_over_time({label=value}[$__auto]))',
queryType: LokiQueryType.Range,
refId: 'log-volume-A',
supportingQueryType: SupportingQueryType.LogsVolume,

View File

@ -204,7 +204,7 @@ export class LokiDatasource
refId: `${REF_ID_STARTER_LOG_VOLUME}${normalizedQuery.refId}`,
queryType: LokiQueryType.Range,
supportingQueryType: SupportingQueryType.LogsVolume,
expr: `sum by (level) (count_over_time(${expr}[$__interval]))`,
expr: `sum by (level) (count_over_time(${expr}[$__auto]))`,
};
case SupplementaryQueryType.LogsSample:
@ -888,7 +888,7 @@ export class LokiDatasource
applyTemplateVariables(target: LokiQuery, scopedVars: ScopedVars): LokiQuery {
// We want to interpolate these variables on backend because we support using them in
// alerting/ML queries and we want to have consistent interpolation for all queries
const { __interval, __interval_ms, __range, __range_s, __range_ms, ...rest } = scopedVars || {};
const { __auto, __interval, __interval_ms, __range, __range_s, __range_ms, ...rest } = scopedVars || {};
const exprWithAdHoc = this.addAdHocFilters(target.expr);

View File

@ -109,7 +109,7 @@ export function getLokiQueryType(query: LokiQuery): LokiQueryType {
}
const tagsToObscure = ['String', 'Identifier', 'LineComment', 'Number'];
const partsToKeep = ['__error__', '__interval', '__interval_ms'];
const partsToKeep = ['__error__', '__interval', '__interval_ms', '__auto'];
export function obfuscate(query: string): string {
let obfuscatedQuery: string = query;
const tree = parser.parse(query);

View File

@ -115,65 +115,65 @@ export class LokiQueryModeller extends LokiAndPromQueryModellerBase {
{
name: 'Query on value inside a log line',
type: LokiQueryPatternType.Metric,
// sum(sum_over_time({ | logfmt | __error__=`` | unwrap | __error__=`` [$__interval]))
// sum(sum_over_time({ | logfmt | __error__=`` | unwrap | __error__=`` [$__auto]))
operations: [
{ id: LokiOperationId.LineContains, params: [''] },
{ id: LokiOperationId.Logfmt, params: [] },
{ id: LokiOperationId.LabelFilterNoErrors, params: [] },
{ id: LokiOperationId.Unwrap, params: [''] },
{ id: LokiOperationId.LabelFilterNoErrors, params: [] },
{ id: LokiOperationId.SumOverTime, params: ['$__interval'] },
{ id: LokiOperationId.SumOverTime, params: ['$__auto'] },
{ id: LokiOperationId.Sum, params: [] },
],
},
{
name: 'Total requests per label of streams',
type: LokiQueryPatternType.Metric,
// sum by() (count_over_time({}[$__interval)
// sum by() (count_over_time({}[$__auto)
operations: [
{ id: LokiOperationId.LineContains, params: [''] },
{ id: LokiOperationId.CountOverTime, params: ['$__interval'] },
{ id: LokiOperationId.CountOverTime, params: ['$__auto'] },
{ id: LokiOperationId.Sum, params: [] },
],
},
{
name: 'Total requests per parsed label or label of streams',
type: LokiQueryPatternType.Metric,
// sum by() (count_over_time({}| logfmt | __error__=`` [$__interval))
// sum by() (count_over_time({}| logfmt | __error__=`` [$__auto))
operations: [
{ id: LokiOperationId.LineContains, params: [''] },
{ id: LokiOperationId.Logfmt, params: [] },
{ id: LokiOperationId.LabelFilterNoErrors, params: [] },
{ id: LokiOperationId.CountOverTime, params: ['$__interval'] },
{ id: LokiOperationId.CountOverTime, params: ['$__auto'] },
{ id: LokiOperationId.Sum, params: [] },
],
},
{
name: 'Bytes used by a log stream',
type: LokiQueryPatternType.Metric,
// bytes_over_time({}[$__interval])
// bytes_over_time({}[$__auto])
operations: [
{ id: LokiOperationId.LineContains, params: [''] },
{ id: LokiOperationId.BytesOverTime, params: ['$__interval'] },
{ id: LokiOperationId.BytesOverTime, params: ['$__auto'] },
],
},
{
name: 'Count of log lines per stream',
type: LokiQueryPatternType.Metric,
// count_over_time({}[$__interval])
// count_over_time({}[$__auto])
operations: [
{ id: LokiOperationId.LineContains, params: [''] },
{ id: LokiOperationId.CountOverTime, params: ['$__interval'] },
{ id: LokiOperationId.CountOverTime, params: ['$__auto'] },
],
},
{
name: 'Top N results by label or parsed label',
type: LokiQueryPatternType.Metric,
// topk(10, sum by () (count_over_time({} | logfmt | __error__=`` [$__interval])))
// topk(10, sum by () (count_over_time({} | logfmt | __error__=`` [$__auto])))
operations: [
{ id: LokiOperationId.Logfmt, params: [] },
{ id: LokiOperationId.LabelFilterNoErrors, params: [] },
{ id: LokiOperationId.CountOverTime, params: ['$__interval'] },
{ id: LokiOperationId.CountOverTime, params: ['$__auto'] },
{ id: LokiOperationId.Sum, params: [] },
{ id: LokiOperationId.TopK, params: [10] },
],
@ -181,13 +181,13 @@ export class LokiQueryModeller extends LokiAndPromQueryModellerBase {
{
name: 'Extracted quantile',
type: LokiQueryPatternType.Metric,
// quantile_over_time(0.5,{} | logfmt | unwrap latency[$__interval]) by ()
// quantile_over_time(0.5,{} | logfmt | unwrap latency[$__auto]) by ()
operations: [
{ id: LokiOperationId.Logfmt, params: [] },
{ id: LokiOperationId.LabelFilterNoErrors, params: [] },
{ id: LokiOperationId.Unwrap, params: ['latency'] },
{ id: LokiOperationId.LabelFilterNoErrors, params: [] },
{ id: LokiOperationId.QuantileOverTime, params: ['$__interval', 0.5] },
{ id: LokiOperationId.QuantileOverTime, params: ['$__auto', 0.5] },
{ id: LokiOperationId.Sum, params: [] },
],
},

View File

@ -42,7 +42,7 @@ describe('LokiQueryBuilderContainer', () => {
await addOperation('Range functions', 'Rate');
expect(await screen.findByText('Rate')).toBeInTheDocument();
expect(props.onChange).toBeCalledWith({
expr: 'rate({job="testjob"} [$__interval])',
expr: 'rate({job="testjob"} [$__auto])',
refId: 'A',
});
});

View File

@ -15,7 +15,7 @@ describe('createRangeOperation', () => {
id: 'test_range_operation',
name: 'Test range operation',
params: [{ name: 'Range', type: 'string' }],
defaultParams: ['$__interval'],
defaultParams: ['$__auto'],
alternativesKey: 'range function',
category: LokiVisualQueryOperationCategory.RangeFunctions,
});
@ -34,7 +34,7 @@ describe('createRangeOperation', () => {
optional: true,
},
],
defaultParams: ['$__interval'],
defaultParams: ['$__auto'],
alternativesKey: 'range function',
category: LokiVisualQueryOperationCategory.RangeFunctions,
});
@ -49,7 +49,7 @@ describe('createRangeOperation', () => {
{ name: 'Quantile', type: 'number' },
{ name: 'By label', type: 'string', restParam: true, optional: true },
],
defaultParams: ['$__interval', '0.95'],
defaultParams: ['$__auto', '0.95'],
alternativesKey: 'range function',
category: LokiVisualQueryOperationCategory.RangeFunctions,
});
@ -68,7 +68,7 @@ describe('createRangeOperationWithGrouping', () => {
{ name: 'Quantile', type: 'number' },
{ name: 'By label', type: 'string', restParam: true, optional: true },
],
defaultParams: ['$__interval', '0.95'],
defaultParams: ['$__auto', '0.95'],
alternativesKey: 'range function',
category: LokiVisualQueryOperationCategory.RangeFunctions,
});
@ -81,7 +81,7 @@ describe('createRangeOperationWithGrouping', () => {
{ name: 'Quantile', type: 'number' },
{ name: 'Label', type: 'string', restParam: true, optional: true },
],
defaultParams: ['$__interval', '0.95', ''],
defaultParams: ['$__auto', '0.95', ''],
alternativesKey: 'range function with grouping',
category: LokiVisualQueryOperationCategory.RangeFunctions,
});
@ -94,7 +94,7 @@ describe('createRangeOperationWithGrouping', () => {
{ name: 'Quantile', type: 'number' },
{ name: 'Label', type: 'string', restParam: true, optional: true },
],
defaultParams: ['$__interval', '0.95', ''],
defaultParams: ['$__auto', '0.95', ''],
alternativesKey: 'range function with grouping',
category: LokiVisualQueryOperationCategory.RangeFunctions,
});

View File

@ -17,7 +17,7 @@ import { LokiOperationId, LokiOperationOrder, LokiVisualQuery, LokiVisualQueryOp
export function createRangeOperation(name: string, isRangeOperationWithGrouping?: boolean): QueryBuilderOperationDef {
const params = [getRangeVectorParamDef()];
const defaultParams = ['$__interval'];
const defaultParams = ['$__auto'];
let paramChangedHandler = undefined;
if (name === LokiOperationId.QuantileOverTime) {
@ -53,8 +53,8 @@ export function createRangeOperation(name: string, isRangeOperationWithGrouping?
explainHandler: (op, def) => {
let opDocs = FUNCTIONS.find((x) => x.insertText === op.id)?.documentation ?? '';
if (op.params[0] === '$__interval') {
return `${opDocs} \`$__interval\` is a variable that will be replaced with the [calculated interval](https://grafana.com/docs/grafana/latest/dashboards/variables/add-template-variables/#__interval) based on the time range and width of the graph. In Dashboards, you can affect the interval variable using **Max data points** and **Min interval**. You can find these options under **Query options** right of the data source select dropdown.`;
if (op.params[0] === '$__auto') {
return `${opDocs} \`$__auto\` is a variable that will be replaced with the [value of step](https://grafana.com/docs/grafana/next/datasources/loki/query-editor/#options) for range queries and with the value of the selected time range (calculated to - from) for instant queries.`;
} else {
return `${opDocs} The [range vector](https://grafana.com/docs/loki/latest/logql/metric_queries/#range-vector-aggregation) is set to \`${op.params[0]}\`.`;
}
@ -137,14 +137,14 @@ function operationWithRangeVectorRenderer(
innerExpr: string
) {
const params = model.params ?? [];
const rangeVector = params[0] ?? '$__interval';
const rangeVector = params[0] ?? '$__auto';
// QuantileOverTime is only range vector with more than one param
if (params.length === 2 && model.id === LokiOperationId.QuantileOverTime) {
const quantile = params[1];
return `${model.id}(${quantile}, ${innerExpr} [${rangeVector}])`;
}
return `${model.id}(${innerExpr} [${params[0] ?? '$__interval'}])`;
return `${model.id}(${innerExpr} [${params[0] ?? '$__auto'}])`;
}
export function labelFilterRenderer(model: QueryBuilderOperation, def: QueryBuilderOperationDef, innerExpr: string) {
@ -237,7 +237,7 @@ export function addLokiOperation(
modeller,
(def) => def.category === LokiVisualQueryOperationCategory.Functions
);
operations.splice(placeToInsert, 0, { id: LokiOperationId.Rate, params: ['$__interval'] });
operations.splice(placeToInsert, 0, { id: LokiOperationId.Rate, params: ['$__auto'] });
}
operations.push(newOperation);
break;
@ -292,6 +292,6 @@ function getRangeVectorParamDef(): QueryBuilderOperationParamDef {
return {
name: 'Range',
type: 'string',
options: ['$__interval', '$__range', '1m', '5m', '10m', '1h', '24h'],
options: ['$__auto', '1m', '5m', '10m', '1h', '24h'],
};
}