From a1ec5be730e0c6dfdd16bcd157c3c99b3fc022f5 Mon Sep 17 00:00:00 2001 From: Sven Grossmann Date: Thu, 14 Dec 2023 10:14:02 +0100 Subject: [PATCH] Loki: Add timeRange to labels requests in LogContext to reduce loading times (#79478) Loki: Add timeRange to labels requests --- .../loki/LogContextProvider.test.ts | 45 +++++++++++++++++-- .../datasource/loki/LogContextProvider.ts | 15 +++++-- .../loki/components/LokiContextUi.test.tsx | 16 ++++++- .../loki/components/LokiContextUi.tsx | 8 +++- 4 files changed, 74 insertions(+), 10 deletions(-) diff --git a/public/app/plugins/datasource/loki/LogContextProvider.test.ts b/public/app/plugins/datasource/loki/LogContextProvider.test.ts index 82ab9af4745..1aa9aad86ca 100644 --- a/public/app/plugins/datasource/loki/LogContextProvider.test.ts +++ b/public/app/plugins/datasource/loki/LogContextProvider.test.ts @@ -1,6 +1,13 @@ import { of } from 'rxjs'; -import { DataQueryResponse, FieldType, LogRowContextQueryDirection, LogRowModel, createDataFrame } from '@grafana/data'; +import { + DataQueryResponse, + FieldType, + LogRowContextQueryDirection, + LogRowModel, + createDataFrame, + dateTime, +} from '@grafana/data'; import LokiLanguageProvider from './LanguageProvider'; import { @@ -50,6 +57,7 @@ const defaultLogRow = { }), labels: { bar: 'baz', foo: 'uniqueParsedLabel', xyz: 'abc' }, uid: '1', + timeEpochMs: new Date().getTime(), } as unknown as LogRowModel; describe('LogContextProvider', () => { @@ -83,7 +91,12 @@ describe('LogContextProvider', () => { expect(logContextProvider.getInitContextFilters).toBeCalled(); expect(logContextProvider.getInitContextFilters).toHaveBeenCalledWith( { bar: 'baz', foo: 'uniqueParsedLabel', xyz: 'abc' }, - { expr: '{bar="baz"}', refId: 'A' } + { expr: '{bar="baz"}', refId: 'A' }, + { + from: dateTime(defaultLogRow.timeEpochMs), + to: dateTime(defaultLogRow.timeEpochMs), + raw: { from: dateTime(defaultLogRow.timeEpochMs), to: dateTime(defaultLogRow.timeEpochMs) }, + } ); expect(logContextProvider.appliedContextFilters).toHaveLength(1); }); @@ -371,13 +384,24 @@ describe('LogContextProvider', () => { }); }); - describe('getInitContextFiltersFromLabels', () => { + describe('getInitContextFilters', () => { describe('query with no parser', () => { const queryWithoutParser: LokiQuery = { expr: '{bar="baz"}', refId: 'A', }; + const queryWithParser: LokiQuery = { + expr: '{bar="baz"} | logfmt', + refId: 'A', + }; + + const timeRange = { + from: dateTime(defaultLogRow.timeEpochMs), + to: dateTime(defaultLogRow.timeEpochMs), + raw: { from: dateTime(defaultLogRow.timeEpochMs), to: dateTime(defaultLogRow.timeEpochMs) }, + }; + it('should correctly create contextFilters', async () => { const filters = await logContextProvider.getInitContextFilters(defaultLogRow.labels, queryWithoutParser); expect(filters).toEqual([ @@ -396,6 +420,21 @@ describe('LogContextProvider', () => { const filters = await logContextProvider.getInitContextFilters({}, queryWithoutParser); expect(filters).toEqual([]); }); + + it('should call fetchSeriesLabels if parser', async () => { + await logContextProvider.getInitContextFilters(defaultLogRow.labels, queryWithParser); + expect(defaultLanguageProviderMock.fetchSeriesLabels).toBeCalled(); + }); + + it('should call fetchSeriesLabels with given timerange', async () => { + await logContextProvider.getInitContextFilters(defaultLogRow.labels, queryWithParser, timeRange); + expect(defaultLanguageProviderMock.fetchSeriesLabels).toBeCalledWith(`{bar="baz"}`, { timeRange }); + }); + + it('should call `languageProvider.start` if no parser with given timerange', async () => { + await logContextProvider.getInitContextFilters(defaultLogRow.labels, queryWithoutParser, timeRange); + expect(defaultLanguageProviderMock.start).toBeCalledWith(timeRange); + }); }); describe('query with parser', () => { diff --git a/public/app/plugins/datasource/loki/LogContextProvider.ts b/public/app/plugins/datasource/loki/LogContextProvider.ts index 0b770794c24..02506ab82fc 100644 --- a/public/app/plugins/datasource/loki/LogContextProvider.ts +++ b/public/app/plugins/datasource/loki/LogContextProvider.ts @@ -13,6 +13,7 @@ import { toUtc, LogRowContextQueryDirection, LogRowContextOptions, + dateTime, } from '@grafana/data'; import { LabelParser, LabelFilter, LineFilters, PipelineStage, Logfmt, Json } from '@grafana/lezer-logql'; import { Labels } from '@grafana/schema'; @@ -58,7 +59,13 @@ export class LogContextProvider { // This happens only on initial load, when user haven't applied any filters yet // We need to get the initial filters from the row labels if (this.appliedContextFilters.length === 0) { - const filters = (await this.getInitContextFilters(row.labels, origQuery)).filter((filter) => filter.enabled); + const filters = ( + await this.getInitContextFilters(row.labels, origQuery, { + from: dateTime(row.timeEpochMs), + to: dateTime(row.timeEpochMs), + raw: { from: dateTime(row.timeEpochMs), to: dateTime(row.timeEpochMs) }, + }) + ).filter((filter) => filter.enabled); this.appliedContextFilters = filters; } @@ -285,7 +292,7 @@ export class LogContextProvider { ); }; - getInitContextFilters = async (labels: Labels, query?: LokiQuery) => { + getInitContextFilters = async (labels: Labels, query?: LokiQuery, timeRange?: TimeRange) => { if (!query || isEmpty(labels)) { return []; } @@ -296,13 +303,13 @@ export class LogContextProvider { if (!isQueryWithParser(query.expr).queryWithParser) { // If there is no parser, we use getLabelKeys because it has better caching // and all labels should already be fetched - await this.datasource.languageProvider.start(); + await this.datasource.languageProvider.start(timeRange); allLabels = this.datasource.languageProvider.getLabelKeys(); } else { // If we have parser, we use fetchSeriesLabels to fetch actual labels for selected stream const stream = getStreamSelectorsFromQuery(query.expr); // We are using stream[0] as log query can always have just 1 stream selector - const series = await this.datasource.languageProvider.fetchSeriesLabels(stream[0]); + const series = await this.datasource.languageProvider.fetchSeriesLabels(stream[0], { timeRange }); allLabels = Object.keys(series); } diff --git a/public/app/plugins/datasource/loki/components/LokiContextUi.test.tsx b/public/app/plugins/datasource/loki/components/LokiContextUi.test.tsx index 83fd414785b..2ad3c1b4949 100644 --- a/public/app/plugins/datasource/loki/components/LokiContextUi.test.tsx +++ b/public/app/plugins/datasource/loki/components/LokiContextUi.test.tsx @@ -3,7 +3,7 @@ import userEvent from '@testing-library/user-event'; import React from 'react'; import { selectOptionInTest } from 'test/helpers/selectOptionInTest'; -import { LogRowModel } from '@grafana/data'; +import { LogRowModel, dateTime } from '@grafana/data'; import { LogContextProvider, SHOULD_INCLUDE_PIPELINE_OPERATIONS } from '../LogContextProvider'; import { ContextFilter, LokiQuery } from '../types'; @@ -41,6 +41,7 @@ const setupProps = (): LokiContextUiProps => { label1: 'value1', label3: 'value3', }, + timeEpochMs: new Date().getTime(), } as unknown as LogRowModel, onClose: jest.fn(), origQuery: { @@ -128,6 +129,19 @@ describe('LokiContextUi', () => { }); }); + it('calls `getInitContextFilters` with the right set of parameters', async () => { + const props = setupProps(); + render(); + + await waitFor(() => { + expect(props.logContextProvider.getInitContextFilters).toHaveBeenCalledWith(props.row.labels, props.origQuery, { + from: dateTime(props.row.timeEpochMs), + to: dateTime(props.row.timeEpochMs), + raw: { from: dateTime(props.row.timeEpochMs), to: dateTime(props.row.timeEpochMs) }, + }); + }); + }); + it('finds label1 as a real label', async () => { const props = setupProps(); render(); diff --git a/public/app/plugins/datasource/loki/components/LokiContextUi.tsx b/public/app/plugins/datasource/loki/components/LokiContextUi.tsx index 2b278ae3689..1532ef1e8f7 100644 --- a/public/app/plugins/datasource/loki/components/LokiContextUi.tsx +++ b/public/app/plugins/datasource/loki/components/LokiContextUi.tsx @@ -2,7 +2,7 @@ import { css } from '@emotion/css'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { useAsync } from 'react-use'; -import { GrafanaTheme2, LogRowModel, renderMarkdown, SelectableValue } from '@grafana/data'; +import { dateTime, GrafanaTheme2, LogRowModel, renderMarkdown, SelectableValue } from '@grafana/data'; import { reportInteraction } from '@grafana/runtime'; import { Button, @@ -199,7 +199,11 @@ export function LokiContextUi(props: LokiContextUiProps) { useAsync(async () => { setLoading(true); - const initContextFilters = await logContextProvider.getInitContextFilters(row.labels, origQuery); + const initContextFilters = await logContextProvider.getInitContextFilters(row.labels, origQuery, { + from: dateTime(row.timeEpochMs), + to: dateTime(row.timeEpochMs), + raw: { from: dateTime(row.timeEpochMs), to: dateTime(row.timeEpochMs) }, + }); setContextFilters(initContextFilters); setInitialized(true);