From aa670280fc46b2d96e3096f1395e5145c715cb44 Mon Sep 17 00:00:00 2001 From: Matias Chomicki Date: Fri, 12 Jan 2024 12:22:03 +0100 Subject: [PATCH] Logs: add infinite scrolling to Explore (#76348) * Explore: propose action, thunk, and decorators for load more * LogsContainer: add loadMore method * Query: remove unused var * Loading more: use navigation to simulate scrolling * Explore: figure out data combination * Fix imports * Explore: deduplicate results when using query splitting * LogsNavigation: add scroll behavior * Remove old code * Scroll: adjust delta value * Load more: remove refIds from signature We can resolve them inside Explore state * Load more: rename to loadMoreLogs * Infinite scrolling: use scrollElement to listen to scrolling events * Explore logs: add fixed height to scrollable logs container * Logs: make logs container the scrolling element * Logs: remove dynamic logs container size It works very well with 1 query, but breaks with more than 1 query or when Logs is not the last rendered panel * Logs navigation: revert changes * Infinite scroll: create component * Infinite scroll: refactor and clean up effect * Infinite scroll: support oldest first scrolling direction * Infinite scroll: support loading oldest logs in ascending and descending order * Infinite scroll: use scroll to top from logs navigation * Logs: make logs container smaller * Logs: make container smaller * State: integrate explore's loading states * Infinite scroll: add loading to effect dependency array * Infinite scroll: display message when scroll limit is reached * Infinite scroll: add support to scroll in both directions * Infinite scroll: capture wheel events for top scroll * scrollableLogsContainer: deprecate in favor of logsInfiniteScrolling * Infinite scroll: implement timerange limits * Infinite scroll: pass timezone * Fix unused variables and imports * Infinite scroll: implement timerange limits for absolute time * Infinite scroll: fix timerange limits for absolute and relative times * Infinite scroll: reset out-of-bounds message * Logs: make container taller * Line limit: use "displayed" instead of "returned" for infinite scrolling * Infinite scrolling: disable behavior when there is no scroll * Remove console log * Infinite scroll: hide limit reached message when using relative time * Logs: migrate styles to object notation * Prettier formatting * LogsModel: fix import order * Update betterer.results * Logs: remove exploreScrollableLogsContainer test * Infinite scroll: display loader * Infinite scroll: improve wheel handling * Explore: unify correlations code * Explore: move new function to helpers * Remove comment * Fix imports * Formatting * Query: add missing awaits in unit test * Logs model: add unit test * Combine frames: move code to feature/logs * Explore: move getCorrelations call back to query It was causing a weird test failure * Fix imports * Infinite scroll: parametrize scrolling threshold * Logs: fix overflow css * Infinite scroll: add basic unit test * Infinite scroll: add unit test for absolute time ranges * Formatting * Explore query: add custom interaction for scrolling * Query: move correlations before update time * Fix import in test * Update comment * Remove comment * Remove comment * Infinite scroll: report interactions from component * Fix import order * Rename action * Infinite scroll: update limit reached message * Explore logs: remove type assertion * Update betterer --- .betterer.results | 17 +- .../app/features/explore/Logs/Logs.test.tsx | 75 +--- public/app/features/explore/Logs/Logs.tsx | 202 +++++---- .../features/explore/Logs/LogsContainer.tsx | 9 +- .../features/explore/Logs/LogsNavigation.tsx | 30 +- .../features/explore/Logs/LogsSamplePanel.tsx | 10 +- .../app/features/explore/state/query.test.ts | 28 +- public/app/features/explore/state/query.ts | 122 +++++- public/app/features/explore/state/time.ts | 8 +- public/app/features/explore/state/utils.ts | 24 +- .../app/features/explore/utils/decorators.ts | 6 + .../logs/components/InfiniteScroll.test.tsx | 263 +++++++++++ .../logs/components/InfiniteScroll.tsx | 224 ++++++++++ .../logs/components/getLogRowStyles.ts | 3 - public/app/features/logs/logsModel.test.ts | 64 ++- public/app/features/logs/logsModel.ts | 9 +- public/app/features/logs/response.test.ts | 411 +++++++++++++++++ public/app/features/logs/response.ts | 150 +++++++ .../plugins/datasource/loki/querySplitting.ts | 2 +- .../datasource/loki/responseUtils.test.ts | 412 +----------------- .../plugins/datasource/loki/responseUtils.ts | 153 +------ 21 files changed, 1418 insertions(+), 804 deletions(-) create mode 100644 public/app/features/logs/components/InfiniteScroll.test.tsx create mode 100644 public/app/features/logs/components/InfiniteScroll.tsx create mode 100644 public/app/features/logs/response.test.ts create mode 100644 public/app/features/logs/response.ts diff --git a/.betterer.results b/.betterer.results index 583c2b24e30..cd630533c99 100644 --- a/.betterer.results +++ b/.betterer.results @@ -3214,19 +3214,7 @@ exports[`better eslint`] = { [0, 0, 0, "Styles should be written using objects.", "4"] ], "public/app/features/explore/Logs/Logs.tsx:5381": [ - [0, 0, 0, "Unexpected any. Specify a different type.", "0"], - [0, 0, 0, "Do not use any type assertions.", "1"], - [0, 0, 0, "Styles should be written using objects.", "2"], - [0, 0, 0, "Styles should be written using objects.", "3"], - [0, 0, 0, "Styles should be written using objects.", "4"], - [0, 0, 0, "Styles should be written using objects.", "5"], - [0, 0, 0, "Styles should be written using objects.", "6"], - [0, 0, 0, "Styles should be written using objects.", "7"], - [0, 0, 0, "Styles should be written using objects.", "8"], - [0, 0, 0, "Styles should be written using objects.", "9"], - [0, 0, 0, "Styles should be written using objects.", "10"], - [0, 0, 0, "Styles should be written using objects.", "11"], - [0, 0, 0, "Styles should be written using objects.", "12"] + [0, 0, 0, "Unexpected any. Specify a different type.", "0"] ], "public/app/features/explore/Logs/LogsMetaRow.tsx:5381": [ [0, 0, 0, "Styles should be written using objects.", "0"], @@ -3248,8 +3236,7 @@ exports[`better eslint`] = { "public/app/features/explore/Logs/LogsSamplePanel.tsx:5381": [ [0, 0, 0, "Styles should be written using objects.", "0"], [0, 0, 0, "Styles should be written using objects.", "1"], - [0, 0, 0, "Styles should be written using objects.", "2"], - [0, 0, 0, "Styles should be written using objects.", "3"] + [0, 0, 0, "Styles should be written using objects.", "2"] ], "public/app/features/explore/Logs/LogsVolumePanel.tsx:5381": [ [0, 0, 0, "Styles should be written using objects.", "0"], diff --git a/public/app/features/explore/Logs/Logs.test.tsx b/public/app/features/explore/Logs/Logs.test.tsx index efc500d4e56..f2eb2779f97 100644 --- a/public/app/features/explore/Logs/Logs.test.tsx +++ b/public/app/features/explore/Logs/Logs.test.tsx @@ -163,65 +163,24 @@ describe('Logs', () => { window.innerHeight = originalInnerHeight; }); - describe('when `exploreScrollableLogsContainer` is set', () => { - let featureToggle: boolean | undefined; - beforeEach(() => { - featureToggle = config.featureToggles.exploreScrollableLogsContainer; - config.featureToggles.exploreScrollableLogsContainer = true; - }); - afterEach(() => { - config.featureToggles.exploreScrollableLogsContainer = featureToggle; - jest.clearAllMocks(); - }); + it('should call `scrollElement.scroll`', () => { + const logs = []; + for (let i = 0; i < 50; i++) { + logs.push(makeLog({ uid: `uid${i}`, rowId: `id${i}`, timeEpochMs: i })); + } + const scrollElementMock = { + scroll: jest.fn(), + scrollTop: 920, + }; + setup( + { scrollElement: scrollElementMock as unknown as HTMLDivElement, panelState: { logs: { id: 'uid47' } } }, + undefined, + logs + ); - it('should call `this.state.logsContainer.scroll`', () => { - const scrollIntoViewSpy = jest.spyOn(window.HTMLElement.prototype, 'scrollIntoView'); - jest.spyOn(window.HTMLElement.prototype, 'scrollTop', 'get').mockReturnValue(920); - const scrollSpy = jest.spyOn(window.HTMLElement.prototype, 'scroll'); - - const logs = []; - for (let i = 0; i < 50; i++) { - logs.push(makeLog({ uid: `uid${i}`, rowId: `id${i}`, timeEpochMs: i })); - } - - setup({ panelState: { logs: { id: 'uid47' } } }, undefined, logs); - - expect(scrollIntoViewSpy).toBeCalledTimes(1); - // element.getBoundingClientRect().top will always be 0 for jsdom - // calc will be `this.state.logsContainer.scrollTop - window.innerHeight / 2` -> 920 - 500 = 420 - expect(scrollSpy).toBeCalledWith({ behavior: 'smooth', top: 420 }); - }); - }); - - describe('when `exploreScrollableLogsContainer` is not set', () => { - let featureToggle: boolean | undefined; - beforeEach(() => { - featureToggle = config.featureToggles.exploreScrollableLogsContainer; - config.featureToggles.exploreScrollableLogsContainer = false; - }); - afterEach(() => { - config.featureToggles.exploreScrollableLogsContainer = featureToggle; - }); - - it('should call `scrollElement.scroll`', () => { - const logs = []; - for (let i = 0; i < 50; i++) { - logs.push(makeLog({ uid: `uid${i}`, rowId: `id${i}`, timeEpochMs: i })); - } - const scrollElementMock = { - scroll: jest.fn(), - scrollTop: 920, - }; - setup( - { scrollElement: scrollElementMock as unknown as HTMLDivElement, panelState: { logs: { id: 'uid47' } } }, - undefined, - logs - ); - - // element.getBoundingClientRect().top will always be 0 for jsdom - // calc will be `scrollElement.scrollTop - window.innerHeight / 2` -> 920 - 500 = 420 - expect(scrollElementMock.scroll).toBeCalledWith({ behavior: 'smooth', top: 420 }); - }); + // element.getBoundingClientRect().top will always be 0 for jsdom + // calc will be `scrollElement.scrollTop - window.innerHeight / 2` -> 920 - 500 = 420 + expect(scrollElementMock.scroll).toBeCalledWith({ behavior: 'smooth', top: 420 }); }); }); diff --git a/public/app/features/explore/Logs/Logs.tsx b/public/app/features/explore/Logs/Logs.tsx index 7a87d42066b..e47dd07afe3 100644 --- a/public/app/features/explore/Logs/Logs.tsx +++ b/public/app/features/explore/Logs/Logs.tsx @@ -47,6 +47,8 @@ import { } from '@grafana/ui'; import store from 'app/core/store'; import { createAndCopyShortLink } from 'app/core/utils/shortLinks'; +import { InfiniteScroll } from 'app/features/logs/components/InfiniteScroll'; +import { getLogLevelFromKey } from 'app/features/logs/utils'; import { dispatch, getState } from 'app/store/store'; import { ExploreItemState } from '../../../types'; @@ -108,6 +110,7 @@ interface Props extends Themeable2 { range: TimeRange; onClickFilterValue?: (value: string, refId?: string) => void; onClickFilterOutValue?: (value: string, refId?: string) => void; + loadMoreLogs?(range: AbsoluteTimeRange): void; } export type LogsVisualisationType = 'table' | 'logs'; @@ -130,8 +133,6 @@ interface State { logsContainer?: HTMLDivElement; } -const scrollableLogsContainer = config.featureToggles.exploreScrollableLogsContainer; - // we need to define the order of these explicitly const DEDUP_OPTIONS = [ LogsDedupStrategy.none, @@ -203,6 +204,7 @@ class UnthemedLogs extends PureComponent { ); } } + updatePanelState = (logsPanelState: Partial) => { const state: ExploreItemState | undefined = getState().explore.panes[this.props.exploreId]; if (state?.panelsState) { @@ -346,7 +348,7 @@ class UnthemedLogs extends PureComponent { }; onToggleLogLevel = (hiddenRawLevels: string[]) => { - const hiddenLogLevels = hiddenRawLevels.map((level) => LogLevel[level as LogLevel]); + const hiddenLogLevels = hiddenRawLevels.map((level) => getLogLevelFromKey(level)); this.setState({ hiddenLogLevels }); }; @@ -471,7 +473,7 @@ class UnthemedLogs extends PureComponent { }; scrollIntoView = (element: HTMLElement) => { - if (config.featureToggles.exploreScrollableLogsContainer) { + if (config.featureToggles.logsInfiniteScrolling) { if (this.state.logsContainer) { this.topLogsRef.current?.scrollIntoView(); this.state.logsContainer.scroll({ @@ -521,16 +523,15 @@ class UnthemedLogs extends PureComponent { }); scrollToTopLogs = () => { - if (config.featureToggles.exploreScrollableLogsContainer) { + if (config.featureToggles.logsInfiniteScrolling) { if (this.state.logsContainer) { this.state.logsContainer.scroll({ behavior: 'auto', top: 0, }); } - } else { - this.topLogsRef.current?.scrollIntoView(); } + this.topLogsRef.current?.scrollIntoView(); }; render() { @@ -560,6 +561,7 @@ class UnthemedLogs extends PureComponent { getRowContext, getLogRowContextUi, getRowContextQuery, + loadMoreLogs, } = this.props; const { @@ -784,38 +786,52 @@ class UnthemedLogs extends PureComponent { )} {this.state.visualisationType === 'logs' && hasData && ( -
- + + rows={logRows} + scrollElement={this.state.logsContainer} + sortOrder={logsSortOrder} + > + +
)} {!loading && !hasData && !scanning && ( @@ -861,61 +877,65 @@ export const Logs = withTheme2(UnthemedLogs); const getStyles = (theme: GrafanaTheme2, wrapLogMessage: boolean, tableHeight: number) => { return { - noData: css` - > * { - margin-left: 0.5em; - } - `, - logOptions: css` - display: flex; - justify-content: space-between; - align-items: baseline; - flex-wrap: wrap; - background-color: ${theme.colors.background.primary}; - padding: ${theme.spacing(1, 2)}; - border-radius: ${theme.shape.radius.default}; - margin: ${theme.spacing(0, 0, 1)}; - border: 1px solid ${theme.colors.border.medium}; - `, - headerButton: css` - margin: ${theme.spacing(0.5, 0, 0, 1)}; - `, - horizontalInlineLabel: css` - > label { - margin-right: 0; - } - `, - horizontalInlineSwitch: css` - padding: 0 ${theme.spacing(1)} 0 0; - `, - radioButtons: css` - margin: 0; - `, - logsSection: css` - display: flex; - flex-direction: row; - justify-content: space-between; - `, + noData: css({ + '& > *': { + marginLeft: '0.5em', + }, + }), + logOptions: css({ + display: 'flex', + justifyContent: 'space-between', + alignItems: 'baseline', + flexWrap: 'wrap', + backgroundColor: theme.colors.background.primary, + padding: `${theme.spacing(1)} ${theme.spacing(2)}`, + borderRadius: theme.shape.radius.default, + margin: `${theme.spacing(0, 0, 1)}`, + border: `1px solid ${theme.colors.border.medium}`, + }), + headerButton: css({ + margin: `${theme.spacing(0.5, 0, 0, 1)}`, + }), + horizontalInlineLabel: css({ + '& > label': { + marginRight: '0', + }, + }), + horizontalInlineSwitch: css({ + padding: `0 ${theme.spacing(1)} 0 0`, + }), + radioButtons: css({ + margin: '0', + }), + logsSection: css({ + display: 'flex', + flexDirection: 'row', + justifyContent: 'space-between', + }), logsTable: css({ maxHeight: `${tableHeight}px`, }), - logRows: css` - overflow-x: ${scrollableLogsContainer ? 'scroll;' : `${wrapLogMessage ? 'unset' : 'scroll'};`} - overflow-y: visible; - width: 100%; - ${scrollableLogsContainer && 'max-height: calc(100vh - 170px);'} - `, - visualisationType: css` - display: flex; - flex: 1; - justify-content: space-between; - `, - visualisationTypeRadio: css` - margin: 0 0 0 ${theme.spacing(1)}; - `, - stickyNavigation: css` - ${scrollableLogsContainer && 'margin-bottom: 0px'} - overflow: visible; - `, + scrollableLogRows: css({ + overflowY: 'scroll', + width: '100%', + maxHeight: '75vh', + }), + logRows: css({ + overflowX: `${wrapLogMessage ? 'unset' : 'scroll'}`, + overflowY: 'visible', + width: '100%', + }), + visualisationType: css({ + display: 'flex', + flex: '1', + justifyContent: 'space-between', + }), + visualisationTypeRadio: css({ + margin: `0 0 0 ${theme.spacing(1)}`, + }), + stickyNavigation: css({ + overflow: 'visible', + ...(config.featureToggles.logsInfiniteScrolling && { marginBottom: '0px' }), + }), }; }; diff --git a/public/app/features/explore/Logs/LogsContainer.tsx b/public/app/features/explore/Logs/LogsContainer.tsx index ab0f2cb65b6..48a173cd064 100644 --- a/public/app/features/explore/Logs/LogsContainer.tsx +++ b/public/app/features/explore/Logs/LogsContainer.tsx @@ -36,7 +36,7 @@ import { selectIsWaitingForData, setSupplementaryQueryEnabled, } from '../state/query'; -import { updateTimeRange } from '../state/time'; +import { updateTimeRange, loadMoreLogs } from '../state/time'; import { LiveTailControls } from '../useLiveTailControls'; import { getFieldLinksForExplore } from '../utils/links'; @@ -140,6 +140,11 @@ class LogsContainer extends PureComponent { + const { exploreId, loadMoreLogs } = this.props; + loadMoreLogs({ exploreId, absoluteRange }); + }; + private getQuery( logsQueries: DataQuery[] | undefined, row: LogRowModel, @@ -322,6 +327,7 @@ class LogsContainer extends PureComponent loadSupplementaryQueryData(exploreId, SupplementaryQueryType.LogsVolume)} onChangeTime={this.onChangeTime} + loadMoreLogs={this.loadMoreLogs} onClickFilterLabel={this.logDetailsFilterAvailable() ? onClickFilterLabel : undefined} onClickFilterOutLabel={this.logDetailsFilterAvailable() ? onClickFilterOutLabel : undefined} onStartScanning={onStartScanning} @@ -395,6 +401,7 @@ function mapStateToProps(state: StoreState, { exploreId }: { exploreId: string } const mapDispatchToProps = { updateTimeRange, + loadMoreLogs, addResultsToCache, clearCache, loadSupplementaryQueryData, diff --git a/public/app/features/explore/Logs/LogsNavigation.tsx b/public/app/features/explore/Logs/LogsNavigation.tsx index e053660260a..6257569baa7 100644 --- a/public/app/features/explore/Logs/LogsNavigation.tsx +++ b/public/app/features/explore/Logs/LogsNavigation.tsx @@ -3,7 +3,7 @@ import { isEqual } from 'lodash'; import React, { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { AbsoluteTimeRange, GrafanaTheme2, LogsSortOrder } from '@grafana/data'; -import { reportInteraction } from '@grafana/runtime'; +import { config, reportInteraction } from '@grafana/runtime'; import { DataQuery, TimeZone } from '@grafana/schema'; import { Button, Icon, Spinner, useTheme2 } from '@grafana/ui'; import { TOP_BAR_LEVEL_HEIGHT } from 'app/core/components/AppChrome/types'; @@ -175,16 +175,20 @@ function LogsNavigation({ return (
- {oldestLogsFirst ? olderLogsButton : newerLogsButton} - - {oldestLogsFirst ? newerLogsButton : olderLogsButton} + {!config.featureToggles.logsInfiniteScrolling && ( + <> + {oldestLogsFirst ? olderLogsButton : newerLogsButton} + + {oldestLogsFirst ? newerLogsButton : olderLogsButton} + + )}