Logs: Add pagination range to prevent pagination from skipping entries

This commit is contained in:
Ivana Huckova 2022-04-28 12:31:05 +02:00
parent ab144bf05e
commit 2e7a6b3870
8 changed files with 38 additions and 27 deletions

View File

@ -85,7 +85,8 @@ export interface LogsModel {
meta?: LogsMetaItem[];
rows: LogRowModel[];
series?: DataFrame[];
visibleRange?: AbsoluteTimeRange;
histogramRange?: AbsoluteTimeRange;
logsRange?: AbsoluteTimeRange;
queries?: DataQuery[];
}

View File

@ -1078,7 +1078,7 @@ describe('getSeriesProperties()', () => {
const range = { from: 10, to: 20 };
const result = getSeriesProperties(rows, 1, range, 2, 1);
expect(result.bucketSize).toBe(2);
expect(result.visibleRange).toMatchObject(range);
expect(result.histogramRange).toMatchObject(range);
});
it('clamps the range and adjusts the bucketSize if the logs row times do not completely cover the given range', () => {
@ -1088,10 +1088,10 @@ describe('getSeriesProperties()', () => {
] as any;
const range = { from: 0, to: 30 };
const result = getSeriesProperties(rows, 3, range, 2, 1);
// Bucketsize 6 gets shortened to 4 because of new visible range is 20ms vs original range being 30ms
// Bucket size 6 gets shortened to 4 because of new visible range is 20ms vs original range being 30ms
expect(result.bucketSize).toBe(4);
// From time is also aligned to bucketSize (divisible by 4)
expect(result.visibleRange).toMatchObject({ from: 8, to: 30 });
expect(result.histogramRange).toMatchObject({ from: 8, to: 30 });
});
});

View File

@ -207,12 +207,13 @@ export function dataFrameToLogsModel(
// Create histogram metrics from logs using the interval as bucket size for the line count
if (intervalMs && logsModel.rows.length > 0) {
const sortedRows = logsModel.rows.sort(sortInAscendingOrder);
const { visibleRange, bucketSize, visibleRangeMs, requestedRangeMs } = getSeriesProperties(
const { histogramRange, bucketSize, visibleRangeMs, requestedRangeMs } = getSeriesProperties(
sortedRows,
intervalMs,
absoluteRange
);
logsModel.visibleRange = visibleRange;
logsModel.histogramRange = histogramRange;
logsModel.logsRange = { from: sortedRows[0].timeEpochMs, to: sortedRows[sortedRows.length - 1].timeEpochMs };
logsModel.series = makeDataFramesForLogs(sortedRows, bucketSize);
if (logsModel.meta) {
@ -249,7 +250,7 @@ export function getSeriesProperties(
pxPerBar = 20,
minimumBucketSize = 1000
) {
let visibleRange = absoluteRange;
let histogramRange = absoluteRange;
let resolutionIntervalMs = intervalMs;
let bucketSize = Math.max(resolutionIntervalMs * pxPerBar, minimumBucketSize);
let visibleRangeMs;
@ -265,18 +266,18 @@ export function getSeriesProperties(
// Adjust interval bucket size for potentially shorter visible range
const clampingFactor = visibleRangeMs / requestedRangeMs;
resolutionIntervalMs *= clampingFactor;
// Minimum bucketsize of 1s for nicer graphing
// Minimum bucket size of 1s for nicer graphing
bucketSize = Math.max(Math.ceil(resolutionIntervalMs * pxPerBar), minimumBucketSize);
// makeSeriesForLogs() aligns dataspoints with time buckets, so we do the same here to not cut off data
// makeSeriesForLogs() aligns datapoints with time buckets, so we do the same here to not cut off data
const adjustedEarliest = Math.floor(earliestTsLogs / bucketSize) * bucketSize;
visibleRange = { from: adjustedEarliest, to: absoluteRange.to };
histogramRange = { from: adjustedEarliest, to: absoluteRange.to };
} else {
// We use visibleRangeMs to calculate range coverage of received logs. However, some data sources are rounding up range in requests. This means that received logs
// can (in edge cases) be outside of the requested range and visibleRangeMs < 0. In that case, we want to change visibleRangeMs to be 1 so we can calculate coverage.
visibleRangeMs = 1;
}
}
return { bucketSize, visibleRange, visibleRangeMs, requestedRangeMs };
return { bucketSize, histogramRange, visibleRangeMs, requestedRangeMs };
}
function separateLogsAndMetrics(dataFrames: DataFrame[]) {

View File

@ -55,7 +55,8 @@ interface Props extends Themeable2 {
logsMeta?: LogsMetaItem[];
logsSeries?: DataFrame[];
logsQueries?: DataQuery[];
visibleRange?: AbsoluteTimeRange;
histogramRange?: AbsoluteTimeRange;
logsRange?: AbsoluteTimeRange;
theme: GrafanaTheme2;
loading: boolean;
loadingState: LoadingState;
@ -257,7 +258,8 @@ class UnthemedLogs extends PureComponent<Props, State> {
logRows,
logsMeta,
logsSeries,
visibleRange,
histogramRange,
logsRange,
loading = false,
loadingState,
onClickFilterLabel,
@ -311,7 +313,7 @@ class UnthemedLogs extends PureComponent<Props, State> {
height={150}
width={width}
tooltipDisplayMode={TooltipDisplayMode.Multi}
absoluteRange={visibleRange || absoluteRange}
absoluteRange={histogramRange || absoluteRange}
timeZone={timeZone}
loadingState={loadingState}
onChangeTime={onChangeTime}
@ -430,7 +432,7 @@ class UnthemedLogs extends PureComponent<Props, State> {
</div>
<LogsNavigation
logsSortOrder={logsSortOrder}
visibleRange={visibleRange ?? absoluteRange}
logsRange={logsRange ?? absoluteRange}
absoluteRange={absoluteRange}
timeZone={timeZone}
onChangeTime={onChangeTime}

View File

@ -82,7 +82,8 @@ class LogsContainer extends PureComponent<LogsContainerProps> {
onStopScanning,
absoluteRange,
timeZone,
visibleRange,
histogramRange,
logsRange,
scanning,
range,
width,
@ -141,7 +142,8 @@ class LogsContainer extends PureComponent<LogsContainerProps> {
onStartScanning={onStartScanning}
onStopScanning={onStopScanning}
absoluteRange={absoluteRange}
visibleRange={visibleRange}
histogramRange={histogramRange}
logsRange={logsRange}
timeZone={timeZone}
scanning={scanning}
scanRange={range.raw}
@ -182,7 +184,8 @@ function mapStateToProps(state: StoreState, { exploreId }: { exploreId: string }
logsMeta: logsResult?.meta,
logsSeries: logsResult?.series,
logsQueries: logsResult?.queries,
visibleRange: logsResult?.visibleRange,
histogramRange: logsResult?.histogramRange,
logsRange: logsResult?.logsRange,
scanning,
timeZone,
datasourceInstance,

View File

@ -12,7 +12,7 @@ const defaultProps: LogsNavigationProps = {
queries: [],
loading: false,
logsSortOrder: undefined,
visibleRange: { from: 1637322959000, to: 1637322981811 },
logsRange: { from: 1637322959000, to: 1637322981811 },
onChangeTime: jest.fn(),
scrollToTopLogs: jest.fn(),
addResultsToCache: jest.fn(),
@ -77,7 +77,7 @@ describe('LogsNavigation', () => {
<LogsNavigation
{...defaultProps}
absoluteRange={{ from: 1637319359000, to: 1637322959000 }}
visibleRange={{ from: 1637322938000, to: 1637322959000 }}
logsRange={{ from: 1637322938000, to: 1637322959000 }}
onChangeTime={onChangeTimeMock}
logsSortOrder={LogsSortOrder.Ascending}
/>

View File

@ -12,7 +12,7 @@ type Props = {
timeZone: TimeZone;
queries: DataQuery[];
loading: boolean;
visibleRange: AbsoluteTimeRange;
logsRange: AbsoluteTimeRange;
logsSortOrder?: LogsSortOrder | null;
onChangeTime: (range: AbsoluteTimeRange) => void;
scrollToTopLogs: () => void;
@ -32,7 +32,7 @@ function LogsNavigation({
loading,
onChangeTime,
scrollToTopLogs,
visibleRange,
logsRange,
queries,
clearCache,
addResultsToCache,
@ -43,7 +43,7 @@ function LogsNavigation({
// These refs are to determine, if we want to clear up logs navigation when totally new query is run
const expectedQueriesRef = useRef<DataQuery[]>();
const expectedRangeRef = useRef<AbsoluteTimeRange>();
// This ref is to store range span for future queres based on firstly selected time range
// This ref is to store range span for future queries based on firstly selected time range
// e.g. if last 5 min selected, always run 5 min range
const rangeSpanRef = useRef(0);
@ -55,7 +55,7 @@ function LogsNavigation({
// Main effect to set pages and index
useEffect(() => {
const newPage = { logsRange: visibleRange, queryRange: absoluteRange };
const newPage = { logsRange, queryRange: absoluteRange };
let newPages: LogsPage[] = [];
// We want to start new pagination if queries change or if absolute range is different than expected
if (!isEqual(expectedRangeRef.current, absoluteRange) || !isEqual(expectedQueriesRef.current, queries)) {
@ -80,7 +80,7 @@ function LogsNavigation({
setCurrentPageIndex(index);
}
addResultsToCache();
}, [visibleRange, absoluteRange, logsSortOrder, queries, clearCache, addResultsToCache]);
}, [logsRange, absoluteRange, logsSortOrder, queries, clearCache, addResultsToCache]);
useEffect(() => {
clearCache();
@ -115,7 +115,7 @@ function LogsNavigation({
});
} else {
//If we are on the last page, create new range
changeTime({ from: visibleRange.from - rangeSpanRef.current, to: visibleRange.from });
changeTime({ from: logsRange.from - rangeSpanRef.current, to: logsRange.from });
}
}}
disabled={loading}

View File

@ -387,7 +387,11 @@ describe('decorateWithLogsResult', () => {
],
},
],
visibleRange: undefined,
histogramRange: undefined,
logsRange: {
from: 100,
to: 100,
},
});
});