Logs: Add pagination range to prevent pagination from skipping entries

packages/grafana-data/src/types/logs.ts

@@ -85,7 +85,8 @@ export interface LogsModel {
   meta?: LogsMetaItem[];
   rows: LogRowModel[];
   series?: DataFrame[];
-  visibleRange?: AbsoluteTimeRange;
+  histogramRange?: AbsoluteTimeRange;
+  logsRange?: AbsoluteTimeRange;
   queries?: DataQuery[];
 }
 

public/app/core/logs_model.test.ts

@@ -1078,7 +1078,7 @@ describe('getSeriesProperties()', () => {
     const range = { from: 10, to: 20 };
     const result = getSeriesProperties(rows, 1, range, 2, 1);
     expect(result.bucketSize).toBe(2);
-    expect(result.visibleRange).toMatchObject(range);
+    expect(result.histogramRange).toMatchObject(range);
   });
 
   it('clamps the range and adjusts the bucketSize if the logs row times do not completely cover the given range', () => {
@@ -1088,10 +1088,10 @@ describe('getSeriesProperties()', () => {
     ] as any;
     const range = { from: 0, to: 30 };
     const result = getSeriesProperties(rows, 3, range, 2, 1);
-    // Bucketsize 6 gets shortened to 4 because of new visible range is 20ms vs original range being 30ms
+    // Bucket size 6 gets shortened to 4 because of new visible range is 20ms vs original range being 30ms
     expect(result.bucketSize).toBe(4);
     // From time is also aligned to bucketSize (divisible by 4)
-    expect(result.visibleRange).toMatchObject({ from: 8, to: 30 });
+    expect(result.histogramRange).toMatchObject({ from: 8, to: 30 });
   });
 });
 

public/app/core/logs_model.ts

@@ -207,12 +207,13 @@ export function dataFrameToLogsModel(
     // Create histogram metrics from logs using the interval as bucket size for the line count
     if (intervalMs && logsModel.rows.length > 0) {
       const sortedRows = logsModel.rows.sort(sortInAscendingOrder);
-      const { visibleRange, bucketSize, visibleRangeMs, requestedRangeMs } = getSeriesProperties(
+      const { histogramRange, bucketSize, visibleRangeMs, requestedRangeMs } = getSeriesProperties(
         sortedRows,
         intervalMs,
         absoluteRange
       );
-      logsModel.visibleRange = visibleRange;
+      logsModel.histogramRange = histogramRange;
+      logsModel.logsRange = { from: sortedRows[0].timeEpochMs, to: sortedRows[sortedRows.length - 1].timeEpochMs };
       logsModel.series = makeDataFramesForLogs(sortedRows, bucketSize);
 
       if (logsModel.meta) {
@@ -249,7 +250,7 @@ export function getSeriesProperties(
   pxPerBar = 20,
   minimumBucketSize = 1000
 ) {
-  let visibleRange = absoluteRange;
+  let histogramRange = absoluteRange;
   let resolutionIntervalMs = intervalMs;
   let bucketSize = Math.max(resolutionIntervalMs * pxPerBar, minimumBucketSize);
   let visibleRangeMs;
@@ -265,18 +266,18 @@ export function getSeriesProperties(
       // Adjust interval bucket size for potentially shorter visible range
       const clampingFactor = visibleRangeMs / requestedRangeMs;
       resolutionIntervalMs *= clampingFactor;
-      // Minimum bucketsize of 1s for nicer graphing
+      // Minimum bucket size of 1s for nicer graphing
       bucketSize = Math.max(Math.ceil(resolutionIntervalMs * pxPerBar), minimumBucketSize);
-      // makeSeriesForLogs() aligns dataspoints with time buckets, so we do the same here to not cut off data
+      // makeSeriesForLogs() aligns datapoints with time buckets, so we do the same here to not cut off data
       const adjustedEarliest = Math.floor(earliestTsLogs / bucketSize) * bucketSize;
-      visibleRange = { from: adjustedEarliest, to: absoluteRange.to };
+      histogramRange = { from: adjustedEarliest, to: absoluteRange.to };
     } else {
       // We use visibleRangeMs to calculate range coverage of received logs. However, some data sources are rounding up range in requests. This means that received logs
       // can (in edge cases) be outside of the requested range and visibleRangeMs < 0. In that case, we want to change visibleRangeMs to be 1 so we can calculate coverage.
       visibleRangeMs = 1;
     }
   }
-  return { bucketSize, visibleRange, visibleRangeMs, requestedRangeMs };
+  return { bucketSize, histogramRange, visibleRangeMs, requestedRangeMs };
 }
 
 function separateLogsAndMetrics(dataFrames: DataFrame[]) {

public/app/features/explore/Logs.tsx

@@ -55,7 +55,8 @@ interface Props extends Themeable2 {
   logsMeta?: LogsMetaItem[];
   logsSeries?: DataFrame[];
   logsQueries?: DataQuery[];
-  visibleRange?: AbsoluteTimeRange;
+  histogramRange?: AbsoluteTimeRange;
+  logsRange?: AbsoluteTimeRange;
   theme: GrafanaTheme2;
   loading: boolean;
   loadingState: LoadingState;
@@ -257,7 +258,8 @@ class UnthemedLogs extends PureComponent<Props, State> {
       logRows,
       logsMeta,
       logsSeries,
-      visibleRange,
+      histogramRange,
+      logsRange,
       loading = false,
       loadingState,
       onClickFilterLabel,
@@ -311,7 +313,7 @@ class UnthemedLogs extends PureComponent<Props, State> {
               height={150}
               width={width}
               tooltipDisplayMode={TooltipDisplayMode.Multi}
-              absoluteRange={visibleRange || absoluteRange}
+              absoluteRange={histogramRange || absoluteRange}
               timeZone={timeZone}
               loadingState={loadingState}
               onChangeTime={onChangeTime}
@@ -430,7 +432,7 @@ class UnthemedLogs extends PureComponent<Props, State> {
           </div>
           <LogsNavigation
             logsSortOrder={logsSortOrder}
-            visibleRange={visibleRange ?? absoluteRange}
+            logsRange={logsRange ?? absoluteRange}
             absoluteRange={absoluteRange}
             timeZone={timeZone}
             onChangeTime={onChangeTime}

public/app/features/explore/LogsContainer.tsx

@@ -82,7 +82,8 @@ class LogsContainer extends PureComponent<LogsContainerProps> {
       onStopScanning,
       absoluteRange,
       timeZone,
-      visibleRange,
+      histogramRange,
+      logsRange,
       scanning,
       range,
       width,
@@ -141,7 +142,8 @@ class LogsContainer extends PureComponent<LogsContainerProps> {
               onStartScanning={onStartScanning}
               onStopScanning={onStopScanning}
               absoluteRange={absoluteRange}
-              visibleRange={visibleRange}
+              histogramRange={histogramRange}
+              logsRange={logsRange}
               timeZone={timeZone}
               scanning={scanning}
               scanRange={range.raw}
@@ -182,7 +184,8 @@ function mapStateToProps(state: StoreState, { exploreId }: { exploreId: string }
     logsMeta: logsResult?.meta,
     logsSeries: logsResult?.series,
     logsQueries: logsResult?.queries,
-    visibleRange: logsResult?.visibleRange,
+    histogramRange: logsResult?.histogramRange,
+    logsRange: logsResult?.logsRange,
     scanning,
     timeZone,
     datasourceInstance,

public/app/features/explore/LogsNavigation.test.tsx

@@ -12,7 +12,7 @@ const defaultProps: LogsNavigationProps = {
   queries: [],
   loading: false,
   logsSortOrder: undefined,
-  visibleRange: { from: 1637322959000, to: 1637322981811 },
+  logsRange: { from: 1637322959000, to: 1637322981811 },
   onChangeTime: jest.fn(),
   scrollToTopLogs: jest.fn(),
   addResultsToCache: jest.fn(),
@@ -77,7 +77,7 @@ describe('LogsNavigation', () => {
       <LogsNavigation
         {...defaultProps}
         absoluteRange={{ from: 1637319359000, to: 1637322959000 }}
-        visibleRange={{ from: 1637322938000, to: 1637322959000 }}
+        logsRange={{ from: 1637322938000, to: 1637322959000 }}
         onChangeTime={onChangeTimeMock}
         logsSortOrder={LogsSortOrder.Ascending}
       />

public/app/features/explore/LogsNavigation.tsx

@@ -12,7 +12,7 @@ type Props = {
   timeZone: TimeZone;
   queries: DataQuery[];
   loading: boolean;
-  visibleRange: AbsoluteTimeRange;
+  logsRange: AbsoluteTimeRange;
   logsSortOrder?: LogsSortOrder | null;
   onChangeTime: (range: AbsoluteTimeRange) => void;
   scrollToTopLogs: () => void;
@@ -32,7 +32,7 @@ function LogsNavigation({
   loading,
   onChangeTime,
   scrollToTopLogs,
-  visibleRange,
+  logsRange,
   queries,
   clearCache,
   addResultsToCache,
@@ -43,7 +43,7 @@ function LogsNavigation({
   // These refs are to determine, if we want to clear up logs navigation when totally new query is run
   const expectedQueriesRef = useRef<DataQuery[]>();
   const expectedRangeRef = useRef<AbsoluteTimeRange>();
-  // This ref is to store range span for future queres based on firstly selected time range
+  // This ref is to store range span for future queries based on firstly selected time range
   // e.g. if last 5 min selected, always run 5 min range
   const rangeSpanRef = useRef(0);
 
@@ -55,7 +55,7 @@ function LogsNavigation({
 
   // Main effect to set pages and index
   useEffect(() => {
-    const newPage = { logsRange: visibleRange, queryRange: absoluteRange };
+    const newPage = { logsRange, queryRange: absoluteRange };
     let newPages: LogsPage[] = [];
     // We want to start new pagination if queries change or if absolute range is different than expected
     if (!isEqual(expectedRangeRef.current, absoluteRange) || !isEqual(expectedQueriesRef.current, queries)) {
@@ -80,7 +80,7 @@ function LogsNavigation({
       setCurrentPageIndex(index);
     }
     addResultsToCache();
-  }, [visibleRange, absoluteRange, logsSortOrder, queries, clearCache, addResultsToCache]);
+  }, [logsRange, absoluteRange, logsSortOrder, queries, clearCache, addResultsToCache]);
 
   useEffect(() => {
     clearCache();
@@ -115,7 +115,7 @@ function LogsNavigation({
           });
         } else {
           //If we are on the last page, create new range
-          changeTime({ from: visibleRange.from - rangeSpanRef.current, to: visibleRange.from });
+          changeTime({ from: logsRange.from - rangeSpanRef.current, to: logsRange.from });
         }
       }}
       disabled={loading}

public/app/features/explore/utils/decorators.test.ts

@@ -387,7 +387,11 @@ describe('decorateWithLogsResult', () => {
           ],
         },
       ],
-      visibleRange: undefined,
+      histogramRange: undefined,
+      logsRange: {
+        from: 100,
+        to: 100,
+      },
     });
   });