Loki: Show loki datasource stats in panel inspector (#24190)

* Loki: Show loki datasource stats in panel inspector

- puts the loki query result stats into the query results meta stat API
of Grafana, this allows the display of all backend loki stats in the
panel inspector in the dashboards
- added a hack to also display one of those values in Explore as a meta
label using the dataframe meta `custom` mechanims to point to a single
stat entry for each series which is then added together to show total
bytes processed across all query row results (this should be changed for
7.1 to make full use of the panel inspector in Explore)

* Fix test

* nicer stats labels for loki stats with units

public/app/core/logs_model.ts

@@ -32,6 +32,7 @@ import { getThemeColor } from 'app/core/utils/colors';
 
 import { sortInAscendingOrder, deduplicateLogRowsById } from 'app/core/utils/explore';
 import { getGraphSeriesModel } from 'app/plugins/panel/graph2/getGraphSeriesModel';
+import { decimalSIPrefix } from '@grafana/data/src/valueFormats/symbolFormatters';
 
 export const LogLevelColor = {
   [LogLevel.critical]: colors[7],
@@ -375,6 +376,26 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi
     });
   }
 
+  // Hack to print loki stats in Explore. Should be using proper stats display via drawer in Explore (rework in 7.1)
+  let totalBytes = 0;
+  for (const series of logSeries) {
+    const totalBytesKey = series.meta?.custom?.lokiQueryStatKey;
+    if (totalBytesKey && series.meta.stats) {
+      const byteStat = series.meta.stats.find(stat => stat.title === totalBytesKey);
+      if (byteStat) {
+        totalBytes += byteStat.value;
+      }
+    }
+  }
+  if (totalBytes > 0) {
+    const { text, suffix } = decimalSIPrefix('B')(totalBytes);
+    meta.push({
+      label: 'Total bytes processed',
+      value: `${text} ${suffix}`,
+      kind: LogsMetaKind.String,
+    });
+  }
+
   return {
     hasUniqueLabels,
     meta,

public/app/plugins/datasource/loki/result_transformer.test.ts

@@ -1,5 +1,5 @@
 import { CircularDataFrame, FieldCache, FieldType, MutableDataFrame } from '@grafana/data';
-import { LokiStreamResult, LokiTailResponse } from './types';
+import { LokiStreamResult, LokiTailResponse, LokiStreamResponse, LokiResultType } from './types';
 import * as ResultTransformer from './result_transformer';
 import { enhanceDataFrame } from './result_transformer';
 
@@ -18,6 +18,19 @@ const streamResult: LokiStreamResult[] = [
   },
 ];
 
+const lokiResponse: LokiStreamResponse = {
+  status: 'success',
+  data: {
+    result: streamResult,
+    resultType: LokiResultType.Stream,
+    stats: {
+      summary: {
+        bytesTotal: 900,
+      },
+    },
+  },
+};
+
 describe('loki result transformer', () => {
   afterAll(() => {
     jest.restoreAllMocks();
@@ -45,7 +58,7 @@ describe('loki result transformer', () => {
   describe('lokiStreamsToDataframes', () => {
     it('should enhance data frames', () => {
       jest.spyOn(ResultTransformer, 'enhanceDataFrame');
-      const dataFrames = ResultTransformer.lokiStreamsToDataframes(streamResult, { refId: 'B' }, 500, {
+      const dataFrames = ResultTransformer.lokiStreamsToDataframes(lokiResponse, { refId: 'B' }, 500, {
         derivedFields: [
           {
             matcherRegex: 'trace=(w+)',

public/app/plugins/datasource/loki/result_transformer.ts

@@ -14,6 +14,7 @@ import {
   DataFrameView,
   DataLink,
   Field,
+  QueryResultMetaStat,
 } from '@grafana/data';
 
 import templateSrv from 'app/features/templating/template_srv';
@@ -31,6 +32,8 @@ import {
   LokiQuery,
   LokiOptions,
   DerivedFieldConfig,
+  LokiStreamResponse,
+  LokiStats,
 } from './types';
 
 /**
@@ -257,13 +260,48 @@ function getOriginalMetricName(labelData: { [key: string]: string }) {
   return `${metricName}{${labelPart}}`;
 }
 
+export function decamelize(s: string): string {
+  return s.replace(/[A-Z]/g, m => ` ${m.toLowerCase()}`);
+}
+
+// Turn loki stats { metric: value } into meta stat { title: metric, value: value }
+function lokiStatsToMetaStat(stats: LokiStats): QueryResultMetaStat[] {
+  const result: QueryResultMetaStat[] = [];
+  if (!stats) {
+    return result;
+  }
+  for (const section in stats) {
+    const values = stats[section];
+    for (const label in values) {
+      const value = values[label];
+      let unit;
+      if (/time/i.test(label) && value) {
+        unit = 's';
+      } else if (/bytes.*persecond/i.test(label)) {
+        unit = 'Bps';
+      } else if (/bytes/i.test(label)) {
+        unit = 'decbytes';
+      }
+      const title = `${_.capitalize(section)}: ${decamelize(label)}`;
+      result.push({ title, value, unit });
+    }
+  }
+  return result;
+}
+
 export function lokiStreamsToDataframes(
-  data: LokiStreamResult[],
+  response: LokiStreamResponse,
   target: { refId: string; expr?: string; regexp?: string },
   limit: number,
   config: LokiOptions,
   reverse = false
 ): DataFrame[] {
+  const data = limit > 0 ? response.data.result : [];
+  const stats: QueryResultMetaStat[] = lokiStatsToMetaStat(response.data.stats);
+  // Use custom mechanism to identify which stat we want to promote to label
+  const custom = {
+    lokiQueryStatKey: 'Summary: totalBytesProcessed',
+  };
   const series: DataFrame[] = data.map(stream => {
     const dataFrame = lokiStreamResultToDataFrame(stream, reverse);
     enhanceDataFrame(dataFrame, config);
@@ -273,6 +311,8 @@ export function lokiStreamsToDataframes(
       meta: {
         searchWords: getHighlighterExpressionsFromQuery(formatQuery(target.expr, target.regexp)),
         limit,
+        stats,
+        custom,
       },
     };
   });
@@ -378,7 +418,7 @@ export function processRangeQueryResponse(
   switch (response.data.resultType) {
     case LokiResultType.Stream:
       return of({
-        data: lokiStreamsToDataframes(limit > 0 ? response.data.result : [], target, limit, config, reverse),
+        data: lokiStreamsToDataframes(response as LokiStreamResponse, target, limit, config, reverse),
         key: `${target.refId}_log`,
       });
 

public/app/plugins/datasource/loki/types.ts

@@ -39,6 +39,12 @@ export interface LokiOptions extends DataSourceJsonData {
   derivedFields?: DerivedFieldConfig[];
 }
 
+export interface LokiStats {
+  [component: string]: {
+    [label: string]: number;
+  };
+}
+
 export interface LokiVectorResult {
   metric: { [label: string]: string };
   value: [number, string];
@@ -49,6 +55,7 @@ export interface LokiVectorResponse {
   data: {
     resultType: LokiResultType.Vector;
     result: LokiVectorResult[];
+    stats?: LokiStats;
   };
 }
 
@@ -62,6 +69,7 @@ export interface LokiMatrixResponse {
   data: {
     resultType: LokiResultType.Matrix;
     result: LokiMatrixResult[];
+    stats?: LokiStats;
   };
 }
 
@@ -75,6 +83,7 @@ export interface LokiStreamResponse {
   data: {
     resultType: LokiResultType.Stream;
     result: LokiStreamResult[];
+    stats?: LokiStats;
   };
 }