SSE: Localize/Contain Errors within an Expression (#73163)

Changes SSE to not always fail all queries when one fails. Now only the query itself, and nodes that depend on it will error.
---------

Co-authored-by: Gilles De Mey <gilles.de.mey@gmail.com>

public/app/features/alerting/unified/components/expressions/Expression.tsx

@@ -4,7 +4,7 @@ import React, { FC, useCallback, useState } from 'react';
 
 import { DataFrame, dateTimeFormat, GrafanaTheme2, isTimeSeriesFrames, LoadingState, PanelData } from '@grafana/data';
 import { Stack } from '@grafana/experimental';
-import { AutoSizeInput, Button, clearButtonStyles, IconButton, useStyles2 } from '@grafana/ui';
+import { AutoSizeInput, Badge, Button, clearButtonStyles, IconButton, useStyles2 } from '@grafana/ui';
 import { ClassicConditions } from 'app/features/expressions/components/ClassicConditions';
 import { Math } from 'app/features/expressions/components/Math';
 import { Reduce } from 'app/features/expressions/components/Reduce';
@@ -302,6 +302,10 @@ const Header: FC<HeaderProps> = ({
           <div>{getExpressionLabel(queryType)}</div>
         </Stack>
         <Spacer />
+        {/* when we have an evaluation error, we show a badge next to "set as alert condition" */}
+        {!alertCondition && error && (
+          <Badge color="red" icon="exclamation-circle" text="Error" tooltip={error.message} />
+        )}
         <AlertConditionIndicator
           onSetCondition={() => onSetCondition(query.refId)}
           enabled={alertCondition}

public/app/features/alerting/unified/components/rule-editor/ExpressionsEditor.tsx

@@ -9,7 +9,7 @@ import { AlertQuery } from 'app/types/unified-alerting-dto';
 
 import { Expression } from '../expressions/Expression';
 
-import { errorFromSeries, warningFromSeries } from './util';
+import { errorFromPreviewData, warningFromSeries } from './util';
 
 interface Props {
   condition: string | null;
@@ -45,7 +45,7 @@ export const ExpressionsEditor = ({
         const data = panelData[query.refId];
 
         const isAlertCondition = condition === query.refId;
-        const error = isAlertCondition && data ? errorFromSeries(data.series) : undefined;
+        const error = data ? errorFromPreviewData(data) : undefined;
         const warning = isAlertCondition && data ? warningFromSeries(data.series) : undefined;
 
         return (

public/app/features/alerting/unified/components/rule-editor/QueryRows.tsx

@@ -18,7 +18,7 @@ import { getDatasourceSrv } from 'app/features/plugins/datasource_srv';
 import { AlertDataQuery, AlertQuery } from 'app/types/unified-alerting-dto';
 
 import { AlertQueryOptions, EmptyQueryWrapper, QueryWrapper } from './QueryWrapper';
-import { errorFromSeries, getThresholdsForQueries } from './util';
+import { errorFromPreviewData, getThresholdsForQueries } from './util';
 
 interface Props {
   // The query configuration
@@ -161,9 +161,7 @@ export class QueryRows extends PureComponent<Props> {
                       state: LoadingState.NotStarted,
                     };
                     const dsSettings = this.getDataSourceSettings(query);
-
-                    const isAlertCondition = this.props.condition === query.refId;
-                    const error = isAlertCondition ? errorFromSeries(data.series) : undefined;
+                    const error = data ? errorFromPreviewData(data) : undefined;
 
                     if (!dsSettings) {
                       return (

public/app/features/alerting/unified/components/rule-editor/QueryWrapper.tsx

@@ -14,7 +14,7 @@ import {
 } from '@grafana/data';
 import { Stack } from '@grafana/experimental';
 import { DataQuery } from '@grafana/schema';
-import { GraphTresholdsStyleMode, Icon, InlineField, Input, Tooltip, useStyles2 } from '@grafana/ui';
+import { Badge, GraphTresholdsStyleMode, Icon, InlineField, Input, Tooltip, useStyles2 } from '@grafana/ui';
 import { QueryEditorRow } from 'app/features/query/components/QueryEditorRow';
 import { AlertQuery } from 'app/types/unified-alerting-dto';
 
@@ -119,6 +119,8 @@ export const QueryWrapper = ({
       minInterval: queryOptions.minInterval,
     };
 
+    const isAlertCondition = condition === query.refId;
+
     return (
       <Stack direction="row" alignItems="baseline" gap={1}>
         <SelectingDataSourceTooltip />
@@ -132,9 +134,12 @@ export const QueryWrapper = ({
 
         <AlertConditionIndicator
           onSetCondition={() => onSetCondition(query.refId)}
-          enabled={condition === query.refId}
+          enabled={isAlertCondition}
           error={error}
         />
+        {!isAlertCondition && error && (
+          <Badge color="red" icon="exclamation-circle" text="Error" tooltip={error.message} />
+        )}
       </Stack>
     );
   }

public/app/features/alerting/unified/components/rule-editor/util.ts

@@ -110,6 +110,16 @@ export function errorFromSeries(series: DataFrame[]): Error | undefined {
   return error;
 }
 
+export function errorFromPreviewData(data: PanelData): Error | undefined {
+  // give preference to QueryErrors
+  if (data.errors?.length) {
+    return new Error(data.errors[0].message);
+  }
+
+  // if none, return errors from series
+  return errorFromSeries(data.series);
+}
+
 export function warningFromSeries(series: DataFrame[]): Error | undefined {
   const notices = series[0]?.meta?.notices ?? [];
   const warning = notices.find((notice) => notice.severity === 'warning')?.text;

public/app/features/alerting/unified/state/AlertingQueryRunner.test.ts

@@ -45,6 +45,7 @@ describe('AlertingQueryRunner', () => {
         A: {
           annotations: [],
           state: LoadingState.Done,
+          errors: [],
           series: [
             expectDataFrameWithValues({
               time: [1620051612238, 1620051622238, 1620051632238],
@@ -60,6 +61,7 @@ describe('AlertingQueryRunner', () => {
         B: {
           annotations: [],
           state: LoadingState.Done,
+          errors: [],
           series: [
             expectDataFrameWithValues({
               time: [1620051612238, 1620051622238],
@@ -136,6 +138,7 @@ describe('AlertingQueryRunner', () => {
         A: {
           annotations: [],
           state: LoadingState.Done,
+          errors: [],
           series: [
             expectDataFrameWithValues({
               time: [1620051612238, 1620051622238, 1620051632238],
@@ -151,6 +154,7 @@ describe('AlertingQueryRunner', () => {
         B: {
           annotations: [],
           state: LoadingState.Done,
+          errors: [],
           series: [
             expectDataFrameWithValues({
               time: [1620051612238, 1620051622238],

public/app/features/alerting/unified/state/AlertingQueryRunner.ts

@@ -24,6 +24,8 @@ import { AlertQuery } from 'app/types/unified-alerting-dto';
 import { getTimeRangeForExpression } from '../utils/timeRange';
 
 export interface AlertingQueryResult {
+  error?: string;
+  status?: number; // HTTP status error
   frames: DataFrameJSON[];
 }
 
@@ -183,10 +185,16 @@ const mapToPanelData = (
     const results: Record<string, PanelData> = {};
 
     for (const [refId, result] of Object.entries(data.results)) {
+      const { error, status, frames = [] } = result;
+
+      // extract errors from the /eval results
+      const errors = error ? [{ message: error, refId, status }] : [];
+
       results[refId] = {
+        errors,
         timeRange: dataByQuery[refId].timeRange,
         state: LoadingState.Done,
-        series: result.frames.map(dataFrameFromJSON),
+        series: frames.map(dataFrameFromJSON),
       };
     }