Alerting: Allow executing "hidden" queries (#70064)

.betterer.results

@@ -1905,8 +1905,7 @@ exports[`better eslint`] = {
       [0, 0, 0, "Unexpected any. Specify a different type.", "1"]
     ],
     "public/app/features/alerting/unified/components/rule-editor/AlertRuleForm.tsx:5381": [
-      [0, 0, 0, "Do not use any type assertions.", "0"],
-      [0, 0, 0, "Do not use any type assertions.", "1"]
+      [0, 0, 0, "Do not use any type assertions.", "0"]
     ],
     "public/app/features/alerting/unified/components/rule-editor/AnnotationKeyInput.tsx:5381": [
       [0, 0, 0, "Do not use any type assertions.", "0"]

public/app/features/alerting/unified/__snapshots__/PanelAlertTabContent.test.tsx.snap

@@ -67,7 +67,6 @@ exports[`PanelAlertTabContent Will render alerts belonging to panel and a button
           "uid": "__expr__",
         },
         "expression": "A",
-        "hide": false,
         "reducer": "last",
         "refId": "B",
         "type": "reduce",
@@ -106,7 +105,6 @@ exports[`PanelAlertTabContent Will render alerts belonging to panel and a button
           "uid": "__expr__",
         },
         "expression": "B",
-        "hide": false,
         "refId": "C",
         "type": "threshold",
       },

public/app/features/alerting/unified/components/rule-editor/AlertRuleForm.tsx

@@ -1,4 +1,5 @@
 import { css } from '@emotion/css';
+import { omit } from 'lodash';
 import React, { useEffect, useMemo, useState } from 'react';
 import { DeepMap, FieldError, FormProvider, useForm, useFormContext, UseFormWatch } from 'react-hook-form';
 import { Link, useParams } from 'react-router-dom';
@@ -12,6 +13,7 @@ import { useCleanup } from 'app/core/hooks/useCleanup';
 import { useQueryParams } from 'app/core/hooks/useQueryParams';
 import { useDispatch } from 'app/types';
 import { RuleWithLocation } from 'app/types/unified-alerting';
+import { RulerRuleDTO } from 'app/types/unified-alerting-dto';
 
 import { LogMessages, trackNewAlerRuleFormError } from '../../Analytics';
 import { useUnifiedAlertingSelector } from '../../hooks/useUnifiedAlertingSelector';
@@ -90,21 +92,21 @@ export const AlertRuleForm = ({ existing, prefill }: Props) => {
 
   const defaultValues: RuleFormValues = useMemo(() => {
     if (existing) {
-      return rulerRuleToFormValues(existing);
+      return formValuesFromExistingRule(existing);
     }
 
     if (prefill) {
-      return {
-        ...getDefaultFormValues(),
-        ...prefill,
-      };
+      return formValuesFromPrefill(prefill);
+    }
+
+    if (typeof queryParams['defaults'] === 'string') {
+      return formValuesFromQueryParams(queryParams['defaults'], ruleType);
     }
 
     return {
       ...getDefaultFormValues(),
-      queries: getDefaultQueries(),
       condition: 'C',
-      ...(queryParams['defaults'] ? JSON.parse(queryParams['defaults'] as string) : {}),
+      queries: getDefaultQueries(),
       type: ruleType || RuleFormType.grafana,
       evaluateEvery: evaluateEvery,
     };
@@ -281,6 +283,49 @@ const isCortexLokiOrRecordingRule = (watch: UseFormWatch<RuleFormValues>) => {
   return (ruleType === RuleFormType.cloudAlerting || ruleType === RuleFormType.cloudRecording) && dataSourceName !== '';
 };
 
+// the backend will always execute "hidden" queries, so we have no choice but to remove the property in the front-end
+// to avoid confusion. The query editor shows them as "disabled" and that's a different semantic meaning.
+// furthermore the "AlertingQueryRunner" calls `filterQuery` on each data source and those will skip running queries that are "hidden"."
+// It seems like we have no choice but to act like "hidden" queries don't exist in alerting.
+const ignoreHiddenQueries = (ruleDefinition: RuleFormValues): RuleFormValues => {
+  return {
+    ...ruleDefinition,
+    queries: ruleDefinition.queries?.map((query) => omit(query, 'model.hide')),
+  };
+};
+
+function formValuesFromQueryParams(ruleDefinition: string, type: RuleFormType): RuleFormValues {
+  let ruleFromQueryParams: Partial<RuleFormValues>;
+
+  try {
+    ruleFromQueryParams = JSON.parse(ruleDefinition);
+  } catch (err) {
+    return {
+      ...getDefaultFormValues(),
+      queries: getDefaultQueries(),
+    };
+  }
+
+  return ignoreHiddenQueries({
+    ...getDefaultFormValues(),
+    ...ruleFromQueryParams,
+    queries: ruleFromQueryParams.queries ?? getDefaultQueries(),
+    type: type || RuleFormType.grafana,
+    evaluateEvery: MINUTE,
+  });
+}
+
+function formValuesFromPrefill(rule: Partial<RuleFormValues>): RuleFormValues {
+  return ignoreHiddenQueries({
+    ...getDefaultFormValues(),
+    ...rule,
+  });
+}
+
+function formValuesFromExistingRule(rule: RuleWithLocation<RulerRuleDTO>) {
+  return ignoreHiddenQueries(rulerRuleToFormValues(rule));
+}
+
 const getStyles = (theme: GrafanaTheme2) => {
   return {
     buttonSpinner: css`

public/app/features/alerting/unified/components/rule-editor/QueryRows.tsx

@@ -3,6 +3,7 @@ import React, { PureComponent, useState } from 'react';
 import { DragDropContext, Droppable, DropResult } from 'react-beautiful-dnd';
 
 import { DataQuery, DataSourceInstanceSettings, LoadingState, PanelData, RelativeTimeRange } from '@grafana/data';
+import { Stack } from '@grafana/experimental';
 import { getDataSourceSrv } from '@grafana/runtime';
 import { Button, Card, Icon } from '@grafana/ui';
 import { QueryOperationRow } from 'app/core/components/QueryOperationRow/QueryOperationRow';
@@ -142,6 +143,7 @@ export class QueryRows extends PureComponent<Props> {
           {(provided) => {
             return (
               <div ref={provided.innerRef} {...provided.droppableProps}>
+                <Stack direction="column">
                   {queries.map((query, index) => {
                     const data: PanelData = this.props.data?.[query.refId] ?? {
                       series: [],
@@ -195,6 +197,7 @@ export class QueryRows extends PureComponent<Props> {
                     );
                   })}
                   {provided.placeholder}
+                </Stack>
               </div>
             );
           }}

public/app/features/alerting/unified/components/rule-editor/QueryWrapper.tsx

@@ -4,7 +4,6 @@ import React, { ChangeEvent, useState } from 'react';
 
 import {
   CoreApp,
-  DataQuery,
   DataSourceApi,
   DataSourceInstanceSettings,
   getDefaultRelativeTimeRange,
@@ -15,6 +14,7 @@ import {
   ThresholdsConfig,
 } from '@grafana/data';
 import { Stack } from '@grafana/experimental';
+import { DataQuery } from '@grafana/schema';
 import {
   GraphTresholdsStyleMode,
   Icon,
@@ -82,6 +82,11 @@ export const QueryWrapper = ({
   const [dsInstance, setDsInstance] = useState<DataSourceApi>();
   const defaults = dsInstance?.getDefaultQuery ? dsInstance.getDefaultQuery(CoreApp.UnifiedAlerting) : {};
 
+  const queryWithDefaults = {
+    ...defaults,
+    ...cloneDeep(query.model),
+  };
+
   function SelectingDataSourceTooltip() {
     const styles = useStyles2(getStyles);
     return (
@@ -139,6 +144,8 @@ export const QueryWrapper = ({
     );
   }
 
+  const showVizualisation = data.state !== LoadingState.NotStarted;
+
   return (
     <Stack direction="column" gap={0.5}>
       <div className={styles.wrapper}>
@@ -151,10 +158,7 @@ export const QueryWrapper = ({
           index={index}
           key={query.refId}
           data={data}
-          query={{
-            ...defaults,
-            ...cloneDeep(query.model),
-          }}
+          query={queryWithDefaults}
           onChange={(query) => onChangeQuery(query, index)}
           onRemoveQuery={onRemoveQuery}
           onAddQuery={() => onDuplicateQuery(cloneDeep(query))}
@@ -165,7 +169,7 @@ export const QueryWrapper = ({
           hideDisableQuery={true}
         />
       </div>
-      {data.state !== LoadingState.NotStarted && (
+      {showVizualisation && (
         <VizWrapper
           data={data}
           thresholds={thresholds}

public/app/features/alerting/unified/state/AlertingQueryRunner.test.ts

@@ -1,3 +1,4 @@
+import { defaultsDeep } from 'lodash';
 import { Observable, of, throwError } from 'rxjs';
 import { delay, take } from 'rxjs/operators';
 import { createFetchResponse } from 'test/helpers/createFetchResponse';
@@ -14,7 +15,7 @@ import {
 } from '@grafana/data';
 import { DataSourceSrv, FetchResponse } from '@grafana/runtime';
 import { BackendSrv } from 'app/core/services/backend_srv';
-import { AlertQuery } from 'app/types/unified-alerting-dto';
+import { AlertDataQuery, AlertQuery } from 'app/types/unified-alerting-dto';
 
 import { AlertingQueryResponse, AlertingQueryRunner } from './AlertingQueryRunner';
 
@@ -188,7 +189,7 @@ describe('AlertingQueryRunner', () => {
     });
   });
 
-  it('should not execute if a query fails filterQuery check', async () => {
+  it('should not execute if all queries fail filterQuery check', async () => {
     const runner = new AlertingQueryRunner(
       mockBackendSrv({
         fetch: () => throwError(new Error("shouldn't happen")),
@@ -209,6 +210,39 @@ describe('AlertingQueryRunner', () => {
       expect(data.B.series).toHaveLength(0);
     });
   });
+
+  it('should skip hidden queries', async () => {
+    const results = createFetchResponse<AlertingQueryResponse>({
+      results: {
+        B: { frames: [createDataFrameJSON([1, 2, 3])] },
+      },
+    });
+
+    const runner = new AlertingQueryRunner(
+      mockBackendSrv({
+        fetch: () => of(results),
+      }),
+      mockDataSourceSrv({ filterQuery: (model: AlertDataQuery) => model.hide !== true })
+    );
+
+    const data = runner.get();
+    runner.run([
+      createQuery('A', {
+        model: {
+          refId: 'A',
+          hide: true,
+        },
+      }),
+      createQuery('B'),
+    ]);
+
+    await expect(data.pipe(take(1))).toEmitValuesWith((values) => {
+      const [loading, _data] = values;
+
+      expect(loading.A).toBeUndefined();
+      expect(loading.B.state).toEqual(LoadingState.Done);
+    });
+  });
 });
 
 type MockBackendSrvConfig = {
@@ -269,12 +303,12 @@ const createDataFrameJSON = (values: number[]): DataFrameJSON => {
   };
 };
 
-const createQuery = (refId: string): AlertQuery => {
-  return {
+const createQuery = (refId: string, options?: Partial<AlertQuery>): AlertQuery => {
+  return defaultsDeep(options, {
     refId,
     queryType: '',
     datasourceUid: '',
     model: { refId },
     relativeTimeRange: getDefaultRelativeTimeRange(),
-  };
+  });
 };

public/app/features/alerting/unified/state/AlertingQueryRunner.ts

@@ -1,3 +1,4 @@
+import { reject } from 'lodash';
 import { Observable, of, OperatorFunction, ReplaySubject, Unsubscribable } from 'rxjs';
 import { catchError, map, share } from 'rxjs/operators';
 import { v4 as uuidv4 } from 'uuid';
@@ -44,24 +45,33 @@ export class AlertingQueryRunner {
   }
 
   async run(queries: AlertQuery[]) {
-    if (queries.length === 0) {
     const empty = initialState(queries, LoadingState.Done);
-      return this.subject.next(empty);
-    }
+    const queriesToExclude: string[] = [];
 
     // do not execute if one more of the queries are not runnable,
     // for example not completely configured
     for (const query of queries) {
-      if (!isExpressionQuery(query.model)) {
-        const ds = await this.dataSourceSrv.get(query.datasourceUid);
-        if (ds.filterQuery && !ds.filterQuery(query.model)) {
-          const empty = initialState(queries, LoadingState.Done);
-          return this.subject.next(empty);
+      const refId = query.model.refId;
+
+      if (isExpressionQuery(query.model)) {
+        continue;
       }
+
+      const dataSourceInstance = await this.dataSourceSrv.get(query.datasourceUid);
+      const skipRunningQuery = dataSourceInstance.filterQuery && !dataSourceInstance.filterQuery(query.model);
+
+      if (skipRunningQuery) {
+        queriesToExclude.push(refId);
       }
     }
 
-    this.subscription = runRequest(this.backendSrv, queries).subscribe({
+    const queriesToRun = reject(queries, (q) => queriesToExclude.includes(q.model.refId));
+
+    if (queriesToRun.length === 0) {
+      return this.subject.next(empty);
+    }
+
+    this.subscription = runRequest(this.backendSrv, queriesToRun).subscribe({
       next: (dataPerQuery) => {
         const nextResult = applyChange(dataPerQuery, (refId, data) => {
           const previous = this.lastResult[refId];

public/app/features/alerting/unified/utils/rule-form.ts

@@ -127,6 +127,7 @@ export function rulerRuleToFormValues(ruleWithLocation: RuleWithLocation): RuleF
   if (isGrafanaRulesSource(ruleSourceName)) {
     if (isGrafanaRulerRule(rule)) {
       const ga = rule.grafana_alert;
+
       return {
         ...defaultFormValues,
         name: ga.title,
@@ -220,7 +221,6 @@ export const getDefaultQueries = (): AlertQuery[] => {
       relativeTimeRange,
       model: {
         refId: 'A',
-        hide: false,
       },
     },
     ...getDefaultExpressions('B', 'C'),
@@ -240,7 +240,6 @@ export const getDefaultRecordingRulesQueries = (
       relativeTimeRange,
       model: {
         refId: 'A',
-        hide: false,
       },
     },
   ];
@@ -252,7 +251,6 @@ const getDefaultExpressions = (...refIds: [string, string]): AlertQuery[] => {
 
   const reduceExpression: ExpressionQuery = {
     refId: refIds[0],
-    hide: false,
     type: ExpressionQueryType.reduce,
     datasource: {
       uid: ExpressionDatasourceUID,
@@ -283,7 +281,6 @@ const getDefaultExpressions = (...refIds: [string, string]): AlertQuery[] => {
 
   const thresholdExpression: ExpressionQuery = {
     refId: refTwo,
-    hide: false,
     type: ExpressionQueryType.threshold,
     datasource: {
       uid: ExpressionDatasourceUID,