Alerting: Show evaluation interval global limit warning (#52942)

Co-authored-by: George Robinson <george.robinson@grafana.com>

packages/grafana-data/src/types/config.ts

@@ -73,6 +73,10 @@ export interface GrafanaJavascriptAgentConfig {
   apiKey: string;
 }
 
+export interface UnifiedAlertingConfig {
+  minInterval: string;
+}
+
 /**
  * Describes the plugins that should be preloaded prior to start Grafana.
  *
@@ -201,6 +205,7 @@ export interface GrafanaConfig {
   geomapDefaultBaseLayer?: MapLayerOptions;
   geomapDisableCustomBaseLayer?: boolean;
   unifiedAlertingEnabled: boolean;
+  unifiedAlerting: UnifiedAlertingConfig;
   angularSupportEnabled: boolean;
   feedbackLinksEnabled: boolean;
   secretsManagerPluginEnabled: boolean;

packages/grafana-runtime/src/config.ts

@@ -119,6 +119,7 @@ export class GrafanaBootConfig implements GrafanaConfig {
   geomapDefaultBaseLayerConfig?: MapLayerOptions;
   geomapDisableCustomBaseLayer?: boolean;
   unifiedAlertingEnabled = false;
+  unifiedAlerting = { minInterval: '' };
   applicationInsightsConnectionString?: string;
   applicationInsightsEndpointUrl?: string;
   recordedQueries = {

packages/grafana-ui/src/types/icon.ts

@@ -166,6 +166,7 @@ export const getAvailableIcons = () =>
     'square-shape',
     'star',
     'step-backward',
+    'stopwatch-slash',
     'sync',
     'table',
     'tag-alt',

pkg/api/frontendsettings.go

@@ -179,6 +179,9 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *models.ReqContext) (map[string]i
 			"enabled": hs.Cfg.SectionWithEnvOverrides("reporting").Key("enabled").MustBool(true),
 		},
 		"unifiedAlertingEnabled": hs.Cfg.UnifiedAlerting.Enabled,
+		"unifiedAlerting": map[string]interface{}{
+			"minInterval": hs.Cfg.UnifiedAlerting.MinInterval.String(),
+		},
 	}
 
 	if hs.ThumbService != nil {

public/app/features/alerting/unified/PanelAlertTabContent.test.tsx

@@ -27,14 +27,15 @@ import {
   mockPromRuleNamespace,
   mockRulerGrafanaRule,
 } from './mocks';
-import { getAllDataSources } from './utils/config';
+import * as config from './utils/config';
 import { Annotation } from './utils/constants';
 import { DataSourceType, GRAFANA_RULES_SOURCE_NAME } from './utils/datasource';
 import * as ruleFormUtils from './utils/rule-form';
 
 jest.mock('./api/prometheus');
 jest.mock('./api/ruler');
-jest.mock('./utils/config');
+
+jest.spyOn(config, 'getAllDataSources');
 
 const dataSources = {
   prometheus: mockDataSource<PromOptions>({
@@ -52,7 +53,7 @@ dataSources.prometheus.meta.alerting = true;
 dataSources.default.meta.alerting = true;
 
 const mocks = {
-  getAllDataSources: jest.mocked(getAllDataSources),
+  getAllDataSources: jest.mocked(config.getAllDataSources),
   api: {
     fetchRules: jest.mocked(fetchRules),
     fetchRulerRules: jest.mocked(fetchRulerRules),

public/app/features/alerting/unified/RuleEditor.test.tsx

@@ -22,7 +22,7 @@ import { discoverFeatures } from './api/buildInfo';
 import { fetchRulerRules, fetchRulerRulesGroup, fetchRulerRulesNamespace, setRulerRuleGroup } from './api/ruler';
 import { ExpressionEditorProps } from './components/rule-editor/ExpressionEditor';
 import { disableRBAC, mockDataSource, MockDataSourceSrv, mockFolder } from './mocks';
-import { getAllDataSources } from './utils/config';
+import * as config from './utils/config';
 import { DataSourceType, GRAFANA_RULES_SOURCE_NAME } from './utils/datasource';
 import { getDefaultQueries } from './utils/rule-form';
 
@@ -35,7 +35,6 @@ jest.mock('./components/rule-editor/ExpressionEditor', () => ({
 
 jest.mock('./api/buildInfo');
 jest.mock('./api/ruler');
-jest.mock('./utils/config');
 jest.mock('../../../../app/features/manage-dashboards/state/actions');
 
 // there's no angular scope in test and things go terribly wrong when trying to render the query editor row.
@@ -45,8 +44,10 @@ jest.mock('app/features/query/components/QueryEditorRow', () => ({
   QueryEditorRow: () => <p>hi</p>,
 }));
 
+jest.spyOn(config, 'getAllDataSources');
+
 const mocks = {
-  getAllDataSources: jest.mocked(getAllDataSources),
+  getAllDataSources: jest.mocked(config.getAllDataSources),
   searchFolders: jest.mocked(searchFolders),
   api: {
     discoverFeatures: jest.mocked(discoverFeatures),

public/app/features/alerting/unified/RuleList.test.tsx

@@ -30,13 +30,12 @@ import {
   somePromRules,
   someRulerRules,
 } from './mocks';
-import { getAllDataSources } from './utils/config';
+import * as config from './utils/config';
 import { DataSourceType, GRAFANA_RULES_SOURCE_NAME } from './utils/datasource';
 
 jest.mock('./api/buildInfo');
 jest.mock('./api/prometheus');
 jest.mock('./api/ruler');
-jest.mock('./utils/config');
 jest.mock('app/core/core', () => ({
   appEvents: {
     subscribe: () => {
@@ -46,8 +45,10 @@ jest.mock('app/core/core', () => ({
   },
 }));
 
+jest.spyOn(config, 'getAllDataSources');
+
 const mocks = {
-  getAllDataSourcesMock: jest.mocked(getAllDataSources),
+  getAllDataSourcesMock: jest.mocked(config.getAllDataSources),
 
   api: {
     discoverFeatures: jest.mocked(discoverFeatures),

public/app/features/alerting/unified/components/rule-editor/GrafanaEvaluationBehavior.tsx

@@ -1,12 +1,18 @@
 import { css } from '@emotion/css';
 import React, { FC, useState } from 'react';
-import { useFormContext, RegisterOptions } from 'react-hook-form';
+import { RegisterOptions, useFormContext } from 'react-hook-form';
 
-import { parseDuration, durationToMilliseconds, GrafanaTheme2 } from '@grafana/data';
-import { Field, InlineLabel, Input, InputControl, useStyles2 } from '@grafana/ui';
+import { durationToMilliseconds, GrafanaTheme2, parseDuration } from '@grafana/data';
+import { config } from '@grafana/runtime';
+import { Alert, Field, InlineLabel, Input, InputControl, useStyles2 } from '@grafana/ui';
 
 import { RuleFormValues } from '../../types/rule-form';
-import { positiveDurationValidationPattern, durationValidationPattern } from '../../utils/time';
+import { checkEvaluationIntervalGlobalLimit } from '../../utils/config';
+import {
+  durationValidationPattern,
+  parseDurationToMilliseconds,
+  positiveDurationValidationPattern,
+} from '../../utils/time';
 import { CollapseToggle } from '../CollapseToggle';
 
 import { GrafanaAlertStatePicker } from './GrafanaAlertStatePicker';
@@ -22,10 +28,13 @@ const forValidationOptions = (evaluateEvery: string): RegisterOptions => ({
   },
   pattern: durationValidationPattern,
   validate: (value) => {
-    const evaluateEveryDuration = parseDuration(evaluateEvery);
-    const forDuration = parseDuration(value);
-    const millisFor = durationToMilliseconds(forDuration);
-    const millisEvery = durationToMilliseconds(evaluateEveryDuration);
+    const millisFor = parseDurationToMilliseconds(value);
+    const millisEvery = parseDurationToMilliseconds(evaluateEvery);
+
+    // 0 is a special value meaning for equals evaluation interval
+    if (millisFor === 0) {
+      return true;
+    }
 
     return millisFor >= millisEvery ? true : 'For must be greater than or equal to evaluate every.';
   },
@@ -61,6 +70,8 @@ export const GrafanaEvaluationBehavior: FC = () => {
     watch,
   } = useFormContext<RuleFormValues>();
 
+  const { exceedsLimit: exceedsGlobalEvaluationLimit } = checkEvaluationIntervalGlobalLimit(watch('evaluateEvery'));
+
   const evaluateEveryId = 'eval-every-input';
   const evaluateForId = 'eval-for-input';
 
@@ -79,7 +90,15 @@ export const GrafanaEvaluationBehavior: FC = () => {
           >
             Evaluate every
           </InlineLabel>
-          <Input id={evaluateEveryId} width={8} {...register('evaluateEvery', evaluateEveryValidationOptions)} />
+          <Field
+            className={styles.inlineField}
+            error={errors.evaluateEvery?.message}
+            invalid={!!errors.evaluateEvery}
+            validationMessageHorizontalOverflow={true}
+          >
+            <Input id={evaluateEveryId} width={8} {...register('evaluateEvery', evaluateEveryValidationOptions)} />
+          </Field>
+
           <InlineLabel
             htmlFor={evaluateForId}
             width={7}
@@ -101,6 +120,14 @@ export const GrafanaEvaluationBehavior: FC = () => {
           </Field>
         </div>
       </Field>
+      {exceedsGlobalEvaluationLimit && (
+        <Alert severity="warning" title="Global evalutation interval limit exceeded">
+          A minimum evaluation interval of{' '}
+          <span className={styles.globalLimitValue}>{config.unifiedAlerting.minInterval}</span> has been configured in
+          Grafana. <br />
+          Please contact the administrator to configure a lower interval.
+        </Alert>
+      )}
       <CollapseToggle
         isCollapsed={!showErrorHandling}
         onToggle={(collapsed) => setShowErrorHandling(!collapsed)}
@@ -159,4 +186,7 @@ const getStyles = (theme: GrafanaTheme2) => ({
   collapseToggle: css`
     margin: ${theme.spacing(2, 0, 2, -1)};
   `,
+  globalLimitValue: css`
+    font-weight: ${theme.typography.fontWeightBold};
+  `,
 });

public/app/features/alerting/unified/components/rules/RuleConfigStatus.tsx

@@ -0,0 +1,49 @@
+import { css } from '@emotion/css';
+import React from 'react';
+
+import { GrafanaTheme2 } from '@grafana/data/src';
+import { config } from '@grafana/runtime/src';
+import { Icon, Tooltip, useStyles2 } from '@grafana/ui/src';
+
+import { CombinedRule } from '../../../../../types/unified-alerting';
+import { checkEvaluationIntervalGlobalLimit } from '../../utils/config';
+
+interface RuleConfigStatusProps {
+  rule: CombinedRule;
+}
+
+export function RuleConfigStatus({ rule }: RuleConfigStatusProps) {
+  const styles = useStyles2(getStyles);
+
+  const { exceedsLimit } = checkEvaluationIntervalGlobalLimit(rule.group.interval ?? '');
+
+  if (!exceedsLimit) {
+    return null;
+  }
+
+  return (
+    <Tooltip
+      theme="error"
+      content={
+        <div>
+          A minimum evaluation interval of{' '}
+          <span className={styles.globalLimitValue}>{config.unifiedAlerting.minInterval}</span> has been configured in
+          Grafana and will be used instead of the {rule.group.interval} interval configured for the Rule Group.
+        </div>
+      }
+    >
+      <Icon name="stopwatch-slash" className={styles.icon} />
+    </Tooltip>
+  );
+}
+
+function getStyles(theme: GrafanaTheme2) {
+  return {
+    globalLimitValue: css`
+      font-weight: ${theme.typography.fontWeightBold};
+    `,
+    icon: css`
+      fill: ${theme.colors.warning.text};
+    `,
+  };
+}

public/app/features/alerting/unified/components/rules/RuleHealth.tsx

@@ -11,6 +11,7 @@ interface Prom {
 
 export const RuleHealth: FC<Prom> = ({ rule }) => {
   const style = useStyles2(getStyle);
+
   if (rule.health === 'err' || rule.health === 'error') {
     return (
       <Tooltip theme="error" content={rule.lastError || 'No error message provided.'}>
@@ -21,6 +22,7 @@ export const RuleHealth: FC<Prom> = ({ rule }) => {
       </Tooltip>
     );
   }
+
   return <>{rule.health}</>;
 };
 
@@ -29,6 +31,7 @@ const getStyle = (theme: GrafanaTheme2) => ({
     display: inline-flex;
     flex-direction: row;
     color: ${theme.colors.warning.text};
+
     & > * + * {
       margin-left: ${theme.spacing(1)};
     }

public/app/features/alerting/unified/components/rules/RulesTable.tsx

@@ -15,6 +15,7 @@ import { ProvisioningBadge } from '../Provisioning';
 import { RuleLocation } from '../RuleLocation';
 import { Tokenize } from '../Tokenize';
 
+import { RuleConfigStatus } from './RuleConfigStatus';
 import { RuleDetails } from './RuleDetails';
 import { RuleHealth } from './RuleHealth';
 import { RuleState } from './RuleState';
@@ -148,11 +149,17 @@ function useColumns(showSummaryColumn: boolean, showGroupColumn: boolean) {
         },
         size: '100px',
       },
+      {
+        id: 'warnings',
+        label: '',
+        renderCell: ({ data: combinedRule }) => <RuleConfigStatus rule={combinedRule} />,
+        size: '45px',
+      },
       {
         id: 'health',
         label: 'Health',
         // eslint-disable-next-line react/display-name
-        renderCell: ({ data: { promRule } }) => (promRule ? <RuleHealth rule={promRule} /> : null),
+        renderCell: ({ data: { promRule, group } }) => (promRule ? <RuleHealth rule={promRule} /> : null),
         size: '75px',
       },
     ];

public/app/features/alerting/unified/utils/config.ts

@@ -1,6 +1,19 @@
-import { DataSourceInstanceSettings, DataSourceJsonData } from '@grafana/data';
+import { DataSourceInstanceSettings, DataSourceJsonData, isValidGoDuration, rangeUtil } from '@grafana/data';
 import { config } from '@grafana/runtime';
 
 export function getAllDataSources(): Array<DataSourceInstanceSettings<DataSourceJsonData>> {
   return Object.values(config.datasources);
 }
+
+export function checkEvaluationIntervalGlobalLimit(alertGroupEvaluateEvery: string) {
+  if (!isValidGoDuration(config.unifiedAlerting.minInterval)) {
+    return { globalLimit: 0, exceedsLimit: false };
+  }
+
+  const evaluateEveryMs = rangeUtil.intervalToMs(alertGroupEvaluateEvery);
+  const evaluateEveryGlobalLimitMs = rangeUtil.intervalToMs(config.unifiedAlerting.minInterval);
+
+  const exceedsLimit = evaluateEveryGlobalLimitMs > evaluateEveryMs && evaluateEveryMs > 0;
+
+  return { globalLimit: evaluateEveryGlobalLimitMs, exceedsLimit };
+}

public/app/features/alerting/unified/utils/time.ts

@@ -1,3 +1,4 @@
+import { durationToMilliseconds, parseDuration } from '@grafana/data';
 import { describeInterval } from '@grafana/data/src/datetime/rangeutil';
 
 import { TimeOptions } from '../types/time';
@@ -35,3 +36,7 @@ export const durationValidationPattern = {
     TimeOptions
   ).join(', ')}`,
 };
+
+export function parseDurationToMilliseconds(duration: string) {
+  return durationToMilliseconds(parseDuration(duration));
+}