Alerting: Allow more characters in label names so notifications are sent (#38629)

Remove validation for labels to be accepted in the Alertmanager, This helps with datasources that produce non-compatible labels.

Adds an "object_matchers" to alert manager routers so we can support labels names with extended characters beyond prometheus/openmetrics. It only does this for the internal Grafana managed Alert Manager.

This requires a change to alert manager, so for now we use grafana/alertmanager which is a slight fork, with the intention of going back to upstream.

The frontend handles the migration of "matchers" -> "object_matchers" when the route is edited and saved. Once this is done, downgrades will not work old versions will not recognize the "object_matchers".

Co-authored-by: Kyle Brandt <kyle@grafana.com>
Co-authored-by: Nathan Rodman <nathanrodman@gmail.com>

public/app/features/alerting/unified/AmRoutes.test.tsx

@@ -36,6 +36,7 @@ const mocks = {
 
 const renderAmRoutes = (alertManagerSourceName?: string) => {
   const store = configureStore();
+  locationService.push(location);
 
   locationService.push(
     '/alerting/routes' + (alertManagerSourceName ? `?${ALERTMANAGER_NAME_QUERY_KEY}=${alertManagerSourceName}` : '')
@@ -144,6 +145,11 @@ describe('AmRoutes', () => {
     },
   ];
 
+  const simpleRoute: Route = {
+    receiver: 'simple-receiver',
+    matchers: ['hello=world', 'foo!=bar'],
+  };
+
   const rootRoute: Route = {
     receiver: 'default-receiver',
     group_by: ['a-group', 'another-group'],
@@ -349,6 +355,142 @@ describe('AmRoutes', () => {
     expect(ui.editButton.query()).not.toBeInTheDocument();
   });
 
+  it('Converts matchers to object_matchers for grafana alertmanager', async () => {
+    const defaultConfig: AlertManagerCortexConfig = {
+      alertmanager_config: {
+        receivers: [{ name: 'default' }, { name: 'critical' }],
+        route: {
+          continue: false,
+          receiver: 'default',
+          group_by: ['alertname'],
+          routes: [simpleRoute],
+          group_interval: '4m',
+          group_wait: '1m',
+          repeat_interval: '5h',
+        },
+        templates: [],
+      },
+      template_files: {},
+    };
+
+    const currentConfig = { current: defaultConfig };
+    mocks.api.updateAlertManagerConfig.mockImplementation((amSourceName, newConfig) => {
+      currentConfig.current = newConfig;
+      return Promise.resolve();
+    });
+
+    mocks.api.fetchAlertManagerConfig.mockImplementation(() => {
+      return Promise.resolve(currentConfig.current);
+    });
+
+    await renderAmRoutes(GRAFANA_RULES_SOURCE_NAME);
+    expect(await ui.rootReceiver.find()).toHaveTextContent('default');
+    expect(mocks.api.fetchAlertManagerConfig).toHaveBeenCalled();
+
+    // Toggle a save to test new object_matchers
+    const rootRouteContainer = await ui.rootRouteContainer.find();
+    userEvent.click(ui.editButton.get(rootRouteContainer));
+    userEvent.click(ui.saveButton.get(rootRouteContainer));
+
+    await waitFor(() => expect(ui.editButton.query(rootRouteContainer)).not.toBeInTheDocument());
+
+    expect(mocks.api.updateAlertManagerConfig).toHaveBeenCalled();
+    expect(mocks.api.updateAlertManagerConfig).toHaveBeenCalledWith(GRAFANA_RULES_SOURCE_NAME, {
+      alertmanager_config: {
+        receivers: [{ name: 'default' }, { name: 'critical' }],
+        route: {
+          continue: false,
+          group_by: ['alertname'],
+          group_interval: '4m',
+          group_wait: '1m',
+          receiver: 'default',
+          repeat_interval: '5h',
+          routes: [
+            {
+              continue: false,
+              group_by: [],
+              object_matchers: [
+                ['hello', '=', 'world'],
+                ['foo', '!=', 'bar'],
+              ],
+              receiver: 'simple-receiver',
+              routes: [],
+            },
+          ],
+        },
+        templates: [],
+      },
+      template_files: {},
+    });
+  });
+
+  it('Keeps matchers for non-grafana alertmanager sources', async () => {
+    const defaultConfig: AlertManagerCortexConfig = {
+      alertmanager_config: {
+        receivers: [{ name: 'default' }, { name: 'critical' }],
+        route: {
+          continue: false,
+          receiver: 'default',
+          group_by: ['alertname'],
+          routes: [simpleRoute],
+          group_interval: '4m',
+          group_wait: '1m',
+          repeat_interval: '5h',
+        },
+        templates: [],
+      },
+      template_files: {},
+    };
+
+    const currentConfig = { current: defaultConfig };
+    mocks.api.updateAlertManagerConfig.mockImplementation((amSourceName, newConfig) => {
+      currentConfig.current = newConfig;
+      return Promise.resolve();
+    });
+
+    mocks.api.fetchAlertManagerConfig.mockImplementation(() => {
+      return Promise.resolve(currentConfig.current);
+    });
+
+    await renderAmRoutes(dataSources.am.name);
+    expect(await ui.rootReceiver.find()).toHaveTextContent('default');
+    expect(mocks.api.fetchAlertManagerConfig).toHaveBeenCalled();
+
+    // Toggle a save to test new object_matchers
+    const rootRouteContainer = await ui.rootRouteContainer.find();
+    userEvent.click(ui.editButton.get(rootRouteContainer));
+    userEvent.click(ui.saveButton.get(rootRouteContainer));
+
+    await waitFor(() => expect(ui.editButton.query(rootRouteContainer)).not.toBeInTheDocument());
+
+    expect(mocks.api.updateAlertManagerConfig).toHaveBeenCalled();
+    expect(mocks.api.updateAlertManagerConfig).toHaveBeenCalledWith(dataSources.am.name, {
+      alertmanager_config: {
+        receivers: [{ name: 'default' }, { name: 'critical' }],
+        route: {
+          continue: false,
+          group_by: ['alertname'],
+          group_interval: '4m',
+          group_wait: '1m',
+          matchers: [],
+          receiver: 'default',
+          repeat_interval: '5h',
+          routes: [
+            {
+              continue: false,
+              group_by: [],
+              matchers: ['hello=world', 'foo!=bar'],
+              receiver: 'simple-receiver',
+              routes: [],
+            },
+          ],
+        },
+        templates: [],
+      },
+      template_files: {},
+    });
+  });
+
   it('Prometheus Alertmanager routes cannot be edited', async () => {
     mocks.api.fetchStatus.mockResolvedValue({
       ...someCloudAlertManagerStatus,

public/app/features/alerting/unified/AmRoutes.tsx

@@ -59,6 +59,7 @@ const AmRoutes: FC = () => {
   useCleanup((state) => state.unifiedAlerting.saveAMConfig);
   const handleSave = (data: Partial<FormAmRoute>) => {
     const newData = formAmRouteToAmRoute(
+      alertManagerSourceName,
       {
         ...rootRoute,
         ...data,

public/app/features/alerting/unified/components/amroutes/AmRoutesExpandedForm.tsx

@@ -51,19 +51,19 @@ export const AmRoutesExpandedForm: FC<AmRoutesExpandedFormProps> = ({ onCancel,
           {/* @ts-ignore-check: react-hook-form made me do this */}
           <input type="hidden" {...register('id')} />
           {/* @ts-ignore-check: react-hook-form made me do this */}
-          <FieldArray name="matchers" control={control}>
+          <FieldArray name="object_matchers" control={control}>
             {({ fields, append, remove }) => (
               <>
                 <div>Matching labels</div>
                 <div className={styles.matchersContainer}>
                   {fields.map((field, index) => {
-                    const localPath = `matchers[${index}]`;
+                    const localPath = `object_matchers[${index}]`;
                     return (
                       <HorizontalGroup key={field.id} align="flex-start">
                         <Field
                           label="Label"
-                          invalid={!!errors.matchers?.[index]?.name}
-                          error={errors.matchers?.[index]?.name?.message}
+                          invalid={!!errors.object_matchers?.[index]?.name}
+                          error={errors.object_matchers?.[index]?.name?.message}
                         >
                           <Input
                             {...register(`${localPath}.name`, { required: 'Field is required' })}
@@ -89,8 +89,8 @@ export const AmRoutesExpandedForm: FC<AmRoutesExpandedFormProps> = ({ onCancel,
                         </Field>
                         <Field
                           label="Value"
-                          invalid={!!errors.matchers?.[index]?.value}
-                          error={errors.matchers?.[index]?.value?.message}
+                          invalid={!!errors.object_matchers?.[index]?.value}
+                          error={errors.object_matchers?.[index]?.value?.message}
                         >
                           <Input
                             {...register(`${localPath}.value`, { required: 'Field is required' })}

public/app/features/alerting/unified/components/amroutes/AmRoutesTable.tsx

@@ -41,7 +41,7 @@ export const AmRoutesTable: FC<AmRoutesTableProps> = ({
       id: 'matchingCriteria',
       label: 'Matching labels',
       // eslint-disable-next-line react/display-name
-      renderCell: (item) => <Matchers matchers={item.data.matchers.map(matcherFieldToMatcher)} />,
+      renderCell: (item) => <Matchers matchers={item.data.object_matchers.map(matcherFieldToMatcher)} />,
       size: 10,
     },
     {

public/app/features/alerting/unified/types/amroutes.ts

@@ -2,7 +2,7 @@ import { MatcherFieldValue } from './silence-form';
 
 export interface FormAmRoute {
   id: string;
-  matchers: MatcherFieldValue[];
+  object_matchers: MatcherFieldValue[];
   continue: boolean;
   receiver: string;
   groupBy: string[];

public/app/features/alerting/unified/utils/amroutes.ts

@@ -3,9 +3,10 @@ import { Validate } from 'react-hook-form';
 import { MatcherOperator, Route } from 'app/plugins/datasource/alertmanager/types';
 import { FormAmRoute } from '../types/amroutes';
 import { parseInterval, timeOptions } from './time';
-import { matcherToMatcherField, matcherFieldToMatcher, parseMatcher, stringifyMatcher } from './alertmanager';
 import { isUndefined, omitBy } from 'lodash';
 import { MatcherFieldValue } from '../types/silence-form';
+import { matcherToMatcherField, parseMatcher } from './alertmanager';
+import { GRAFANA_RULES_SOURCE_NAME } from './datasource';
 
 const defaultValueAndType: [string, string] = ['', timeOptions[0].value];
 
@@ -55,7 +56,7 @@ export const emptyArrayFieldMatcher: MatcherFieldValue = {
 export const emptyRoute: FormAmRoute = {
   id: '',
   groupBy: [],
-  matchers: [],
+  object_matchers: [],
   routes: [],
   continue: false,
   receiver: '',
@@ -88,11 +89,18 @@ export const amRouteToFormAmRoute = (route: Route | undefined): [FormAmRoute, Re
     Object.assign(id2route, subId2Route);
   });
 
+  // Frontend migration to use object_matchers instead of matchers
+  const matchers = route.matchers
+    ? route.matchers?.map((matcher) => matcherToMatcherField(parseMatcher(matcher))) ?? []
+    : route.object_matchers?.map(
+        (matcher) => ({ name: matcher[0], operator: matcher[1], value: matcher[2] } as MatcherFieldValue)
+      ) ?? [];
+
   return [
     {
       id,
-      matchers: [
-        ...(route.matchers?.map((matcher) => matcherToMatcherField(parseMatcher(matcher))) ?? []),
+      object_matchers: [
+        ...matchers,
         ...matchersToArrayFieldMatchers(route.match, false),
         ...matchersToArrayFieldMatchers(route.match_re, true),
       ],
@@ -111,14 +119,18 @@ export const amRouteToFormAmRoute = (route: Route | undefined): [FormAmRoute, Re
   ];
 };
 
-export const formAmRouteToAmRoute = (formAmRoute: FormAmRoute, id2ExistingRoute: Record<string, Route>): Route => {
+export const formAmRouteToAmRoute = (
+  alertManagerSourceName: string | undefined,
+  formAmRoute: FormAmRoute,
+  id2ExistingRoute: Record<string, Route>
+): Route => {
   const existing: Route | undefined = id2ExistingRoute[formAmRoute.id];
   const amRoute: Route = {
     ...(existing ?? {}),
     continue: formAmRoute.continue,
     group_by: formAmRoute.groupBy,
-    matchers: formAmRoute.matchers.length
-      ? formAmRoute.matchers.map((matcher) => stringifyMatcher(matcherFieldToMatcher(matcher)))
+    object_matchers: formAmRoute.object_matchers.length
+      ? formAmRoute.object_matchers.map((matcher) => [matcher.name, matcher.operator, matcher.value])
       : undefined,
     match: undefined,
     match_re: undefined,
@@ -131,9 +143,18 @@ export const formAmRouteToAmRoute = (formAmRoute: FormAmRoute, id2ExistingRoute:
     repeat_interval: formAmRoute.repeatIntervalValue
       ? `${formAmRoute.repeatIntervalValue}${formAmRoute.repeatIntervalValueType}`
       : undefined,
-    routes: formAmRoute.routes.map((subRoute) => formAmRouteToAmRoute(subRoute, id2ExistingRoute)),
+    routes: formAmRoute.routes.map((subRoute) =>
+      formAmRouteToAmRoute(alertManagerSourceName, subRoute, id2ExistingRoute)
+    ),
   };
 
+  if (alertManagerSourceName !== GRAFANA_RULES_SOURCE_NAME) {
+    amRoute.matchers = formAmRoute.object_matchers.map(({ name, operator, value }) => `${name}${operator}${value}`);
+    amRoute.object_matchers = undefined;
+  } else {
+    amRoute.matchers = undefined;
+  }
+
   if (formAmRoute.receiver) {
     amRoute.receiver = formAmRoute.receiver;
   }

public/app/plugins/datasource/alertmanager/types.ts

@@ -89,12 +89,17 @@ export type Receiver = {
   [key: string]: any;
 };
 
+type ObjectMatcher = [name: string, operator: MatcherOperator, value: string];
+
 export type Route = {
   receiver?: string;
   group_by?: string[];
   continue?: boolean;
+  object_matchers?: ObjectMatcher[];
   matchers?: string[];
+  /** @deprecated use `object_matchers` */
   match?: Record<string, string>;
+  /** @deprecated use `object_matchers` */
   match_re?: Record<string, string>;
   group_wait?: string;
   group_interval?: string;