CI: Notify failures in separate pipeline (#27936)

* Notify failures in separate pipeline

* Include branch and event in triggers

* Use snake case

* Use snake case

* Fix enterprise/downstream

* Apply suggestions from code review

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>

.drone.yml

@@ -276,17 +276,6 @@ steps:
   environment:
     DOCKERIZE_VERSION: 0.6.1
 
-- name: slack
-  image: plugins/slack
-  settings:
-    channel: grafana-ci-failures
-    template: "Build {{build.number}} failed: {{build.link}}"
-    webhook:
-      from_secret: slack_webhook
-  when:
-    status:
-    - failure
-
 - name: trigger-enterprise-downstream
   image: grafana/drone-downstream
   settings:
@@ -652,17 +641,6 @@ steps:
   environment:
     DOCKERIZE_VERSION: 0.6.1
 
-- name: slack
-  image: plugins/slack
-  settings:
-    channel: grafana-ci-failures
-    template: "Build {{build.number}} failed: {{build.link}}"
-    webhook:
-      from_secret: slack_webhook
-  when:
-    status:
-    - failure
-
 - name: publish-packages
   image: grafana/grafana-ci-deploy:1.2.6
   commands:
@@ -683,4 +661,35 @@ depends_on:
 - build-master
 - windows-master
 
+---
+kind: pipeline
+type: docker
+name: notify-master
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: slack
+  image: plugins/slack
+  settings:
+    channel: grafana-ci-notifications
+    template: "Build {{build.number}} failed: {{build.link}}"
+    webhook:
+      from_secret: slack_webhook
+
+trigger:
+  branch:
+  - master
+  event:
+  - push
+  status:
+  - failure
+
+depends_on:
+- build-master
+- windows-master
+- publish-master
+
 ...

scripts/lib.star

@@ -153,6 +153,12 @@ def master_pipelines(edition):
             name='publish-master', edition=edition, trigger=trigger, steps=publish_steps,
             depends_on=['build-master', 'windows-master',], install_deps=False, version_mode=version_mode,
         ))
+
+        notify_trigger = dict(trigger, status = ['failure'])
+        pipelines.append(notify_pipeline(
+            name='notify-master', slack_channel='grafana-ci-notifications', trigger=notify_trigger,
+            depends_on=['build-master', 'windows-master', 'publish-master'],
+        ))
     if edition == 'enterprise':
         # Add downstream enterprise pipelines triggerable from OSS builds
         trigger = {
@@ -173,6 +179,12 @@ def master_pipelines(edition):
             version_mode=version_mode,
         ))
 
+        notify_trigger = dict(trigger, status = ['failure'])
+        pipelines.append(notify_pipeline(
+            name='notify-master-downstream', slack_channel='grafana-enterprise-ci-notifications', trigger=notify_trigger,
+            depends_on=['build-master-downstream', 'windows-master-downstream', 'publish-master-downstream'],
+        ))
+
     return pipelines
 
 def pipeline(
@@ -212,6 +224,22 @@ def pipeline(
 
     return pipeline
 
+def notify_pipeline(name, slack_channel, trigger, depends_on=[]):
+    return {
+        'kind': 'pipeline',
+        'type': 'docker',
+        'platform': {
+            'os': 'linux',
+            'arch': 'amd64',
+        },
+        'name': name,
+        'trigger': trigger,
+        'steps': [
+            slack_step(slack_channel),
+        ],
+        'depends_on': depends_on,
+    }
+
 def slack_step(channel):
     return {
         'name': 'slack',
@@ -223,9 +251,6 @@ def slack_step(channel):
             'channel': channel,
             'template': 'Build {{build.number}} failed: {{build.link}}',
         },
-        'when': {
-            'status': ['failure',],
-        },
     }
 
 def init_steps(edition, platform, version_mode, is_downstream=False, install_deps=True):
@@ -300,8 +325,7 @@ def init_steps(edition, platform, version_mode, is_downstream=False, install_dep
                 ] + common_cmds,
             },
         ]
-        if version_mode == 'master':
+
-            steps.append(slack_step(channel='grafana-enterprise-ci-failures'))
         return steps
 
     steps = [
@@ -323,9 +347,6 @@ def init_steps(edition, platform, version_mode, is_downstream=False, install_dep
         },
     ]
 
-    if version_mode == 'master':
-        steps.append(slack_step(channel='grafana-ci-failures'))
-
     return steps
 
 def enterprise_downstream_step(edition):