Alerting: Send alerts to external Alertmanager(s) (#37298)

* Alerting: Send alerts to external Alertmanager(s)

Within this PR we're adding support for registering or unregistering
sending to a set of external alertmanagers. A few of the things that are
going are:

- Introduce a new table to hold "admin" (either org or global)
  configuration we can change at runtime.
- A new periodic check that polls for this configuration and adjusts the
  "senders" accordingly.
- Introduces a new concept of "senders" that are responsible for
  shipping the alerts to the external Alertmanager(s). In a nutshell,
this is the Prometheus notifier (the one in charge of sending the alert)
mapped to a multi-tenant map.

There are a few code movements here and there but those are minor, I
tried to keep things intact as much as possible so that we could have an
easier diff.

pkg/services/ngalert/api/api.go

@@ -3,18 +3,16 @@ package api
 import (
 	"time"
 
-	"github.com/grafana/grafana/pkg/services/quota"
-
-	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
-	"github.com/grafana/grafana/pkg/services/ngalert/state"
-
 	"github.com/grafana/grafana/pkg/api/routing"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/services/datasourceproxy"
 	"github.com/grafana/grafana/pkg/services/datasources"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	"github.com/grafana/grafana/pkg/services/ngalert/schedule"
+	"github.com/grafana/grafana/pkg/services/ngalert/state"
 	"github.com/grafana/grafana/pkg/services/ngalert/store"
+	"github.com/grafana/grafana/pkg/services/quota"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb"
 )
@@ -41,18 +39,19 @@ type Alertmanager interface {
 
 // API handlers.
 type API struct {
-	Cfg             *setting.Cfg
-	DatasourceCache datasources.CacheService
-	RouteRegister   routing.RouteRegister
-	DataService     *tsdb.Service
-	QuotaService    *quota.QuotaService
-	Schedule        schedule.ScheduleService
-	RuleStore       store.RuleStore
-	InstanceStore   store.InstanceStore
-	AlertingStore   store.AlertingStore
-	DataProxy       *datasourceproxy.DatasourceProxyService
-	Alertmanager    Alertmanager
-	StateManager    *state.Manager
+	Cfg              *setting.Cfg
+	DatasourceCache  datasources.CacheService
+	RouteRegister    routing.RouteRegister
+	DataService      *tsdb.Service
+	QuotaService     *quota.QuotaService
+	Schedule         schedule.ScheduleService
+	RuleStore        store.RuleStore
+	InstanceStore    store.InstanceStore
+	AlertingStore    store.AlertingStore
+	AdminConfigStore store.AdminConfigurationStore
+	DataProxy        *datasourceproxy.DatasourceProxyService
+	Alertmanager     Alertmanager
+	StateManager     *state.Manager
 }
 
 // RegisterAPIEndpoints registers API handlers
@@ -87,4 +86,8 @@ func (api *API) RegisterAPIEndpoints(m *metrics.Metrics) {
 		DatasourceCache: api.DatasourceCache,
 		log:             logger,
 	}, m)
+	api.RegisterConfigurationApiEndpoints(AdminSrv{
+		store: api.AdminConfigStore,
+		log:   logger,
+	}, m)
 }

pkg/services/ngalert/api/api_admin.go

@@ -0,0 +1,74 @@
+package api
+
+import (
+	"errors"
+	"net/http"
+
+	"github.com/grafana/grafana/pkg/api/response"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/models"
+	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+	ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
+	"github.com/grafana/grafana/pkg/services/ngalert/store"
+)
+
+type AdminSrv struct {
+	store store.AdminConfigurationStore
+	log   log.Logger
+}
+
+func (srv AdminSrv) RouteGetNGalertConfig(c *models.ReqContext) response.Response {
+	if c.OrgRole != models.ROLE_ADMIN {
+		return accessForbiddenResp()
+	}
+
+	cfg, err := srv.store.GetAdminConfiguration(c.OrgId)
+	if err != nil {
+		if errors.Is(err, store.ErrNoAdminConfiguration) {
+			return ErrResp(http.StatusNotFound, err, "")
+		}
+
+		msg := "failed to fetch admin configuration from the database"
+		srv.log.Error(msg, "err", err)
+		return ErrResp(http.StatusInternalServerError, err, msg)
+	}
+
+	resp := apimodels.GettableNGalertConfig{
+		Alertmanagers: cfg.Alertmanagers,
+	}
+	return response.JSON(http.StatusOK, resp)
+}
+
+func (srv AdminSrv) RoutePostNGalertConfig(c *models.ReqContext, body apimodels.PostableNGalertConfig) response.Response {
+	if c.OrgRole != models.ROLE_ADMIN {
+		return accessForbiddenResp()
+	}
+
+	cfg := &ngmodels.AdminConfiguration{
+		Alertmanagers: body.Alertmanagers,
+		OrgID:         c.OrgId,
+	}
+
+	cmd := store.UpdateAdminConfigurationCmd{AdminConfiguration: cfg}
+	if err := srv.store.UpdateAdminConfiguration(cmd); err != nil {
+		msg := "failed to save the admin configuration to the database"
+		srv.log.Error(msg, "err", err)
+		return ErrResp(http.StatusBadRequest, err, msg)
+	}
+
+	return response.JSON(http.StatusCreated, "admin configuration updated")
+}
+
+func (srv AdminSrv) RouteDeleteNGalertConfig(c *models.ReqContext) response.Response {
+	if c.OrgRole != models.ROLE_ADMIN {
+		return accessForbiddenResp()
+	}
+
+	err := srv.store.DeleteAdminConfiguration(c.OrgId)
+	if err != nil {
+		srv.log.Error("unable to delete configuration", "err", err)
+		return ErrResp(http.StatusInternalServerError, err, "")
+	}
+
+	return response.JSON(http.StatusOK, "admin configuration deleted")
+}

pkg/services/ngalert/api/generated_base_api_alertmanager.go

@@ -4,7 +4,6 @@
  *
  *Do not manually edit these files, please find ngalert/api/swagger-codegen/ for commands on how to generate them.
  */
-
 package api
 
 import (

pkg/services/ngalert/api/generated_base_api_configuration.go

@@ -0,0 +1,59 @@
+/*Package api contains base API implementation of unified alerting
+ *
+ *Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ *
+ *Do not manually edit these files, please find ngalert/api/swagger-codegen/ for commands on how to generate them.
+ */
+package api
+
+import (
+	"net/http"
+
+	"github.com/go-macaron/binding"
+
+	"github.com/grafana/grafana/pkg/api/response"
+	"github.com/grafana/grafana/pkg/api/routing"
+	"github.com/grafana/grafana/pkg/middleware"
+	"github.com/grafana/grafana/pkg/models"
+	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
+)
+
+type ConfigurationApiService interface {
+	RouteDeleteNGalertConfig(*models.ReqContext) response.Response
+	RouteGetNGalertConfig(*models.ReqContext) response.Response
+	RoutePostNGalertConfig(*models.ReqContext, apimodels.PostableNGalertConfig) response.Response
+}
+
+func (api *API) RegisterConfigurationApiEndpoints(srv ConfigurationApiService, m *metrics.Metrics) {
+	api.RouteRegister.Group("", func(group routing.RouteRegister) {
+		group.Delete(
+			toMacaronPath("/api/v1/ngalert/admin_config"),
+			metrics.Instrument(
+				http.MethodDelete,
+				"/api/v1/ngalert/admin_config",
+				srv.RouteDeleteNGalertConfig,
+				m,
+			),
+		)
+		group.Get(
+			toMacaronPath("/api/v1/ngalert/admin_config"),
+			metrics.Instrument(
+				http.MethodGet,
+				"/api/v1/ngalert/admin_config",
+				srv.RouteGetNGalertConfig,
+				m,
+			),
+		)
+		group.Post(
+			toMacaronPath("/api/v1/ngalert/admin_config"),
+			binding.Bind(apimodels.PostableNGalertConfig{}),
+			metrics.Instrument(
+				http.MethodPost,
+				"/api/v1/ngalert/admin_config",
+				srv.RoutePostNGalertConfig,
+				m,
+			),
+		)
+	}, middleware.ReqSignedIn)
+}

pkg/services/ngalert/api/generated_base_api_prometheus.go

@@ -4,7 +4,6 @@
  *
  *Do not manually edit these files, please find ngalert/api/swagger-codegen/ for commands on how to generate them.
  */
-
 package api
 
 import (

pkg/services/ngalert/api/generated_base_api_ruler.go

@@ -4,7 +4,6 @@
  *
  *Do not manually edit these files, please find ngalert/api/swagger-codegen/ for commands on how to generate them.
  */
-
 package api
 
 import (

pkg/services/ngalert/api/generated_base_api_testing.go

@@ -4,7 +4,6 @@
  *
  *Do not manually edit these files, please find ngalert/api/swagger-codegen/ for commands on how to generate them.
  */
-
 package api
 
 import (

pkg/services/ngalert/api/tooling/definitions/admin.go

@@ -0,0 +1,51 @@
+package definitions
+
+// swagger:route GET /api/v1/ngalert/admin_config configuration RouteGetNGalertConfig
+//
+//  Get the NGalert configuration of the user's organization, returns 404 if no configuration is present.
+//
+//     Produces:
+//     - application/json
+//
+//     Responses:
+//		 200: GettableNGalertConfig
+//		 404: Failure
+//		 500: Failure
+
+// swagger:route POST /api/v1/ngalert/admin_config configuration RoutePostNGalertConfig
+//
+// Creates or updates the NGalert configuration of the user's organization.
+//
+//     Consumes:
+//     - application/json
+//
+//     Responses:
+//       201: Ack
+//       400: ValidationError
+
+// swagger:route DELETE /api/v1/ngalert/admin_config configuration RouteDeleteNGalertConfig
+//
+// Deletes the NGalert configuration of the user's organization.
+//
+//     Consumes:
+//     - application/json
+//
+//     Responses:
+//       200: Ack
+//       500: Failure
+
+// swagger:parameters RoutePostNGalertConfig
+type NGalertConfig struct {
+	// in:body
+	Body PostableNGalertConfig
+}
+
+// swagger:model
+type PostableNGalertConfig struct {
+	Alertmanagers []string `json:"alertmanagers"`
+}
+
+// swagger:model
+type GettableNGalertConfig struct {
+	Alertmanagers []string `json:"alertmanagers"`
+}

pkg/services/ngalert/api/tooling/post.json

@@ -57,7 +57,9 @@
    "type": "object",
    "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
   },
-  "AlertGroup": {},
+  "AlertGroup": {
+   "$ref": "#/definitions/alertGroup"
+  },
   "AlertGroups": {
    "$ref": "#/definitions/alertGroups"
   },
@@ -751,6 +753,19 @@
    "type": "object",
    "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
   },
+  "GettableNGalertConfig": {
+   "properties": {
+    "alertmanagers": {
+     "items": {
+      "type": "string"
+     },
+     "type": "array",
+     "x-go-name": "Alertmanagers"
+    }
+   },
+   "type": "object",
+   "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+  },
   "GettableRuleGroupConfig": {
    "properties": {
     "interval": {
@@ -771,12 +786,8 @@
    "type": "object",
    "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
   },
-  "GettableSilence": {
-   "$ref": "#/definitions/gettableSilence"
-  },
-  "GettableSilences": {
-   "$ref": "#/definitions/gettableSilences"
-  },
+  "GettableSilence": {},
+  "GettableSilences": {},
   "GettableStatus": {
    "properties": {
     "cluster": {
@@ -1545,6 +1556,19 @@
    "type": "object",
    "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
   },
+  "PostableNGalertConfig": {
+   "properties": {
+    "alertmanagers": {
+     "items": {
+      "type": "string"
+     },
+     "type": "array",
+     "x-go-name": "Alertmanagers"
+    }
+   },
+   "type": "object",
+   "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+  },
   "PostableRuleGroupConfig": {
    "properties": {
     "interval": {
@@ -2425,7 +2449,7 @@
     "alerts": {
      "description": "alerts",
      "items": {
-      "$ref": "#/definitions/GettableAlert"
+      "$ref": "#/definitions/gettableAlert"
      },
      "type": "array",
      "x-go-name": "Alerts"
@@ -2434,7 +2458,7 @@
      "$ref": "#/definitions/labelSet"
     },
     "receiver": {
-     "$ref": "#/definitions/receiver"
+     "$ref": "#/definitions/Receiver"
     }
    },
    "required": [
@@ -2601,7 +2625,7 @@
     "receivers": {
      "description": "receivers",
      "items": {
-      "$ref": "#/definitions/receiver"
+      "$ref": "#/definitions/Receiver"
      },
      "type": "array",
      "x-go-name": "Receivers"
@@ -2639,7 +2663,7 @@
   "gettableAlerts": {
    "description": "GettableAlerts gettable alerts",
    "items": {
-    "$ref": "#/definitions/GettableAlert"
+    "$ref": "#/definitions/gettableAlert"
    },
    "type": "array",
    "x-go-name": "GettableAlerts",
@@ -2705,7 +2729,7 @@
   "gettableSilences": {
    "description": "GettableSilences gettable silences",
    "items": {
-    "$ref": "#/definitions/gettableSilence"
+    "$ref": "#/definitions/GettableSilence"
    },
    "type": "array",
    "x-go-name": "GettableSilences",
@@ -3775,6 +3799,95 @@
     ]
    }
   },
+  "/api/v1/ngalert/admin_config": {
+   "delete": {
+    "consumes": [
+     "application/json"
+    ],
+    "operationId": "RouteDeleteNGalertConfig",
+    "responses": {
+     "200": {
+      "description": "Ack",
+      "schema": {
+       "$ref": "#/definitions/Ack"
+      }
+     },
+     "500": {
+      "description": "Failure",
+      "schema": {
+       "$ref": "#/definitions/Failure"
+      }
+     }
+    },
+    "summary": "Deletes the NGalert configuration of the user's organization.",
+    "tags": [
+     "configuration"
+    ]
+   },
+   "get": {
+    "operationId": "RouteGetNGalertConfig",
+    "produces": [
+     "application/json"
+    ],
+    "responses": {
+     "200": {
+      "description": "GettableNGalertConfig",
+      "schema": {
+       "$ref": "#/definitions/GettableNGalertConfig"
+      }
+     },
+     "404": {
+      "description": "Failure",
+      "schema": {
+       "$ref": "#/definitions/Failure"
+      }
+     },
+     "500": {
+      "description": "Failure",
+      "schema": {
+       "$ref": "#/definitions/Failure"
+      }
+     }
+    },
+    "summary": "Get the NGalert configuration of the user's organization, returns 404 if no configuration is present.",
+    "tags": [
+     "configuration"
+    ]
+   },
+   "post": {
+    "consumes": [
+     "application/json"
+    ],
+    "operationId": "RoutePostNGalertConfig",
+    "parameters": [
+     {
+      "in": "body",
+      "name": "Body",
+      "schema": {
+       "$ref": "#/definitions/PostableNGalertConfig"
+      }
+     }
+    ],
+    "responses": {
+     "201": {
+      "description": "Ack",
+      "schema": {
+       "$ref": "#/definitions/Ack"
+      }
+     },
+     "400": {
+      "description": "ValidationError",
+      "schema": {
+       "$ref": "#/definitions/ValidationError"
+      }
+     }
+    },
+    "summary": "Creates or updates the NGalert configuration of the user's organization.",
+    "tags": [
+     "configuration"
+    ]
+   }
+  },
   "/api/v1/receiver/test/{Recipient}": {
    "post": {
     "consumes": [

pkg/services/ngalert/api/tooling/spec.json

@@ -781,6 +781,95 @@
         }
       }
     },
+    "/api/v1/ngalert/admin_config": {
+      "get": {
+        "produces": [
+          "application/json"
+        ],
+        "tags": [
+          "configuration"
+        ],
+        "summary": "Get the NGalert configuration of the user's organization, returns 404 if no configuration is present.",
+        "operationId": "RouteGetNGalertConfig",
+        "responses": {
+          "200": {
+            "description": "GettableNGalertConfig",
+            "schema": {
+              "$ref": "#/definitions/GettableNGalertConfig"
+            }
+          },
+          "404": {
+            "description": "Failure",
+            "schema": {
+              "$ref": "#/definitions/Failure"
+            }
+          },
+          "500": {
+            "description": "Failure",
+            "schema": {
+              "$ref": "#/definitions/Failure"
+            }
+          }
+        }
+      },
+      "post": {
+        "consumes": [
+          "application/json"
+        ],
+        "tags": [
+          "configuration"
+        ],
+        "summary": "Creates or updates the NGalert configuration of the user's organization.",
+        "operationId": "RoutePostNGalertConfig",
+        "parameters": [
+          {
+            "name": "Body",
+            "in": "body",
+            "schema": {
+              "$ref": "#/definitions/PostableNGalertConfig"
+            }
+          }
+        ],
+        "responses": {
+          "201": {
+            "description": "Ack",
+            "schema": {
+              "$ref": "#/definitions/Ack"
+            }
+          },
+          "400": {
+            "description": "ValidationError",
+            "schema": {
+              "$ref": "#/definitions/ValidationError"
+            }
+          }
+        }
+      },
+      "delete": {
+        "consumes": [
+          "application/json"
+        ],
+        "tags": [
+          "configuration"
+        ],
+        "summary": "Deletes the NGalert configuration of the user's organization.",
+        "operationId": "RouteDeleteNGalertConfig",
+        "responses": {
+          "200": {
+            "description": "Ack",
+            "schema": {
+              "$ref": "#/definitions/Ack"
+            }
+          },
+          "500": {
+            "description": "Failure",
+            "schema": {
+              "$ref": "#/definitions/Failure"
+            }
+          }
+        }
+      }
+    },
     "/api/v1/receiver/test/{Recipient}": {
       "post": {
         "description": "Test receiver",
@@ -927,7 +1016,7 @@
       "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
     },
     "AlertGroup": {
-      "$ref": "#/definitions/AlertGroup"
+      "$ref": "#/definitions/alertGroup"
     },
     "AlertGroups": {
       "$ref": "#/definitions/alertGroups"
@@ -1625,6 +1714,19 @@
       },
       "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
     },
+    "GettableNGalertConfig": {
+      "type": "object",
+      "properties": {
+        "alertmanagers": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "x-go-name": "Alertmanagers"
+        }
+      },
+      "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+    },
     "GettableRuleGroupConfig": {
       "type": "object",
       "properties": {
@@ -1646,10 +1748,10 @@
       "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
     },
     "GettableSilence": {
-      "$ref": "#/definitions/gettableSilence"
+      "$ref": "#/definitions/GettableSilence"
     },
     "GettableSilences": {
-      "$ref": "#/definitions/gettableSilences"
+      "$ref": "#/definitions/GettableSilences"
     },
     "GettableStatus": {
       "type": "object",
@@ -2420,6 +2522,19 @@
       },
       "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
     },
+    "PostableNGalertConfig": {
+      "type": "object",
+      "properties": {
+        "alertmanagers": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "x-go-name": "Alertmanagers"
+        }
+      },
+      "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+    },
     "PostableRuleGroupConfig": {
       "type": "object",
       "properties": {
@@ -3310,7 +3425,7 @@
           "description": "alerts",
           "type": "array",
           "items": {
-            "$ref": "#/definitions/GettableAlert"
+            "$ref": "#/definitions/gettableAlert"
           },
           "x-go-name": "Alerts"
         },
@@ -3318,7 +3433,7 @@
           "$ref": "#/definitions/labelSet"
         },
         "receiver": {
-          "$ref": "#/definitions/receiver"
+          "$ref": "#/definitions/Receiver"
         }
       },
       "x-go-name": "AlertGroup",
@@ -3491,7 +3606,7 @@
           "description": "receivers",
           "type": "array",
           "items": {
-            "$ref": "#/definitions/receiver"
+            "$ref": "#/definitions/Receiver"
           },
           "x-go-name": "Receivers"
         },
@@ -3518,7 +3633,7 @@
       "description": "GettableAlerts gettable alerts",
       "type": "array",
       "items": {
-        "$ref": "#/definitions/GettableAlert"
+        "$ref": "#/definitions/gettableAlert"
       },
       "x-go-name": "GettableAlerts",
       "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
@@ -3584,7 +3699,7 @@
       "description": "GettableSilences gettable silences",
       "type": "array",
       "items": {
-        "$ref": "#/definitions/gettableSilence"
+        "$ref": "#/definitions/GettableSilence"
       },
       "x-go-name": "GettableSilences",
       "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"

pkg/services/ngalert/api/util.go

@@ -257,3 +257,8 @@ func ErrResp(status int, err error, msg string, args ...interface{}) *response.N
 	}
 	return response.Error(status, err.Error(), nil)
 }
+
+// accessForbiddenResp creates a response of forbidden access.
+func accessForbiddenResp() response.Response {
+	return ErrResp(http.StatusForbidden, errors.New("Permission denied"), "")
+}