Alerting: send alerts to external, internal, or both alertmanagers (#40341)

* (WIP) send alerts to external, internal, or both alertmanagers

* Modify admin configuration endpoint, update swagger docs

* Integration test for admin config updated

* Code review changes

* Fix alertmanagers choice not changing bug, add unit test

* Add AlertmanagersChoice as enum in swagger, code review changes

* Fix API and tests errors

* Change enum from int to string, use 'SendAlertsTo' instead of 'AlertmanagerChoice' where necessary

* Fix tests to reflect last changes

* Keep senders running when alerts are handled just internally

* Check if any external AM has been discovered before sending alerts, update tests

* remove duplicate data from logs

* update comment

* represent alertmanagers choice as an int instead of a string

* default alertmanagers choice to all alertmanagers, test cases

* update definitions and generate spec

pkg/services/ngalert/api/api_admin.go

@@ -55,7 +55,8 @@ func (srv AdminSrv) RouteGetNGalertConfig(c *models.ReqContext) response.Respons
 	}
 
 	resp := apimodels.GettableNGalertConfig{
-		Alertmanagers: cfg.Alertmanagers,
+		Alertmanagers:       cfg.Alertmanagers,
+		AlertmanagersChoice: apimodels.AlertmanagersChoice(cfg.SendAlertsTo.String()),
 	}
 	return response.JSON(http.StatusOK, resp)
 }
@@ -65,8 +66,18 @@ func (srv AdminSrv) RoutePostNGalertConfig(c *models.ReqContext, body apimodels.
 		return accessForbiddenResp()
 	}
 
+	sendAlertsTo, err := ngmodels.StringToAlertmanagersChoice(string(body.AlertmanagersChoice))
+	if err != nil {
+		return response.Error(400, "Invalid alertmanager choice specified", nil)
+	}
+
+	if sendAlertsTo == ngmodels.ExternalAlertmanagers && len(body.Alertmanagers) == 0 {
+		return response.Error(400, "At least one Alertmanager must be provided to choose this option", nil)
+	}
+
 	cfg := &ngmodels.AdminConfiguration{
 		Alertmanagers: body.Alertmanagers,
+		SendAlertsTo:  sendAlertsTo,
 		OrgID:         c.OrgId,
 	}
 

pkg/services/ngalert/api/tooling/definitions/admin.go

@@ -1,6 +1,8 @@
 package definitions
 
-import v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+import (
+	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+)
 
 // swagger:route GET /api/v1/ngalert/alertmanagers configuration RouteGetAlertmanagers
 //
@@ -26,7 +28,7 @@ import v1 "github.com/prometheus/client_golang/api/prometheus/v1"
 
 // swagger:route POST /api/v1/ngalert/admin_config configuration RoutePostNGalertConfig
 //
-// Creates or updates the NGalert configuration of the user's organization.
+// Creates or updates the NGalert configuration of the user's organization. If no value is sent for alertmanagersChoice, it defaults to "all".
 //
 //     Consumes:
 //     - application/json
@@ -52,14 +54,25 @@ type NGalertConfig struct {
 	Body PostableNGalertConfig
 }
 
+// swagger:enum AlertmanagersChoice
+type AlertmanagersChoice string
+
+const (
+	AllAlertmanagers      AlertmanagersChoice = "all"
+	InternalAlertmanager  AlertmanagersChoice = "internal"
+	ExternalAlertmanagers AlertmanagersChoice = "external"
+)
+
 // swagger:model
 type PostableNGalertConfig struct {
-	Alertmanagers []string `json:"alertmanagers"`
+	Alertmanagers       []string            `json:"alertmanagers"`
+	AlertmanagersChoice AlertmanagersChoice `json:"alertmanagersChoice"`
 }
 
 // swagger:model
 type GettableNGalertConfig struct {
-	Alertmanagers []string `json:"alertmanagers"`
+	Alertmanagers       []string            `json:"alertmanagers"`
+	AlertmanagersChoice AlertmanagersChoice `json:"alertmanagersChoice"`
 }
 
 // swagger:model

pkg/services/ngalert/api/tooling/post.json

@@ -769,6 +769,7 @@
       "Alerting"
      ],
      "type": "string",
+     "x-go-enum-desc": "Alerting AlertingErrState",
      "x-go-name": "ExecErrState"
     },
     "id": {
@@ -797,6 +798,7 @@
       "OK"
      ],
      "type": "string",
+     "x-go-enum-desc": "Alerting Alerting\nNoData NoData\nOK OK",
      "x-go-name": "NoDataState"
     },
     "orgId": {
@@ -838,6 +840,16 @@
      },
      "type": "array",
      "x-go-name": "Alertmanagers"
+    },
+    "alertmanagersChoice": {
+     "enum": [
+      "all",
+      "internal",
+      "external"
+     ],
+     "type": "string",
+     "x-go-enum-desc": "all AllAlertmanagers\ninternal InternalAlertmanager\nexternal ExternalAlertmanagers",
+     "x-go-name": "AlertmanagersChoice"
     }
    },
    "type": "object",
@@ -1718,6 +1730,7 @@
       "Alerting"
      ],
      "type": "string",
+     "x-go-enum-desc": "Alerting AlertingErrState",
      "x-go-name": "ExecErrState"
     },
     "no_data_state": {
@@ -1727,6 +1740,7 @@
       "OK"
      ],
      "type": "string",
+     "x-go-enum-desc": "Alerting Alerting\nNoData NoData\nOK OK",
      "x-go-name": "NoDataState"
     },
     "title": {
@@ -1749,6 +1763,16 @@
      },
      "type": "array",
      "x-go-name": "Alertmanagers"
+    },
+    "alertmanagersChoice": {
+     "enum": [
+      "all",
+      "internal",
+      "external"
+     ],
+     "type": "string",
+     "x-go-enum-desc": "all AllAlertmanagers\ninternal InternalAlertmanager\nexternal ExternalAlertmanagers",
+     "x-go-name": "AlertmanagersChoice"
     }
    },
    "type": "object",
@@ -2884,6 +2908,7 @@
    "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
   },
   "alertGroup": {
+   "description": "AlertGroup alert group",
    "properties": {
     "alerts": {
      "description": "alerts",
@@ -2905,16 +2930,15 @@
     "labels",
     "receiver"
    ],
-   "type": "object",
-   "x-go-name": "AlertGroup",
-   "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
+   "type": "object"
   },
   "alertGroups": {
-   "description": "AlertGroups alert groups",
    "items": {
     "$ref": "#/definitions/alertGroup"
    },
-   "type": "array"
+   "type": "array",
+   "x-go-name": "AlertGroups",
+   "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
   },
   "alertStatus": {
    "description": "AlertStatus alert status",
@@ -3096,13 +3120,15 @@
    "type": "object"
   },
   "gettableAlerts": {
-   "description": "GettableAlerts gettable alerts",
    "items": {
     "$ref": "#/definitions/gettableAlert"
    },
-   "type": "array"
+   "type": "array",
+   "x-go-name": "GettableAlerts",
+   "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
   },
   "gettableSilence": {
+   "description": "GettableSilence gettable silence",
    "properties": {
     "comment": {
      "description": "comment",
@@ -3154,9 +3180,7 @@
     "status",
     "updatedAt"
    ],
-   "type": "object",
-   "x-go-name": "GettableSilence",
-   "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
+   "type": "object"
   },
   "gettableSilences": {
    "description": "GettableSilences gettable silences",
@@ -3335,6 +3359,7 @@
    "type": "object"
   },
   "receiver": {
+   "description": "Receiver receiver",
    "properties": {
     "name": {
      "description": "name",
@@ -3345,9 +3370,7 @@
    "required": [
     "name"
    ],
-   "type": "object",
-   "x-go-name": "Receiver",
-   "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
+   "type": "object"
   },
   "silence": {
    "description": "Silence silence",
@@ -3456,8 +3479,8 @@
   }
  },
  "info": {
-  "description": "Package definitions includes the types required for generating or consuming an OpenAPI\nspec for the Grafana 8 Alerts API.",
-  "title": "Grafana 8 Alerts API.",
+  "description": "Package definitions includes the types required for generating or consuming an OpenAPI\nspec for the Grafana Alerting API.",
+  "title": "Grafana Alerting API.",
   "version": "1.1.0"
  },
  "paths": {
@@ -4400,7 +4423,7 @@
       }
      }
     },
-    "summary": "Creates or updates the NGalert configuration of the user's organization.",
+    "summary": "Creates or updates the NGalert configuration of the user's organization. If no value is sent for alertmanagersChoice, it defaults to \"all\".",
     "tags": [
      "configuration"
     ]

pkg/services/ngalert/api/tooling/spec.json

@@ -911,7 +911,7 @@
         "tags": [
           "configuration"
         ],
-        "summary": "Creates or updates the NGalert configuration of the user's organization.",
+        "summary": "Creates or updates the NGalert configuration of the user's organization. If no value is sent for alertmanagersChoice, it defaults to \"all\".",
         "operationId": "RoutePostNGalertConfig",
         "parameters": [
           {
@@ -1863,6 +1863,16 @@
             "type": "string"
           },
           "x-go-name": "Alertmanagers"
+        },
+        "alertmanagersChoice": {
+          "type": "string",
+          "enum": [
+            "all",
+            "internal",
+            "external"
+          ],
+          "x-go-enum-desc": "all AllAlertmanagers\ninternal InternalAlertmanager\nexternal ExternalAlertmanagers",
+          "x-go-name": "AlertmanagersChoice"
         }
       },
       "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
@@ -2777,6 +2787,16 @@
             "type": "string"
           },
           "x-go-name": "Alertmanagers"
+        },
+        "alertmanagersChoice": {
+          "type": "string",
+          "enum": [
+            "all",
+            "internal",
+            "external"
+          ],
+          "x-go-enum-desc": "all AllAlertmanagers\ninternal InternalAlertmanager\nexternal ExternalAlertmanagers",
+          "x-go-name": "AlertmanagersChoice"
         }
       },
       "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
@@ -3680,9 +3700,8 @@
       "x-go-package": "github.com/prometheus/alertmanager/timeinterval"
     },
     "URL": {
-      "description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use RawPath, an optional field which only gets\nset if the default encoding is different from Path.\n\nURL's String method uses the EscapedPath method to obtain the path. See the\nEscapedPath method for more details.",
       "type": "object",
-      "title": "A URL represents a parsed URL (technically, a URI reference).",
+      "title": "URL is a custom URL type that allows validation at configuration load time.",
       "properties": {
         "ForceQuery": {
           "type": "boolean"
@@ -3715,7 +3734,7 @@
           "$ref": "#/definitions/Userinfo"
         }
       },
-      "x-go-package": "net/url"
+      "x-go-package": "github.com/prometheus/common/config"
     },
     "Userinfo": {
       "description": "The Userinfo type is an immutable encapsulation of username and\npassword details for a URL. An existing Userinfo value is guaranteed\nto have a username set (potentially empty, as allowed by RFC 2396),\nand optionally a password.",
@@ -3912,6 +3931,7 @@
       "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
     },
     "alertGroup": {
+      "description": "AlertGroup alert group",
       "type": "object",
       "required": [
         "alerts",
@@ -3934,8 +3954,6 @@
           "$ref": "#/definitions/receiver"
         }
       },
-      "x-go-name": "AlertGroup",
-      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
       "$ref": "#/definitions/alertGroup"
     },
     "alertGroups": {
@@ -4128,11 +4146,12 @@
       "$ref": "#/definitions/gettableAlert"
     },
     "gettableAlerts": {
-      "description": "GettableAlerts gettable alerts",
       "type": "array",
       "items": {
         "$ref": "#/definitions/gettableAlert"
       },
+      "x-go-name": "GettableAlerts",
+      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
       "$ref": "#/definitions/gettableAlerts"
     },
     "gettableSilence": {
@@ -4192,12 +4211,11 @@
       "$ref": "#/definitions/gettableSilence"
     },
     "gettableSilences": {
+      "description": "GettableSilences gettable silences",
       "type": "array",
       "items": {
         "$ref": "#/definitions/gettableSilence"
       },
-      "x-go-name": "GettableSilences",
-      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
       "$ref": "#/definitions/gettableSilences"
     },
     "labelSet": {
@@ -4371,6 +4389,7 @@
       "$ref": "#/definitions/postableSilence"
     },
     "receiver": {
+      "description": "Receiver receiver",
       "type": "object",
       "required": [
         "name"
@@ -4382,8 +4401,6 @@
           "x-go-name": "Name"
         }
       },
-      "x-go-name": "Receiver",
-      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
       "$ref": "#/definitions/receiver"
     },
     "silence": {

pkg/services/ngalert/models/admin_configuration.go

@@ -2,10 +2,25 @@ package models
 
 import (
 	"crypto/sha256"
+	"errors"
 	"fmt"
 	"net/url"
 )
 
+type AlertmanagersChoice int
+
+const (
+	AllAlertmanagers AlertmanagersChoice = iota
+	InternalAlertmanager
+	ExternalAlertmanagers
+)
+
+var alertmanagersChoiceMap = map[AlertmanagersChoice]string{
+	AllAlertmanagers:      "all",
+	InternalAlertmanager:  "internal",
+	ExternalAlertmanagers: "external",
+}
+
 // AdminConfiguration represents the ngalert administration configuration settings.
 type AdminConfiguration struct {
 	ID    int64 `xorm:"pk autoincr 'id'"`
@@ -14,6 +29,9 @@ type AdminConfiguration struct {
 	// List of Alertmanager(s) URL to push alerts to.
 	Alertmanagers []string
 
+	// SendAlertsTo indicates which set of alertmanagers will handle the alert.
+	SendAlertsTo AlertmanagersChoice `xorm:"send_alerts_to"`
+
 	CreatedAt int64 `xorm:"created"`
 	UpdatedAt int64 `xorm:"updated"`
 }
@@ -34,3 +52,21 @@ func (ac *AdminConfiguration) Validate() error {
 
 	return nil
 }
+
+// String implements the Stringer interface
+func (amc AlertmanagersChoice) String() string {
+	return alertmanagersChoiceMap[amc]
+}
+
+func StringToAlertmanagersChoice(str string) (AlertmanagersChoice, error) {
+	if str == "" {
+		return AllAlertmanagers, nil
+	}
+
+	for k, v := range alertmanagersChoiceMap {
+		if str == v {
+			return k, nil
+		}
+	}
+	return 0, errors.New("invalid alertmanager choice")
+}

pkg/services/ngalert/models/admin_configuration_test.go

@@ -1,6 +1,7 @@
 package models
 
 import (
+	"errors"
 	"fmt"
 	"testing"
 
@@ -56,3 +57,56 @@ func TestAdminConfiguration_Validate(t *testing.T) {
 		})
 	}
 }
+
+func TestStringToAlertmanagersChoice(t *testing.T) {
+	tests := []struct {
+		name                string
+		str                 string
+		alertmanagersChoice AlertmanagersChoice
+		err                 error
+	}{
+		{
+			"all alertmanagers",
+			"all",
+			AllAlertmanagers,
+			nil,
+		},
+		{
+			"internal alertmanager",
+			"internal",
+			InternalAlertmanager,
+			nil,
+		},
+		{
+			"external alertmanagers",
+			"external",
+			ExternalAlertmanagers,
+			nil,
+		},
+		{
+			"empty string value",
+			"",
+			AllAlertmanagers,
+			nil,
+		},
+		{
+			"invalid string",
+			"invalid",
+			0,
+			errors.New("invalid alertmanager choice"),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(tt *testing.T) {
+			amc, err := StringToAlertmanagersChoice(test.str)
+			if test.err != nil {
+				require.EqualError(tt, err, test.err.Error())
+			} else {
+				require.NoError(tt, err)
+			}
+
+			require.Equal(tt, amc, test.alertmanagersChoice)
+		})
+	}
+}

pkg/services/ngalert/schedule/schedule.go

@@ -91,7 +91,8 @@ type schedule struct {
 	metrics          *metrics.Scheduler
 
 	// Senders help us send alerts to external Alertmanagers.
-	sendersMtx              sync.RWMutex
+	adminConfigMtx          sync.RWMutex
+	sendAlertsTo            map[int64]models.AlertmanagersChoice
 	sendersCfgHash          map[int64]string
 	senders                 map[int64]*sender.Sender
 	adminConfigPollInterval time.Duration
@@ -142,6 +143,7 @@ func NewScheduler(cfg SchedulerCfg, expressionService *expr.Service, appURL *url
 		metrics:                 cfg.Metrics,
 		appURL:                  appURL,
 		stateManager:            stateManager,
+		sendAlertsTo:            map[int64]models.AlertmanagersChoice{},
 		senders:                 map[int64]*sender.Sender{},
 		sendersCfgHash:          map[int64]string{},
 		adminConfigPollInterval: cfg.AdminConfigPollInterval,
@@ -191,7 +193,8 @@ func (sch *schedule) Run(ctx context.Context) error {
 	return nil
 }
 
-// SyncAndApplyConfigFromDatabase looks for the admin configuration in the database and adjusts the sender(s) accordingly.
+// SyncAndApplyConfigFromDatabase looks for the admin configuration in the database
+// and adjusts the sender(s) and alert handling mechanism accordingly.
 func (sch *schedule) SyncAndApplyConfigFromDatabase() error {
 	sch.log.Debug("start of admin configuration sync")
 	cfgs, err := sch.adminConfigStore.GetAdminConfigurations()
@@ -202,7 +205,7 @@ func (sch *schedule) SyncAndApplyConfigFromDatabase() error {
 	sch.log.Debug("found admin configurations", "count", len(cfgs))
 
 	orgsFound := make(map[int64]struct{}, len(cfgs))
-	sch.sendersMtx.Lock()
+	sch.adminConfigMtx.Lock()
 	for _, cfg := range cfgs {
 		_, isDisabledOrg := sch.disabledOrgs[cfg.OrgID]
 		if isDisabledOrg {
@@ -210,15 +213,23 @@ func (sch *schedule) SyncAndApplyConfigFromDatabase() error {
 			continue
 		}
 
+		// Update the Alertmanagers choice for the organization.
+		sch.sendAlertsTo[cfg.OrgID] = cfg.SendAlertsTo
+
 		orgsFound[cfg.OrgID] = struct{}{} // keep track of the which senders we need to keep.
 
 		existing, ok := sch.senders[cfg.OrgID]
 
-		// If the tenant has no Alertmanager(s) configured and no running sender no-op.
+		// We have no running sender and no Alertmanager(s) configured, no-op.
 		if !ok && len(cfg.Alertmanagers) == 0 {
 			sch.log.Debug("no external alertmanagers configured", "org", cfg.OrgID)
 			continue
 		}
+		//  We have no running sender and alerts are handled internally, no-op.
+		if !ok && cfg.SendAlertsTo == models.InternalAlertmanager {
+			sch.log.Debug("alerts are handled internally", "org", cfg.OrgID)
+			continue
+		}
 
 		// We have a running sender but no Alertmanager(s) configured, shut it down.
 		if ok && len(cfg.Alertmanagers) == 0 {
@@ -273,7 +284,7 @@ func (sch *schedule) SyncAndApplyConfigFromDatabase() error {
 			delete(sch.sendersCfgHash, orgID)
 		}
 	}
-	sch.sendersMtx.Unlock()
+	sch.adminConfigMtx.Unlock()
 
 	// We can now stop these senders w/o having to hold a lock.
 	for orgID, s := range sendersToStop {
@@ -289,8 +300,8 @@ func (sch *schedule) SyncAndApplyConfigFromDatabase() error {
 
 // AlertmanagersFor returns all the discovered Alertmanager(s) for a particular organization.
 func (sch *schedule) AlertmanagersFor(orgID int64) []*url.URL {
-	sch.sendersMtx.RLock()
-	defer sch.sendersMtx.RUnlock()
+	sch.adminConfigMtx.RLock()
+	defer sch.adminConfigMtx.RUnlock()
 	s, ok := sch.senders[orgID]
 	if !ok {
 		return []*url.URL{}
@@ -301,8 +312,8 @@ func (sch *schedule) AlertmanagersFor(orgID int64) []*url.URL {
 
 // DroppedAlertmanagersFor returns all the dropped Alertmanager(s) for a particular organization.
 func (sch *schedule) DroppedAlertmanagersFor(orgID int64) []*url.URL {
-	sch.sendersMtx.RLock()
-	defer sch.sendersMtx.RUnlock()
+	sch.adminConfigMtx.RLock()
+	defer sch.adminConfigMtx.RUnlock()
 	s, ok := sch.senders[orgID]
 	if !ok {
 		return []*url.URL{}
@@ -340,12 +351,12 @@ func (sch *schedule) adminConfigSync(ctx context.Context) error {
 			}
 		case <-ctx.Done():
 			// Stop sending alerts to all external Alertmanager(s).
-			sch.sendersMtx.Lock()
+			sch.adminConfigMtx.Lock()
 			for orgID, s := range sch.senders {
 				delete(sch.senders, orgID) // delete before we stop to make sure we don't accept any more alerts.
 				s.Stop()
 			}
-			sch.sendersMtx.Unlock()
+			sch.adminConfigMtx.Unlock()
 
 			return nil
 		}
@@ -468,32 +479,39 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key models.AlertRul
 
 	notify := func(alerts definitions.PostableAlerts, logger log.Logger) {
 		if len(alerts.PostableAlerts) == 0 {
-			logger.Debug("no alerts to put in the notifier")
+			logger.Debug("no alerts to put in the notifier or to send to external Alertmanager(s)")
 			return
 		}
 
+		// Send alerts to local notifier if they need to be handled internally
+		// or if no external AMs have been discovered yet.
 		var localNotifierExist, externalNotifierExist bool
-		logger.Debug("sending alerts to notifier", "count", len(alerts.PostableAlerts), "alerts", alerts.PostableAlerts)
-		n, err := sch.multiOrgNotifier.AlertmanagerFor(key.OrgID)
-		if err == nil {
-			localNotifierExist = true
-			if err := n.PutAlerts(alerts); err != nil {
-				logger.Error("failed to put alerts in the local notifier", "count", len(alerts.PostableAlerts), "err", err)
-			}
+		if sch.sendAlertsTo[key.OrgID] == models.ExternalAlertmanagers && len(sch.AlertmanagersFor(key.OrgID)) > 0 {
+			logger.Debug("no alerts to put in the notifier")
 		} else {
-			if errors.Is(err, notifier.ErrNoAlertmanagerForOrg) {
-				logger.Debug("local notifier was not found")
+			logger.Debug("sending alerts to local notifier", "count", len(alerts.PostableAlerts), "alerts", alerts.PostableAlerts)
+			n, err := sch.multiOrgNotifier.AlertmanagerFor(key.OrgID)
+			if err == nil {
+				localNotifierExist = true
+				if err := n.PutAlerts(alerts); err != nil {
+					logger.Error("failed to put alerts in the local notifier", "count", len(alerts.PostableAlerts), "err", err)
+				}
 			} else {
-				logger.Error("local notifier is not available", "err", err)
+				if errors.Is(err, notifier.ErrNoAlertmanagerForOrg) {
+					logger.Debug("local notifier was not found")
+				} else {
+					logger.Error("local notifier is not available", "err", err)
+				}
 			}
 		}
 
-		// Send alerts to external Alertmanager(s) if we have a sender for this organization.
-		sch.sendersMtx.RLock()
-		defer sch.sendersMtx.RUnlock()
+		// Send alerts to external Alertmanager(s) if we have a sender for this organization
+		// and alerts are not being handled just internally.
+		sch.adminConfigMtx.RLock()
+		defer sch.adminConfigMtx.RUnlock()
 		s, ok := sch.senders[key.OrgID]
-		if ok {
-			logger.Debug("sending alerts to external notifier", "count", len(alerts.PostableAlerts))
+		if ok && sch.sendAlertsTo[key.OrgID] != models.InternalAlertmanager {
+			logger.Debug("sending alerts to external notifier", "count", len(alerts.PostableAlerts), "alerts", alerts.PostableAlerts)
 			s.SendAlerts(alerts)
 			externalNotifierExist = true
 		}

pkg/services/ngalert/schedule/schedule_unit_test.go

@@ -48,7 +48,7 @@ func TestSendingToExternalAlertmanager(t *testing.T) {
 	alertRule := CreateTestAlertRule(t, fakeRuleStore, 1, 1, eval.Alerting)
 
 	// First, let's create an admin configuration that holds an alertmanager.
-	adminConfig := &models.AdminConfiguration{OrgID: 1, Alertmanagers: []string{fakeAM.server.URL}}
+	adminConfig := &models.AdminConfiguration{OrgID: 1, Alertmanagers: []string{fakeAM.server.URL}, SendAlertsTo: models.AllAlertmanagers}
 	cmd := store.UpdateAdminConfigurationCmd{AdminConfiguration: adminConfig}
 	require.NoError(t, fakeAdminConfigStore.UpdateAdminConfiguration(cmd))
 
@@ -57,10 +57,10 @@ func TestSendingToExternalAlertmanager(t *testing.T) {
 	// Make sure we sync the configuration at least once before the evaluation happens to guarantee the sender is running
 	// when the first alert triggers.
 	require.NoError(t, sched.SyncAndApplyConfigFromDatabase())
-	sched.sendersMtx.Lock()
+	sched.adminConfigMtx.Lock()
 	require.Equal(t, 1, len(sched.senders))
 	require.Equal(t, 1, len(sched.sendersCfgHash))
-	sched.sendersMtx.Unlock()
+	sched.adminConfigMtx.Unlock()
 
 	// Then, ensure we've discovered the Alertmanager.
 	require.Eventually(t, func() bool {
@@ -91,10 +91,10 @@ func TestSendingToExternalAlertmanager(t *testing.T) {
 
 	// Again, make sure we sync and verify the senders.
 	require.NoError(t, sched.SyncAndApplyConfigFromDatabase())
-	sched.sendersMtx.Lock()
+	sched.adminConfigMtx.Lock()
 	require.Equal(t, 0, len(sched.senders))
 	require.Equal(t, 0, len(sched.sendersCfgHash))
-	sched.sendersMtx.Unlock()
+	sched.adminConfigMtx.Unlock()
 
 	// Then, ensure we've dropped the Alertmanager.
 	require.Eventually(t, func() bool {
@@ -119,10 +119,10 @@ func TestSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T) {
 	// Make sure we sync the configuration at least once before the evaluation happens to guarantee the sender is running
 	// when the first alert triggers.
 	require.NoError(t, sched.SyncAndApplyConfigFromDatabase())
-	sched.sendersMtx.Lock()
+	sched.adminConfigMtx.Lock()
 	require.Equal(t, 1, len(sched.senders))
 	require.Equal(t, 1, len(sched.sendersCfgHash))
-	sched.sendersMtx.Unlock()
+	sched.adminConfigMtx.Unlock()
 
 	// Then, ensure we've discovered the Alertmanager.
 	require.Eventuallyf(t, func() bool {
@@ -145,10 +145,10 @@ func TestSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T) {
 
 	// If we sync again, new senders must have spawned.
 	require.NoError(t, sched.SyncAndApplyConfigFromDatabase())
-	sched.sendersMtx.Lock()
+	sched.adminConfigMtx.Lock()
 	require.Equal(t, 2, len(sched.senders))
 	require.Equal(t, 2, len(sched.sendersCfgHash))
-	sched.sendersMtx.Unlock()
+	sched.adminConfigMtx.Unlock()
 
 	// Then, ensure we've discovered the Alertmanager for the new organization.
 	require.Eventuallyf(t, func() bool {
@@ -179,19 +179,19 @@ func TestSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T) {
 	require.NoError(t, fakeAdminConfigStore.UpdateAdminConfiguration(cmd))
 
 	// Before we sync, let's grab the existing hash of this particular org.
-	sched.sendersMtx.Lock()
+	sched.adminConfigMtx.Lock()
 	currentHash := sched.sendersCfgHash[2]
-	sched.sendersMtx.Unlock()
+	sched.adminConfigMtx.Unlock()
 
 	// Now, sync again.
 	require.NoError(t, sched.SyncAndApplyConfigFromDatabase())
 
 	// The hash for org two should not be the same and we should still have two senders.
-	sched.sendersMtx.Lock()
+	sched.adminConfigMtx.Lock()
 	require.NotEqual(t, sched.sendersCfgHash[2], currentHash)
 	require.Equal(t, 2, len(sched.senders))
 	require.Equal(t, 2, len(sched.sendersCfgHash))
-	sched.sendersMtx.Unlock()
+	sched.adminConfigMtx.Unlock()
 
 	// Wait for the discovery of the new Alertmanager for orgID = 2.
 	require.Eventuallyf(t, func() bool {
@@ -204,17 +204,17 @@ func TestSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T) {
 	require.NoError(t, fakeAdminConfigStore.UpdateAdminConfiguration(cmd))
 
 	// Before we sync, let's get the current config hash.
-	sched.sendersMtx.Lock()
+	sched.adminConfigMtx.Lock()
 	currentHash = sched.sendersCfgHash[1]
-	sched.sendersMtx.Unlock()
+	sched.adminConfigMtx.Unlock()
 
 	// Now, sync again.
 	require.NoError(t, sched.SyncAndApplyConfigFromDatabase())
 
 	// The old configuration should still be running.
-	sched.sendersMtx.Lock()
+	sched.adminConfigMtx.Lock()
 	require.Equal(t, sched.sendersCfgHash[1], currentHash)
-	sched.sendersMtx.Unlock()
+	sched.adminConfigMtx.Unlock()
 	require.Equal(t, 1, len(sched.AlertmanagersFor(1)))
 
 	// If we fix it - it should be applied.
@@ -222,18 +222,18 @@ func TestSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T) {
 	cmd = store.UpdateAdminConfigurationCmd{AdminConfiguration: adminConfig2}
 	require.NoError(t, fakeAdminConfigStore.UpdateAdminConfiguration(cmd))
 	require.NoError(t, sched.SyncAndApplyConfigFromDatabase())
-	sched.sendersMtx.Lock()
+	sched.adminConfigMtx.Lock()
 	require.NotEqual(t, sched.sendersCfgHash[1], currentHash)
-	sched.sendersMtx.Unlock()
+	sched.adminConfigMtx.Unlock()
 
 	// Finally, remove everything.
 	require.NoError(t, fakeAdminConfigStore.DeleteAdminConfiguration(1))
 	require.NoError(t, fakeAdminConfigStore.DeleteAdminConfiguration(2))
 	require.NoError(t, sched.SyncAndApplyConfigFromDatabase())
-	sched.sendersMtx.Lock()
+	sched.adminConfigMtx.Lock()
 	require.Equal(t, 0, len(sched.senders))
 	require.Equal(t, 0, len(sched.sendersCfgHash))
-	sched.sendersMtx.Unlock()
+	sched.adminConfigMtx.Unlock()
 
 	require.Eventuallyf(t, func() bool {
 		NoAlertmanagerOrgOne := len(sched.AlertmanagersFor(1)) == 0 && len(sched.DroppedAlertmanagersFor(1)) == 0
@@ -243,6 +243,97 @@ func TestSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T) {
 	}, 10*time.Second, 200*time.Millisecond, "Alertmanager for org 1 and 2 were never removed")
 }
 
+func TestChangingAlertmanagersChoice(t *testing.T) {
+	fakeAM := NewFakeExternalAlertmanager(t)
+	defer fakeAM.Close()
+	fakeRuleStore := newFakeRuleStore(t)
+	fakeInstanceStore := &FakeInstanceStore{}
+	fakeAdminConfigStore := newFakeAdminConfigStore(t)
+
+	// create alert rule with one second interval and an Alertmanagers choice.
+	alertRule := CreateTestAlertRule(t, fakeRuleStore, 1, 1, eval.Alerting)
+
+	// First, let's create an admin configuration that holds an alertmanager
+	// and sends alerts to both internal and external alertmanagers (default).
+	adminConfig := &models.AdminConfiguration{OrgID: 1, Alertmanagers: []string{fakeAM.server.URL}}
+	cmd := store.UpdateAdminConfigurationCmd{AdminConfiguration: adminConfig}
+	require.NoError(t, fakeAdminConfigStore.UpdateAdminConfiguration(cmd))
+
+	sched, mockedClock := setupScheduler(t, fakeRuleStore, fakeInstanceStore, fakeAdminConfigStore, nil)
+
+	// Make sure we sync the configuration at least once before the evaluation happens to guarantee the sender is running
+	// when the first alert triggers.
+	require.NoError(t, sched.SyncAndApplyConfigFromDatabase())
+	sched.adminConfigMtx.Lock()
+	require.Equal(t, 1, len(sched.senders))
+	require.Equal(t, 1, len(sched.sendersCfgHash))
+	sched.adminConfigMtx.Unlock()
+
+	// Then, ensure we've discovered the Alertmanager and the Alertmanagers choice is correct.
+	require.Eventually(t, func() bool {
+		return len(sched.AlertmanagersFor(1)) == 1 &&
+			len(sched.DroppedAlertmanagersFor(1)) == 0 &&
+			sched.sendAlertsTo[1] == adminConfig.SendAlertsTo
+	}, 10*time.Second, 200*time.Millisecond)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(func() {
+		cancel()
+	})
+	go func() {
+		err := sched.Run(ctx)
+		require.NoError(t, err)
+	}()
+
+	// With everything up and running, let's advance the time to make sure we get at least one alert iteration.
+	mockedClock.Add(2 * time.Second)
+
+	// Eventually, our Alertmanager should have received alerts.
+	require.Eventually(t, func() bool {
+		return fakeAM.AlertsCount() >= 1 &&
+			fakeAM.AlertNamesCompare([]string{alertRule.Title})
+	}, 10*time.Second, 200*time.Millisecond)
+
+	// Now, let's change the Alertmanagers choice to send only to the external Alertmanager.
+	adminConfig.SendAlertsTo = models.ExternalAlertmanagers
+	cmd = store.UpdateAdminConfigurationCmd{AdminConfiguration: adminConfig}
+	require.NoError(t, fakeAdminConfigStore.UpdateAdminConfiguration(cmd))
+
+	// Again, make sure we sync and verify the senders.
+	require.NoError(t, sched.SyncAndApplyConfigFromDatabase())
+	sched.adminConfigMtx.Lock()
+	require.Equal(t, 1, len(sched.senders))
+	require.Equal(t, 1, len(sched.sendersCfgHash))
+	sched.adminConfigMtx.Unlock()
+
+	// Then, ensure we still have the Alertmanager but the Alertmanagers choice has changed.
+	require.Eventually(t, func() bool {
+		return len(sched.AlertmanagersFor(1)) == 1 &&
+			len(sched.DroppedAlertmanagersFor(1)) == 0 &&
+			sched.sendAlertsTo[1] == adminConfig.SendAlertsTo
+	}, 10*time.Second, 200*time.Millisecond)
+
+	// Finally, let's change the Alertmanagers choice to send only to the internal Alertmanager.
+	adminConfig.SendAlertsTo = models.InternalAlertmanager
+	cmd = store.UpdateAdminConfigurationCmd{AdminConfiguration: adminConfig}
+	require.NoError(t, fakeAdminConfigStore.UpdateAdminConfiguration(cmd))
+
+	// Again, make sure we sync and verify the senders.
+	// Senders should be running even though alerts are being handled externally.
+	require.NoError(t, sched.SyncAndApplyConfigFromDatabase())
+	sched.adminConfigMtx.Lock()
+	require.Equal(t, 1, len(sched.senders))
+	require.Equal(t, 1, len(sched.sendersCfgHash))
+	sched.adminConfigMtx.Unlock()
+
+	// Then, ensure the Alertmanager is still listed and the Alertmanagers choice has changed.
+	require.Eventually(t, func() bool {
+		return len(sched.AlertmanagersFor(1)) == 1 &&
+			len(sched.DroppedAlertmanagersFor(1)) == 0 &&
+			sched.sendAlertsTo[1] == adminConfig.SendAlertsTo
+	}, 10*time.Second, 200*time.Millisecond)
+}
+
 func TestSchedule_ruleRoutine(t *testing.T) {
 	createSchedule := func(
 		evalAppliedChan chan time.Time,

pkg/services/sqlstore/migrations/ualert/tables.go

@@ -323,4 +323,7 @@ func AddAlertAdminConfigMigrations(mg *migrator.Migrator) {
 
 	mg.AddMigration("create_ngalert_configuration_table", migrator.NewAddTableMigration(adminConfiguration))
 	mg.AddMigration("add index in ngalert_configuration on org_id column", migrator.NewAddIndexMigration(adminConfiguration, adminConfiguration.Indices[0]))
+	mg.AddMigration("add column send_alerts_to in ngalert_configuration", migrator.NewAddColumnMigration(adminConfiguration, &migrator.Column{
+		Name: "send_alerts_to", Type: migrator.DB_SmallInt, Nullable: false,
+	}))
 }

pkg/tests/api/alerting/api_admin_configuration_test.go

@@ -70,10 +70,47 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
 		require.JSONEq(t, string(b), "{\"message\": \"no admin configuration available\",\"error\": \"no admin configuration available\"}")
 	}
 
-	// Now, lets re-set external Alertmanagers for main organisation.
+	// An invalid alertmanager choice should return an error.
 	{
 		ac := apimodels.PostableNGalertConfig{
-			Alertmanagers: []string{fakeAM1.URL(), fakeAM2.URL()},
+			AlertmanagersChoice: apimodels.AlertmanagersChoice("invalid"),
+		}
+		buf := bytes.Buffer{}
+		enc := json.NewEncoder(&buf)
+		err := enc.Encode(&ac)
+		require.NoError(t, err)
+
+		alertsURL := fmt.Sprintf("http://grafana:password@%s/api/v1/ngalert/admin_config", grafanaListedAddr)
+		resp := postRequest(t, alertsURL, buf.String(), http.StatusBadRequest) // nolint
+		b, err := ioutil.ReadAll(resp.Body)
+		require.NoError(t, err)
+		require.JSONEq(t, string(b), "{\"message\": \"Invalid alertmanager choice specified\"}")
+	}
+
+	// Let's try to send all the alerts to an external Alertmanager
+	// but never specify any. This should return an error.
+	{
+		ac := apimodels.PostableNGalertConfig{
+			AlertmanagersChoice: apimodels.AlertmanagersChoice(ngmodels.ExternalAlertmanagers.String()),
+		}
+		buf := bytes.Buffer{}
+		enc := json.NewEncoder(&buf)
+		err := enc.Encode(&ac)
+		require.NoError(t, err)
+
+		alertsURL := fmt.Sprintf("http://grafana:password@%s/api/v1/ngalert/admin_config", grafanaListedAddr)
+		resp := postRequest(t, alertsURL, buf.String(), http.StatusBadRequest) // nolint
+		b, err := ioutil.ReadAll(resp.Body)
+		require.NoError(t, err)
+		require.JSONEq(t, string(b), "{\"message\": \"At least one Alertmanager must be provided to choose this option\"}")
+	}
+
+	// Now, lets re-set external Alertmanagers for main organisation
+	// and make it so that only the external Alertmanagers handle the alerts.
+	{
+		ac := apimodels.PostableNGalertConfig{
+			Alertmanagers:       []string{fakeAM1.URL(), fakeAM2.URL()},
+			AlertmanagersChoice: apimodels.AlertmanagersChoice(ngmodels.ExternalAlertmanagers.String()),
 		}
 		buf := bytes.Buffer{}
 		enc := json.NewEncoder(&buf)
@@ -93,7 +130,7 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
 		resp := getRequest(t, alertsURL, http.StatusOK) // nolint
 		b, err := ioutil.ReadAll(resp.Body)
 		require.NoError(t, err)
-		require.JSONEq(t, string(b), fmt.Sprintf("{\"alertmanagers\":[\"%s\",\"%s\"]}\n", fakeAM1.URL(), fakeAM2.URL()))
+		require.JSONEq(t, string(b), fmt.Sprintf("{\"alertmanagers\":[\"%s\",\"%s\"], \"alertmanagersChoice\": %q}\n", fakeAM1.URL(), fakeAM2.URL(), ngmodels.ExternalAlertmanagers))
 	}
 
 	// With the configuration set, we should eventually discover those Alertmanagers.
@@ -170,6 +207,7 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
 	}
 
 	// Now, lets re-set external Alertmanagers for the other organisation.
+	// Sending an empty value for AlertmanagersChoice should default to AllAlertmanagers.
 	{
 		ac := apimodels.PostableNGalertConfig{
 			Alertmanagers: []string{fakeAM3.URL()},
@@ -192,7 +230,7 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
 		resp := getRequest(t, alertsURL, http.StatusOK) // nolint
 		b, err := ioutil.ReadAll(resp.Body)
 		require.NoError(t, err)
-		require.JSONEq(t, string(b), fmt.Sprintf("{\"alertmanagers\":[\"%s\"]}\n", fakeAM3.URL()))
+		require.JSONEq(t, string(b), fmt.Sprintf("{\"alertmanagers\":[\"%s\"], \"alertmanagersChoice\": %q}\n", fakeAM3.URL(), ngmodels.AllAlertmanagers))
 	}
 
 	// With the configuration set, we should eventually not discover Alertmanagers.