mirror of
https://github.com/grafana/grafana.git
synced 2025-02-11 08:05:43 -06:00
0726c7c3fa
This commit prevents saving configurations containing inhibition rules in Grafana Alertmanager. It does not reject inhibition rules when using external Alertmanagers, such as Mimir. This meant the validation had to be put in the MultiOrgAlertmanager instead of in the validation of PostableUserConfig. We can remove this when inhibition rules are supported in Grafana Managed Alerts.
403 lines
14 KiB
Go
403 lines
14 KiB
Go
package alerting
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/prometheus/alertmanager/config"
|
|
"github.com/prometheus/alertmanager/pkg/labels"
|
|
"github.com/prometheus/common/model"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/grafana/grafana/pkg/components/simplejson"
|
|
"github.com/grafana/grafana/pkg/expr"
|
|
"github.com/grafana/grafana/pkg/services/datasources"
|
|
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
|
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
|
|
"github.com/grafana/grafana/pkg/services/ngalert/sender"
|
|
"github.com/grafana/grafana/pkg/services/org"
|
|
"github.com/grafana/grafana/pkg/services/org/orgimpl"
|
|
"github.com/grafana/grafana/pkg/services/quota/quotatest"
|
|
"github.com/grafana/grafana/pkg/services/user"
|
|
"github.com/grafana/grafana/pkg/tests/testinfra"
|
|
)
|
|
|
|
func TestIntegrationAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
|
|
testinfra.SQLiteIntegrationTest(t)
|
|
|
|
const disableOrgID int64 = 3
|
|
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
|
|
DisableLegacyAlerting: true,
|
|
EnableUnifiedAlerting: true,
|
|
DisableAnonymous: true,
|
|
NGAlertAdminConfigPollInterval: 2 * time.Second,
|
|
UnifiedAlertingDisabledOrgs: []int64{disableOrgID}, // disable unified alerting for organisation 3
|
|
AppModeProduction: true,
|
|
})
|
|
|
|
grafanaListedAddr, s := testinfra.StartGrafana(t, dir, path)
|
|
|
|
orgService, err := orgimpl.ProvideService(s, s.Cfg, quotatest.New(false, nil))
|
|
require.NoError(t, err)
|
|
|
|
// Create a user to make authenticated requests
|
|
userID := createUser(t, s, user.CreateUserCommand{
|
|
DefaultOrgRole: string(org.RoleAdmin),
|
|
Login: "grafana",
|
|
Password: "password",
|
|
})
|
|
apiClient := newAlertingApiClient(grafanaListedAddr, "grafana", "password")
|
|
|
|
// create another organisation
|
|
newOrg, err := orgService.CreateWithMember(context.Background(), &org.CreateOrgCommand{Name: "another org", UserID: userID})
|
|
require.NoError(t, err)
|
|
orgID := newOrg.ID
|
|
|
|
// ensure that the orgID is 3 (the disabled org)
|
|
require.Equal(t, disableOrgID, orgID)
|
|
|
|
// create user under different organisation
|
|
createUser(t, s, user.CreateUserCommand{
|
|
DefaultOrgRole: string(org.RoleAdmin),
|
|
Password: "admin-42",
|
|
Login: "admin-42",
|
|
OrgID: orgID,
|
|
})
|
|
|
|
// Create a couple of "fake" Alertmanagers
|
|
fakeAM1 := sender.NewFakeExternalAlertmanager(t)
|
|
fakeAM2 := sender.NewFakeExternalAlertmanager(t)
|
|
fakeAM3 := sender.NewFakeExternalAlertmanager(t)
|
|
|
|
// Now, let's test the configuration API.
|
|
{
|
|
alertsURL := fmt.Sprintf("http://grafana:password@%s/api/v1/ngalert/admin_config", grafanaListedAddr)
|
|
resp := getRequest(t, alertsURL, http.StatusNotFound) // nolint
|
|
b, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
var res map[string]any
|
|
err = json.Unmarshal(b, &res)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "no admin configuration available", res["message"])
|
|
}
|
|
|
|
// An invalid alertmanager choice should return an error.
|
|
{
|
|
ac := apimodels.PostableNGalertConfig{
|
|
AlertmanagersChoice: apimodels.AlertmanagersChoice("invalid"),
|
|
}
|
|
buf := bytes.Buffer{}
|
|
enc := json.NewEncoder(&buf)
|
|
err := enc.Encode(&ac)
|
|
require.NoError(t, err)
|
|
|
|
alertsURL := fmt.Sprintf("http://grafana:password@%s/api/v1/ngalert/admin_config", grafanaListedAddr)
|
|
resp := postRequest(t, alertsURL, buf.String(), http.StatusBadRequest) // nolint
|
|
b, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
var res map[string]any
|
|
err = json.Unmarshal(b, &res)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "Invalid alertmanager choice specified", res["message"])
|
|
}
|
|
|
|
// Let's try to send all the alerts to an external Alertmanager
|
|
// but never specify any. This should return an error.
|
|
{
|
|
ac := apimodels.PostableNGalertConfig{
|
|
AlertmanagersChoice: apimodels.AlertmanagersChoice(ngmodels.ExternalAlertmanagers.String()),
|
|
}
|
|
buf := bytes.Buffer{}
|
|
enc := json.NewEncoder(&buf)
|
|
err := enc.Encode(&ac)
|
|
require.NoError(t, err)
|
|
|
|
alertsURL := fmt.Sprintf("http://grafana:password@%s/api/v1/ngalert/admin_config", grafanaListedAddr)
|
|
resp := postRequest(t, alertsURL, buf.String(), http.StatusBadRequest) // nolint
|
|
b, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
var res map[string]any
|
|
err = json.Unmarshal(b, &res)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "At least one Alertmanager must be provided or configured as a datasource that handles alerts to choose this option", res["message"])
|
|
}
|
|
|
|
// Add an alertmanager datasource
|
|
{
|
|
cmd := datasources.AddDataSourceCommand{
|
|
OrgID: 1,
|
|
Name: "AM1",
|
|
Type: datasources.DS_ALERTMANAGER,
|
|
Access: "proxy",
|
|
URL: fakeAM1.URL(),
|
|
JsonData: simplejson.NewFromAny(map[string]any{
|
|
"handleGrafanaManagedAlerts": true,
|
|
"implementation": "prometheus",
|
|
}),
|
|
}
|
|
buf := bytes.Buffer{}
|
|
enc := json.NewEncoder(&buf)
|
|
err := enc.Encode(&cmd)
|
|
require.NoError(t, err)
|
|
dataSourcesUrl := fmt.Sprintf("http://grafana:password@%s/api/datasources", grafanaListedAddr)
|
|
resp := postRequest(t, dataSourcesUrl, buf.String(), http.StatusOK) // nolint
|
|
b, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
var res map[string]any
|
|
err = json.Unmarshal(b, &res)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "Datasource added", res["message"])
|
|
}
|
|
|
|
// Add another alertmanager datasource
|
|
{
|
|
cmd := datasources.AddDataSourceCommand{
|
|
OrgID: 1,
|
|
Name: "AM2",
|
|
Type: datasources.DS_ALERTMANAGER,
|
|
Access: "proxy",
|
|
URL: fakeAM2.URL(),
|
|
JsonData: simplejson.NewFromAny(map[string]any{
|
|
"handleGrafanaManagedAlerts": true,
|
|
"implementation": "prometheus",
|
|
}),
|
|
}
|
|
buf := bytes.Buffer{}
|
|
enc := json.NewEncoder(&buf)
|
|
err := enc.Encode(&cmd)
|
|
require.NoError(t, err)
|
|
dataSourcesUrl := fmt.Sprintf("http://grafana:password@%s/api/datasources", grafanaListedAddr)
|
|
resp := postRequest(t, dataSourcesUrl, buf.String(), http.StatusOK) // nolint
|
|
b, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
var res map[string]any
|
|
err = json.Unmarshal(b, &res)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "Datasource added", res["message"])
|
|
}
|
|
|
|
// Now, lets re-set external Alertmanagers for main organisation
|
|
// and make it so that only the external Alertmanagers handle the alerts.
|
|
{
|
|
ac := apimodels.PostableNGalertConfig{
|
|
AlertmanagersChoice: apimodels.AlertmanagersChoice(ngmodels.ExternalAlertmanagers.String()),
|
|
}
|
|
buf := bytes.Buffer{}
|
|
enc := json.NewEncoder(&buf)
|
|
err := enc.Encode(&ac)
|
|
require.NoError(t, err)
|
|
|
|
alertsURL := fmt.Sprintf("http://grafana:password@%s/api/v1/ngalert/admin_config", grafanaListedAddr)
|
|
resp := postRequest(t, alertsURL, buf.String(), http.StatusCreated) // nolint
|
|
b, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
var res map[string]any
|
|
err = json.Unmarshal(b, &res)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "admin configuration updated", res["message"])
|
|
}
|
|
|
|
// If we get the configuration again, it shows us what we've set.
|
|
{
|
|
alertsURL := fmt.Sprintf("http://grafana:password@%s/api/v1/ngalert/admin_config", grafanaListedAddr)
|
|
resp := getRequest(t, alertsURL, http.StatusOK) // nolint
|
|
b, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
require.JSONEq(t, fmt.Sprintf("{\"alertmanagersChoice\": %q}\n", ngmodels.ExternalAlertmanagers), string(b))
|
|
}
|
|
|
|
// With the configuration set, we should eventually discover those Alertmanagers.
|
|
{
|
|
alertsURL := fmt.Sprintf("http://grafana:password@%s/api/v1/ngalert/alertmanagers", grafanaListedAddr)
|
|
require.Eventually(t, func() bool {
|
|
resp := getRequest(t, alertsURL, http.StatusOK) // nolint
|
|
b, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
|
|
var alertmanagers apimodels.GettableAlertmanagers
|
|
require.NoError(t, json.Unmarshal(b, &alertmanagers))
|
|
|
|
return len(alertmanagers.Data.Active) == 2
|
|
}, 16*time.Second, 8*time.Second) // the sync interval is 2s so after 8s all alertmanagers most probably are started
|
|
}
|
|
|
|
// Now, let's set an alert that should fire as quickly as possible.
|
|
{
|
|
// Create the namespace we'll save our alerts to
|
|
apiClient.CreateFolder(t, "default", "default")
|
|
interval, err := model.ParseDuration("10s")
|
|
require.NoError(t, err)
|
|
|
|
rules := apimodels.PostableRuleGroupConfig{
|
|
Name: "arulegroup",
|
|
Interval: interval,
|
|
Rules: []apimodels.PostableExtendedRuleNode{
|
|
{
|
|
ApiRuleNode: &apimodels.ApiRuleNode{
|
|
For: &interval,
|
|
Labels: map[string]string{"label1": "val1"},
|
|
Annotations: map[string]string{"annotation1": "val1"},
|
|
},
|
|
// this rule does not explicitly set no data and error states
|
|
// therefore it should get the default values
|
|
GrafanaManagedAlert: &apimodels.PostableGrafanaRule{
|
|
Title: "AlwaysFiring",
|
|
Condition: "A",
|
|
Data: []apimodels.AlertQuery{
|
|
{
|
|
RefID: "A",
|
|
RelativeTimeRange: apimodels.RelativeTimeRange{
|
|
From: apimodels.Duration(time.Duration(5) * time.Hour),
|
|
To: apimodels.Duration(time.Duration(3) * time.Hour),
|
|
},
|
|
DatasourceUID: expr.DatasourceUID,
|
|
Model: json.RawMessage(`{
|
|
"type": "math",
|
|
"expression": "2 + 3 > 1"
|
|
}`),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
buf := bytes.Buffer{}
|
|
enc := json.NewEncoder(&buf)
|
|
err = enc.Encode(&rules)
|
|
require.NoError(t, err)
|
|
|
|
ruleURL := fmt.Sprintf("http://grafana:password@%s/api/ruler/grafana/api/v1/rules/default", grafanaListedAddr)
|
|
// nolint
|
|
_ = postRequest(t, ruleURL, buf.String(), http.StatusAccepted)
|
|
}
|
|
|
|
// Eventually, our Alertmanagers should receiver the alert.
|
|
{
|
|
require.Eventually(t, func() bool {
|
|
return fakeAM1.AlertsCount() == 1 && fakeAM2.AlertsCount() == 1
|
|
}, time.Minute, 5*time.Second)
|
|
}
|
|
|
|
// Add an alertmanager datasource fot the other organisation
|
|
{
|
|
cmd := datasources.AddDataSourceCommand{
|
|
OrgID: 2,
|
|
Name: "AM3",
|
|
Type: datasources.DS_ALERTMANAGER,
|
|
Access: "proxy",
|
|
URL: fakeAM3.URL(),
|
|
JsonData: simplejson.NewFromAny(map[string]any{
|
|
"handleGrafanaManagedAlerts": true,
|
|
"implementation": "prometheus",
|
|
}),
|
|
}
|
|
buf := bytes.Buffer{}
|
|
enc := json.NewEncoder(&buf)
|
|
err := enc.Encode(&cmd)
|
|
require.NoError(t, err)
|
|
dataSourcesUrl := fmt.Sprintf("http://admin-42:admin-42@%s/api/datasources", grafanaListedAddr)
|
|
resp := postRequest(t, dataSourcesUrl, buf.String(), http.StatusOK) // nolint
|
|
b, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
var res map[string]any
|
|
err = json.Unmarshal(b, &res)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "Datasource added", res["message"])
|
|
}
|
|
|
|
// Now, lets re-set external Alertmanagers for the other organisation.
|
|
// Sending an empty value for AlertmanagersChoice should default to AllAlertmanagers.
|
|
{
|
|
ac := apimodels.PostableNGalertConfig{}
|
|
buf := bytes.Buffer{}
|
|
enc := json.NewEncoder(&buf)
|
|
err := enc.Encode(&ac)
|
|
require.NoError(t, err)
|
|
|
|
alertsURL := fmt.Sprintf("http://admin-42:admin-42@%s/api/v1/ngalert/admin_config", grafanaListedAddr)
|
|
resp := postRequest(t, alertsURL, buf.String(), http.StatusCreated) // nolint
|
|
b, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
var res map[string]any
|
|
err = json.Unmarshal(b, &res)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "admin configuration updated", res["message"])
|
|
}
|
|
|
|
// If we get the configuration again, it shows us what we've set.
|
|
{
|
|
alertsURL := fmt.Sprintf("http://admin-42:admin-42@%s/api/v1/ngalert/admin_config", grafanaListedAddr)
|
|
resp := getRequest(t, alertsURL, http.StatusOK) // nolint
|
|
b, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
require.JSONEq(t, fmt.Sprintf("{\"alertmanagersChoice\": %q}\n", ngmodels.AllAlertmanagers), string(b))
|
|
}
|
|
|
|
// With the configuration set, we should eventually not discover Alertmanagers.
|
|
{
|
|
alertsURL := fmt.Sprintf("http://admin-42:admin-42@%s/api/v1/ngalert/alertmanagers", grafanaListedAddr)
|
|
require.Eventually(t, func() bool {
|
|
resp := getRequest(t, alertsURL, http.StatusOK) // nolint
|
|
b, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
|
|
var alertmanagers apimodels.GettableAlertmanagers
|
|
require.NoError(t, json.Unmarshal(b, &alertmanagers))
|
|
|
|
return len(alertmanagers.Data.Active) == 0
|
|
}, 16*time.Second, 8*time.Second) // the sync interval is 2s so after 8s all alertmanagers (if any) most probably are started
|
|
}
|
|
}
|
|
|
|
func TestIntegrationAdminConfiguration_CannotCreateInhibitionRules(t *testing.T) {
|
|
testinfra.SQLiteIntegrationTest(t)
|
|
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
|
|
DisableLegacyAlerting: true,
|
|
EnableUnifiedAlerting: true,
|
|
AppModeProduction: true,
|
|
})
|
|
grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)
|
|
createUser(t, store, user.CreateUserCommand{
|
|
DefaultOrgRole: string(org.RoleAdmin),
|
|
Password: "admin",
|
|
Login: "admin",
|
|
})
|
|
client := newAlertingApiClient(grafanaListedAddr, "admin", "admin")
|
|
|
|
cfg := apimodels.PostableUserConfig{
|
|
AlertmanagerConfig: apimodels.PostableApiAlertingConfig{
|
|
Config: apimodels.Config{
|
|
Route: &apimodels.Route{
|
|
Receiver: "test",
|
|
},
|
|
InhibitRules: []config.InhibitRule{{
|
|
SourceMatchers: config.Matchers{{
|
|
Type: labels.MatchEqual,
|
|
Name: "foo",
|
|
Value: "bar",
|
|
}},
|
|
TargetMatchers: config.Matchers{{
|
|
Type: labels.MatchEqual,
|
|
Name: "bar",
|
|
Value: "baz",
|
|
}},
|
|
}},
|
|
},
|
|
Receivers: []*apimodels.PostableApiReceiver{{
|
|
Receiver: config.Receiver{
|
|
Name: "test",
|
|
},
|
|
}},
|
|
},
|
|
}
|
|
ok, err := client.PostConfiguration(t, cfg)
|
|
require.False(t, ok)
|
|
require.EqualError(t, err, "inhibition rules are not supported")
|
|
}
|