2022-08-01 18:28:38 -05:00
|
|
|
package ngalert
|
|
|
|
|
|
|
|
import (
|
2023-03-13 15:54:46 -05:00
|
|
|
"bytes"
|
2022-08-01 18:28:38 -05:00
|
|
|
"context"
|
|
|
|
"math/rand"
|
|
|
|
"testing"
|
|
|
|
"time"
|
|
|
|
|
2023-03-13 15:54:46 -05:00
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"github.com/prometheus/client_golang/prometheus/testutil"
|
2022-08-01 18:28:38 -05:00
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
|
2022-10-18 08:31:56 -05:00
|
|
|
"github.com/grafana/grafana/pkg/bus"
|
2022-08-01 18:28:38 -05:00
|
|
|
"github.com/grafana/grafana/pkg/events"
|
|
|
|
"github.com/grafana/grafana/pkg/infra/log"
|
2022-10-18 08:31:56 -05:00
|
|
|
"github.com/grafana/grafana/pkg/infra/tracing"
|
2022-11-11 07:28:24 -06:00
|
|
|
"github.com/grafana/grafana/pkg/services/folder"
|
2023-03-13 15:54:46 -05:00
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
|
2022-08-01 18:28:38 -05:00
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/models"
|
2022-09-30 14:36:51 -05:00
|
|
|
"github.com/grafana/grafana/pkg/services/ngalert/tests/fakes"
|
2023-03-13 15:54:46 -05:00
|
|
|
"github.com/grafana/grafana/pkg/setting"
|
2022-08-01 18:28:38 -05:00
|
|
|
"github.com/grafana/grafana/pkg/util"
|
|
|
|
)
|
|
|
|
|
|
|
|
func Test_subscribeToFolderChanges(t *testing.T) {
|
|
|
|
orgID := rand.Int63()
|
2022-11-11 07:28:24 -06:00
|
|
|
folder := &folder.Folder{
|
|
|
|
UID: util.GenerateShortUID(),
|
2022-08-01 18:28:38 -05:00
|
|
|
Title: "Folder" + util.GenerateShortUID(),
|
|
|
|
}
|
|
|
|
rules := models.GenerateAlertRules(5, models.AlertRuleGen(models.WithOrgID(orgID), models.WithNamespace(folder)))
|
|
|
|
|
2022-10-18 08:31:56 -05:00
|
|
|
bus := bus.ProvideBus(tracing.InitializeTracerForTest())
|
2022-09-30 14:36:51 -05:00
|
|
|
db := fakes.NewRuleStore(t)
|
2022-08-01 18:28:38 -05:00
|
|
|
db.Folders[orgID] = append(db.Folders[orgID], folder)
|
|
|
|
db.PutRule(context.Background(), rules...)
|
|
|
|
|
2023-03-14 17:02:51 -05:00
|
|
|
subscribeToFolderChanges(log.New("test"), bus, db)
|
2022-08-01 18:28:38 -05:00
|
|
|
|
|
|
|
err := bus.Publish(context.Background(), &events.FolderTitleUpdated{
|
|
|
|
Timestamp: time.Now(),
|
|
|
|
Title: "Folder" + util.GenerateShortUID(),
|
2022-11-11 07:28:24 -06:00
|
|
|
UID: folder.UID,
|
2022-08-01 18:28:38 -05:00
|
|
|
OrgID: orgID,
|
|
|
|
})
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
require.Eventuallyf(t, func() bool {
|
2023-08-30 10:46:47 -05:00
|
|
|
return len(db.GetRecordedCommands(func(cmd any) (any, bool) {
|
2022-09-30 14:36:51 -05:00
|
|
|
c, ok := cmd.(fakes.GenericRecordedQuery)
|
2022-08-01 18:28:38 -05:00
|
|
|
if !ok || c.Name != "IncreaseVersionForAllRulesInNamespace" {
|
|
|
|
return nil, false
|
|
|
|
}
|
|
|
|
return c, true
|
|
|
|
})) > 0
|
|
|
|
}, time.Second, 10*time.Millisecond, "expected to call db store method but nothing was called")
|
|
|
|
}
|
2023-03-13 15:54:46 -05:00
|
|
|
|
|
|
|
func TestConfigureHistorianBackend(t *testing.T) {
|
|
|
|
t.Run("fail initialization if invalid backend", func(t *testing.T) {
|
2023-11-27 08:20:50 -06:00
|
|
|
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
|
2023-03-13 15:54:46 -05:00
|
|
|
logger := log.NewNopLogger()
|
|
|
|
cfg := setting.UnifiedAlertingStateHistorySettings{
|
|
|
|
Enabled: true,
|
|
|
|
Backend: "invalid-backend",
|
|
|
|
}
|
|
|
|
|
|
|
|
_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
|
|
|
|
|
|
|
|
require.ErrorContains(t, err, "unrecognized")
|
|
|
|
})
|
|
|
|
|
2023-03-17 12:41:18 -05:00
|
|
|
t.Run("fail initialization if invalid multi-backend primary", func(t *testing.T) {
|
2023-11-27 08:20:50 -06:00
|
|
|
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
|
2023-03-17 12:41:18 -05:00
|
|
|
logger := log.NewNopLogger()
|
|
|
|
cfg := setting.UnifiedAlertingStateHistorySettings{
|
|
|
|
Enabled: true,
|
|
|
|
Backend: "multiple",
|
|
|
|
MultiPrimary: "invalid-backend",
|
|
|
|
}
|
|
|
|
|
|
|
|
_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
|
|
|
|
|
|
|
|
require.ErrorContains(t, err, "multi-backend target")
|
|
|
|
require.ErrorContains(t, err, "unrecognized")
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("fail initialization if invalid multi-backend secondary", func(t *testing.T) {
|
2023-11-27 08:20:50 -06:00
|
|
|
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
|
2023-03-17 12:41:18 -05:00
|
|
|
logger := log.NewNopLogger()
|
|
|
|
cfg := setting.UnifiedAlertingStateHistorySettings{
|
|
|
|
Enabled: true,
|
|
|
|
Backend: "multiple",
|
|
|
|
MultiPrimary: "annotations",
|
2023-03-31 11:15:56 -05:00
|
|
|
MultiSecondaries: []string{"annotations", "invalid-backend"},
|
2023-03-17 12:41:18 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
|
|
|
|
|
|
|
|
require.ErrorContains(t, err, "multi-backend target")
|
|
|
|
require.ErrorContains(t, err, "unrecognized")
|
|
|
|
})
|
|
|
|
|
2023-03-13 15:54:46 -05:00
|
|
|
t.Run("do not fail initialization if pinging Loki fails", func(t *testing.T) {
|
2023-11-27 08:20:50 -06:00
|
|
|
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
|
2023-03-13 15:54:46 -05:00
|
|
|
logger := log.NewNopLogger()
|
|
|
|
cfg := setting.UnifiedAlertingStateHistorySettings{
|
|
|
|
Enabled: true,
|
|
|
|
Backend: "loki",
|
|
|
|
// Should never resolve at the DNS level: https://www.rfc-editor.org/rfc/rfc6761#section-6.4
|
|
|
|
LokiReadURL: "http://gone.invalid",
|
|
|
|
LokiWriteURL: "http://gone.invalid",
|
|
|
|
}
|
|
|
|
|
|
|
|
h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
|
|
|
|
|
|
|
|
require.NotNil(t, h)
|
|
|
|
require.NoError(t, err)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("emit metric describing chosen backend", func(t *testing.T) {
|
|
|
|
reg := prometheus.NewRegistry()
|
2023-11-27 08:20:50 -06:00
|
|
|
met := metrics.NewHistorianMetrics(reg, metrics.Subsystem)
|
2023-03-13 15:54:46 -05:00
|
|
|
logger := log.NewNopLogger()
|
|
|
|
cfg := setting.UnifiedAlertingStateHistorySettings{
|
|
|
|
Enabled: true,
|
|
|
|
Backend: "annotations",
|
|
|
|
}
|
|
|
|
|
|
|
|
h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
|
|
|
|
|
|
|
|
require.NotNil(t, h)
|
|
|
|
require.NoError(t, err)
|
|
|
|
exp := bytes.NewBufferString(`
|
|
|
|
# HELP grafana_alerting_state_history_info Information about the state history store.
|
|
|
|
# TYPE grafana_alerting_state_history_info gauge
|
|
|
|
grafana_alerting_state_history_info{backend="annotations"} 1
|
|
|
|
`)
|
|
|
|
err = testutil.GatherAndCompare(reg, exp, "grafana_alerting_state_history_info")
|
|
|
|
require.NoError(t, err)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("emit special zero metric if state history disabled", func(t *testing.T) {
|
|
|
|
reg := prometheus.NewRegistry()
|
2023-11-27 08:20:50 -06:00
|
|
|
met := metrics.NewHistorianMetrics(reg, metrics.Subsystem)
|
2023-03-13 15:54:46 -05:00
|
|
|
logger := log.NewNopLogger()
|
|
|
|
cfg := setting.UnifiedAlertingStateHistorySettings{
|
|
|
|
Enabled: false,
|
|
|
|
}
|
|
|
|
|
|
|
|
h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
|
|
|
|
|
|
|
|
require.NotNil(t, h)
|
|
|
|
require.NoError(t, err)
|
|
|
|
exp := bytes.NewBufferString(`
|
|
|
|
# HELP grafana_alerting_state_history_info Information about the state history store.
|
|
|
|
# TYPE grafana_alerting_state_history_info gauge
|
|
|
|
grafana_alerting_state_history_info{backend="noop"} 0
|
|
|
|
`)
|
|
|
|
err = testutil.GatherAndCompare(reg, exp, "grafana_alerting_state_history_info")
|
|
|
|
require.NoError(t, err)
|
|
|
|
})
|
|
|
|
}
|