Alerting: Make ticker to tick at predictable time (#50197)

pkg/services/alerting/ticker.go

@@ -32,7 +32,7 @@ func NewTicker(c clock.Clock, interval time.Duration, metric *metrics.Ticker) *T
 	t := &Ticker{
 		C:        make(chan time.Time),
 		clock:    c,
-		last:     c.Now(),
+		last:     getStartTick(c, interval),
 		interval: interval,
 		metrics:  metric,
 		stopCh:   make(chan struct{}),
@@ -42,9 +42,14 @@ func NewTicker(c clock.Clock, interval time.Duration, metric *metrics.Ticker) *T
 	return t
 }
 
+func getStartTick(clk clock.Clock, interval time.Duration) time.Time {
+	nano := clk.Now().UnixNano()
+	return time.Unix(0, nano-(nano%interval.Nanoseconds()))
+}
+
 func (t *Ticker) run() {
 	logger := log.New("ticker")
-	logger.Info("starting")
+	logger.Info("starting", "first_tick", t.last.Add(t.interval))
 LOOP:
 	for {
 		next := t.last.Add(t.interval) // calculate the time of the next tick

pkg/services/alerting/ticker_test.go

@@ -35,9 +35,24 @@ func TestTicker(t *testing.T) {
 		}
 		return time.Time{}
 	}
-	t.Run("should not drop ticks", func(t *testing.T) {
+	t.Run("should align with clock", func(t *testing.T) {
+		interval := 10 * time.Second
 		clk := clock.NewMock()
+		clk.Add(1 * time.Minute)
+
+		require.Equal(t, clk.Now(), getStartTick(clk, interval))
+		now := clk.Now()
+		for i := 0; i < 100; i++ {
+			delta := time.Duration(rand.Int63n(interval.Nanoseconds()))
+			clk.Set(now.Add(delta))
+			require.Equal(t, now, getStartTick(clk, interval))
+		}
+	})
+
+	t.Run("should not drop ticks", func(t *testing.T) {
 		interval := time.Duration(rand.Int63n(100)+10) * time.Second
+		clk := clock.NewMock()
+		clk.Add(interval) // align clock with the start tick
 		ticker := NewTicker(clk, interval, metrics.NewTickerMetrics(prometheus.NewRegistry()))
 
 		ticks := rand.Intn(9) + 1
@@ -119,7 +134,7 @@ func TestTicker(t *testing.T) {
 		interval := time.Duration(rand.Int63n(9)+1) * time.Second
 		registry := prometheus.NewPedanticRegistry()
 		ticker := NewTicker(clk, interval, metrics.NewTickerMetrics(registry))
-		expectedTick := clk.Now().Add(interval)
+		expectedTick := getStartTick(clk, interval).Add(interval)
 
 		expectedMetricFmt := `# HELP grafana_alerting_ticker_interval_seconds Interval at which the ticker is meant to tick.
                     # TYPE grafana_alerting_ticker_interval_seconds gauge

pkg/services/ngalert/CHANGELOG.md

@@ -46,6 +46,7 @@ Scopes must have an order to ensure consistency and ease of search, this helps u
 ## Grafana Alerting - main / unreleased
 
 - [ENHANCEMENT] Scheduler: Drop ticks if rule evaluation is too slow and adds a metric grafana_alerting_schedule_rule_evaluations_missed_total to track missed evaluations per rule #48885
+- [ENHANCEMENT] Ticker to tick at predictable time #50197
 
 ## 9.0.0