IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Expect 406s from the remote Alertmanager during the readiness check (#99507)

Browse Source 
* Alerting: Expect 406s from the remote Alertmanager during the readiness check

* make it clear in the warning logs that we'll attempt to send the confgiuration/state without comparing in case of error pulling the current state/config
This commit is contained in:
Santiago
2025-01-24 16:26:57 +01:00
committed by GitHub
parent 6284dd61ae
commit 361312bbd7
2 changed files with 10 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/services/ngalert/remote/alertmanager.go
View File

@@ -642,7 +642,7 @@ func (am *Alertmanager) shouldSendConfig(ctx context.Context, config *apimodels.
rc, err := am.mimirClient.GetGrafanaAlertmanagerConfig(ctx)
if err != nil {
// Log the error and return true so we try to upload our config anyway.
am.log.Error("Unable to get the remote Alertmanager configuration for comparison", "err", err)
am.log.Warn("Unable to get the remote Alertmanager configuration for comparison, sending the configuration without comparing", "err", err)
return true
}
@@ -669,7 +669,7 @@ func (am *Alertmanager) shouldSendState(ctx context.Context, state string) bool
rs, err := am.mimirClient.GetGrafanaAlertmanagerState(ctx)
if err != nil {
// Log the error and return true so we try to upload our state anyway.
am.log.Error("Unable to get the remote Alertmanager state for comparison", "err", err)
am.log.Warn("Unable to get the remote Alertmanager state for comparison, sending the state without comparing", "err", err)
return true
}

8
pkg/services/ngalert/remote/client/alertmanager.go
View File

@@ -110,6 +110,14 @@ func (am *Alertmanager) IsReadyWithBackoff(ctx context.Context) (bool, error) {
if status != http.StatusOK {
if status >= 400 && status < 500 {
if status == http.StatusNotAcceptable {
// Mimir returns a 406 when the Alertmanager for the tenant is not running.
// This is expected if the Grafana Alertmanager configuration is default or not promoted.
// We can still use the endpoints to store and retrieve configuration/state.
am.logger.Debug("Remote Alertmanager not initialized for tenant", "attempt", attempts, "status", status)
return true, nil
}
am.logger.Debug("Ready check failed with non-retriable status code", "attempt", attempts, "status", status)
return false, fmt.Errorf("ready check failed with non-retriable status code %d", status)
}