Alerting/consistent api errors (#34858)

* consolidates alertmanager api errors

* util & testing consistent errors

* consistent errors for rest of ngalert apis

* updates expected errors in testware

* bump ci

* linting

* unrelated: dashboard.go lint

pkg/api/dashboard.go

@@ -51,8 +51,7 @@ func (hs *HTTPServer) TrimDashboard(c *models.ReqContext, cmd models.TrimDashboa
 	dash := cmd.Dashboard
 	meta := cmd.Meta
 
-	trimedResult := *dash
+	trimedResult, err := hs.LoadSchemaService.DashboardTrimDefaults(*dash)
-	trimedResult, err = hs.LoadSchemaService.DashboardTrimDefaults(*dash)
 	if err != nil {
 		return response.Error(500, "Error while trim default value from dashboard json", err)
 	}

pkg/services/ngalert/api/api_alertmanager.go

@@ -23,38 +23,38 @@ type AlertmanagerSrv struct {
 
 func (srv AlertmanagerSrv) RouteCreateSilence(c *models.ReqContext, postableSilence apimodels.PostableSilence) response.Response {
 	if !c.HasUserRole(models.ROLE_EDITOR) {
-		return response.Error(http.StatusForbidden, "Permission denied", nil)
+		return ErrResp(http.StatusForbidden, errors.New("permission denied"), "")
 	}
 	silenceID, err := srv.am.CreateSilence(&postableSilence)
 	if err != nil {
 		if errors.Is(err, notifier.ErrSilenceNotFound) {
-			return response.Error(http.StatusNotFound, err.Error(), nil)
+			return ErrResp(http.StatusNotFound, err, "")
 		}
 
 		if errors.Is(err, notifier.ErrCreateSilenceBadPayload) {
-			return response.Error(http.StatusBadRequest, err.Error(), nil)
+			return ErrResp(http.StatusBadRequest, err, "")
 		}
 
-		return response.Error(http.StatusInternalServerError, "failed to create silence", err)
+		return ErrResp(http.StatusInternalServerError, err, "failed to create silence")
 	}
 	return response.JSON(http.StatusAccepted, util.DynMap{"message": "silence created", "id": silenceID})
 }
 
 func (srv AlertmanagerSrv) RouteDeleteAlertingConfig(c *models.ReqContext) response.Response {
 	// not implemented
-	return response.Error(http.StatusNotImplemented, "", nil)
+	return NotImplementedResp
 }
 
 func (srv AlertmanagerSrv) RouteDeleteSilence(c *models.ReqContext) response.Response {
 	if !c.HasUserRole(models.ROLE_EDITOR) {
-		return response.Error(http.StatusForbidden, "Permission denied", nil)
+		return ErrResp(http.StatusForbidden, errors.New("permission denied"), "")
 	}
 	silenceID := c.Params(":SilenceId")
 	if err := srv.am.DeleteSilence(silenceID); err != nil {
 		if errors.Is(err, notifier.ErrSilenceNotFound) {
-			return response.Error(http.StatusNotFound, err.Error(), nil)
+			return ErrResp(http.StatusNotFound, err, "")
 		}
-		return response.Error(http.StatusInternalServerError, err.Error(), nil)
+		return ErrResp(http.StatusInternalServerError, err, "")
 	}
 	return response.JSON(http.StatusOK, util.DynMap{"message": "silence deleted"})
 }
@@ -63,14 +63,14 @@ func (srv AlertmanagerSrv) RouteGetAlertingConfig(c *models.ReqContext) response
 	query := ngmodels.GetLatestAlertmanagerConfigurationQuery{}
 	if err := srv.store.GetLatestAlertmanagerConfiguration(&query); err != nil {
 		if errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
-			return response.Error(http.StatusNotFound, err.Error(), nil)
+			return ErrResp(http.StatusNotFound, err, "")
 		}
-		return response.Error(http.StatusInternalServerError, "failed to get latest configuration", err)
+		return ErrResp(http.StatusInternalServerError, err, "failed to get latest configuration")
 	}
 
 	cfg, err := notifier.Load([]byte(query.Result.AlertmanagerConfiguration))
 	if err != nil {
-		return response.Error(http.StatusInternalServerError, "failed to unmarshal alertmanager configuration", err)
+		return ErrResp(http.StatusInternalServerError, err, "failed to unmarshal alertmanager configuration")
 	}
 
 	result := apimodels.GettableUserConfig{
@@ -86,7 +86,7 @@ func (srv AlertmanagerSrv) RouteGetAlertingConfig(c *models.ReqContext) response
 			for k := range pr.SecureSettings {
 				decryptedValue, err := pr.GetDecryptedSecret(k)
 				if err != nil {
-					return response.Error(http.StatusInternalServerError, fmt.Sprintf("failed to decrypt stored secure setting: %s", k), err)
+					return ErrResp(http.StatusInternalServerError, err, "failed to decrypt stored secure setting: %s", k)
 				}
 				if decryptedValue == "" {
 					continue
@@ -125,10 +125,10 @@ func (srv AlertmanagerSrv) RouteGetAMAlertGroups(c *models.ReqContext) response.
 	)
 	if err != nil {
 		if errors.Is(err, notifier.ErrGetAlertGroupsBadPayload) {
-			return response.Error(http.StatusBadRequest, err.Error(), nil)
+			return ErrResp(http.StatusBadRequest, err, "")
 		}
 		// any other error here should be an unexpected failure and thus an internal error
-		return response.Error(http.StatusInternalServerError, err.Error(), nil)
+		return ErrResp(http.StatusInternalServerError, err, "")
 	}
 
 	return response.JSON(http.StatusOK, groups)
@@ -144,10 +144,10 @@ func (srv AlertmanagerSrv) RouteGetAMAlerts(c *models.ReqContext) response.Respo
 	)
 	if err != nil {
 		if errors.Is(err, notifier.ErrGetAlertsBadPayload) {
-			return response.Error(http.StatusBadRequest, err.Error(), nil)
+			return ErrResp(http.StatusBadRequest, err, "")
 		}
 		// any other error here should be an unexpected failure and thus an internal error
-		return response.Error(http.StatusInternalServerError, err.Error(), nil)
+		return ErrResp(http.StatusInternalServerError, err, "")
 	}
 
 	return response.JSON(http.StatusOK, alerts)
@@ -158,10 +158,10 @@ func (srv AlertmanagerSrv) RouteGetSilence(c *models.ReqContext) response.Respon
 	gettableSilence, err := srv.am.GetSilence(silenceID)
 	if err != nil {
 		if errors.Is(err, notifier.ErrSilenceNotFound) {
-			return response.Error(http.StatusNotFound, err.Error(), nil)
+			return ErrResp(http.StatusNotFound, err, "")
 		}
 		// any other error here should be an unexpected failure and thus an internal error
-		return response.Error(http.StatusInternalServerError, err.Error(), nil)
+		return ErrResp(http.StatusInternalServerError, err, "")
 	}
 	return response.JSON(http.StatusOK, gettableSilence)
 }
@@ -170,17 +170,17 @@ func (srv AlertmanagerSrv) RouteGetSilences(c *models.ReqContext) response.Respo
 	gettableSilences, err := srv.am.ListSilences(c.QueryStrings("filter"))
 	if err != nil {
 		if errors.Is(err, notifier.ErrListSilencesBadPayload) {
-			return response.Error(http.StatusBadRequest, err.Error(), nil)
+			return ErrResp(http.StatusBadRequest, err, "")
 		}
 		// any other error here should be an unexpected failure and thus an internal error
-		return response.Error(http.StatusInternalServerError, err.Error(), nil)
+		return ErrResp(http.StatusInternalServerError, err, "")
 	}
 	return response.JSON(http.StatusOK, gettableSilences)
 }
 
 func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body apimodels.PostableUserConfig) response.Response {
 	if !c.HasUserRole(models.ROLE_EDITOR) {
-		return response.Error(http.StatusForbidden, "Permission denied", nil)
+		return ErrResp(http.StatusForbidden, errors.New("permission denied"), "")
 	}
 
 	// Get the last known working configuration
@@ -188,13 +188,13 @@ func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body ap
 	if err := srv.store.GetLatestAlertmanagerConfiguration(&query); err != nil {
 		// If we don't have a configuration there's nothing for us to know and we should just continue saving the new one
 		if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
-			return response.Error(http.StatusInternalServerError, "failed to get latest configuration", err)
+			return ErrResp(http.StatusInternalServerError, err, "failed to get latest configuration")
 		}
 	}
 
 	currentConfig, err := notifier.Load([]byte(query.Result.AlertmanagerConfiguration))
 	if err != nil {
-		return response.Error(http.StatusInternalServerError, "failed to load lastest configuration", err)
+		return ErrResp(http.StatusInternalServerError, err, "failed to load lastest configuration")
 	}
 	currentReceiverMap := currentConfig.GetGrafanaReceiverMap()
 
@@ -208,7 +208,7 @@ func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body ap
 			cgmr, ok := currentReceiverMap[gr.UID]
 			if !ok {
 				// it tries to update a receiver that didn't previously exist
-				return response.Error(http.StatusBadRequest, fmt.Sprintf("unknown receiver: %s", gr.UID), nil)
+				return ErrResp(http.StatusBadRequest, fmt.Errorf("unknown receiver: %s", gr.UID), "")
 			}
 
 			// frontend sends only the secure settings that have to be updated
@@ -218,7 +218,7 @@ func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body ap
 				if !ok {
 					decryptedValue, err := cgmr.GetDecryptedSecret(key)
 					if err != nil {
-						return response.Error(http.StatusInternalServerError, fmt.Sprintf("failed to decrypt stored secure setting: %s", key), err)
+						return ErrResp(http.StatusInternalServerError, err, "failed to decrypt stored secure setting: %s", key)
 					}
 
 					if body.AlertmanagerConfig.Receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings == nil {
@@ -232,12 +232,12 @@ func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body ap
 	}
 
 	if err := body.ProcessConfig(); err != nil {
-		return response.Error(http.StatusInternalServerError, "failed to post process Alertmanager configuration", err)
+		return ErrResp(http.StatusInternalServerError, err, "failed to post process Alertmanager configuration")
 	}
 
 	if err := srv.am.SaveAndApplyConfig(&body); err != nil {
 		srv.log.Error("unable to save and apply alertmanager configuration", "err", err)
-		return response.Error(http.StatusBadRequest, "failed to save and apply Alertmanager configuration", err)
+		return ErrResp(http.StatusBadRequest, err, "failed to save and apply Alertmanager configuration")
 	}
 
 	return response.JSON(http.StatusAccepted, util.DynMap{"message": "configuration created"})
@@ -245,5 +245,5 @@ func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body ap
 
 func (srv AlertmanagerSrv) RoutePostAMAlerts(c *models.ReqContext, body apimodels.PostableAlerts) response.Response {
 	// not implemented
-	return response.Error(http.StatusNotImplemented, "", nil)
+	return NotImplementedResp
 }

pkg/services/ngalert/api/api_ruler.go

@@ -2,7 +2,6 @@ package api
 
 import (
 	"errors"
-	"fmt"
 	"net/http"
 	"time"
 
@@ -38,7 +37,7 @@ func (srv RulerSrv) RouteDeleteNamespaceRulesConfig(c *models.ReqContext) respon
 
 	uids, err := srv.store.DeleteNamespaceAlertRules(c.SignedInUser.OrgId, namespace.Uid)
 	if err != nil {
-		return response.Error(http.StatusInternalServerError, "failed to delete namespace alert rules", err)
+		return ErrResp(http.StatusInternalServerError, err, "failed to delete namespace alert rules")
 	}
 
 	for _, uid := range uids {
@@ -59,9 +58,9 @@ func (srv RulerSrv) RouteDeleteRuleGroupConfig(c *models.ReqContext) response.Re
 
 	if err != nil {
 		if errors.Is(err, ngmodels.ErrRuleGroupNamespaceNotFound) {
-			return response.Error(http.StatusNotFound, "failed to delete rule group", err)
+			return ErrResp(http.StatusNotFound, err, "failed to delete rule group")
 		}
-		return response.Error(http.StatusInternalServerError, "failed to delete rule group", err)
+		return ErrResp(http.StatusInternalServerError, err, "failed to delete rule group")
 	}
 
 	for _, uid := range uids {
@@ -83,7 +82,7 @@ func (srv RulerSrv) RouteGetNamespaceRulesConfig(c *models.ReqContext) response.
 		NamespaceUID: namespace.Uid,
 	}
 	if err := srv.store.GetNamespaceAlertRules(&q); err != nil {
-		return response.Error(http.StatusInternalServerError, "failed to update rule group", err)
+		return ErrResp(http.StatusInternalServerError, err, "failed to update rule group")
 	}
 
 	result := apimodels.NamespaceConfigResponse{}
@@ -126,7 +125,7 @@ func (srv RulerSrv) RouteGetRulegGroupConfig(c *models.ReqContext) response.Resp
 		RuleGroup:    ruleGroup,
 	}
 	if err := srv.store.GetRuleGroupAlertRules(&q); err != nil {
-		return response.Error(http.StatusInternalServerError, "failed to get group alert rules", err)
+		return ErrResp(http.StatusInternalServerError, err, "failed to get group alert rules")
 	}
 
 	var ruleGroupInterval model.Duration
@@ -151,7 +150,7 @@ func (srv RulerSrv) RouteGetRulesConfig(c *models.ReqContext) response.Response
 		OrgID: c.SignedInUser.OrgId,
 	}
 	if err := srv.store.GetOrgAlertRules(&q); err != nil {
-		return response.Error(http.StatusInternalServerError, "failed to get alert rules", err)
+		return ErrResp(http.StatusInternalServerError, err, "failed to get alert rules")
 	}
 
 	configs := make(map[string]map[string]apimodels.GettableRuleGroupConfig)
@@ -217,17 +216,17 @@ func (srv RulerSrv) RoutePostNameRulesConfig(c *models.ReqContext, ruleGroupConf
 	// and rollback the transaction in case of violation
 	limitReached, err := srv.QuotaService.QuotaReached(c, "alert_rule")
 	if err != nil {
-		return response.Error(http.StatusInternalServerError, "failed to get quota", err)
+		return ErrResp(http.StatusInternalServerError, err, "failed to get quota")
 	}
 	if limitReached {
-		return response.Error(http.StatusForbidden, "quota reached", nil)
+		return ErrResp(http.StatusForbidden, errors.New("quota reached"), "")
 	}
 
 	// TODO validate UID uniqueness in the payload
 
 	//TODO: Should this belong in alerting-api?
 	if ruleGroupConfig.Name == "" {
-		return response.Error(http.StatusBadRequest, "rule group name is not valid", nil)
+		return ErrResp(http.StatusBadRequest, errors.New("rule group name is not valid"), "")
 	}
 
 	var alertRuleUIDs []string
@@ -238,7 +237,7 @@ func (srv RulerSrv) RoutePostNameRulesConfig(c *models.ReqContext, ruleGroupConf
 			Data:      r.GrafanaManagedAlert.Data,
 		}
 		if err := validateCondition(cond, c.SignedInUser, c.SkipCache, srv.DatasourceCache); err != nil {
-			return response.Error(http.StatusBadRequest, fmt.Sprintf("failed to validate alert rule %s", r.GrafanaManagedAlert.Title), err)
+			return ErrResp(http.StatusBadRequest, err, "failed to validate alert rule %s", r.GrafanaManagedAlert.Title)
 		}
 		alertRuleUIDs = append(alertRuleUIDs, r.GrafanaManagedAlert.UID)
 	}
@@ -249,11 +248,11 @@ func (srv RulerSrv) RoutePostNameRulesConfig(c *models.ReqContext, ruleGroupConf
 		RuleGroupConfig: ruleGroupConfig,
 	}); err != nil {
 		if errors.Is(err, ngmodels.ErrAlertRuleNotFound) {
-			return response.Error(http.StatusNotFound, "failed to update rule group", err)
+			return ErrResp(http.StatusNotFound, err, "failed to update rule group")
 		} else if errors.Is(err, ngmodels.ErrAlertRuleFailedValidation) {
-			return response.Error(http.StatusBadRequest, "failed to update rule group", err)
+			return ErrResp(http.StatusBadRequest, err, "failed to update rule group")
 		}
-		return response.Error(http.StatusInternalServerError, "failed to update rule group", err)
+		return ErrResp(http.StatusInternalServerError, err, "failed to update rule group")
 	}
 
 	for _, uid := range alertRuleUIDs {
@@ -292,10 +291,10 @@ func toGettableExtendedRuleNode(r ngmodels.AlertRule, namespaceID int64) apimode
 
 func toNamespaceErrorResponse(err error) response.Response {
 	if errors.Is(err, ngmodels.ErrCannotEditNamespace) {
-		return response.Error(http.StatusForbidden, err.Error(), err)
+		return ErrResp(http.StatusForbidden, err, err.Error())
 	}
 	if errors.Is(err, models.ErrDashboardIdentifierNotSet) {
-		return response.Error(http.StatusBadRequest, err.Error(), err)
+		return ErrResp(http.StatusBadRequest, err, err.Error())
 	}
 	return coreapi.ToFolderErrorResponse(err)
 }

pkg/services/ngalert/api/api_testing.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -34,20 +35,20 @@ func (srv TestingApiSrv) RouteTestRuleConfig(c *models.ReqContext, body apimodel
 	recipient := c.Params("Recipient")
 	if recipient == apimodels.GrafanaBackend.String() {
 		if body.Type() != apimodels.GrafanaBackend || body.GrafanaManagedCondition == nil {
-			return response.Error(http.StatusBadRequest, "unexpected payload", nil)
+			return ErrResp(http.StatusBadRequest, errors.New("unexpected payload"), "")
 		}
 		return conditionEval(c, *body.GrafanaManagedCondition, srv.DatasourceCache, srv.DataService, srv.Cfg)
 	}
 
 	if body.Type() != apimodels.LoTexRulerBackend {
-		return response.Error(http.StatusBadRequest, "unexpected payload", nil)
+		return ErrResp(http.StatusBadRequest, errors.New("unexpected payload"), "")
 	}
 
 	var path string
 	if datasourceID, err := strconv.ParseInt(recipient, 10, 64); err == nil {
 		ds, err := srv.DatasourceCache.GetDatasource(datasourceID, c.SignedInUser, c.SkipCache)
 		if err != nil {
-			return response.Error(http.StatusInternalServerError, "failed to get datasource", err)
+			return ErrResp(http.StatusInternalServerError, err, "failed to get datasource")
 		}
 
 		switch ds.Type {
@@ -56,14 +57,14 @@ func (srv TestingApiSrv) RouteTestRuleConfig(c *models.ReqContext, body apimodel
 		case "prometheus":
 			path = "api/v1/query"
 		default:
-			return response.Error(http.StatusBadRequest, fmt.Sprintf("unexpected recipient type %s", ds.Type), nil)
+			return ErrResp(http.StatusBadRequest, fmt.Errorf("unexpected recipient type %s", ds.Type), "")
 		}
 	}
 
 	t := timeNow()
 	queryURL, err := url.Parse(path)
 	if err != nil {
-		return response.Error(http.StatusInternalServerError, "failed to parse url", err)
+		return ErrResp(http.StatusInternalServerError, err, "failed to parse url")
 	}
 	params := queryURL.Query()
 	params.Set("query", body.Expr)
@@ -86,13 +87,13 @@ func (srv TestingApiSrv) RouteEvalQueries(c *models.ReqContext, cmd apimodels.Ev
 	}
 
 	if _, err := validateQueriesAndExpressions(cmd.Data, c.SignedInUser, c.SkipCache, srv.DatasourceCache); err != nil {
-		return response.Error(http.StatusBadRequest, "invalid queries or expressions", err)
+		return ErrResp(http.StatusBadRequest, err, "invalid queries or expressions")
 	}
 
 	evaluator := eval.Evaluator{Cfg: srv.Cfg}
 	evalResults, err := evaluator.QueriesAndExpressionsEval(c.SignedInUser.OrgId, cmd.Data, now, srv.DataService)
 	if err != nil {
-		return response.Error(http.StatusBadRequest, "Failed to evaluate queries and expressions", err)
+		return ErrResp(http.StatusBadRequest, err, "Failed to evaluate queries and expressions")
 	}
 
 	return response.JSONStreaming(http.StatusOK, evalResults)

pkg/services/ngalert/api/fork_ruler.go

@@ -27,7 +27,7 @@ func NewForkedRuler(datasourceCache datasources.CacheService, lotex, grafana Rul
 func (r *ForkedRuler) RouteDeleteNamespaceRulesConfig(ctx *models.ReqContext) response.Response {
 	t, err := backendType(ctx, r.DatasourceCache)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 	switch t {
 	case apimodels.GrafanaBackend:
@@ -35,14 +35,14 @@ func (r *ForkedRuler) RouteDeleteNamespaceRulesConfig(ctx *models.ReqContext) re
 	case apimodels.LoTexRulerBackend:
 		return r.LotexRuler.RouteDeleteNamespaceRulesConfig(ctx)
 	default:
-		return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
+		return ErrResp(400, fmt.Errorf("unexpected backend type (%v)", t), "")
 	}
 }
 
 func (r *ForkedRuler) RouteDeleteRuleGroupConfig(ctx *models.ReqContext) response.Response {
 	t, err := backendType(ctx, r.DatasourceCache)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 	switch t {
 	case apimodels.GrafanaBackend:
@@ -50,14 +50,14 @@ func (r *ForkedRuler) RouteDeleteRuleGroupConfig(ctx *models.ReqContext) respons
 	case apimodels.LoTexRulerBackend:
 		return r.LotexRuler.RouteDeleteRuleGroupConfig(ctx)
 	default:
-		return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
+		return ErrResp(400, fmt.Errorf("unexpected backend type (%v)", t), "")
 	}
 }
 
 func (r *ForkedRuler) RouteGetNamespaceRulesConfig(ctx *models.ReqContext) response.Response {
 	t, err := backendType(ctx, r.DatasourceCache)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 	switch t {
 	case apimodels.GrafanaBackend:
@@ -65,14 +65,14 @@ func (r *ForkedRuler) RouteGetNamespaceRulesConfig(ctx *models.ReqContext) respo
 	case apimodels.LoTexRulerBackend:
 		return r.LotexRuler.RouteGetNamespaceRulesConfig(ctx)
 	default:
-		return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
+		return ErrResp(400, fmt.Errorf("unexpected backend type (%v)", t), "")
 	}
 }
 
 func (r *ForkedRuler) RouteGetRulegGroupConfig(ctx *models.ReqContext) response.Response {
 	t, err := backendType(ctx, r.DatasourceCache)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 	switch t {
 	case apimodels.GrafanaBackend:
@@ -80,14 +80,14 @@ func (r *ForkedRuler) RouteGetRulegGroupConfig(ctx *models.ReqContext) response.
 	case apimodels.LoTexRulerBackend:
 		return r.LotexRuler.RouteGetRulegGroupConfig(ctx)
 	default:
-		return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
+		return ErrResp(400, fmt.Errorf("unexpected backend type (%v)", t), "")
 	}
 }
 
 func (r *ForkedRuler) RouteGetRulesConfig(ctx *models.ReqContext) response.Response {
 	t, err := backendType(ctx, r.DatasourceCache)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 	switch t {
 	case apimodels.GrafanaBackend:
@@ -95,27 +95,19 @@ func (r *ForkedRuler) RouteGetRulesConfig(ctx *models.ReqContext) response.Respo
 	case apimodels.LoTexRulerBackend:
 		return r.LotexRuler.RouteGetRulesConfig(ctx)
 	default:
-		return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
+		return ErrResp(400, fmt.Errorf("unexpected backend type (%v)", t), "")
 	}
 }
 
 func (r *ForkedRuler) RoutePostNameRulesConfig(ctx *models.ReqContext, conf apimodels.PostableRuleGroupConfig) response.Response {
 	backendType, err := backendType(ctx, r.DatasourceCache)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 	payloadType := conf.Type()
 
 	if backendType != payloadType {
-		return response.Error(
+		return ErrResp(400, fmt.Errorf("unexpected backend type (%v) vs payload type (%v)", backendType, payloadType), "")
-			400,
-			fmt.Sprintf(
-				"unexpected backend type (%v) vs payload type (%v)",
-				backendType,
-				payloadType,
-			),
-			nil,
-		)
 	}
 
 	switch backendType {
@@ -124,6 +116,6 @@ func (r *ForkedRuler) RoutePostNameRulesConfig(ctx *models.ReqContext, conf apim
 	case apimodels.LoTexRulerBackend:
 		return r.LotexRuler.RoutePostNameRulesConfig(ctx, conf)
 	default:
-		return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", backendType), nil)
+		return ErrResp(400, fmt.Errorf("unexpected backend type (%v)", backendType), "")
 	}
 }

pkg/services/ngalert/api/forked_am.go

@@ -42,7 +42,7 @@ func (am *ForkedAMSvc) getService(ctx *models.ReqContext) (AlertmanagerApiServic
 func (am *ForkedAMSvc) RouteCreateSilence(ctx *models.ReqContext, body apimodels.PostableSilence) response.Response {
 	s, err := am.getService(ctx)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 
 	return s.RouteCreateSilence(ctx, body)
@@ -51,7 +51,7 @@ func (am *ForkedAMSvc) RouteCreateSilence(ctx *models.ReqContext, body apimodels
 func (am *ForkedAMSvc) RouteDeleteAlertingConfig(ctx *models.ReqContext) response.Response {
 	s, err := am.getService(ctx)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 
 	return s.RouteDeleteAlertingConfig(ctx)
@@ -60,7 +60,7 @@ func (am *ForkedAMSvc) RouteDeleteAlertingConfig(ctx *models.ReqContext) respons
 func (am *ForkedAMSvc) RouteDeleteSilence(ctx *models.ReqContext) response.Response {
 	s, err := am.getService(ctx)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 
 	return s.RouteDeleteSilence(ctx)
@@ -69,7 +69,7 @@ func (am *ForkedAMSvc) RouteDeleteSilence(ctx *models.ReqContext) response.Respo
 func (am *ForkedAMSvc) RouteGetAlertingConfig(ctx *models.ReqContext) response.Response {
 	s, err := am.getService(ctx)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 
 	return s.RouteGetAlertingConfig(ctx)
@@ -78,7 +78,7 @@ func (am *ForkedAMSvc) RouteGetAlertingConfig(ctx *models.ReqContext) response.R
 func (am *ForkedAMSvc) RouteGetAMAlertGroups(ctx *models.ReqContext) response.Response {
 	s, err := am.getService(ctx)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 
 	return s.RouteGetAMAlertGroups(ctx)
@@ -87,7 +87,7 @@ func (am *ForkedAMSvc) RouteGetAMAlertGroups(ctx *models.ReqContext) response.Re
 func (am *ForkedAMSvc) RouteGetAMAlerts(ctx *models.ReqContext) response.Response {
 	s, err := am.getService(ctx)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 
 	return s.RouteGetAMAlerts(ctx)
@@ -96,7 +96,7 @@ func (am *ForkedAMSvc) RouteGetAMAlerts(ctx *models.ReqContext) response.Respons
 func (am *ForkedAMSvc) RouteGetSilence(ctx *models.ReqContext) response.Response {
 	s, err := am.getService(ctx)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 
 	return s.RouteGetSilence(ctx)
@@ -105,7 +105,7 @@ func (am *ForkedAMSvc) RouteGetSilence(ctx *models.ReqContext) response.Response
 func (am *ForkedAMSvc) RouteGetSilences(ctx *models.ReqContext) response.Response {
 	s, err := am.getService(ctx)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 
 	return s.RouteGetSilences(ctx)
@@ -114,20 +114,16 @@ func (am *ForkedAMSvc) RouteGetSilences(ctx *models.ReqContext) response.Respons
 func (am *ForkedAMSvc) RoutePostAlertingConfig(ctx *models.ReqContext, body apimodels.PostableUserConfig) response.Response {
 	s, err := am.getService(ctx)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 
 	b, err := backendType(ctx, am.DatasourceCache)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 
 	if err := body.AlertmanagerConfig.ReceiverType().MatchesBackend(b); err != nil {
-		return response.Error(
+		return ErrResp(400, err, "bad match")
-			400,
-			"bad match",
-			err,
-		)
 	}
 
 	return s.RoutePostAlertingConfig(ctx, body)
@@ -136,7 +132,7 @@ func (am *ForkedAMSvc) RoutePostAlertingConfig(ctx *models.ReqContext, body apim
 func (am *ForkedAMSvc) RoutePostAMAlerts(ctx *models.ReqContext, body apimodels.PostableAlerts) response.Response {
 	s, err := am.getService(ctx)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 
 	return s.RoutePostAMAlerts(ctx, body)

pkg/services/ngalert/api/forked_prom.go

@@ -26,7 +26,7 @@ func NewForkedProm(datasourceCache datasources.CacheService, proxy, grafana Prom
 func (p *ForkedPromSvc) RouteGetAlertStatuses(ctx *models.ReqContext) response.Response {
 	t, err := backendType(ctx, p.DatasourceCache)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 
 	switch t {
@@ -35,14 +35,14 @@ func (p *ForkedPromSvc) RouteGetAlertStatuses(ctx *models.ReqContext) response.R
 	case apimodels.LoTexRulerBackend:
 		return p.ProxySvc.RouteGetAlertStatuses(ctx)
 	default:
-		return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
+		return ErrResp(400, fmt.Errorf("unexpected backend type (%v)", t), "")
 	}
 }
 
 func (p *ForkedPromSvc) RouteGetRuleStatuses(ctx *models.ReqContext) response.Response {
 	t, err := backendType(ctx, p.DatasourceCache)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(400, err, "")
 	}
 
 	switch t {
@@ -51,6 +51,6 @@ func (p *ForkedPromSvc) RouteGetRuleStatuses(ctx *models.ReqContext) response.Re
 	case apimodels.LoTexRulerBackend:
 		return p.ProxySvc.RouteGetRuleStatuses(ctx)
 	default:
-		return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
+		return ErrResp(400, fmt.Errorf("unexpected backend type (%v)", t), "")
 	}
 }

pkg/services/ngalert/api/lotex_am.go

@@ -36,7 +36,7 @@ func NewLotexAM(proxy *AlertingProxy, log log.Logger) *LotexAM {
 func (am *LotexAM) RouteCreateSilence(ctx *models.ReqContext, silenceBody apimodels.PostableSilence) response.Response {
 	blob, err := json.Marshal(silenceBody)
 	if err != nil {
-		return response.Error(500, "Failed marshal silence", err)
+		return ErrResp(500, err, "Failed marshal silence")
 	}
 	return am.withReq(
 		ctx,
@@ -149,7 +149,7 @@ func (am *LotexAM) RouteGetSilences(ctx *models.ReqContext) response.Response {
 func (am *LotexAM) RoutePostAlertingConfig(ctx *models.ReqContext, config apimodels.PostableUserConfig) response.Response {
 	yml, err := yaml.Marshal(&config)
 	if err != nil {
-		return response.Error(500, "Failed marshal alert manager configuration ", err)
+		return ErrResp(500, err, "Failed marshal alert manager configuration ")
 	}
 
 	return am.withReq(
@@ -165,7 +165,7 @@ func (am *LotexAM) RoutePostAlertingConfig(ctx *models.ReqContext, config apimod
 func (am *LotexAM) RoutePostAMAlerts(ctx *models.ReqContext, alerts apimodels.PostableAlerts) response.Response {
 	yml, err := yaml.Marshal(alerts)
 	if err != nil {
-		return response.Error(500, "Failed marshal postable alerts", err)
+		return ErrResp(500, err, "Failed marshal postable alerts")
 	}
 
 	return am.withReq(

pkg/services/ngalert/api/lotex_prom.go

@@ -40,7 +40,7 @@ func NewLotexProm(proxy *AlertingProxy, log log.Logger) *LotexProm {
 func (p *LotexProm) RouteGetAlertStatuses(ctx *models.ReqContext) response.Response {
 	endpoints, err := p.getEndpoints(ctx)
 	if err != nil {
-		return response.Error(500, err.Error(), nil)
+		return ErrResp(http.StatusInternalServerError, err, "")
 	}
 
 	return p.withReq(
@@ -59,7 +59,7 @@ func (p *LotexProm) RouteGetAlertStatuses(ctx *models.ReqContext) response.Respo
 func (p *LotexProm) RouteGetRuleStatuses(ctx *models.ReqContext) response.Response {
 	endpoints, err := p.getEndpoints(ctx)
 	if err != nil {
-		return response.Error(500, err.Error(), nil)
+		return ErrResp(http.StatusInternalServerError, err, "")
 	}
 
 	return p.withReq(

pkg/services/ngalert/api/lotex_ruler.go

@@ -34,7 +34,7 @@ func NewLotexRuler(proxy *AlertingProxy, log log.Logger) *LotexRuler {
 func (r *LotexRuler) RouteDeleteNamespaceRulesConfig(ctx *models.ReqContext) response.Response {
 	legacyRulerPrefix, err := r.getPrefix(ctx)
 	if err != nil {
-		return response.Error(500, err.Error(), nil)
+		return ErrResp(500, err, "")
 	}
 	return r.withReq(
 		ctx,
@@ -52,7 +52,7 @@ func (r *LotexRuler) RouteDeleteNamespaceRulesConfig(ctx *models.ReqContext) res
 func (r *LotexRuler) RouteDeleteRuleGroupConfig(ctx *models.ReqContext) response.Response {
 	legacyRulerPrefix, err := r.getPrefix(ctx)
 	if err != nil {
-		return response.Error(500, err.Error(), nil)
+		return ErrResp(500, err, "")
 	}
 	return r.withReq(
 		ctx,
@@ -75,7 +75,7 @@ func (r *LotexRuler) RouteDeleteRuleGroupConfig(ctx *models.ReqContext) response
 func (r *LotexRuler) RouteGetNamespaceRulesConfig(ctx *models.ReqContext) response.Response {
 	legacyRulerPrefix, err := r.getPrefix(ctx)
 	if err != nil {
-		return response.Error(500, err.Error(), nil)
+		return ErrResp(500, err, "")
 	}
 	return r.withReq(
 		ctx,
@@ -97,7 +97,7 @@ func (r *LotexRuler) RouteGetNamespaceRulesConfig(ctx *models.ReqContext) respon
 func (r *LotexRuler) RouteGetRulegGroupConfig(ctx *models.ReqContext) response.Response {
 	legacyRulerPrefix, err := r.getPrefix(ctx)
 	if err != nil {
-		return response.Error(500, err.Error(), nil)
+		return ErrResp(500, err, "")
 	}
 	return r.withReq(
 		ctx,
@@ -120,7 +120,7 @@ func (r *LotexRuler) RouteGetRulegGroupConfig(ctx *models.ReqContext) response.R
 func (r *LotexRuler) RouteGetRulesConfig(ctx *models.ReqContext) response.Response {
 	legacyRulerPrefix, err := r.getPrefix(ctx)
 	if err != nil {
-		return response.Error(500, err.Error(), nil)
+		return ErrResp(500, err, "")
 	}
 	return r.withReq(
 		ctx,
@@ -138,11 +138,11 @@ func (r *LotexRuler) RouteGetRulesConfig(ctx *models.ReqContext) response.Respon
 func (r *LotexRuler) RoutePostNameRulesConfig(ctx *models.ReqContext, conf apimodels.PostableRuleGroupConfig) response.Response {
 	legacyRulerPrefix, err := r.getPrefix(ctx)
 	if err != nil {
-		return response.Error(500, err.Error(), nil)
+		return ErrResp(500, err, "")
 	}
 	yml, err := yaml.Marshal(conf)
 	if err != nil {
-		return response.Error(500, "Failed marshal rule group", err)
+		return ErrResp(500, err, "Failed marshal rule group")
 	}
 	ns := ctx.Params("Namespace")
 	u := withPath(*ctx.Req.URL, fmt.Sprintf("%s/%s", legacyRulerPrefix, ns))

pkg/services/ngalert/api/util.go

@@ -22,12 +22,15 @@ import (
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb"
 	"github.com/grafana/grafana/pkg/util"
+	"github.com/pkg/errors"
 	"gopkg.in/macaron.v1"
 	"gopkg.in/yaml.v3"
 )
 
 var searchRegex = regexp.MustCompile(`\{(\w+)\}`)
 
+var NotImplementedResp = ErrResp(http.StatusNotImplemented, errors.New("endpoint not implemented"), "")
+
 func toMacaronPath(path string) string {
 	return string(searchRegex.ReplaceAllFunc([]byte(path), func(s []byte) []byte {
 		m := string(s[1 : len(s)-1])
@@ -93,7 +96,7 @@ func (p *AlertingProxy) withReq(
 ) response.Response {
 	req, err := http.NewRequest(method, u.String(), body)
 	if err != nil {
-		return response.Error(400, err.Error(), nil)
+		return ErrResp(http.StatusBadRequest, err, "")
 	}
 	for h, v := range headers {
 		req.Header.Add(h, v)
@@ -116,17 +119,17 @@ func (p *AlertingProxy) withReq(
 				}
 			}
 		}
-		return response.Error(status, errMessage, nil)
+		return ErrResp(status, errors.New(errMessage), "")
 	}
 
 	t, err := extractor(resp)
 	if err != nil {
-		return response.Error(500, err.Error(), nil)
+		return ErrResp(http.StatusInternalServerError, err, "")
 	}
 
 	b, err := json.Marshal(t)
 	if err != nil {
-		return response.Error(500, err.Error(), nil)
+		return ErrResp(http.StatusInternalServerError, err, "")
 	}
 
 	return response.JSON(status, b)
@@ -222,7 +225,7 @@ func conditionEval(c *models.ReqContext, cmd ngmodels.EvalAlertConditionCommand,
 		Data:      cmd.Data,
 	}
 	if err := validateCondition(evalCond, c.SignedInUser, c.SkipCache, datasourceCache); err != nil {
-		return response.Error(400, "invalid condition", err)
+		return ErrResp(http.StatusBadRequest, err, "invalid condition")
 	}
 
 	now := cmd.Now
@@ -233,7 +236,7 @@ func conditionEval(c *models.ReqContext, cmd ngmodels.EvalAlertConditionCommand,
 	evaluator := eval.Evaluator{Cfg: cfg}
 	evalResults, err := evaluator.ConditionEval(&evalCond, now, dataService)
 	if err != nil {
-		return response.Error(http.StatusBadRequest, "Failed to evaluate conditions", err)
+		return ErrResp(http.StatusBadRequest, err, "Failed to evaluate conditions")
 	}
 
 	frame := evalResults.AsDataFrame()
@@ -241,3 +244,11 @@ func conditionEval(c *models.ReqContext, cmd ngmodels.EvalAlertConditionCommand,
 		"instances": []*data.Frame{&frame},
 	})
 }
+
+// ErrorResp creates a response with a visible error
+func ErrResp(status int, err error, msg string, args ...interface{}) *response.NormalResponse {
+	if msg != "" {
+		err = errors.WithMessagef(err, msg, args...)
+	}
+	return response.Error(status, err.Error(), nil)
+}

pkg/tests/api/alerting/api_alertmanager_configuration_test.go

@@ -61,7 +61,7 @@ func TestAlertmanagerConfigurationIsTransactional(t *testing.T) {
 }
 `
 		resp := postRequest(t, alertConfigURL, payload, http.StatusBadRequest) // nolint
-		require.JSONEq(t, "{\"error\":\"alert validation error: token must be specified when using the Slack chat API\", \"message\":\"failed to save and apply Alertmanager configuration\"}", getBody(t, resp.Body))
+		require.JSONEq(t, "{\"message\":\"failed to save and apply Alertmanager configuration: alert validation error: token must be specified when using the Slack chat API\"}", getBody(t, resp.Body))
 
 		resp = getRequest(t, alertConfigURL, http.StatusOK) // nolint
 		require.JSONEq(t, defaultAlertmanagerConfigJSON, getBody(t, resp.Body))

pkg/tests/api/alerting/api_alertmanager_test.go

@@ -84,7 +84,7 @@ func TestAMConfigAccess(t *testing.T) {
 				desc:      "viewer request should fail",
 				url:       "http://viewer:viewer@%s/api/alertmanager/grafana/config/api/v1/alerts",
 				expStatus: http.StatusForbidden,
-				expBody:   `{"message": "Permission denied"}`,
+				expBody:   `{"message": "permission denied"}`,
 			},
 			{
 				desc:      "editor request should succeed",
@@ -146,7 +146,7 @@ func TestAMConfigAccess(t *testing.T) {
 				desc:      "viewer request should fail",
 				url:       "http://viewer:viewer@%s/api/alertmanager/grafana/api/v2/silences",
 				expStatus: http.StatusForbidden,
-				expBody:   `{"message": "Permission denied"}`,
+				expBody:   `{"message": "permission denied"}`,
 			},
 			{
 				desc:      "editor request should succeed",
@@ -252,7 +252,7 @@ func TestAMConfigAccess(t *testing.T) {
 				desc:      "viewer request should fail",
 				url:       "http://viewer:viewer@%s/api/alertmanager/grafana/api/v2/silence/%s",
 				expStatus: http.StatusForbidden,
-				expBody:   `{"message": "Permission denied"}`,
+				expBody:   `{"message": "permission denied"}`,
 			},
 			{
 				desc:      "editor request should succeed",
@@ -504,7 +504,7 @@ func TestRulerAccess(t *testing.T) {
 			desc:             "viewer request should fail",
 			url:              "http://viewer:viewer@%s/api/ruler/grafana/api/v1/rules/default",
 			expStatus:        http.StatusForbidden,
-			expectedResponse: `{"error":"user does not have permissions to edit the namespace", "message":"user does not have permissions to edit the namespace"}`,
+			expectedResponse: `{"message":"user does not have permissions to edit the namespace: user does not have permissions to edit the namespace"}`,
 		},
 		{
 			desc:             "editor request should succeed",
@@ -763,7 +763,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						Data:  []ngmodels.AlertQuery{},
 					},
 				},
-				expectedResponse: `{"error":"invalid alert rule: no queries or expressions are found", "message":"failed to update rule group"}`,
+				expectedResponse: `{"message":"failed to update rule group: invalid alert rule: no queries or expressions are found"}`,
 			},
 			{
 				desc:      "alert rule with empty title",
@@ -793,7 +793,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"error":"invalid alert rule: title is empty", "message":"failed to update rule group"}`,
+				expectedResponse: `{"message":"failed to update rule group: invalid alert rule: title is empty"}`,
 			},
 			{
 				desc:      "alert rule with too long name",
@@ -823,7 +823,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"error":"invalid alert rule: name length should not be greater than 190", "message":"failed to update rule group"}`,
+				expectedResponse: `{"message":"failed to update rule group: invalid alert rule: name length should not be greater than 190"}`,
 			},
 			{
 				desc:      "alert rule with too long rulegroup",
@@ -853,7 +853,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"error":"invalid alert rule: rule group name length should not be greater than 190", "message":"failed to update rule group"}`,
+				expectedResponse: `{"message":"failed to update rule group: invalid alert rule: rule group name length should not be greater than 190"}`,
 			},
 			{
 				desc:      "alert rule with invalid interval",
@@ -884,7 +884,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"error":"invalid alert rule: interval (1s) should be non-zero and divided exactly by scheduler interval: 10s", "message":"failed to update rule group"}`,
+				expectedResponse: `{"message":"failed to update rule group: invalid alert rule: interval (1s) should be non-zero and divided exactly by scheduler interval: 10s"}`,
 			},
 			{
 				desc:      "alert rule with unknown datasource",
@@ -914,7 +914,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"error":"invalid query A: data source not found: unknown", "message":"failed to validate alert rule AlwaysFiring"}`,
+				expectedResponse: `{"message":"failed to validate alert rule AlwaysFiring: invalid query A: data source not found: unknown"}`,
 			},
 			{
 				desc:      "alert rule with invalid condition",
@@ -944,7 +944,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"error":"condition B not found in any query or expression: it should be one of: [A]", "message":"failed to validate alert rule AlwaysFiring"}`,
+				expectedResponse: `{"message":"failed to validate alert rule AlwaysFiring: condition B not found in any query or expression: it should be one of: [A]"}`,
 			},
 		}
 
@@ -1233,7 +1233,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 		require.NoError(t, err)
 
 		assert.Equal(t, http.StatusNotFound, resp.StatusCode)
-		require.JSONEq(t, `{"error":"failed to get alert rule unknown: could not find alert rule", "message": "failed to update rule group"}`, string(b))
+		require.JSONEq(t, `{"message":"failed to update rule group: failed to get alert rule unknown: could not find alert rule"}`, string(b))
 
 		// let's make sure that rule definitions are not affected by the failed POST request.
 		u = fmt.Sprintf("http://grafana:password@%s/api/ruler/grafana/api/v1/rules/default", grafanaListedAddr)
@@ -1412,7 +1412,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 			require.NoError(t, err)
 
 			require.Equal(t, http.StatusNotFound, resp.StatusCode)
-			require.JSONEq(t, `{"error":"rule group not found under this namespace", "message": "failed to delete rule group"}`, string(b))
+			require.JSONEq(t, `{"message":"failed to delete rule group: rule group not found under this namespace"}`, string(b))
 		})
 
 		t.Run("succeed if the rule group name does exist", func(t *testing.T) {
@@ -1707,7 +1707,7 @@ func TestEval(t *testing.T) {
 			}
 			`,
 			expectedStatusCode: http.StatusBadRequest,
-			expectedResponse:   `{"error":"condition B not found in any query or expression: it should be one of: [A]","message":"invalid condition"}`,
+			expectedResponse:   `{"message":"invalid condition: condition B not found in any query or expression: it should be one of: [A]"}`,
 		},
 		{
 			desc: "unknown query datasource",
@@ -1732,7 +1732,7 @@ func TestEval(t *testing.T) {
 			}
 			`,
 			expectedStatusCode: http.StatusBadRequest,
-			expectedResponse:   `{"error":"invalid query A: data source not found: unknown","message":"invalid condition"}`,
+			expectedResponse:   `{"message":"invalid condition: invalid query A: data source not found: unknown"}`,
 		},
 	}
 
@@ -1888,7 +1888,7 @@ func TestEval(t *testing.T) {
 			}
 			`,
 			expectedStatusCode: http.StatusBadRequest,
-			expectedResponse:   `{"error":"invalid query A: data source not found: unknown","message":"invalid queries or expressions"}`,
+			expectedResponse:   `{"message":"invalid queries or expressions: invalid query A: data source not found: unknown"}`,
 		},
 	}