[Alerting]: Add alerting endpoint for Query Evaluation (#33174)

* [Alerting]: Add alerting endpoint for Query Evaluation

* Fix passing down now parameter

* Add validations and test

* Fix eval queries and expressions test

* Add eval tests

pkg/expr/translate/translate.go

@@ -34,7 +34,7 @@ func DashboardAlertConditions(rawDCondJSON []byte, orgID int64) (*ngmodels.Condi
 		return nil, err
 	}
 
-	backendReq, err := eval.GetQueryDataRequest(eval.AlertExecCtx{ExpressionsEnabled: true}, ngCond, time.Unix(500, 0))
+	backendReq, err := eval.GetQueryDataRequest(eval.AlertExecCtx{ExpressionsEnabled: true}, ngCond.Data, time.Unix(500, 0))
 
 	if err != nil {
 		return nil, err

pkg/services/ngalert/api/api_testing.go

@@ -11,6 +11,7 @@ import (
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/datasources"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+	"github.com/grafana/grafana/pkg/services/ngalert/eval"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb"
 	"github.com/grafana/grafana/pkg/util"
@@ -77,3 +78,21 @@ func (srv TestingApiSrv) RouteTestRuleConfig(c *models.ReqContext, body apimodel
 		nil,
 	)
 }
+
+func (srv TestingApiSrv) RouteEvalQueries(c *models.ReqContext, cmd apimodels.EvalQueriesPayload) response.Response {
+	now := cmd.Now
+	if now.IsZero() {
+		now = timeNow()
+	}
+	if err := validateQueriesAndExpressions(cmd.Data, c.SignedInUser, c.SkipCache, srv.DatasourceCache); err != nil {
+		return response.Error(http.StatusBadRequest, "invalid queries or expressions", err)
+	}
+
+	evaluator := eval.Evaluator{Cfg: srv.Cfg}
+	evalResults, err := evaluator.QueriesAndExpressionsEval(c.SignedInUser.OrgId, cmd.Data, now, srv.DataService)
+	if err != nil {
+		return response.Error(http.StatusBadRequest, "Failed to evaluate queries and expressions", err)
+	}
+
+	return response.JSONStreaming(http.StatusOK, evalResults)
+}

pkg/services/ngalert/api/generated_base_api_testing.go

@@ -18,12 +18,14 @@ import (
 )
 
 type TestingApiService interface {
+	RouteEvalQueries(*models.ReqContext, apimodels.EvalQueriesPayload) response.Response
 	RouteTestReceiverConfig(*models.ReqContext, apimodels.ExtendedReceiver) response.Response
 	RouteTestRuleConfig(*models.ReqContext, apimodels.TestRulePayload) response.Response
 }
 
 func (api *API) RegisterTestingApiEndpoints(srv TestingApiService) {
 	api.RouteRegister.Group("", func(group routing.RouteRegister) {
+		group.Post(toMacaronPath("/api/v1/eval"), binding.Bind(apimodels.EvalQueriesPayload{}), routing.Wrap(srv.RouteEvalQueries))
 		group.Post(toMacaronPath("/api/v1/receiver/test/{Recipient}"), binding.Bind(apimodels.ExtendedReceiver{}), routing.Wrap(srv.RouteTestReceiverConfig))
 		group.Post(toMacaronPath("/api/v1/rule/test/{Recipient}"), binding.Bind(apimodels.TestRulePayload{}), routing.Wrap(srv.RouteTestRuleConfig))
 	}, middleware.ReqSignedIn)

pkg/services/ngalert/api/test-data/test.http

@@ -25,6 +25,52 @@ content-type: application/json
   }
 }
 
+###
+
+POST http://admin:admin@localhost:3000/api/v1/eval
+content-type: application/json
+
+{
+"data": [
+  {
+    "refId": "A",
+    "queryType": "",
+    "relativeTimeRange": {
+      "from": 18000,
+      "to": 10800
+    },
+    "model": {
+      "datasourceUid": "000000004",
+      "intervalMs": 1000,
+      "maxDataPoints": 100,
+      "orgId": 0,
+      "refId": "A",
+      "scenarioId": "csv_metric_values",
+      "stringInput": "1,20,90,30,5,0"
+    }
+  },
+  {
+    "refId": "B",
+    "queryType": "",
+    "relativeTimeRange": {
+      "from": 18000,
+      "to": 10800
+    },
+    "model": {
+      "datasourceUid": "-100",
+      "expression": "$A",
+      "intervalMs": 2000,
+      "maxDataPoints": 200,
+      "orgId": 0,
+      "reducer": "mean",
+      "refId": "B",
+      "type": "reduce"
+    }
+  }
+],
+"now": "2021-04-11T14:38:14Z"
+}
+
 ###
 POST http://admin:admin@localhost:3000/api/v1/rule/test/{{lokiDatasourceID}}
 content-type: application/json

pkg/services/ngalert/api/tooling/definitions/testing.go

@@ -3,6 +3,9 @@ package definitions
 import (
 	"encoding/json"
 	"fmt"
+	"time"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
 
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/prometheus/alertmanager/config"
@@ -37,6 +40,19 @@ import (
 //     Responses:
 //       200: TestRuleResponse
 
+// swagger:route Post /api/v1/eval testing RouteEvalQueries
+//
+// Test rule
+//
+//     Consumes:
+//     - application/json
+//
+//     Produces:
+//     - application/json
+//
+//     Responses:
+//       200: EvalQueriesResponse
+
 // swagger:parameters RouteTestReceiverConfig
 type TestReceiverRequest struct {
 	// in:body
@@ -57,6 +73,18 @@ type TestRulePayload struct {
 	GrafanaManagedCondition *models.EvalAlertConditionCommand `json:"grafana_condition,omitempty"`
 }
 
+// swagger:parameters RouteEvalQueries
+type EvalQueriesRequest struct {
+	// in:body
+	Body EvalQueriesPayload
+}
+
+// swagger:model
+type EvalQueriesPayload struct {
+	Data []models.AlertQuery `json:"data"`
+	Now  time.Time           `json:"now"`
+}
+
 func (p *TestRulePayload) UnmarshalJSON(b []byte) error {
 	type plain TestRulePayload
 	if err := json.Unmarshal(b, (*plain)(p)); err != nil {
@@ -96,6 +124,9 @@ type TestRuleResponse struct {
 	GrafanaAlertInstances AlertInstancesResponse `json:"grafana_alert_instances"`
 }
 
+// swagger:model
+type EvalQueriesResponse = backend.QueryDataResponse
+
 // swagger:model
 type AlertInstancesResponse struct {
 	// Instances is an array of arrow encoded dataframes

pkg/services/ngalert/api/tooling/spec.json

@@ -1,14 +1,4 @@
 {
-  "consumes": [
-    "application/json"
-  ],
-  "produces": [
-    "application/json"
-  ],
-  "schemes": [
-    "http",
-    "https"
-  ],
   "swagger": "2.0",
   "info": {
     "description": "Package definitions includes the types required for generating or consuming an OpenAPI\nspec for the Unified Alerting API.\nDocumentation of the API.",
@@ -717,6 +707,38 @@
         }
       }
     },
+    "/api/v1/eval": {
+      "post": {
+        "description": "Test rule",
+        "consumes": [
+          "application/json"
+        ],
+        "produces": [
+          "application/json"
+        ],
+        "tags": [
+          "testing"
+        ],
+        "operationId": "RouteEvalQueries",
+        "parameters": [
+          {
+            "name": "Body",
+            "in": "body",
+            "schema": {
+              "$ref": "#/definitions/EvalQueriesPayload"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "EvalQueriesResponse",
+            "schema": {
+              "$ref": "#/definitions/EvalQueriesResponse"
+            }
+          }
+        }
+      }
+    },
     "/api/v1/receiver/test/{Recipient}": {
       "post": {
         "description": "Test receiver",
@@ -1332,6 +1354,27 @@
       },
       "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/models"
     },
+    "EvalQueriesPayload": {
+      "type": "object",
+      "properties": {
+        "data": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/AlertQuery"
+          },
+          "x-go-name": "Data"
+        },
+        "now": {
+          "type": "string",
+          "format": "date-time",
+          "x-go-name": "Now"
+        }
+      },
+      "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+    },
+    "EvalQueriesResponse": {
+      "$ref": "#/definitions/EvalQueriesResponse"
+    },
     "ExtendedReceiver": {
       "type": "object",
       "properties": {
@@ -1369,7 +1412,7 @@
       "$ref": "#/definitions/ResponseDetails"
     },
     "GettableAlert": {
-      "$ref": "#/definitions/GettableAlert"
+      "$ref": "#/definitions/gettableAlert"
     },
     "GettableAlerts": {
       "$ref": "#/definitions/GettableAlerts"
@@ -1638,7 +1681,7 @@
       "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
     },
     "GettableSilence": {
-      "$ref": "#/definitions/gettableSilence"
+      "$ref": "#/definitions/GettableSilence"
     },
     "GettableSilences": {
       "$ref": "#/definitions/GettableSilences"
@@ -3204,7 +3247,7 @@
           "description": "alerts",
           "type": "array",
           "items": {
-            "$ref": "#/definitions/gettableAlert"
+            "$ref": "#/definitions/GettableAlert"
           },
           "x-go-name": "Alerts"
         },
@@ -3775,10 +3818,5 @@
       "x-go-name": "VersionInfo",
       "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
     }
-  },
-  "securityDefinitions": {
-    "basic": {
-      "type": "basic"
-    }
   }
 }

pkg/services/ngalert/api/util.go

@@ -194,6 +194,33 @@ func validateCondition(c ngmodels.Condition, user *models.SignedInUser, skipCach
 	return nil
 }
 
+func validateQueriesAndExpressions(data []ngmodels.AlertQuery, user *models.SignedInUser, skipCache bool, datasourceCache datasources.CacheService) error {
+	if len(data) == 0 {
+		return nil
+	}
+
+	for _, query := range data {
+		datasourceUID, err := query.GetDatasource()
+		if err != nil {
+			return err
+		}
+
+		isExpression, err := query.IsExpression()
+		if err != nil {
+			return err
+		}
+		if isExpression {
+			continue
+		}
+
+		_, err = datasourceCache.GetDatasourceByUID(datasourceUID, user, skipCache)
+		if err != nil {
+			return fmt.Errorf("failed to get datasource: %s: %w", datasourceUID, err)
+		}
+	}
+	return nil
+}
+
 func conditionEval(c *models.ReqContext, cmd ngmodels.EvalAlertConditionCommand, datasourceCache datasources.CacheService, dataService *tsdb.Service, cfg *setting.Cfg) response.Response {
 	evalCond := ngmodels.Condition{
 		Condition: cmd.Condition,
@@ -210,14 +237,14 @@ func conditionEval(c *models.ReqContext, cmd ngmodels.EvalAlertConditionCommand,
 	}
 
 	evaluator := eval.Evaluator{Cfg: cfg}
-	evalResults, err := evaluator.ConditionEval(&evalCond, timeNow(), dataService)
+	evalResults, err := evaluator.ConditionEval(&evalCond, now, dataService)
 	if err != nil {
-		return response.Error(400, "Failed to evaluate conditions", err)
+		return response.Error(http.StatusBadRequest, "Failed to evaluate conditions", err)
 	}
 
 	frame := evalResults.AsDataFrame()
 
-	return response.JSONStreaming(200, util.DynMap{
+	return response.JSONStreaming(http.StatusOK, util.DynMap{
 		"instances": []*data.Frame{&frame},
 	})
 }

pkg/services/ngalert/eval/eval.go

@@ -103,12 +103,7 @@ type AlertExecCtx struct {
 }
 
 // GetQueryDataRequest validates the condition and creates a backend.QueryDataRequest from it.
-func GetQueryDataRequest(ctx AlertExecCtx, c *models.Condition, now time.Time) (*backend.QueryDataRequest, error) {
+func GetQueryDataRequest(ctx AlertExecCtx, data []models.AlertQuery, now time.Time) (*backend.QueryDataRequest, error) {
-	if !c.IsValid() {
-		return nil, fmt.Errorf("invalid conditions")
-		// TODO: Things probably
-	}
-
 	queryDataReq := &backend.QueryDataRequest{
 		PluginContext: backend.PluginContext{
 			OrgID: ctx.OrgID,
@@ -116,8 +111,8 @@ func GetQueryDataRequest(ctx AlertExecCtx, c *models.Condition, now time.Time) (
 		Queries: []backend.DataQuery{},
 	}
 
-	for i := range c.Data {
+	for i := range data {
-		q := c.Data[i]
+		q := data[i]
 		model, err := q.GetModel()
 		if err != nil {
 			return nil, fmt.Errorf("failed to get query model: %w", err)
@@ -144,25 +139,16 @@ func GetQueryDataRequest(ctx AlertExecCtx, c *models.Condition, now time.Time) (
 	return queryDataReq, nil
 }
 
-// execute runs the Condition's expressions or queries.
+func executeCondition(ctx AlertExecCtx, c *models.Condition, now time.Time, dataService *tsdb.Service) (*ExecutionResults, error) {
-func execute(ctx AlertExecCtx, c *models.Condition, now time.Time, dataService *tsdb.Service) (*ExecutionResults, error) {
 	result := ExecutionResults{}
 
-	queryDataReq, err := GetQueryDataRequest(ctx, c, now)
+	execResp, err := executeQueriesAndExpressions(ctx, c.Data, now, dataService)
+
 	if err != nil {
 		return &result, err
 	}
 
-	exprService := expr.Service{
+	for refID, res := range execResp.Responses {
-		Cfg:         &setting.Cfg{ExpressionsEnabled: ctx.ExpressionsEnabled},
-		DataService: dataService,
-	}
-	pbRes, err := exprService.TransformData(ctx.Ctx, queryDataReq)
-	if err != nil {
-		return &result, err
-	}
-
-	for refID, res := range pbRes.Responses {
 		if refID != c.Condition {
 			continue
 		}
@@ -178,6 +164,19 @@ func execute(ctx AlertExecCtx, c *models.Condition, now time.Time, dataService *
 	return &result, nil
 }
 
+func executeQueriesAndExpressions(ctx AlertExecCtx, data []models.AlertQuery, now time.Time, dataService *tsdb.Service) (*backend.QueryDataResponse, error) {
+	queryDataReq, err := GetQueryDataRequest(ctx, data, now)
+	if err != nil {
+		return nil, err
+	}
+
+	exprService := expr.Service{
+		Cfg:         &setting.Cfg{ExpressionsEnabled: ctx.ExpressionsEnabled},
+		DataService: dataService,
+	}
+	return exprService.TransformData(ctx.Ctx, queryDataReq)
+}
+
 // evaluateExecutionResult takes the ExecutionResult, and returns a frame where
 // each column is a string type that holds a string representing its State.
 func evaluateExecutionResult(results *ExecutionResults, ts time.Time) (Results, error) {
@@ -275,7 +274,7 @@ func (e *Evaluator) ConditionEval(condition *models.Condition, now time.Time, da
 
 	alertExecCtx := AlertExecCtx{OrgID: condition.OrgID, Ctx: alertCtx, ExpressionsEnabled: e.Cfg.ExpressionsEnabled}
 
-	execResult, err := execute(alertExecCtx, condition, now, dataService)
+	execResult, err := executeCondition(alertExecCtx, condition, now, dataService)
 	if err != nil {
 		return nil, fmt.Errorf("failed to execute conditions: %w", err)
 	}
@@ -286,3 +285,18 @@ func (e *Evaluator) ConditionEval(condition *models.Condition, now time.Time, da
 	}
 	return evalResults, nil
 }
+
+// QueriesAndExpressionsEval executes queries and expressions and returns the result.
+func (e *Evaluator) QueriesAndExpressionsEval(orgID int64, data []models.AlertQuery, now time.Time, dataService *tsdb.Service) (*backend.QueryDataResponse, error) {
+	alertCtx, cancelFn := context.WithTimeout(context.Background(), alertingEvaluationTimeout)
+	defer cancelFn()
+
+	alertExecCtx := AlertExecCtx{OrgID: orgID, Ctx: alertCtx, ExpressionsEnabled: e.Cfg.ExpressionsEnabled}
+
+	execResult, err := executeQueriesAndExpressions(alertExecCtx, data, now, dataService)
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute conditions: %w", err)
+	}
+
+	return execResult, nil
+}

pkg/tests/api/alerting/api_alertmanager_test.go

@@ -797,6 +797,344 @@ func TestAlertRuleCRUD(t *testing.T) {
 			require.JSONEq(t, `{"message":"rule group deleted"}`, string(b))
 		})
 	}
+
+	// test eval conditions
+	testCases := []struct {
+		desc               string
+		payload            string
+		expectedStatusCode int
+		expectedResponse   string
+	}{
+		{
+			desc: "alerting condition",
+			payload: `
+			{
+				"grafana_condition": {
+				"condition": "A",
+				"data": [
+					{
+						"refId": "A",
+						"relativeTimeRange": {
+							"from": 18000,
+							"to": 10800
+						},
+						"model": {
+							"datasourceUid": "-100",
+							"type":"math",
+							"expression":"1 < 2"
+						}
+					}
+				],
+				"now": "2021-04-11T14:38:14Z"
+				}
+			}
+			`,
+			expectedStatusCode: http.StatusOK,
+			expectedResponse: `{
+			"instances": [
+			  {
+				"schema": {
+				  "name": "evaluation results",
+				  "fields": [
+					{
+					  "name": "State",
+					  "type": "string",
+					  "typeInfo": {
+						"frame": "string"
+					  }
+					}
+				  ]
+				},
+				"data": {
+				  "values": [
+					[
+					  "Alerting"
+					]
+				  ]
+				}
+			  }
+			]
+		  }`,
+		},
+		{
+			desc: "normal condition",
+			payload: `
+			{
+				"grafana_condition": {
+				"condition": "A",
+				"data": [
+					{
+						"refId": "A",
+						"relativeTimeRange": {
+							"from": 18000,
+							"to": 10800
+						},
+						"model": {
+							"datasourceUid": "-100",
+							"type":"math",
+							"expression":"1 > 2"
+						}
+					}
+				],
+				"now": "2021-04-11T14:38:14Z"
+				}
+			}
+			`,
+			expectedStatusCode: http.StatusOK,
+			expectedResponse: `{
+			"instances": [
+			  {
+				"schema": {
+				  "name": "evaluation results",
+				  "fields": [
+					{
+					  "name": "State",
+					  "type": "string",
+					  "typeInfo": {
+						"frame": "string"
+					  }
+					}
+				  ]
+				},
+				"data": {
+				  "values": [
+					[
+					  "Normal"
+					]
+				  ]
+				}
+			  }
+			]
+		  }`,
+		},
+		{
+			desc: "condition not found in any query or expression",
+			payload: `
+			{
+				"grafana_condition": {
+				"condition": "B",
+				"data": [
+					{
+						"refId": "A",
+						"relativeTimeRange": {
+							"from": 18000,
+							"to": 10800
+						},
+						"model": {
+							"datasourceUid": "-100",
+							"type":"math",
+							"expression":"1 > 2"
+						}
+					}
+				],
+				"now": "2021-04-11T14:38:14Z"
+				}
+			}
+			`,
+			expectedStatusCode: http.StatusBadRequest,
+			expectedResponse:   `{"error":"condition B not found in any query or expression","message":"invalid condition"}`,
+		},
+		{
+			desc: "unknown query datasource",
+			payload: `
+			{
+				"grafana_condition": {
+				"condition": "A",
+				"data": [
+					{
+						"refId": "A",
+						"relativeTimeRange": {
+							"from": 18000,
+							"to": 10800
+						},
+						"model": {
+							"datasourceUid": "unknown"
+						}
+					}
+				],
+				"now": "2021-04-11T14:38:14Z"
+				}
+			}
+			`,
+			expectedStatusCode: http.StatusBadRequest,
+			expectedResponse:   `{"error":"failed to get datasource: unknown: data source not found","message":"invalid condition"}`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			u := fmt.Sprintf("http://%s/api/v1/rule/test/grafana", grafanaListedAddr)
+			r := strings.NewReader(tc.payload)
+			// nolint:gosec
+			resp, err := http.Post(u, "application/json", r)
+			require.NoError(t, err)
+			t.Cleanup(func() {
+				err := resp.Body.Close()
+				require.NoError(t, err)
+			})
+			b, err := ioutil.ReadAll(resp.Body)
+			require.NoError(t, err)
+
+			assert.Equal(t, tc.expectedStatusCode, resp.StatusCode)
+			require.JSONEq(t, tc.expectedResponse, string(b))
+		})
+	}
+
+	// test eval queries and expressions
+	testCases = []struct {
+		desc               string
+		payload            string
+		expectedStatusCode int
+		expectedResponse   string
+	}{
+		{
+			desc: "alerting condition",
+			payload: `
+			{
+				"data": [
+						{
+							"refId": "A",
+							"relativeTimeRange": {
+								"from": 18000,
+								"to": 10800
+							},
+							"model": {
+								"datasourceUid": "-100",
+								"type":"math",
+								"expression":"1 < 2"
+							}
+						}
+					],
+				"now": "2021-04-11T14:38:14Z"
+			}
+			`,
+			expectedStatusCode: http.StatusOK,
+			expectedResponse: `{
+				"results": {
+				  "A": {
+					"frames": [
+					  {
+						"schema": {
+						  "refId": "A",
+						  "fields": [
+							{
+							  "name": "A",
+							  "type": "number",
+							  "typeInfo": {
+								"frame": "float64",
+								"nullable": true
+							  }
+							}
+						  ]
+						},
+						"data": {
+						  "values": [
+							[
+							  1
+							]
+						  ]
+						}
+					  }
+					]
+				  }
+				}
+			}`,
+		},
+		{
+			desc: "normal condition",
+			payload: `
+			{
+				"data": [
+						{
+							"refId": "A",
+							"relativeTimeRange": {
+								"from": 18000,
+								"to": 10800
+							},
+							"model": {
+								"datasourceUid": "-100",
+								"type":"math",
+								"expression":"1 > 2"
+							}
+						}
+					],
+				"now": "2021-04-11T14:38:14Z"
+			}
+			`,
+			expectedStatusCode: http.StatusOK,
+			expectedResponse: `{
+				"results": {
+				  "A": {
+					"frames": [
+					  {
+						"schema": {
+						  "refId": "A",
+						  "fields": [
+							{
+							  "name": "A",
+							  "type": "number",
+							  "typeInfo": {
+								"frame": "float64",
+								"nullable": true
+							  }
+							}
+						  ]
+						},
+						"data": {
+						  "values": [
+							[
+							  0
+							]
+						  ]
+						}
+					  }
+					]
+				  }
+				}
+			}`,
+		},
+		{
+			desc: "unknown query datasource",
+			payload: `
+			{
+				"data": [
+						{
+							"refId": "A",
+							"relativeTimeRange": {
+								"from": 18000,
+								"to": 10800
+							},
+							"model": {
+								"datasourceUid": "unknown"
+							}
+						}
+					],
+				"now": "2021-04-11T14:38:14Z"
+			}
+			`,
+			expectedStatusCode: http.StatusBadRequest,
+			expectedResponse:   `{"error":"failed to get datasource: unknown: data source not found","message":"invalid queries or expressions"}`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			u := fmt.Sprintf("http://%s/api/v1/eval", grafanaListedAddr)
+			r := strings.NewReader(tc.payload)
+			// nolint:gosec
+			resp, err := http.Post(u, "application/json", r)
+			require.NoError(t, err)
+			t.Cleanup(func() {
+				err := resp.Body.Close()
+				require.NoError(t, err)
+			})
+			b, err := ioutil.ReadAll(resp.Body)
+			require.NoError(t, err)
+
+			assert.Equal(t, tc.expectedStatusCode, resp.StatusCode)
+			require.JSONEq(t, tc.expectedResponse, string(b))
+		})
+	}
 }
 
 // createFolder creates a folder for storing our alerts under. Grafana uses folders as a replacement for alert namespaces to match its permission model.