Tracing: Use common traceID context value for opentracing and opentelemetry (#46411)

* use common traceID context value for opentracing and opentelemetry

* support sampled trace IDs as well

* inject traceID into NormalResponse on errors

* Finally the test passed

* fix the test

* fix linter

* change the function parameter

Co-authored-by: Ying WANG <ying.wang@grafana.com>

pkg/api/ldap_debug_test.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"encoding/json"
 	"errors"
 	"net/http"
 	"net/http/httptest"
@@ -146,13 +147,11 @@ func TestGetUserFromLDAPAPIEndpoint_OrgNotfound(t *testing.T) {
 
 	require.Equal(t, http.StatusBadRequest, sc.resp.Code)
 
-	expected := `
-	{
-		"error": "unable to find organization with ID '2'",
-		"message": "An organization was not found - Please verify your LDAP configuration"
-	}
-	`
-	assert.JSONEq(t, expected, sc.resp.Body.String())
+	var res map[string]interface{}
+	err := json.Unmarshal(sc.resp.Body.Bytes(), &res)
+	assert.NoError(t, err)
+	assert.Equal(t, "unable to find organization with ID '2'", res["error"])
+	assert.Equal(t, "An organization was not found - Please verify your LDAP configuration", res["message"])
 }
 
 func TestGetUserFromLDAPAPIEndpoint(t *testing.T) {
@@ -470,14 +469,11 @@ func TestPostSyncUserWithLDAPAPIEndpoint_WhenGrafanaAdmin(t *testing.T) {
 	}, &sqlstoremock)
 	assert.Equal(t, http.StatusBadRequest, sc.resp.Code)
 
-	expected := `
-	{
-		"error": "did not find a user",
-		"message": "Refusing to sync grafana super admin \"ldap-daniel\" - it would be disabled"
-	}
-	`
-
-	assert.JSONEq(t, expected, sc.resp.Body.String())
+	var res map[string]interface{}
+	err := json.Unmarshal(sc.resp.Body.Bytes(), &res)
+	assert.NoError(t, err)
+	assert.Equal(t, "did not find a user", res["error"])
+	assert.Equal(t, "Refusing to sync grafana super admin \"ldap-daniel\" - it would be disabled", res["message"])
 }
 
 func TestPostSyncUserWithLDAPAPIEndpoint_WhenUserNotInLDAP(t *testing.T) {

pkg/api/metrics_test.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"strconv"
@@ -241,15 +242,14 @@ func TestAPIEndpoint_Metrics_QueryMetricsFromDashboard(t *testing.T) {
 			strings.NewReader(queryDatasourceInput),
 			t,
 		)
+
 		assert.Equal(t, http.StatusBadRequest, response.Code)
-		assert.JSONEq(
-			t,
-			fmt.Sprintf(
-				"{\"error\":\"%[1]s\",\"message\":\"%[1]s\"}",
-				models.ErrDashboardOrPanelIdentifierNotSet,
-			),
-			response.Body.String(),
-		)
+
+		var res map[string]interface{}
+		err := json.Unmarshal(response.Body.Bytes(), &res)
+		assert.NoError(t, err)
+		assert.Equal(t, models.ErrDashboardOrPanelIdentifierNotSet.Error(), res["error"])
+		assert.Equal(t, models.ErrDashboardOrPanelIdentifierNotSet.Error(), res["message"])
 	})
 
 	t.Run("Cannot query without a valid orgid", func(t *testing.T) {
@@ -261,14 +261,10 @@ func TestAPIEndpoint_Metrics_QueryMetricsFromDashboard(t *testing.T) {
 			t,
 		)
 		assert.Equal(t, http.StatusBadRequest, response.Code)
-		assert.JSONEq(
-			t,
-			fmt.Sprintf(
-				"{\"error\":\"%[1]s\",\"message\":\"%[1]s\"}",
-				models.ErrDashboardOrPanelIdentifierNotSet,
-			),
-			response.Body.String(),
-		)
+		var res map[string]interface{}
+		assert.NoError(t, json.Unmarshal(response.Body.Bytes(), &res))
+		assert.Equal(t, models.ErrDashboardOrPanelIdentifierNotSet.Error(), res["error"])
+		assert.Equal(t, models.ErrDashboardOrPanelIdentifierNotSet.Error(), res["message"])
 	})
 
 	t.Run("Cannot query without a valid dashboard or panel ID", func(t *testing.T) {
@@ -280,14 +276,11 @@ func TestAPIEndpoint_Metrics_QueryMetricsFromDashboard(t *testing.T) {
 			t,
 		)
 		assert.Equal(t, http.StatusBadRequest, response.Code)
-		assert.JSONEq(
-			t,
-			fmt.Sprintf(
-				"{\"error\":\"%[1]s\",\"message\":\"%[1]s\"}",
-				models.ErrDashboardOrPanelIdentifierNotSet,
-			),
-			response.Body.String(),
-		)
+
+		var res map[string]interface{}
+		assert.NoError(t, json.Unmarshal(response.Body.Bytes(), &res))
+		assert.Equal(t, models.ErrDashboardOrPanelIdentifierNotSet.Error(), res["error"])
+		assert.Equal(t, models.ErrDashboardOrPanelIdentifierNotSet.Error(), res["message"])
 	})
 
 	t.Run("Cannot query when ValidatedQueries is disabled", func(t *testing.T) {

pkg/api/response/response.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"net/http"
 
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/setting"
 	jsoniter "github.com/json-iterator/go"
@@ -73,7 +74,15 @@ func (r *NormalResponse) ErrMessage() string {
 
 func (r *NormalResponse) WriteTo(ctx *models.ReqContext) {
 	if r.err != nil {
-		ctx.Logger.Error(r.errMessage, "error", r.err, "remote_addr", ctx.RemoteAddr())
+		v := map[string]interface{}{}
+		traceID := tracing.TraceIDFromContext(ctx.Req.Context(), false)
+		if err := json.Unmarshal(r.body.Bytes(), &v); err == nil {
+			v["traceID"] = traceID
+			if b, err := json.Marshal(v); err == nil {
+				r.body = bytes.NewBuffer(b)
+			}
+		}
+		ctx.Logger.Error(r.errMessage, "error", r.err, "remote_addr", ctx.RemoteAddr(), "traceID", traceID)
 	}
 
 	header := ctx.Resp.Header()

pkg/infra/tracing/opentelemetry_tracing.go

@@ -126,6 +126,7 @@ func (ots *Opentelemetry) Start(ctx context.Context, spanName string, opts ...tr
 	opentelemetrySpan := OpentelemetrySpan{
 		span: span,
 	}
+	ctx = context.WithValue(ctx, traceKey{}, traceValue{span.SpanContext().TraceID().String(), span.SpanContext().IsSampled()})
 	return ctx, opentelemetrySpan
 }
 

pkg/infra/tracing/tracing.go

@@ -14,6 +14,7 @@ import (
 	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go/ext"
 	ol "github.com/opentracing/opentracing-go/log"
+	"github.com/uber/jaeger-client-go"
 	jaegercfg "github.com/uber/jaeger-client-go/config"
 	"github.com/uber/jaeger-client-go/zipkin"
 	"go.opentelemetry.io/otel/attribute"
@@ -52,6 +53,21 @@ func ProvideService(cfg *setting.Cfg) (Tracer, error) {
 	return ots, ots.initOpentelemetryTracer()
 }
 
+type traceKey struct{}
+type traceValue struct {
+	ID        string
+	IsSampled bool
+}
+
+func TraceIDFromContext(c context.Context, requireSampled bool) string {
+	v := c.Value(traceKey{})
+	// Return traceID if a) it is present and b) it is sampled when requireSampled param is true
+	if trace, ok := v.(traceValue); ok && (!requireSampled || trace.IsSampled) {
+		return trace.ID
+	}
+	return ""
+}
+
 type Opentracing struct {
 	enabled                  bool
 	address                  string
@@ -172,6 +188,9 @@ func (ts *Opentracing) Run(ctx context.Context) error {
 func (ts *Opentracing) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) {
 	span, ctx := opentracing.StartSpanFromContext(ctx, spanName)
 	opentracingSpan := OpentracingSpan{span: span}
+	if sctx, ok := span.Context().(jaeger.SpanContext); ok {
+		ctx = context.WithValue(ctx, traceKey{}, traceValue{sctx.TraceID().String(), sctx.IsSampled()})
+	}
 	return ctx, opentracingSpan
 }
 

pkg/middleware/logger.go

@@ -19,10 +19,10 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/contexthandler"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/web"
-	cw "github.com/weaveworks/common/tracing"
 )
 
 func Logger(cfg *setting.Cfg) web.Handler {
@@ -58,8 +58,8 @@ func Logger(cfg *setting.Cfg) web.Handler {
 				"referer", req.Referer(),
 			}
 
-			traceID, exist := cw.ExtractTraceID(ctx.Req.Context())
-			if exist {
+			traceID := tracing.TraceIDFromContext(ctx.Req.Context(), false)
+			if traceID != "" {
 				logParams = append(logParams, "traceID", traceID)
 			}
 

pkg/middleware/request_metrics.go

@@ -7,10 +7,10 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/infra/metrics"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/web"
 	"github.com/prometheus/client_golang/prometheus"
-	cw "github.com/weaveworks/common/tracing"
 )
 
 var (
@@ -80,7 +80,7 @@ func RequestMetrics(features featuremgmt.FeatureToggles) web.Handler {
 			// since they dont make much sense. We should remove them later.
 			histogram := httpRequestDurationHistogram.
 				WithLabelValues(handler, code, req.Method)
-			if traceID, ok := cw.ExtractSampledTraceID(c.Req.Context()); ok {
+			if traceID := tracing.TraceIDFromContext(c.Req.Context(), true); traceID != "" {
 				// Need to type-convert the Observer to an
 				// ExemplarObserver. This will always work for a
 				// HistogramVec.

pkg/models/context.go

@@ -4,6 +4,7 @@ import (
 	"strings"
 
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/web"
 	"github.com/prometheus/client_golang/prometheus"
@@ -51,9 +52,11 @@ func (ctx *ReqContext) IsApiRequest() bool {
 
 func (ctx *ReqContext) JsonApiErr(status int, message string, err error) {
 	resp := make(map[string]interface{})
+	traceID := tracing.TraceIDFromContext(ctx.Req.Context(), false)
 
 	if err != nil {
-		ctx.Logger.Error(message, "error", err)
+		resp["traceID"] = traceID
+		ctx.Logger.Error(message, "error", err, "traceID", traceID)
 		if setting.Env != setting.Prod {
 			resp["error"] = err.Error()
 		}

pkg/services/contexthandler/contexthandler.go

@@ -25,7 +25,6 @@ import (
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/util"
 	"github.com/grafana/grafana/pkg/web"
-	cw "github.com/weaveworks/common/tracing"
 )
 
 const (
@@ -98,8 +97,8 @@ func (h *ContextHandler) Middleware(mContext *web.Context) {
 	mContext.Req = mContext.Req.WithContext(context.WithValue(mContext.Req.Context(), reqContextKey{}, reqContext))
 	mContext.Map(mContext.Req)
 
-	traceID, exists := cw.ExtractTraceID(mContext.Req.Context())
-	if exists {
+	traceID := tracing.TraceIDFromContext(mContext.Req.Context(), false)
+	if traceID != "" {
 		reqContext.Logger = reqContext.Logger.New("traceID", traceID)
 	}
 

pkg/services/sqlstore/database_wrapper.go

@@ -16,7 +16,6 @@ import (
 	"github.com/lib/pq"
 	"github.com/mattn/go-sqlite3"
 	"github.com/prometheus/client_golang/prometheus"
-	cw "github.com/weaveworks/common/tracing"
 	"xorm.io/core"
 )
 
@@ -83,7 +82,7 @@ func (h *databaseQueryWrapper) instrument(ctx context.Context, status string, qu
 	elapsed := time.Since(begin)
 
 	histogram := databaseQueryHistogram.WithLabelValues(status)
-	if traceID, ok := cw.ExtractSampledTraceID(ctx); ok {
+	if traceID := tracing.TraceIDFromContext(ctx, true); traceID != "" {
 		// Need to type-convert the Observer to an
 		// ExemplarObserver. This will always work for a
 		// HistogramVec.

pkg/tests/api/alerting/api_admin_configuration_test.go

@@ -66,7 +66,10 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
 		resp := getRequest(t, alertsURL, http.StatusNotFound) // nolint
 		b, err := ioutil.ReadAll(resp.Body)
 		require.NoError(t, err)
-		require.JSONEq(t, `{"message": "no admin configuration available"}`, string(b))
+		var res map[string]interface{}
+		err = json.Unmarshal(b, &res)
+		require.NoError(t, err)
+		require.Equal(t, "no admin configuration available", res["message"])
 	}
 
 	// An invalid alertmanager choice should return an error.

pkg/tests/api/alerting/api_alertmanager_configuration_test.go

@@ -3,6 +3,7 @@ package alerting
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"regexp"
 	"testing"
@@ -89,9 +90,13 @@ func TestAlertmanagerConfigurationIsTransactional(t *testing.T) {
 }
 `
 		resp := postRequest(t, alertConfigURL, payload, http.StatusBadRequest) // nolint
-		require.JSONEq(t, `{"message": "failed to save and apply Alertmanager configuration: failed to build integration map: the receiver is invalid: failed to validate receiver \"slack.receiver\" of type \"slack\": token must be specified when using the Slack chat API"}`, getBody(t, resp.Body))
-
+		b, err := ioutil.ReadAll(resp.Body)
+		require.NoError(t, err)
+		var res map[string]interface{}
+		require.NoError(t, json.Unmarshal(b, &res))
+		require.Equal(t, `failed to save and apply Alertmanager configuration: failed to build integration map: the receiver is invalid: failed to validate receiver "slack.receiver" of type "slack": token must be specified when using the Slack chat API`, res["message"])
 		resp = getRequest(t, alertConfigURL, http.StatusOK) // nolint
+
 		require.JSONEq(t, defaultAlertmanagerConfigJSON, getBody(t, resp.Body))
 	}
 
@@ -210,7 +215,10 @@ func TestAlertmanagerConfigurationPersistSecrets(t *testing.T) {
 	`
 
 		resp := postRequest(t, alertConfigURL, payload, http.StatusBadRequest) // nolint
-		require.JSONEq(t, `{"message": "unknown receiver: invalid"}`, getBody(t, resp.Body))
+		s := getBody(t, resp.Body)
+		var res map[string]interface{}
+		require.NoError(t, json.Unmarshal([]byte(s), &res))
+		require.Equal(t, "unknown receiver: invalid", res["message"])
 	}
 
 	// The secure settings must be present

pkg/tests/api/alerting/api_alertmanager_test.go

@@ -437,7 +437,10 @@ func TestAlertAndGroupsQuery(t *testing.T) {
 		b, err := ioutil.ReadAll(resp.Body)
 		require.NoError(t, err)
 		require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
-		require.JSONEq(t, `{"message": "invalid username or password"}`, string(b))
+
+		var res map[string]interface{}
+		require.NoError(t, json.Unmarshal(b, &res))
+		require.Equal(t, "invalid username or password", res["message"])
 	}
 
 	// When there are no alerts available, it returns an empty list.
@@ -897,7 +900,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						Data:  []ngmodels.AlertQuery{},
 					},
 				},
-				expectedResponse: `{"message": "invalid rule specification at index [0]: invalid alert rule: no queries or expressions are found"}`,
+				expectedResponse: `{"message": "invalid rule specification at index [0]: invalid alert rule: no queries or expressions are found", "traceID":"00000000000000000000000000000000"}`,
 			},
 			{
 				desc:      "alert rule with empty title",
@@ -927,7 +930,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"message": "invalid rule specification at index [0]: alert rule title cannot be empty"}`,
+				expectedResponse: `{"message": "invalid rule specification at index [0]: alert rule title cannot be empty", "traceID":"00000000000000000000000000000000"}`,
 			},
 			{
 				desc:      "alert rule with too long name",
@@ -957,7 +960,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"message": "invalid rule specification at index [0]: alert rule title is too long. Max length is 190"}`,
+				expectedResponse: `{"message": "invalid rule specification at index [0]: alert rule title is too long. Max length is 190", "traceID":"00000000000000000000000000000000"}`,
 			},
 			{
 				desc:      "alert rule with too long rulegroup",
@@ -987,7 +990,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"message": "rule group name is too long. Max length is 190"}`,
+				expectedResponse: `{"message": "rule group name is too long. Max length is 190", "traceID":"00000000000000000000000000000000"}`,
 			},
 			{
 				desc:      "alert rule with invalid interval",
@@ -1018,7 +1021,8 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"message": "rule evaluation interval (1 second) should be positive number that is multiple of the base interval of 10 seconds"}`,
+				expectedResponse: `{"message": "rule evaluation interval (1 second) should be positive ` +
+					`number that is multiple of the base interval of 10 seconds", "traceID":"00000000000000000000000000000000"}`,
 			},
 			{
 				desc:      "alert rule with unknown datasource",
@@ -1048,7 +1052,8 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"message": "invalid rule specification at index [0]: failed to validate condition of alert rule AlwaysFiring: invalid query A: data source not found: unknown"}`,
+				expectedResponse: `{"message": "invalid rule specification at index [0]: failed to validate condition of alert rule AlwaysFiring:` +
+					` invalid query A: data source not found: unknown", "traceID":"00000000000000000000000000000000"}`,
 			},
 			{
 				desc:      "alert rule with invalid condition",
@@ -1078,7 +1083,8 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"message": "invalid rule specification at index [0]: failed to validate condition of alert rule AlwaysFiring: condition B not found in any query or expression: it should be one of: [A]"}`,
+				expectedResponse: `{"message": "invalid rule specification at index [0]: failed to validate condition of alert rule AlwaysFiring: ` +
+					`condition B not found in any query or expression: it should be one of: [A]", "traceID":"00000000000000000000000000000000"}`,
 			},
 		}
 
@@ -1368,7 +1374,9 @@ func TestAlertRuleCRUD(t *testing.T) {
 		require.NoError(t, err)
 
 		assert.Equal(t, http.StatusNotFound, resp.StatusCode)
-		require.JSONEq(t, `{"message": "failed to update rule group: failed to update rule with UID unknown because could not find alert rule"}`, string(b))
+		var res map[string]interface{}
+		assert.NoError(t, json.Unmarshal(b, &res))
+		require.Equal(t, "failed to update rule group: failed to update rule with UID unknown because could not find alert rule", res["message"])
 
 		// let's make sure that rule definitions are not affected by the failed POST request.
 		u = fmt.Sprintf("http://grafana:password@%s/api/ruler/grafana/api/v1/rules/default", grafanaListedAddr)
@@ -1487,7 +1495,9 @@ func TestAlertRuleCRUD(t *testing.T) {
 		require.NoError(t, err)
 
 		assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
-		require.JSONEq(t, fmt.Sprintf(`{"message": "rule [1] has UID %s that is already assigned to another rule at index 0"}`, ruleUID), string(b))
+		var res map[string]interface{}
+		require.NoError(t, json.Unmarshal(b, &res))
+		require.Equal(t, fmt.Sprintf("rule [1] has UID %s that is already assigned to another rule at index 0", ruleUID), res["message"])
 
 		// let's make sure that rule definitions are not affected by the failed POST request.
 		u = fmt.Sprintf("http://grafana:password@%s/api/ruler/grafana/api/v1/rules/default", grafanaListedAddr)
@@ -1893,7 +1903,9 @@ func TestAlertRuleCRUD(t *testing.T) {
 			require.NoError(t, err)
 
 			require.Equal(t, http.StatusAccepted, resp.StatusCode)
-			require.JSONEq(t, `{"message":"rules deleted"}`, string(b))
+			var res map[string]interface{}
+			require.NoError(t, json.Unmarshal(b, &res))
+			require.Equal(t, "rules deleted", res["message"])
 		})
 
 		t.Run("succeed if the rule group name does exist", func(t *testing.T) {
@@ -2103,7 +2115,9 @@ func TestQuota(t *testing.T) {
 		b, err := ioutil.ReadAll(resp.Body)
 		require.NoError(t, err)
 		assert.Equal(t, http.StatusForbidden, resp.StatusCode)
-		require.JSONEq(t, `{"message": "quota has been exceeded"}`, string(b))
+		var res map[string]interface{}
+		require.NoError(t, json.Unmarshal(b, &res))
+		require.Equal(t, "quota has been exceeded", res["message"])
 	})
 
 	t.Run("when quota limit exceed updating existing rule should succeed", func(t *testing.T) {
@@ -2400,7 +2414,8 @@ func TestEval(t *testing.T) {
 			}
 			`,
 			expectedStatusCode: http.StatusBadRequest,
-			expectedResponse:   `{"message": "invalid condition: condition B not found in any query or expression: it should be one of: [A]"}`,
+			expectedResponse: `{"message": "invalid condition: condition B not found in any query or expression: it should be one of: [A]",` +
+				`"traceID": "00000000000000000000000000000000"}`,
 		},
 		{
 			desc: "unknown query datasource",
@@ -2425,7 +2440,7 @@ func TestEval(t *testing.T) {
 			}
 			`,
 			expectedStatusCode: http.StatusBadRequest,
-			expectedResponse:   `{"message": "invalid condition: invalid query A: data source not found: unknown"}`,
+			expectedResponse:   `{"message": "invalid condition: invalid query A: data source not found: unknown", "traceID": "00000000000000000000000000000000"}`,
 		},
 	}
 
@@ -2581,7 +2596,8 @@ func TestEval(t *testing.T) {
 			}
 			`,
 			expectedStatusCode: http.StatusBadRequest,
-			expectedResponse:   `{"message": "invalid queries or expressions: invalid query A: data source not found: unknown"}`,
+			expectedResponse: `{"message": "invalid queries or expressions: invalid query A: data source not found: unknown",` +
+				`"traceID": "00000000000000000000000000000000"}`,
 		},
 	}
 

pkg/tests/api/alerting/api_notification_channel_test.go

@@ -61,7 +61,7 @@ func TestTestReceivers(t *testing.T) {
 
 		b, err := ioutil.ReadAll(resp.Body)
 		require.NoError(t, err)
-		require.JSONEq(t, `{}`, string(b))
+		require.JSONEq(t, `{"traceID":"00000000000000000000000000000000"}`, string(b))
 	})
 
 	t.Run("assert working receiver returns OK", func(t *testing.T) {

pkg/tests/api/alerting/api_prometheus_test.go

@@ -206,7 +206,9 @@ func TestPrometheusRules(t *testing.T) {
 		require.NoError(t, err)
 
 		assert.Equal(t, 400, resp.StatusCode)
-		require.JSONEq(t, `{"message": "invalid rule specification at index [0]: both annotations __dashboardUid__ and __panelId__ must be specified"}`, string(b))
+		var res map[string]interface{}
+		require.NoError(t, json.Unmarshal(b, &res))
+		require.Equal(t, "invalid rule specification at index [0]: both annotations __dashboardUid__ and __panelId__ must be specified", res["message"])
 	}
 
 	// Now, let's see how this looks like.
@@ -587,7 +589,9 @@ func TestPrometheusRulesFilterByDashboard(t *testing.T) {
 		require.Equal(t, http.StatusBadRequest, resp.StatusCode)
 		b, err := ioutil.ReadAll(resp.Body)
 		require.NoError(t, err)
-		require.JSONEq(t, `{"message": "invalid panel_id: strconv.ParseInt: parsing \"invalid\": invalid syntax"}`, string(b))
+		var res map[string]interface{}
+		require.NoError(t, json.Unmarshal(b, &res))
+		require.Equal(t, `invalid panel_id: strconv.ParseInt: parsing "invalid": invalid syntax`, res["message"])
 	}
 
 	// Now, let's check a panel_id without dashboard_uid returns a 400 Bad Request response
@@ -603,7 +607,9 @@ func TestPrometheusRulesFilterByDashboard(t *testing.T) {
 		require.Equal(t, http.StatusBadRequest, resp.StatusCode)
 		b, err := ioutil.ReadAll(resp.Body)
 		require.NoError(t, err)
-		require.JSONEq(t, `{"message": "panel_id must be set with dashboard_uid"}`, string(b))
+		var res map[string]interface{}
+		require.NoError(t, json.Unmarshal(b, &res))
+		require.Equal(t, "panel_id must be set with dashboard_uid", res["message"])
 	}
 }
 

pkg/tests/api/alerting/api_ruler_test.go

@@ -410,7 +410,10 @@ func TestAlertRuleConflictingTitle(t *testing.T) {
 		require.NoError(t, err)
 
 		assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
-		require.JSONEq(t, `{"message": "failed to update rule group: failed to add rules: a conflicting alert rule is found: rule title under the same organisation and folder should be unique"}`, string(b))
+
+		var res map[string]interface{}
+		require.NoError(t, json.Unmarshal(b, &res))
+		require.Equal(t, "failed to update rule group: failed to add rules: a conflicting alert rule is found: rule title under the same organisation and folder should be unique", res["message"])
 	})
 
 	t.Run("trying to update an alert to the title of an existing alert in the same folder should fail", func(t *testing.T) {
@@ -435,7 +438,10 @@ func TestAlertRuleConflictingTitle(t *testing.T) {
 		require.NoError(t, err)
 
 		assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
-		require.JSONEq(t, `{"message": "failed to update rule group: failed to update rules: a conflicting alert rule is found: rule title under the same organisation and folder should be unique"}`, string(b))
+
+		var res map[string]interface{}
+		require.NoError(t, json.Unmarshal(b, &res))
+		require.Equal(t, "failed to update rule group: failed to update rules: a conflicting alert rule is found: rule title under the same organisation and folder should be unique", res["message"])
 	})
 
 	t.Run("trying to create alert with same title under another folder should succeed", func(t *testing.T) {
@@ -789,7 +795,9 @@ func TestRulerRulesFilterByDashboard(t *testing.T) {
 		require.Equal(t, http.StatusBadRequest, resp.StatusCode)
 		b, err := ioutil.ReadAll(resp.Body)
 		require.NoError(t, err)
-		require.JSONEq(t, `{"message": "invalid panel_id: strconv.ParseInt: parsing \"invalid\": invalid syntax"}`, string(b))
+		var res map[string]interface{}
+		require.NoError(t, json.Unmarshal(b, &res))
+		require.Equal(t, `invalid panel_id: strconv.ParseInt: parsing "invalid": invalid syntax`, res["message"])
 	}
 
 	// Now, let's check a panel_id without dashboard_uid returns a 400 Bad Request response
@@ -805,7 +813,9 @@ func TestRulerRulesFilterByDashboard(t *testing.T) {
 		require.Equal(t, http.StatusBadRequest, resp.StatusCode)
 		b, err := ioutil.ReadAll(resp.Body)
 		require.NoError(t, err)
-		require.JSONEq(t, `{"message": "panel_id must be set with dashboard_uid"}`, string(b))
+		var res map[string]interface{}
+		require.NoError(t, json.Unmarshal(b, &res))
+		require.Equal(t, "panel_id must be set with dashboard_uid", res["message"])
 	}
 }