API: Query database from /api/health endpoint (#28349)

pkg/api/health.go

@@ -0,0 +1,21 @@
+package api
+
+import (
+	"time"
+
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/models"
+)
+
+func (hs *HTTPServer) databaseHealthy() bool {
+	const cacheKey = "db-healthy"
+
+	if cached, found := hs.CacheService.Get(cacheKey); found {
+		return cached.(bool)
+	}
+
+	healthy := bus.Dispatch(&models.GetDBHealthQuery{}) == nil
+
+	hs.CacheService.Set(cacheKey, healthy, time.Second*5)
+	return healthy
+}

pkg/api/health_test.go

@@ -0,0 +1,190 @@
+package api
+
+import (
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/infra/localcache"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/stretchr/testify/require"
+	macaron "gopkg.in/macaron.v1"
+)
+
+func TestHealthAPI_Version(t *testing.T) {
+	m, _ := setupHealthAPITestEnvironment(t)
+	setting.BuildVersion = "7.4.0"
+	setting.BuildCommit = "59906ab1bf"
+
+	bus.AddHandler("test", func(query *models.GetDBHealthQuery) error {
+		return nil
+	})
+
+	req := httptest.NewRequest(http.MethodGet, "/api/health", nil)
+	rec := httptest.NewRecorder()
+	m.ServeHTTP(rec, req)
+
+	require.Equal(t, 200, rec.Code)
+	expectedBody := `
+		{
+			"database": "ok",
+			"version": "7.4.0",
+			"commit": "59906ab1bf"
+		}
+	`
+	require.JSONEq(t, expectedBody, rec.Body.String())
+}
+
+func TestHealthAPI_AnonymousHideVersion(t *testing.T) {
+	m, hs := setupHealthAPITestEnvironment(t)
+	hs.Cfg.AnonymousHideVersion = true
+
+	bus.AddHandler("test", func(query *models.GetDBHealthQuery) error {
+		return nil
+	})
+
+	req := httptest.NewRequest(http.MethodGet, "/api/health", nil)
+	rec := httptest.NewRecorder()
+	m.ServeHTTP(rec, req)
+
+	require.Equal(t, 200, rec.Code)
+	expectedBody := `
+		{
+			"database": "ok"
+		}
+	`
+	require.JSONEq(t, expectedBody, rec.Body.String())
+}
+
+func TestHealthAPI_DatabaseHealthy(t *testing.T) {
+	const cacheKey = "db-healthy"
+
+	m, hs := setupHealthAPITestEnvironment(t)
+	hs.Cfg.AnonymousHideVersion = true
+
+	bus.AddHandler("test", func(query *models.GetDBHealthQuery) error {
+		return nil
+	})
+
+	healthy, found := hs.CacheService.Get(cacheKey)
+	require.False(t, found)
+	require.Nil(t, healthy)
+
+	req := httptest.NewRequest(http.MethodGet, "/api/health", nil)
+	rec := httptest.NewRecorder()
+	m.ServeHTTP(rec, req)
+
+	require.Equal(t, 200, rec.Code)
+	expectedBody := `
+		{
+			"database": "ok"
+		}
+	`
+	require.JSONEq(t, expectedBody, rec.Body.String())
+
+	healthy, found = hs.CacheService.Get(cacheKey)
+	require.True(t, found)
+	require.True(t, healthy.(bool))
+}
+
+func TestHealthAPI_DatabaseUnhealthy(t *testing.T) {
+	const cacheKey = "db-healthy"
+
+	m, hs := setupHealthAPITestEnvironment(t)
+	hs.Cfg.AnonymousHideVersion = true
+
+	bus.AddHandler("test", func(query *models.GetDBHealthQuery) error {
+		return errors.New("bad")
+	})
+
+	healthy, found := hs.CacheService.Get(cacheKey)
+	require.False(t, found)
+	require.Nil(t, healthy)
+
+	req := httptest.NewRequest(http.MethodGet, "/api/health", nil)
+	rec := httptest.NewRecorder()
+	m.ServeHTTP(rec, req)
+
+	require.Equal(t, 503, rec.Code)
+	expectedBody := `
+		{
+			"database": "failing"
+		}
+	`
+	require.JSONEq(t, expectedBody, rec.Body.String())
+
+	healthy, found = hs.CacheService.Get(cacheKey)
+	require.True(t, found)
+	require.False(t, healthy.(bool))
+}
+
+func TestHealthAPI_DatabaseHealthCached(t *testing.T) {
+	const cacheKey = "db-healthy"
+
+	m, hs := setupHealthAPITestEnvironment(t)
+	hs.Cfg.AnonymousHideVersion = true
+
+	// Database is healthy.
+	bus.AddHandler("test", func(query *models.GetDBHealthQuery) error {
+		return nil
+	})
+
+	// Mock unhealthy database in cache.
+	hs.CacheService.Set(cacheKey, false, 5*time.Minute)
+
+	req := httptest.NewRequest(http.MethodGet, "/api/health", nil)
+	rec := httptest.NewRecorder()
+	m.ServeHTTP(rec, req)
+
+	require.Equal(t, 503, rec.Code)
+	expectedBody := `
+		{
+			"database": "failing"
+		}
+	`
+	require.JSONEq(t, expectedBody, rec.Body.String())
+
+	// Purge cache and redo request.
+	hs.CacheService.Delete(cacheKey)
+	rec = httptest.NewRecorder()
+	m.ServeHTTP(rec, req)
+
+	require.Equal(t, 200, rec.Code)
+	expectedBody = `
+		{
+			"database": "ok"
+		}
+	`
+	require.JSONEq(t, expectedBody, rec.Body.String())
+
+	healthy, found := hs.CacheService.Get(cacheKey)
+	require.True(t, found)
+	require.True(t, healthy.(bool))
+}
+
+func setupHealthAPITestEnvironment(t *testing.T) (*macaron.Macaron, *HTTPServer) {
+	t.Helper()
+
+	oldVersion := setting.BuildVersion
+	oldCommit := setting.BuildCommit
+	t.Cleanup(func() {
+		setting.BuildVersion = oldVersion
+		setting.BuildCommit = oldCommit
+	})
+
+	bus.ClearBusHandlers()
+	t.Cleanup(bus.ClearBusHandlers)
+
+	m := macaron.New()
+	hs := &HTTPServer{
+		CacheService: localcache.New(5*time.Minute, 10*time.Minute),
+		Cfg:          setting.NewCfg(),
+	}
+
+	m.Get("/api/health", hs.apiHealthHandler)
+	return m, hs
+}

pkg/api/http_server.go

@@ -338,7 +338,7 @@ func (hs *HTTPServer) addMiddlewaresAndStaticRoutes() {
 	}))
 
 	// These endpoints are used for monitoring the Grafana instance
-	// and should not be redirect or rejected.
+	// and should not be redirected or rejected.
 	m.Use(hs.healthzHandler)
 	m.Use(hs.apiHealthHandler)
 	m.Use(hs.metricsEndpoint)
@@ -396,7 +396,7 @@ func (hs *HTTPServer) healthzHandler(ctx *macaron.Context) {
 }
 
 // apiHealthHandler will return ok if Grafana's web server is running and it
-// can access the database. If the database cannot be access it will return
+// can access the database. If the database cannot be accessed it will return
 // http status code 503.
 func (hs *HTTPServer) apiHealthHandler(ctx *macaron.Context) {
 	notHeadOrGet := ctx.Req.Method != http.MethodGet && ctx.Req.Method != http.MethodHead
@@ -411,7 +411,7 @@ func (hs *HTTPServer) apiHealthHandler(ctx *macaron.Context) {
 		data.Set("commit", setting.BuildCommit)
 	}
 
-	if err := bus.Dispatch(&models.GetDBHealthQuery{}); err != nil {
+	if !hs.databaseHealthy() {
 		data.Set("database", "failing")
 		ctx.Resp.Header().Set("Content-Type", "application/json; charset=UTF-8")
 		ctx.Resp.WriteHeader(503)

pkg/services/sqlstore/health.go

@@ -10,5 +10,6 @@ func init() {
 }
 
 func GetDBHealthQuery(query *models.GetDBHealthQuery) error {
-	return x.Ping()
+	_, err := x.Exec("SELECT 1")
+	return err
 }

pkg/services/sqlstore/health_test.go

@@ -0,0 +1,18 @@
+// +build integration
+
+package sqlstore
+
+import (
+	"testing"
+
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetDBHealthQuery(t *testing.T) {
+	InitTestDB(t)
+
+	query := models.GetDBHealthQuery{}
+	err := GetDBHealthQuery(&query)
+	require.NoError(t, err)
+}