From 23655315b877c18d0a73e0669e4b8cfea39a2255 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Torkel=20=C3=96degaard?= <torkel@grafana.com>
Date: Wed, 2 May 2018 18:10:21 +0200
Subject: [PATCH] fix: fixed race condition between http.Server ListenAndServe
 & Shutdown, now service crash during startup correctly closes http server
 every time

---
 pkg/api/http_server.go           |  2 ++
 pkg/cmd/grafana-server/main.go   |  2 +-
 pkg/cmd/grafana-server/server.go | 34 ++++++++++++++++++++++++--------
 3 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go
index 2cf8f3796ae..f30e5602484 100644
--- a/pkg/api/http_server.go
+++ b/pkg/api/http_server.go
@@ -71,6 +71,8 @@ func (hs *HTTPServer) Run(ctx context.Context) error {
 	// handle http shutdown on server context done
 	go func() {
 		<-ctx.Done()
+		// Hacky fix for race condition between ListenAndServe and Shutdown
+		time.Sleep(time.Millisecond * 100)
 		if err := hs.httpSrv.Shutdown(context.Background()); err != nil {
 			hs.log.Error("Failed to shutdown server", "error", err)
 		}
diff --git a/pkg/cmd/grafana-server/main.go b/pkg/cmd/grafana-server/main.go
index 23cac74ccdb..02ea7a7f8f1 100644
--- a/pkg/cmd/grafana-server/main.go
+++ b/pkg/cmd/grafana-server/main.go
@@ -86,7 +86,7 @@ func main() {
 
 	go listenToSystemSignals(server)
 
-	err := server.Start()
+	err := server.Run()
 
 	trace.Stop()
 	log.Close()
diff --git a/pkg/cmd/grafana-server/server.go b/pkg/cmd/grafana-server/server.go
index aff40111ae9..43f6bed1646 100644
--- a/pkg/cmd/grafana-server/server.go
+++ b/pkg/cmd/grafana-server/server.go
@@ -54,18 +54,19 @@ func NewGrafanaServer() *GrafanaServerImpl {
 }
 
 type GrafanaServerImpl struct {
-	context        context.Context
-	shutdownFn     context.CancelFunc
-	childRoutines  *errgroup.Group
-	log            log.Logger
-	cfg            *setting.Cfg
-	shutdownReason string
+	context            context.Context
+	shutdownFn         context.CancelFunc
+	childRoutines      *errgroup.Group
+	log                log.Logger
+	cfg                *setting.Cfg
+	shutdownReason     string
+	shutdownInProgress bool
 
 	RouteRegister api.RouteRegister `inject:""`
 	HttpServer    *api.HTTPServer   `inject:""`
 }
 
-func (g *GrafanaServerImpl) Start() error {
+func (g *GrafanaServerImpl) Run() error {
 	g.loadConfiguration()
 	g.writePIDFile()
 
@@ -129,8 +130,24 @@ func (g *GrafanaServerImpl) Start() error {
 		}
 
 		g.childRoutines.Go(func() error {
+			// Skip starting new service is we are shutting down
+			// Ccan happen when service crash during startup
+			if g.shutdownInProgress {
+				return nil
+			}
+
 			err := service.Run(g.context)
-			g.log.Info("Stopped "+reflect.TypeOf(service).Elem().Name(), "reason", err)
+
+			// If error is not canceled then the service crashed
+			if err != context.Canceled {
+				g.log.Error("Stopped "+reflect.TypeOf(service).Elem().Name(), "reason", err)
+			} else {
+				g.log.Info("Stopped "+reflect.TypeOf(service).Elem().Name(), "reason", err)
+			}
+
+			// Mark that we are in shutdown mode
+			// So more services are not started
+			g.shutdownInProgress = true
 			return err
 		})
 	}
@@ -159,6 +176,7 @@ func (g *GrafanaServerImpl) loadConfiguration() {
 func (g *GrafanaServerImpl) Shutdown(reason string) {
 	g.log.Info("Shutdown started", "reason", reason)
 	g.shutdownReason = reason
+	g.shutdownInProgress = true
 
 	// call cancel func on root context
 	g.shutdownFn()