refactoring: lots of refactoring around server shutdown flows, making sure process is terminated when background service has crashed

pkg/api/http_server.go

@@ -64,7 +64,7 @@ func (hs *HTTPServer) Run(ctx context.Context) error {
 	hs.streamManager.Run(ctx)
 
 	listenAddr := fmt.Sprintf("%s:%s", setting.HttpAddr, setting.HttpPort)
-	hs.log.Info("Initializing HTTP Server", "address", listenAddr, "protocol", setting.Protocol, "subUrl", setting.AppSubUrl, "socket", setting.SocketPath)
+	hs.log.Info("HTTP Server Listen", "address", listenAddr, "protocol", setting.Protocol, "subUrl", setting.AppSubUrl, "socket", setting.SocketPath)
 
 	hs.httpSrv = &http.Server{Addr: listenAddr, Handler: hs.macaron}
 
@@ -74,7 +74,6 @@ func (hs *HTTPServer) Run(ctx context.Context) error {
 		if err := hs.httpSrv.Shutdown(context.Background()); err != nil {
 			hs.log.Error("Failed to shutdown server", "error", err)
 		}
-		hs.log.Info("Stopped HTTP Server")
 	}()
 
 	switch setting.Protocol {
@@ -113,12 +112,6 @@ func (hs *HTTPServer) Run(ctx context.Context) error {
 	return err
 }
 
-func (hs *HTTPServer) Shutdown(ctx context.Context) error {
-	err := hs.httpSrv.Shutdown(ctx)
-	hs.log.Info("Stopped HTTP server")
-	return err
-}
-
 func (hs *HTTPServer) listenAndServeTLS(certfile, keyfile string) error {
 	if certfile == "" {
 		return fmt.Errorf("cert_file cannot be empty when using HTTPS")

pkg/cmd/grafana-server/main.go

@@ -81,29 +81,20 @@ func main() {
 	setting.Enterprise, _ = strconv.ParseBool(enterprise)
 
 	metrics.M_Grafana_Version.WithLabelValues(version).Set(1)
-	shutdownCompleted := make(chan int)
+
 	server := NewGrafanaServer()
 
-	go listenToSystemSignals(server, shutdownCompleted)
+	go listenToSystemSignals(server)
 
-	go func() {
-		code := 0
-		if err := server.Start(); err != nil {
-			log.Error2("Startup failed", "error", err)
-			code = 1
-		}
+	err := server.Start()
 
-		exitChan <- code
-	}()
-
-	code := <-shutdownCompleted
-	log.Info2("Grafana shutdown completed.", "code", code)
+	trace.Stop()
 	log.Close()
-	os.Exit(code)
+
+	server.Exit(err)
 }
 
-func listenToSystemSignals(server *GrafanaServerImpl, shutdownCompleted chan int) {
-	var code int
+func listenToSystemSignals(server *GrafanaServerImpl) {
 	signalChan := make(chan os.Signal, 1)
 	ignoreChan := make(chan os.Signal, 1)
 
@@ -112,12 +103,6 @@ func listenToSystemSignals(server *GrafanaServerImpl, shutdownCompleted chan int
 
 	select {
 	case sig := <-signalChan:
-		trace.Stop() // Stops trace if profiling has been enabled
-		server.Shutdown(0, fmt.Sprintf("system signal: %s", sig))
-		shutdownCompleted <- 0
-	case code = <-exitChan:
-		trace.Stop() // Stops trace if profiling has been enabled
-		server.Shutdown(code, "startup error")
-		shutdownCompleted <- code
+		server.Shutdown(fmt.Sprintf("System signal: %s", sig))
 	}
 }

pkg/cmd/grafana-server/server.go

@@ -54,11 +54,12 @@ func NewGrafanaServer() *GrafanaServerImpl {
 }
 
 type GrafanaServerImpl struct {
-	context       context.Context
-	shutdownFn    context.CancelFunc
-	childRoutines *errgroup.Group
-	log           log.Logger
-	cfg           *setting.Cfg
+	context        context.Context
+	shutdownFn     context.CancelFunc
+	childRoutines  *errgroup.Group
+	log            log.Logger
+	cfg            *setting.Cfg
+	shutdownReason string
 
 	RouteRegister api.RouteRegister `inject:""`
 	HttpServer    *api.HTTPServer   `inject:""`
@@ -135,7 +136,8 @@ func (g *GrafanaServerImpl) Start() error {
 	}
 
 	sendSystemdNotification("READY=1")
-	return g.startHttpServer()
+
+	return g.childRoutines.Wait()
 }
 
 func (g *GrafanaServerImpl) loadConfiguration() {
@@ -154,28 +156,28 @@ func (g *GrafanaServerImpl) loadConfiguration() {
 	g.cfg.LogConfigSources()
 }
 
-func (g *GrafanaServerImpl) startHttpServer() error {
-	g.HttpServer.Init()
-
-	err := g.HttpServer.Start(g.context)
-
-	if err != nil {
-		return fmt.Errorf("Fail to start server. error: %v", err)
-	}
-
-	return nil
-}
-
-func (g *GrafanaServerImpl) Shutdown(code int, reason string) {
-	g.log.Info("Shutdown started", "code", code, "reason", reason)
+func (g *GrafanaServerImpl) Shutdown(reason string) {
+	g.log.Info("Shutdown started", "reason", reason)
+	g.shutdownReason = reason
 
 	// call cancel func on root context
 	g.shutdownFn()
 
 	// wait for child routines
-	if err := g.childRoutines.Wait(); err != nil && err != context.Canceled {
-		g.log.Error("Server shutdown completed", "error", err)
+	g.childRoutines.Wait()
+}
+
+func (g *GrafanaServerImpl) Exit(reason error) {
+	// default exit code is 1
+	code := 1
+
+	if reason == context.Canceled && g.shutdownReason != "" {
+		reason = fmt.Errorf(g.shutdownReason)
+		code = 0
 	}
+
+	g.log.Error("Server shutdown", "reason", reason)
+	os.Exit(code)
 }
 
 func (g *GrafanaServerImpl) writePIDFile() {