Chore: Improve grafana-server profiling and tracing (#20593)

Profiling and tracing can now be enabled/disabled separately.
Adds argument for trace file path.
Support overriding profiling and tracing argument using 
environment variables.
Update docs.

Fixes #20576

docs/sources/installation/troubleshooting.md

@@ -30,7 +30,59 @@ If you encounter an error or problem it is a good idea to check the grafana serv
 located at `/var/log/grafana/grafana.log` on Unix systems or in `<grafana_install_dir>/data/log` on
 other platforms and manual installs.
 
-You can enable more logging by changing log level in you grafana configuration file.
+You can enable more logging by changing log level in your grafana configuration file.
+
+## Diagnostics
+
+The `grafana-server` process can be instructued to enable certain diagnostics when it starts. This can be helpful
+when experiencing/investigating certain performance problems. It's `not` recommended to have these enabled per default.
+
+### Profiling
+
+The `grafana-server` can be started with the arguments `-profile` to enable profiling and  `-profile-port` to override
+the default HTTP port (`6060`) where the pprof debugging endpoints will be available, e.g.
+
+```bash
+./grafana-server -profile -profile-port=8080
+```
+
+Note that pprof debugging endpoints are served on a different port than the Grafana HTTP server.
+
+You can configure/override profiling settings using environment variables:
+
+```bash
+export GF_DIAGNOSTICS_PROFILING_ENABLED=true
+export GF_DIAGNOSTICS_PROFILING_PORT=8080
+```
+
+See [Go command pprof](https://golang.org/cmd/pprof/) for more information about how to collect and analyze profiling data.
+
+### Tracing
+
+The `grafana-server` can be started with the arguments `-tracing` to enable tracing and `-tracing-file` to
+override the default trace file (`trace.out`) where trace result will be written to, e.g.
+
+```bash
+./grafana-server -tracing -tracing-file=/tmp/trace.out
+```
+
+You can configure/override profiling settings using environment variables:
+
+```bash
+export GF_DIAGNOSTICS_TRACING_ENABLED=true
+export GF_DIAGNOSTICS_TRACING_FILE=/tmp/trace.out
+```
+
+View the trace in a web browser (Go required to be installed):
+
+```bash
+go tool trace <trace file>
+2019/11/24 22:20:42 Parsing trace...
+2019/11/24 22:20:42 Splitting trace...
+2019/11/24 22:20:42 Opening browser. Trace viewer is listening on http://127.0.0.1:39735
+```
+
+See [Go command trace](https://golang.org/cmd/trace/) for more information about how to analyze trace files.
 
 ## FAQ
 

pkg/cmd/grafana-server/diagnostics.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+)
+
+const (
+	profilingEnabledEnvName = "GF_DIAGNOSTICS_PROFILING_ENABLED"
+	profilingPortEnvName    = "GF_DIAGNOSTICS_PROFILING_PORT"
+	tracingEnabledEnvName   = "GF_DIAGNOSTICS_TRACING_ENABLED"
+	tracingFileEnvName      = "GF_DIAGNOSTICS_TRACING_FILE"
+)
+
+type profilingDiagnostics struct {
+	enabled bool
+	port    uint
+}
+
+func newProfilingDiagnostics(enabled bool, port uint) *profilingDiagnostics {
+	return &profilingDiagnostics{
+		enabled: enabled,
+		port:    port,
+	}
+}
+
+func (pd *profilingDiagnostics) overrideWithEnv() error {
+	enabledEnv := os.Getenv(profilingEnabledEnvName)
+	if enabledEnv != "" {
+		enabled, err := strconv.ParseBool(enabledEnv)
+		if err != nil {
+			return fmt.Errorf("Failed to parse %s environment variable as bool", profilingEnabledEnvName)
+		}
+		pd.enabled = enabled
+	}
+
+	portEnv := os.Getenv(profilingPortEnvName)
+	if portEnv != "" {
+		port, parseErr := strconv.ParseUint(portEnv, 0, 64)
+		if parseErr != nil {
+			return fmt.Errorf("Failed to parse %s enviroment variable to unsigned integer", profilingPortEnvName)
+		}
+		pd.port = uint(port)
+	}
+
+	return nil
+}
+
+type tracingDiagnostics struct {
+	enabled bool
+	file    string
+}
+
+func newTracingDiagnostics(enabled bool, file string) *tracingDiagnostics {
+	return &tracingDiagnostics{
+		enabled: enabled,
+		file:    file,
+	}
+}
+
+func (td *tracingDiagnostics) overrideWithEnv() error {
+	enabledEnv := os.Getenv(tracingEnabledEnvName)
+	if enabledEnv != "" {
+		enabled, err := strconv.ParseBool(enabledEnv)
+		if err != nil {
+			return fmt.Errorf("Failed to parse %s environment variable as bool", tracingEnabledEnvName)
+		}
+		td.enabled = enabled
+	}
+
+	fileEnv := os.Getenv(tracingFileEnvName)
+	if fileEnv != "" {
+		td.file = fileEnv
+	}
+
+	return nil
+}

pkg/cmd/grafana-server/diagnostics_test.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestProfilingDiagnostics(t *testing.T) {
+	tcs := []struct {
+		defaults   *profilingDiagnostics
+		enabledEnv string
+		portEnv    string
+		expected   *profilingDiagnostics
+	}{
+		{defaults: newProfilingDiagnostics(false, 6060), enabledEnv: "", portEnv: "", expected: newProfilingDiagnostics(false, 6060)},
+		{defaults: newProfilingDiagnostics(true, 8080), enabledEnv: "", portEnv: "", expected: newProfilingDiagnostics(true, 8080)},
+		{defaults: newProfilingDiagnostics(false, 6060), enabledEnv: "false", portEnv: "8080", expected: newProfilingDiagnostics(false, 8080)},
+		{defaults: newProfilingDiagnostics(false, 6060), enabledEnv: "true", portEnv: "8080", expected: newProfilingDiagnostics(true, 8080)},
+		{defaults: newProfilingDiagnostics(false, 6060), enabledEnv: "true", portEnv: "", expected: newProfilingDiagnostics(true, 6060)},
+	}
+
+	for i, tc := range tcs {
+		t.Run(fmt.Sprintf("testcase %d", i), func(t *testing.T) {
+			os.Clearenv()
+			if tc.enabledEnv != "" {
+				err := os.Setenv(profilingEnabledEnvName, tc.enabledEnv)
+				assert.NoError(t, err)
+			}
+			if tc.portEnv != "" {
+				err := os.Setenv(profilingPortEnvName, tc.portEnv)
+				assert.NoError(t, err)
+			}
+			err := tc.defaults.overrideWithEnv()
+			assert.NoError(t, err)
+			assert.Exactly(t, tc.expected, tc.defaults)
+		})
+	}
+}
+
+func TestTracingDiagnostics(t *testing.T) {
+	tcs := []struct {
+		defaults   *tracingDiagnostics
+		enabledEnv string
+		fileEnv    string
+		expected   *tracingDiagnostics
+	}{
+		{defaults: newTracingDiagnostics(false, "trace.out"), enabledEnv: "", fileEnv: "", expected: newTracingDiagnostics(false, "trace.out")},
+		{defaults: newTracingDiagnostics(true, "/tmp/trace.out"), enabledEnv: "", fileEnv: "", expected: newTracingDiagnostics(true, "/tmp/trace.out")},
+		{defaults: newTracingDiagnostics(false, "trace.out"), enabledEnv: "false", fileEnv: "/tmp/trace.out", expected: newTracingDiagnostics(false, "/tmp/trace.out")},
+		{defaults: newTracingDiagnostics(false, "trace.out"), enabledEnv: "true", fileEnv: "/tmp/trace.out", expected: newTracingDiagnostics(true, "/tmp/trace.out")},
+		{defaults: newTracingDiagnostics(false, "trace.out"), enabledEnv: "true", fileEnv: "", expected: newTracingDiagnostics(true, "trace.out")},
+	}
+
+	for i, tc := range tcs {
+		t.Run(fmt.Sprintf("testcase %d", i), func(t *testing.T) {
+			os.Clearenv()
+			if tc.enabledEnv != "" {
+				err := os.Setenv(tracingEnabledEnvName, tc.enabledEnv)
+				assert.NoError(t, err)
+			}
+			if tc.fileEnv != "" {
+				err := os.Setenv(tracingFileEnvName, tc.fileEnv)
+				assert.NoError(t, err)
+			}
+			err := tc.defaults.overrideWithEnv()
+			assert.NoError(t, err)
+			assert.Exactly(t, tc.expected, tc.defaults)
+		})
+	}
+}

pkg/cmd/grafana-server/main.go

@@ -46,7 +46,9 @@ func main() {
 
 		v           = flag.Bool("v", false, "prints current version and exits")
 		profile     = flag.Bool("profile", false, "Turn on pprof profiling")
-		profilePort = flag.Int("profile-port", 6060, "Define custom port for profiling")
+		profilePort = flag.Uint("profile-port", 6060, "Define custom port for profiling")
+		tracing     = flag.Bool("tracing", false, "Turn on tracing")
+		tracingFile = flag.String("tracing-file", "trace.out", "Define tracing output file")
 	)
 
 	flag.Parse()
@@ -56,16 +58,32 @@ func main() {
 		os.Exit(0)
 	}
 
-	if *profile {
+	profileDiagnostics := newProfilingDiagnostics(*profile, *profilePort)
+	if err := profileDiagnostics.overrideWithEnv(); err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	traceDiagnostics := newTracingDiagnostics(*tracing, *tracingFile)
+	if err := traceDiagnostics.overrideWithEnv(); err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+
+	if profileDiagnostics.enabled {
+		fmt.Println("diagnostics: pprof profiling enabled", "port", profileDiagnostics.port)
 		runtime.SetBlockProfileRate(1)
 		go func() {
-			err := http.ListenAndServe(fmt.Sprintf("localhost:%d", *profilePort), nil)
+			err := http.ListenAndServe(fmt.Sprintf("localhost:%d", profileDiagnostics.port), nil)
 			if err != nil {
 				panic(err)
 			}
 		}()
+	}
 
-		f, err := os.Create("trace.out")
+	if traceDiagnostics.enabled {
+		fmt.Println("diagnostics: tracing enabled", "file", traceDiagnostics.file)
+		f, err := os.Create(traceDiagnostics.file)
 		if err != nil {
 			panic(err)
 		}