Reworking configuration loading and overriding

.bra.toml

@@ -1,7 +1,7 @@
 [run]
 init_cmds = [
-  ["go", "build", "-o", "./bin/grafana"],
+  ["go", "build", "-o", "./bin/grafana-server"],
-	["./bin/grafana", "web"]
+	["./bin/grafana-server"]
 ]
 watch_all = true
 watch_dirs = [
@@ -12,6 +12,6 @@ watch_dirs = [
 watch_exts = [".go", ".ini"]
 build_delay = 1500
 cmds = [
-  ["go", "build", "-o", "./bin/grafana"],
+  ["go", "build", "-o", "./bin/grafana-server"],
-	["./bin/grafana", "web"]
+	["./bin/grafana-server"]
 ]

build.go

@@ -209,7 +209,7 @@ func test(pkg string) {
 }
 
 func build(pkg string, tags []string) {
-	binary := "./bin/grafana"
+	binary := "./bin/grafana-server"
 	if goos == "windows" {
 		binary += ".exe"
 	}

conf/defaults.ini

@@ -1,13 +1,13 @@
 app_name = Grafana
 app_mode = production
 
+[paths]
 ; data_path
 ; where rendered png images are temporarily stored
 ; file based sessions are stored here (if file based session is configured below)
 ; the database is stored here if sqlite3 database is used
-; can be overriden from command line --data-path
+data = data
-; defaults to `data` path relative to working directory
+logs = data/log
-data_path =
 
 [server]
 ; protocol (http or https)
@@ -21,7 +21,7 @@ domain = localhost
 ; the full public facing url
 root_url = %(protocol)s://%(domain)s:%(http_port)s/
 router_logging = false
-; the path relative to the binary where the static (html/js/css) files are placed
+; the path relative home path where frontend assets are located
 static_root_path = public
 ; enable gzip
 enable_gzip = false
@@ -47,7 +47,7 @@ user = root
 password =
 ; For "postgres" only, either "disable", "require" or "verify-full"
 ssl_mode = disable
-; For "sqlite3" only, path relative to data_dir setting
+; For "sqlite3" only, path relative to data_path setting
 path = grafana.db
 
 [session]
@@ -55,7 +55,7 @@ path = grafana.db
 provider = file
 ; Provider config options
 ; memory: not have any config yet
-; file: session dir path, is relative to grafana data_dir
+; file: session dir path, is relative to grafana data_path
 ; redis: config like redis server addr, poolSize, password, e.g. `127.0.0.1:6379,100,grafana`
 ; mysql: go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1)/database_name`
 provider_config = sessions
@@ -117,11 +117,6 @@ token_url = https://accounts.google.com/o/oauth2/token
 ; allowed_domains = mycompany.com othercompany.com
 
 [log]
-; root_path
-; for deb or rpm package installs this is specified via command line
-; change it in /etc/default/grafana, it defaults to /var/log/grafana
-; for non package installs (running manually) defaults to `log` dir under data_dir
-root_path =
 ; Either "console", "file", default is "console"
 ; Use comma to separate multiple modes, e.g. "console, file"
 mode = console, file

main.go

@@ -64,15 +64,14 @@ func main() {
 }
 
 func initRuntime() {
-	setting.NewConfigContext(&setting.CommandLineArgs{})
+	setting.NewConfigContext(&setting.CommandLineArgs{
+		Config: *configFile,
+		Args:   flag.Args(),
+	})
 
 	log.Info("Starting Grafana")
 	log.Info("Version: %v, Commit: %v, Build date: %v", setting.BuildVersion, setting.BuildCommit, time.Unix(setting.BuildStamp, 0))
-	setting.LogLoadedConfigFiles()
+	setting.LogConfigurationInfo()
-
-	log.Info("Working Path: %s", setting.WorkPath)
-	log.Info("Data Path: %s", setting.DataPath)
-	log.Info("Log Path: %s", setting.LogRootPath)
 
 	sqlstore.NewEngine()
 	sqlstore.EnsureAdminUser()

pkg/services/sqlstore/sqlstore.go

@@ -86,7 +86,7 @@ func SetEngine(engine *xorm.Engine, enableLog bool) (err error) {
 	}
 
 	if enableLog {
-		logPath := path.Join(setting.LogRootPath, "xorm.log")
+		logPath := path.Join(setting.LogsPath, "xorm.log")
 		os.MkdirAll(path.Dir(logPath), os.ModePerm)
 
 		f, err := os.Create(logPath)

pkg/setting/setting.go

@@ -4,15 +4,16 @@
 package setting
 
 import (
+	"bytes"
 	"fmt"
 	"net/url"
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"strings"
 
-	"github.com/Unknwon/com"
 	"github.com/macaron-contrib/session"
 	"gopkg.in/ini.v1"
 
@@ -44,10 +45,14 @@ var (
 	BuildCommit  string
 	BuildStamp   int64
 
+	// Paths
+	LogsPath string
+	HomePath string
+	DataPath string
+
 	// Log settings.
-	LogRootPath string
+	LogModes   []string
-	LogModes    []string
+	LogConfigs []string
-	LogConfigs  []string
 
 	// Http server options
 	Protocol           Scheme
@@ -83,8 +88,6 @@ var (
 	SessionOptions session.Options
 
 	// Global setting objects.
-	DataPath     string
-	WorkPath     string
 	Cfg          *ini.File
 	ConfRootPath string
 	IsWindows    bool
@@ -93,50 +96,24 @@ var (
 	ImagesDir  string
 	PhantomDir string
 
-	configFiles []string
+	// for logging purposes
+	configFiles                  []string
+	appliedCommandLineProperties []string
+	appliedEnvOverrides          []string
 
 	ReportingEnabled  bool
 	GoogleAnalyticsId string
 )
 
 type CommandLineArgs struct {
-	DefaultDataPath string
+	Config string
-	DefaultLogPath  string
+	Args   []string
-	Config          string
 }
 
 func init() {
 	IsWindows = runtime.GOOS == "windows"
 	log.NewLogger(0, "console", `{"level": 0}`)
-	WorkPath, _ = filepath.Abs(".")
+	HomePath, _ = filepath.Abs(".")
-}
-
-func findConfigFiles(customConfigFile string) {
-	ConfRootPath = path.Join(WorkPath, "conf")
-	configFiles = make([]string, 0)
-
-	configFile := path.Join(ConfRootPath, "defaults.ini")
-	if com.IsFile(configFile) {
-		configFiles = append(configFiles, configFile)
-	}
-
-	configFile = path.Join(ConfRootPath, "dev.ini")
-	if com.IsFile(configFile) {
-		configFiles = append(configFiles, configFile)
-	}
-
-	configFile = path.Join(ConfRootPath, "custom.ini")
-	if com.IsFile(configFile) {
-		configFiles = append(configFiles, configFile)
-	}
-
-	if customConfigFile != "" {
-		configFiles = append(configFiles, customConfigFile)
-	}
-
-	if len(configFiles) == 0 {
-		log.Fatal(3, "Could not find any config file")
-	}
 }
 
 func parseAppUrlAndSubUrl(section *ini.Section) (string, string) {
@@ -159,7 +136,8 @@ func ToAbsUrl(relativeUrl string) string {
 	return AppUrl + relativeUrl
 }
 
-func loadEnvVariableOverrides() {
+func applyEnvVariableOverrides() {
+	appliedEnvOverrides = make([]string, 0)
 	for _, section := range Cfg.Sections() {
 		for _, key := range section.Keys() {
 			sectionName := strings.ToUpper(strings.Replace(section.Name(), ".", "_", -1))
@@ -168,46 +146,131 @@ func loadEnvVariableOverrides() {
 			envValue := os.Getenv(envKey)
 
 			if len(envValue) > 0 {
-				log.Info("Setting: ENV override found: %s", envKey)
 				key.SetValue(envValue)
+				appliedEnvOverrides = append(appliedEnvOverrides, fmt.Sprintf("%s=%s", envKey, envValue))
 			}
 		}
 	}
 }
 
-func evalutePathConfig(value, commandLineVal, def string) string {
+func applyCommandLineDefaultProperties(props map[string]string) {
-	if value == "" {
+	appliedCommandLineProperties = make([]string, 0)
-		if commandLineVal != "" {
+	for _, section := range Cfg.Sections() {
-			return commandLineVal
+		for _, key := range section.Keys() {
-		} else {
+			keyString := fmt.Sprintf("default.%s.%s", section.Name(), key.Name())
-			return def
+			value, exists := props[keyString]
+			if exists {
+				key.SetValue(value)
+				appliedCommandLineProperties = append(appliedCommandLineProperties, fmt.Sprintf("%s=%s", keyString, value))
+			}
 		}
 	}
-	return value
+}
+
+func applyCommandLineProperties(props map[string]string) {
+	for _, section := range Cfg.Sections() {
+		for _, key := range section.Keys() {
+			keyString := fmt.Sprintf("%s.%s", section.Name(), key.Name())
+			value, exists := props[keyString]
+			if exists {
+				key.SetValue(value)
+				appliedCommandLineProperties = append(appliedCommandLineProperties, fmt.Sprintf("%s=%s", keyString, value))
+			}
+		}
+	}
+}
+
+func getCommandLineProperties(args []string) map[string]string {
+	props := make(map[string]string)
+
+	for _, arg := range args {
+		if !strings.HasPrefix(arg, "cfg:") {
+			continue
+		}
+
+		trimmed := strings.TrimPrefix(arg, "cfg:")
+		parts := strings.Split(trimmed, "=")
+		if len(parts) != 2 {
+			log.Fatal(3, "Invalid command line argument", arg)
+			return nil
+		}
+
+		props[parts[0]] = parts[1]
+	}
+	return props
+}
+
+func makeAbsolute(path string, root string) string {
+	if filepath.IsAbs(path) {
+		return path
+	}
+	return filepath.Join(root, path)
+}
+
+func evalEnvVarExpression(value string) string {
+	regex := regexp.MustCompile(`\${(\w+)}`)
+	return regex.ReplaceAllStringFunc(value, func(envVar string) string {
+		envVar = strings.TrimPrefix(envVar, "${")
+		envVar = strings.TrimSuffix(envVar, "}")
+		envValue := os.Getenv(envVar)
+		return envValue
+	})
+}
+
+func evalConfigValues() {
+	for _, section := range Cfg.Sections() {
+		for _, key := range section.Keys() {
+			key.SetValue(evalEnvVarExpression(key.Value()))
+		}
+	}
+}
+
+func loadConfiguration(args *CommandLineArgs) {
+	var err error
+
+	args.Config = evalEnvVarExpression(args.Config)
+
+	// load config defaults
+	defaultConfigFile := path.Join(HomePath, "conf/defaults.ini")
+	configFiles = append(configFiles, defaultConfigFile)
+
+	Cfg, err = ini.Load(defaultConfigFile)
+	Cfg.BlockMode = true
+
+	if err != nil {
+		log.Fatal(3, "Failed to parse defaults.ini, %v", err)
+	}
+
+	// command line props
+	commandLineProps := getCommandLineProperties(args.Args)
+
+	// load default overrides
+	applyCommandLineDefaultProperties(commandLineProps)
+
+	// load specified config file
+	if args.Config != "" {
+		err = Cfg.Append(args.Config)
+		if err != nil {
+			log.Fatal(3, "Failed to parse %v, %v", args.Config, err)
+		}
+		configFiles = append(configFiles, args.Config)
+		appliedCommandLineProperties = append(appliedCommandLineProperties, "config="+args.Config)
+	}
+
+	// apply environment overrides
+	applyEnvVariableOverrides()
+
+	// apply command line overrides
+	applyCommandLineProperties(commandLineProps)
+
+	// evaluate config values containing environment variables
+	evalConfigValues()
 }
 
 func NewConfigContext(args *CommandLineArgs) {
-	findConfigFiles(args.Config)
+	loadConfiguration(args)
 
-	var err error
+	DataPath = makeAbsolute(Cfg.Section("paths").Key("data").String(), HomePath)
-
-	for i, file := range configFiles {
-		if i == 0 {
-			Cfg, err = ini.Load(configFiles[i])
-			Cfg.BlockMode = false
-		} else {
-			err = Cfg.Append(configFiles[i])
-		}
-
-		if err != nil {
-			log.Fatal(4, "Fail to parse config file: %v, error: %v", file, err)
-		}
-	}
-
-	loadEnvVariableOverrides()
-
-	DataPath = Cfg.Section("").Key("data_path").String()
-	DataPath = evalutePathConfig(DataPath, args.DefaultDataPath, filepath.Join(WorkPath, "data"))
 
 	initLogging(args)
 
@@ -228,7 +291,7 @@ func NewConfigContext(args *CommandLineArgs) {
 	HttpAddr = server.Key("http_addr").MustString("0.0.0.0")
 	HttpPort = server.Key("http_port").MustString("3000")
 
-	StaticRootPath = server.Key("static_root_path").MustString(path.Join(WorkPath, "public"))
+	StaticRootPath = server.Key("static_root_path").MustString(path.Join(HomePath, "public"))
 	RouterLogging = server.Key("router_logging").MustBool(false)
 	EnableGzip = server.Key("enable_gzip").MustBool(false)
 
@@ -254,7 +317,7 @@ func NewConfigContext(args *CommandLineArgs) {
 
 	// PhantomJS rendering
 	ImagesDir = filepath.Join(DataPath, "png")
-	PhantomDir = filepath.Join(WorkPath, "vendor/phantomjs")
+	PhantomDir = filepath.Join(HomePath, "vendor/phantomjs")
 
 	analytics := Cfg.Section("analytics")
 	ReportingEnabled = analytics.Key("reporting_enabled").MustBool(true)
@@ -276,9 +339,7 @@ func readSessionConfig() {
 	SessionOptions.IDLength = 16
 
 	if SessionOptions.Provider == "file" {
-		if !filepath.IsAbs(SessionOptions.ProviderConfig) {
+		SessionOptions.ProviderConfig = makeAbsolute(SessionOptions.ProviderConfig, DataPath)
-			SessionOptions.ProviderConfig = filepath.Join(DataPath, SessionOptions.ProviderConfig)
-		}
 		os.MkdirAll(path.Dir(SessionOptions.ProviderConfig), os.ModePerm)
 	}
 
@@ -299,14 +360,7 @@ var logLevels = map[string]string{
 func initLogging(args *CommandLineArgs) {
 	// Get and check log mode.
 	LogModes = strings.Split(Cfg.Section("log").Key("mode").MustString("console"), ",")
-	LogRootPath = Cfg.Section("log").Key("root_path").String()
+	LogsPath = makeAbsolute(Cfg.Section("paths").Key("logs").String(), HomePath)
-	if LogRootPath == "" {
-		if args.DefaultLogPath != "" {
-			LogRootPath = args.DefaultLogPath
-		} else {
-			LogRootPath = filepath.Join(DataPath, "log")
-		}
-	}
 
 	LogConfigs = make([]string, len(LogModes))
 	for i, mode := range LogModes {
@@ -329,7 +383,7 @@ func initLogging(args *CommandLineArgs) {
 		case "console":
 			LogConfigs[i] = fmt.Sprintf(`{"level":%s}`, level)
 		case "file":
-			logPath := sec.Key("file_name").MustString(path.Join(LogRootPath, "grafana.log"))
+			logPath := sec.Key("file_name").MustString(path.Join(LogsPath, "grafana.log"))
 			os.MkdirAll(path.Dir(logPath), os.ModePerm)
 			LogConfigs[i] = fmt.Sprintf(
 				`{"level":%s,"filename":"%s","rotate":%v,"maxlines":%d,"maxsize":%d,"daily":%v,"maxdays":%d}`, level,
@@ -362,8 +416,33 @@ func initLogging(args *CommandLineArgs) {
 	}
 }
 
-func LogLoadedConfigFiles() {
+func LogConfigurationInfo() {
-	for _, file := range configFiles {
+	var text bytes.Buffer
-		log.Info("Config: Loaded from %s", file)
+	text.WriteString("Configuration Info\n")
+
+	text.WriteString("Config files:\n")
+	for i, file := range configFiles {
+		text.WriteString(fmt.Sprintf("  [%d]: %s\n", i, file))
 	}
+
+	if len(appliedCommandLineProperties) > 0 {
+		text.WriteString("Command lines overrides:\n")
+		for i, prop := range appliedCommandLineProperties {
+			text.WriteString(fmt.Sprintf("  [%d]: %s\n", i, prop))
+		}
+	}
+
+	if len(appliedEnvOverrides) > 0 {
+		text.WriteString("\tEnvironment variables used:\n")
+		for i, prop := range appliedCommandLineProperties {
+			text.WriteString(fmt.Sprintf("  [%d]: %s\n", i, prop))
+		}
+	}
+
+	text.WriteString("Paths:\n")
+	text.WriteString(fmt.Sprintf("  home: %s\n", HomePath))
+	text.WriteString(fmt.Sprintf("  data: %s\n", DataPath))
+	text.WriteString(fmt.Sprintf("  logs: %s\n", LogsPath))
+
+	log.Info(text.String())
 }

pkg/setting/setting_test.go

@@ -10,12 +10,12 @@ import (
 
 func TestLoadingSettings(t *testing.T) {
 
-	WorkDir, _ = filepath.Abs("../../")
+	HomePath, _ = filepath.Abs("../../")
 
 	Convey("Testing loading settings from ini file", t, func() {
 
 		Convey("Given the default ini files", func() {
-			NewConfigContext("")
+			NewConfigContext(&CommandLineArgs{})
 
 			So(AppName, ShouldEqual, "Grafana")
 			So(AdminUser, ShouldEqual, "admin")
@@ -23,9 +23,63 @@ func TestLoadingSettings(t *testing.T) {
 
 		Convey("Should be able to override via environment variables", func() {
 			os.Setenv("GF_SECURITY_ADMIN_USER", "superduper")
-			NewConfigContext("")
+			NewConfigContext(&CommandLineArgs{})
 
 			So(AdminUser, ShouldEqual, "superduper")
+			So(DataPath, ShouldEqual, filepath.Join(HomePath, "data"))
+			So(LogsPath, ShouldEqual, filepath.Join(DataPath, "log"))
+		})
+
+		Convey("Should get property map from command line args array", func() {
+			props := getCommandLineProperties([]string{"cfg:test=value", "cfg:map.test=1"})
+
+			So(len(props), ShouldEqual, 2)
+			So(props["test"], ShouldEqual, "value")
+			So(props["map.test"], ShouldEqual, "1")
+		})
+
+		Convey("Should be able to override via command line", func() {
+			NewConfigContext(&CommandLineArgs{
+				Args: []string{"cfg:paths.data=/tmp/data", "cfg:paths.logs=/tmp/logs"},
+			})
+
+			So(DataPath, ShouldEqual, "/tmp/data")
+			So(LogsPath, ShouldEqual, "/tmp/logs")
+		})
+
+		Convey("Should be able to override defaults via command line", func() {
+			NewConfigContext(&CommandLineArgs{
+				Args: []string{"cfg:default.paths.data=/tmp/data"},
+			})
+
+			So(DataPath, ShouldEqual, "/tmp/data")
+		})
+
+		Convey("Defaults can be overriden in specified config file", func() {
+			NewConfigContext(&CommandLineArgs{
+				Args:   []string{"cfg:default.paths.data=/tmp/data"},
+				Config: filepath.Join(HomePath, "tests/config-files/override.ini"),
+			})
+
+			So(DataPath, ShouldEqual, "/tmp/override")
+		})
+
+		Convey("Command line overrides specified config file", func() {
+			NewConfigContext(&CommandLineArgs{
+				Args:   []string{"cfg:paths.data=/tmp/data"},
+				Config: filepath.Join(HomePath, "tests/config-files/override.ini"),
+			})
+
+			So(DataPath, ShouldEqual, "/tmp/data")
+		})
+
+		Convey("Can use environment variables in config values", func() {
+			os.Setenv("GF_DATA_PATH", "/tmp/env_override")
+			NewConfigContext(&CommandLineArgs{
+				Args: []string{"cfg:paths.data=${GF_DATA_PATH}"},
+			})
+
+			So(DataPath, ShouldEqual, "/tmp/env_override")
 		})
 
 	})

tasks/build_task.js

@@ -55,7 +55,7 @@ module.exports = function(grunt) {
     grunt.config('copy.backend_bin', {
       cwd: 'bin',
       expand: true,
-      src: ['grafana'],
+      src: ['grafana-server'],
       options: { mode: true},
       dest: '<%= tempDir %>/bin/'
     });

tests/config-files/override.ini

@@ -0,0 +1,2 @@
+[paths]
+data = /tmp/override