From 151b24b95fb52a777533c9fd76db48ae8967a74e Mon Sep 17 00:00:00 2001 From: Andrej Ocenas Date: Mon, 27 May 2019 10:47:21 +0200 Subject: [PATCH] CLI: Add command to migrate all datasources to use encrypted password fields (#17118) closes: #17107 --- pkg/cmd/grafana-cli/commands/commands.go | 23 +++- .../encrypt_datasource_passwords.go | 126 ++++++++++++++++++ .../encrypt_datasource_passwords_test.go | 67 ++++++++++ .../grafana-cli/commands/install_command.go | 7 +- .../commands/listremote_command.go | 3 +- .../commands/listversions_command.go | 5 +- pkg/cmd/grafana-cli/commands/ls_command.go | 3 +- .../grafana-cli/commands/remove_command.go | 5 +- .../commands/reset_password_command.go | 4 +- .../commands/upgrade_all_command.go | 3 +- .../grafana-cli/commands/upgrade_command.go | 3 +- .../{commands => utils}/command_line.go | 16 +-- pkg/util/strings.go | 17 +++ pkg/util/strings_test.go | 9 ++ 14 files changed, 266 insertions(+), 25 deletions(-) create mode 100644 pkg/cmd/grafana-cli/commands/datamigrations/encrypt_datasource_passwords.go create mode 100644 pkg/cmd/grafana-cli/commands/datamigrations/encrypt_datasource_passwords_test.go rename pkg/cmd/grafana-cli/{commands => utils}/command_line.go (64%) diff --git a/pkg/cmd/grafana-cli/commands/commands.go b/pkg/cmd/grafana-cli/commands/commands.go index d5add2b7168..ebaee557348 100644 --- a/pkg/cmd/grafana-cli/commands/commands.go +++ b/pkg/cmd/grafana-cli/commands/commands.go @@ -7,14 +7,16 @@ import ( "github.com/codegangsta/cli" "github.com/fatih/color" "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/commands/datamigrations" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/setting" ) -func runDbCommand(command func(commandLine CommandLine) error) func(context *cli.Context) { +func runDbCommand(command func(commandLine utils.CommandLine, sqlStore *sqlstore.SqlStore) error) func(context *cli.Context) { return func(context *cli.Context) { - cmd := &contextCommandLine{context} + cmd := &utils.ContextCommandLine{Context: context} cfg := setting.NewCfg() cfg.Load(&setting.CommandLineArgs{ @@ -28,7 +30,7 @@ func runDbCommand(command func(commandLine CommandLine) error) func(context *cli engine.Bus = bus.GetBus() engine.Init() - if err := command(cmd); err != nil { + if err := command(cmd, engine); err != nil { logger.Errorf("\n%s: ", color.RedString("Error")) logger.Errorf("%s\n\n", err) @@ -40,10 +42,10 @@ func runDbCommand(command func(commandLine CommandLine) error) func(context *cli } } -func runPluginCommand(command func(commandLine CommandLine) error) func(context *cli.Context) { +func runPluginCommand(command func(commandLine utils.CommandLine) error) func(context *cli.Context) { return func(context *cli.Context) { - cmd := &contextCommandLine{context} + cmd := &utils.ContextCommandLine{Context: context} if err := command(cmd); err != nil { logger.Errorf("\n%s: ", color.RedString("Error")) logger.Errorf("%s %s\n\n", color.RedString("✗"), err) @@ -107,6 +109,17 @@ var adminCommands = []cli.Command{ }, }, }, + { + Name: "data-migration", + Usage: "Runs a script that migrates or cleanups data in your db", + Subcommands: []cli.Command{ + { + Name: "encrypt-datasource-passwords", + Usage: "Migrates passwords from unsecured fields to secure_json_data field. Return ok unless there is an error. Safe to execute multiple times.", + Action: runDbCommand(datamigrations.EncryptDatasourcePaswords), + }, + }, + }, } var Commands = []cli.Command{ diff --git a/pkg/cmd/grafana-cli/commands/datamigrations/encrypt_datasource_passwords.go b/pkg/cmd/grafana-cli/commands/datamigrations/encrypt_datasource_passwords.go new file mode 100644 index 00000000000..e55fa2d70b8 --- /dev/null +++ b/pkg/cmd/grafana-cli/commands/datamigrations/encrypt_datasource_passwords.go @@ -0,0 +1,126 @@ +package datamigrations + +import ( + "context" + "encoding/json" + + "github.com/fatih/color" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" + + "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" + "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/setting" + "github.com/grafana/grafana/pkg/util" + "github.com/grafana/grafana/pkg/util/errutil" +) + +var ( + datasourceTypes = []string{ + "mysql", + "influxdb", + "elasticsearch", + "graphite", + "prometheus", + "opentsdb", + } +) + +// EncryptDatasourcePaswords migrates un-encrypted secrets on datasources +// to the secureJson Column. +func EncryptDatasourcePaswords(c utils.CommandLine, sqlStore *sqlstore.SqlStore) error { + return sqlStore.WithDbSession(context.Background(), func(session *sqlstore.DBSession) error { + passwordsUpdated, err := migrateColumn(session, "password") + if err != nil { + return err + } + + basicAuthUpdated, err := migrateColumn(session, "basic_auth_password") + if err != nil { + return err + } + + logger.Info("\n") + if passwordsUpdated > 0 { + logger.Infof("%s Encrypted password field for %d datasources \n", color.GreenString("✔"), passwordsUpdated) + } + + if basicAuthUpdated > 0 { + logger.Infof("%s Encrypted basic_auth_password field for %d datasources \n", color.GreenString("✔"), basicAuthUpdated) + } + + if passwordsUpdated == 0 && basicAuthUpdated == 0 { + logger.Infof("%s All datasources secrets are allready encrypted\n", color.GreenString("✔")) + } + + logger.Info("\n") + + logger.Warn("Warning: Datasource provisioning files need to be manually changed to prevent overwriting of " + + "the data during provisioning. See https://grafana.com/docs/installation/upgrading/#upgrading-to-v6-2 for " + + "details") + return nil + }) +} + +func migrateColumn(session *sqlstore.DBSession, column string) (int, error) { + var rows []map[string]string + + session.Cols("id", column, "secure_json_data") + session.Table("data_source") + session.In("type", datasourceTypes) + session.Where(column + " IS NOT NULL AND " + column + " != ''") + err := session.Find(&rows) + + if err != nil { + return 0, errutil.Wrapf(err, "failed to select column: %s", column) + } + + rowsUpdated, err := updateRows(session, rows, column) + return rowsUpdated, errutil.Wrapf(err, "failed to update column: %s", column) +} + +func updateRows(session *sqlstore.DBSession, rows []map[string]string, passwordFieldName string) (int, error) { + var rowsUpdated int + + for _, row := range rows { + newSecureJSONData, err := getUpdatedSecureJSONData(row, passwordFieldName) + if err != nil { + return 0, err + } + + data, err := json.Marshal(newSecureJSONData) + if err != nil { + return 0, errutil.Wrap("marshaling newSecureJsonData failed", err) + } + + newRow := map[string]interface{}{"secure_json_data": data, passwordFieldName: ""} + session.Table("data_source") + session.Where("id = ?", row["id"]) + // Setting both columns while having value only for secure_json_data should clear the [passwordFieldName] column + session.Cols("secure_json_data", passwordFieldName) + + _, err = session.Update(newRow) + if err != nil { + return 0, err + } + + rowsUpdated++ + } + return rowsUpdated, nil +} + +func getUpdatedSecureJSONData(row map[string]string, passwordFieldName string) (map[string]interface{}, error) { + encryptedPassword, err := util.Encrypt([]byte(row[passwordFieldName]), setting.SecretKey) + if err != nil { + return nil, err + } + + var secureJSONData map[string]interface{} + + if err := json.Unmarshal([]byte(row["secure_json_data"]), &secureJSONData); err != nil { + return nil, err + } + + jsonFieldName := util.ToCamelCase(passwordFieldName) + secureJSONData[jsonFieldName] = encryptedPassword + return secureJSONData, nil +} diff --git a/pkg/cmd/grafana-cli/commands/datamigrations/encrypt_datasource_passwords_test.go b/pkg/cmd/grafana-cli/commands/datamigrations/encrypt_datasource_passwords_test.go new file mode 100644 index 00000000000..64987423dec --- /dev/null +++ b/pkg/cmd/grafana-cli/commands/datamigrations/encrypt_datasource_passwords_test.go @@ -0,0 +1,67 @@ +package datamigrations + +import ( + "testing" + "time" + + "github.com/grafana/grafana/pkg/cmd/grafana-cli/commands/commandstest" + "github.com/grafana/grafana/pkg/components/securejsondata" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/stretchr/testify/assert" +) + +func TestPasswordMigrationCommand(t *testing.T) { + //setup datasources with password, basic_auth and none + sqlstore := sqlstore.InitTestDB(t) + session := sqlstore.NewSession() + defer session.Close() + + datasources := []*models.DataSource{ + {Type: "influxdb", Name: "influxdb", Password: "foobar"}, + {Type: "graphite", Name: "graphite", BasicAuthPassword: "foobar"}, + {Type: "prometheus", Name: "prometheus", SecureJsonData: securejsondata.GetEncryptedJsonData(map[string]string{})}, + } + + // set required default values + for _, ds := range datasources { + ds.Created = time.Now() + ds.Updated = time.Now() + ds.SecureJsonData = securejsondata.GetEncryptedJsonData(map[string]string{}) + } + + _, err := session.Insert(&datasources) + assert.Nil(t, err) + + //run migration + err = EncryptDatasourcePaswords(&commandstest.FakeCommandLine{}, sqlstore) + assert.Nil(t, err) + + //verify that no datasources still have password or basic_auth + var dss []*models.DataSource + err = session.SQL("select * from data_source").Find(&dss) + assert.Nil(t, err) + assert.Equal(t, len(dss), 3) + + for _, ds := range dss { + sj := ds.SecureJsonData.Decrypt() + + if ds.Name == "influxdb" { + assert.Equal(t, ds.Password, "") + v, exist := sj["password"] + assert.True(t, exist) + assert.Equal(t, v, "foobar", "expected password to be moved to securejson") + } + + if ds.Name == "graphite" { + assert.Equal(t, ds.BasicAuthPassword, "") + v, exist := sj["basicAuthPassword"] + assert.True(t, exist) + assert.Equal(t, v, "foobar", "expected basic_auth_password to be moved to securejson") + } + + if ds.Name == "prometheus" { + assert.Equal(t, len(sj), 0) + } + } +} diff --git a/pkg/cmd/grafana-cli/commands/install_command.go b/pkg/cmd/grafana-cli/commands/install_command.go index 99cef15e50e..db390768263 100644 --- a/pkg/cmd/grafana-cli/commands/install_command.go +++ b/pkg/cmd/grafana-cli/commands/install_command.go @@ -14,13 +14,14 @@ import ( "strings" "github.com/fatih/color" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models" s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" ) -func validateInput(c CommandLine, pluginFolder string) error { +func validateInput(c utils.CommandLine, pluginFolder string) error { arg := c.Args().First() if arg == "" { return errors.New("please specify plugin to install") @@ -46,7 +47,7 @@ func validateInput(c CommandLine, pluginFolder string) error { return nil } -func installCommand(c CommandLine) error { +func installCommand(c utils.CommandLine) error { pluginFolder := c.PluginDirectory() if err := validateInput(c, pluginFolder); err != nil { return err @@ -60,7 +61,7 @@ func installCommand(c CommandLine) error { // InstallPlugin downloads the plugin code as a zip file from the Grafana.com API // and then extracts the zip into the plugins directory. -func InstallPlugin(pluginName, version string, c CommandLine) error { +func InstallPlugin(pluginName, version string, c utils.CommandLine) error { pluginFolder := c.PluginDirectory() downloadURL := c.PluginURL() if downloadURL == "" { diff --git a/pkg/cmd/grafana-cli/commands/listremote_command.go b/pkg/cmd/grafana-cli/commands/listremote_command.go index 4798369def1..7351ee58a37 100644 --- a/pkg/cmd/grafana-cli/commands/listremote_command.go +++ b/pkg/cmd/grafana-cli/commands/listremote_command.go @@ -3,9 +3,10 @@ package commands import ( "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" ) -func listremoteCommand(c CommandLine) error { +func listremoteCommand(c utils.CommandLine) error { plugin, err := s.ListAllPlugins(c.RepoDirectory()) if err != nil { diff --git a/pkg/cmd/grafana-cli/commands/listversions_command.go b/pkg/cmd/grafana-cli/commands/listversions_command.go index 95c536e94f0..78d681c06a3 100644 --- a/pkg/cmd/grafana-cli/commands/listversions_command.go +++ b/pkg/cmd/grafana-cli/commands/listversions_command.go @@ -5,9 +5,10 @@ import ( "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" ) -func validateVersionInput(c CommandLine) error { +func validateVersionInput(c utils.CommandLine) error { arg := c.Args().First() if arg == "" { return errors.New("please specify plugin to list versions for") @@ -16,7 +17,7 @@ func validateVersionInput(c CommandLine) error { return nil } -func listversionsCommand(c CommandLine) error { +func listversionsCommand(c utils.CommandLine) error { if err := validateVersionInput(c); err != nil { return err } diff --git a/pkg/cmd/grafana-cli/commands/ls_command.go b/pkg/cmd/grafana-cli/commands/ls_command.go index 30745ce3172..63492d732e9 100644 --- a/pkg/cmd/grafana-cli/commands/ls_command.go +++ b/pkg/cmd/grafana-cli/commands/ls_command.go @@ -8,6 +8,7 @@ import ( "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models" s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" ) var ls_getPlugins func(path string) []m.InstalledPlugin = s.GetLocalPlugins @@ -31,7 +32,7 @@ var validateLsCommand = func(pluginDir string) error { return nil } -func lsCommand(c CommandLine) error { +func lsCommand(c utils.CommandLine) error { pluginDir := c.PluginDirectory() if err := validateLsCommand(pluginDir); err != nil { return err diff --git a/pkg/cmd/grafana-cli/commands/remove_command.go b/pkg/cmd/grafana-cli/commands/remove_command.go index e51929dc95c..eb536d7b8c7 100644 --- a/pkg/cmd/grafana-cli/commands/remove_command.go +++ b/pkg/cmd/grafana-cli/commands/remove_command.go @@ -5,12 +5,13 @@ import ( "fmt" "strings" - services "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" ) var removePlugin func(pluginPath, id string) error = services.RemoveInstalledPlugin -func removeCommand(c CommandLine) error { +func removeCommand(c utils.CommandLine) error { pluginPath := c.PluginDirectory() plugin := c.Args().First() diff --git a/pkg/cmd/grafana-cli/commands/reset_password_command.go b/pkg/cmd/grafana-cli/commands/reset_password_command.go index af2b8b3f89a..4a6a4b674f2 100644 --- a/pkg/cmd/grafana-cli/commands/reset_password_command.go +++ b/pkg/cmd/grafana-cli/commands/reset_password_command.go @@ -6,13 +6,15 @@ import ( "github.com/fatih/color" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/util" ) const AdminUserId = 1 -func resetPasswordCommand(c CommandLine) error { +func resetPasswordCommand(c utils.CommandLine, sqlStore *sqlstore.SqlStore) error { newPassword := c.Args().First() password := models.Password(newPassword) diff --git a/pkg/cmd/grafana-cli/commands/upgrade_all_command.go b/pkg/cmd/grafana-cli/commands/upgrade_all_command.go index e01df2dab60..a5aadbbb0c2 100644 --- a/pkg/cmd/grafana-cli/commands/upgrade_all_command.go +++ b/pkg/cmd/grafana-cli/commands/upgrade_all_command.go @@ -4,6 +4,7 @@ import ( "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models" s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" "github.com/hashicorp/go-version" ) @@ -27,7 +28,7 @@ func ShouldUpgrade(installed string, remote m.Plugin) bool { return false } -func upgradeAllCommand(c CommandLine) error { +func upgradeAllCommand(c utils.CommandLine) error { pluginsDir := c.PluginDirectory() localPlugins := s.GetLocalPlugins(pluginsDir) diff --git a/pkg/cmd/grafana-cli/commands/upgrade_command.go b/pkg/cmd/grafana-cli/commands/upgrade_command.go index 396371d3577..f32961ce589 100644 --- a/pkg/cmd/grafana-cli/commands/upgrade_command.go +++ b/pkg/cmd/grafana-cli/commands/upgrade_command.go @@ -4,9 +4,10 @@ import ( "github.com/fatih/color" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" + "github.com/grafana/grafana/pkg/cmd/grafana-cli/utils" ) -func upgradeCommand(c CommandLine) error { +func upgradeCommand(c utils.CommandLine) error { pluginsDir := c.PluginDirectory() pluginName := c.Args().First() diff --git a/pkg/cmd/grafana-cli/commands/command_line.go b/pkg/cmd/grafana-cli/utils/command_line.go similarity index 64% rename from pkg/cmd/grafana-cli/commands/command_line.go rename to pkg/cmd/grafana-cli/utils/command_line.go index d487aff8aaa..d3142d0f195 100644 --- a/pkg/cmd/grafana-cli/commands/command_line.go +++ b/pkg/cmd/grafana-cli/utils/command_line.go @@ -1,4 +1,4 @@ -package commands +package utils import ( "github.com/codegangsta/cli" @@ -22,30 +22,30 @@ type CommandLine interface { PluginURL() string } -type contextCommandLine struct { +type ContextCommandLine struct { *cli.Context } -func (c *contextCommandLine) ShowHelp() { +func (c *ContextCommandLine) ShowHelp() { cli.ShowCommandHelp(c.Context, c.Command.Name) } -func (c *contextCommandLine) ShowVersion() { +func (c *ContextCommandLine) ShowVersion() { cli.ShowVersion(c.Context) } -func (c *contextCommandLine) Application() *cli.App { +func (c *ContextCommandLine) Application() *cli.App { return c.App } -func (c *contextCommandLine) PluginDirectory() string { +func (c *ContextCommandLine) PluginDirectory() string { return c.GlobalString("pluginsDir") } -func (c *contextCommandLine) RepoDirectory() string { +func (c *ContextCommandLine) RepoDirectory() string { return c.GlobalString("repo") } -func (c *contextCommandLine) PluginURL() string { +func (c *ContextCommandLine) PluginURL() string { return c.GlobalString("pluginUrl") } diff --git a/pkg/util/strings.go b/pkg/util/strings.go index 9eaa141edbf..9ce5d03e126 100644 --- a/pkg/util/strings.go +++ b/pkg/util/strings.go @@ -4,6 +4,7 @@ import ( "fmt" "math" "regexp" + "strings" "time" ) @@ -66,3 +67,19 @@ func GetAgeString(t time.Time) string { return "< 1m" } + +// ToCamelCase changes kebab case, snake case or mixed strings to camel case. See unit test for examples. +func ToCamelCase(str string) string { + var finalParts []string + parts := strings.Split(str, "_") + + for _, part := range parts { + finalParts = append(finalParts, strings.Split(part, "-")...) + } + + for index, part := range finalParts[1:] { + finalParts[index+1] = strings.Title(part) + } + + return strings.Join(finalParts, "") +} diff --git a/pkg/util/strings_test.go b/pkg/util/strings_test.go index 0cc1905baff..4bc52ee7521 100644 --- a/pkg/util/strings_test.go +++ b/pkg/util/strings_test.go @@ -37,3 +37,12 @@ func TestDateAge(t *testing.T) { So(GetAgeString(time.Now().Add(-time.Hour*24*409)), ShouldEqual, "1y") }) } + +func TestToCamelCase(t *testing.T) { + Convey("ToCamelCase", t, func() { + So(ToCamelCase("kebab-case-string"), ShouldEqual, "kebabCaseString") + So(ToCamelCase("snake_case_string"), ShouldEqual, "snakeCaseString") + So(ToCamelCase("mixed-case_string"), ShouldEqual, "mixedCaseString") + So(ToCamelCase("alreadyCamelCase"), ShouldEqual, "alreadyCamelCase") + }) +}