IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

CLI: Add command to migrate all datasources to use encrypted password fields (#17118)

Browse Source 
closes: #17107
This commit is contained in:
Andrej Ocenas 2019-05-27 10:47:21 +02:00 committed by Carl Bergquist
parent b9181df212
commit 151b24b95f
14 changed files with 266 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
pkg/cmd/grafana-cli/commands/commands.go
View File

@ -7,14 +7,16 @@ import (
"github.com/codegangsta/cli"
"github.com/fatih/color"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/commands/datamigrations"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
)
func runDbCommand(command func(commandLine CommandLine) error) func(context *cli.Context) {
func runDbCommand(command func(commandLine utils.CommandLine, sqlStore *sqlstore.SqlStore) error) func(context *cli.Context) {
return func(context *cli.Context) {
cmd := &contextCommandLine{context}
cmd := &utils.ContextCommandLine{Context: context}
cfg := setting.NewCfg()
cfg.Load(&setting.CommandLineArgs{
@ -28,7 +30,7 @@ func runDbCommand(command func(commandLine CommandLine) error) func(context *cli
engine.Bus = bus.GetBus()
engine.Init()
if err := command(cmd); err != nil {
if err := command(cmd, engine); err != nil {
logger.Errorf("\n%s: ", color.RedString("Error"))
logger.Errorf("%s\n\n", err)
@ -40,10 +42,10 @@ func runDbCommand(command func(commandLine CommandLine) error) func(context *cli
}
}
func runPluginCommand(command func(commandLine CommandLine) error) func(context *cli.Context) {
func runPluginCommand(command func(commandLine utils.CommandLine) error) func(context *cli.Context) {
return func(context *cli.Context) {
cmd := &contextCommandLine{context}
cmd := &utils.ContextCommandLine{Context: context}
if err := command(cmd); err != nil {
logger.Errorf("\n%s: ", color.RedString("Error"))
logger.Errorf("%s %s\n\n", color.RedString("✗"), err)
@ -107,6 +109,17 @@ var adminCommands = []cli.Command{
},
},
},
{
Name: "data-migration",
Usage: "Runs a script that migrates or cleanups data in your db",
Subcommands: []cli.Command{
{
Name: "encrypt-datasource-passwords",
Usage: "Migrates passwords from unsecured fields to secure_json_data field. Return ok unless there is an error. Safe to execute multiple times.",
Action: runDbCommand(datamigrations.EncryptDatasourcePaswords),
},
},
},
}
var Commands = []cli.Command{

126
pkg/cmd/grafana-cli/commands/datamigrations/encrypt_datasource_passwords.go Normal file
View File

@ -0,0 +1,126 @@
package datamigrations
import (
"context"
"encoding/json"
"github.com/fatih/color"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
"github.com/grafana/grafana/pkg/util/errutil"
)
var (
datasourceTypes = []string{
"mysql",
"influxdb",
"elasticsearch",
"graphite",
"prometheus",
"opentsdb",
}
)
// EncryptDatasourcePaswords migrates un-encrypted secrets on datasources
// to the secureJson Column.
func EncryptDatasourcePaswords(c utils.CommandLine, sqlStore *sqlstore.SqlStore) error {
return sqlStore.WithDbSession(context.Background(), func(session *sqlstore.DBSession) error {
passwordsUpdated, err := migrateColumn(session, "password")
if err != nil {
return err
}
basicAuthUpdated, err := migrateColumn(session, "basic_auth_password")
if err != nil {
return err
}
logger.Info("\n")
if passwordsUpdated > 0 {
logger.Infof("%s Encrypted password field for %d datasources \n", color.GreenString("✔"), passwordsUpdated)
}
if basicAuthUpdated > 0 {
logger.Infof("%s Encrypted basic_auth_password field for %d datasources \n", color.GreenString("✔"), basicAuthUpdated)
}
if passwordsUpdated == 0 && basicAuthUpdated == 0 {
logger.Infof("%s All datasources secrets are allready encrypted\n", color.GreenString("✔"))
}
logger.Info("\n")
logger.Warn("Warning: Datasource provisioning files need to be manually changed to prevent overwriting of " +
"the data during provisioning. See https://grafana.com/docs/installation/upgrading/#upgrading-to-v6-2 for " +
"details")
return nil
})
}
func migrateColumn(session *sqlstore.DBSession, column string) (int, error) {
var rows []map[string]string
session.Cols("id", column, "secure_json_data")
session.Table("data_source")
session.In("type", datasourceTypes)
session.Where(column + " IS NOT NULL AND " + column + " != ''")
err := session.Find(&rows)
if err != nil {
return 0, errutil.Wrapf(err, "failed to select column: %s", column)
}
rowsUpdated, err := updateRows(session, rows, column)
return rowsUpdated, errutil.Wrapf(err, "failed to update column: %s", column)
}
func updateRows(session *sqlstore.DBSession, rows []map[string]string, passwordFieldName string) (int, error) {
var rowsUpdated int
for _, row := range rows {
newSecureJSONData, err := getUpdatedSecureJSONData(row, passwordFieldName)
if err != nil {
return 0, err
}
data, err := json.Marshal(newSecureJSONData)
if err != nil {
return 0, errutil.Wrap("marshaling newSecureJsonData failed", err)
}
newRow := map[string]interface{}{"secure_json_data": data, passwordFieldName: ""}
session.Table("data_source")
session.Where("id = ?", row["id"])
// Setting both columns while having value only for secure_json_data should clear the [passwordFieldName] column
session.Cols("secure_json_data", passwordFieldName)
_, err = session.Update(newRow)
if err != nil {
return 0, err
}
rowsUpdated++
}
return rowsUpdated, nil
}
func getUpdatedSecureJSONData(row map[string]string, passwordFieldName string) (map[string]interface{}, error) {
encryptedPassword, err := util.Encrypt([]byte(row[passwordFieldName]), setting.SecretKey)
if err != nil {
return nil, err
}
var secureJSONData map[string]interface{}
if err := json.Unmarshal([]byte(row["secure_json_data"]), &secureJSONData); err != nil {
return nil, err
}
jsonFieldName := util.ToCamelCase(passwordFieldName)
secureJSONData[jsonFieldName] = encryptedPassword
return secureJSONData, nil
}

67
pkg/cmd/grafana-cli/commands/datamigrations/encrypt_datasource_passwords_test.go Normal file
View File

@ -0,0 +1,67 @@
package datamigrations
import (
"testing"
"time"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/commands/commandstest"
"github.com/grafana/grafana/pkg/components/securejsondata"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/stretchr/testify/assert"
)
func TestPasswordMigrationCommand(t *testing.T) {
//setup datasources with password, basic_auth and none
sqlstore := sqlstore.InitTestDB(t)
session := sqlstore.NewSession()
defer session.Close()
datasources := []*models.DataSource{
{Type: "influxdb", Name: "influxdb", Password: "foobar"},
{Type: "graphite", Name: "graphite", BasicAuthPassword: "foobar"},
{Type: "prometheus", Name: "prometheus", SecureJsonData: securejsondata.GetEncryptedJsonData(map[string]string{})},
}
// set required default values
for _, ds := range datasources {
ds.Created = time.Now()
ds.Updated = time.Now()
ds.SecureJsonData = securejsondata.GetEncryptedJsonData(map[string]string{})
}
_, err := session.Insert(&datasources)
assert.Nil(t, err)
//run migration
err = EncryptDatasourcePaswords(&commandstest.FakeCommandLine{}, sqlstore)
assert.Nil(t, err)
//verify that no datasources still have password or basic_auth
var dss []*models.DataSource
err = session.SQL("select * from data_source").Find(&dss)
assert.Nil(t, err)
assert.Equal(t, len(dss), 3)
for _, ds := range dss {
sj := ds.SecureJsonData.Decrypt()
if ds.Name == "influxdb" {
assert.Equal(t, ds.Password, "")
v, exist := sj["password"]
assert.True(t, exist)
assert.Equal(t, v, "foobar", "expected password to be moved to securejson")
}
if ds.Name == "graphite" {
assert.Equal(t, ds.BasicAuthPassword, "")
v, exist := sj["basicAuthPassword"]
assert.True(t, exist)
assert.Equal(t, v, "foobar", "expected basic_auth_password to be moved to securejson")
}
if ds.Name == "prometheus" {
assert.Equal(t, len(sj), 0)
}
}
}

7
pkg/cmd/grafana-cli/commands/install_command.go
View File

@ -14,13 +14,14 @@ import (
"strings"
"github.com/fatih/color"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models"
s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
)
func validateInput(c CommandLine, pluginFolder string) error {
func validateInput(c utils.CommandLine, pluginFolder string) error {
arg := c.Args().First()
if arg == "" {
return errors.New("please specify plugin to install")
@ -46,7 +47,7 @@ func validateInput(c CommandLine, pluginFolder string) error {
return nil
}
func installCommand(c CommandLine) error {
func installCommand(c utils.CommandLine) error {
pluginFolder := c.PluginDirectory()
if err := validateInput(c, pluginFolder); err != nil {
return err
@ -60,7 +61,7 @@ func installCommand(c CommandLine) error {
// InstallPlugin downloads the plugin code as a zip file from the Grafana.com API
// and then extracts the zip into the plugins directory.
func InstallPlugin(pluginName, version string, c CommandLine) error {
func InstallPlugin(pluginName, version string, c utils.CommandLine) error {
pluginFolder := c.PluginDirectory()
downloadURL := c.PluginURL()
if downloadURL == "" {

3
pkg/cmd/grafana-cli/commands/listremote_command.go
View File

@ -3,9 +3,10 @@ package commands
import (
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
)
func listremoteCommand(c CommandLine) error {
func listremoteCommand(c utils.CommandLine) error {
plugin, err := s.ListAllPlugins(c.RepoDirectory())
if err != nil {

5
pkg/cmd/grafana-cli/commands/listversions_command.go
View File

@ -5,9 +5,10 @@ import (
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
)
func validateVersionInput(c CommandLine) error {
func validateVersionInput(c utils.CommandLine) error {
arg := c.Args().First()
if arg == "" {
return errors.New("please specify plugin to list versions for")
@ -16,7 +17,7 @@ func validateVersionInput(c CommandLine) error {
return nil
}
func listversionsCommand(c CommandLine) error {
func listversionsCommand(c utils.CommandLine) error {
if err := validateVersionInput(c); err != nil {
return err
}

3
pkg/cmd/grafana-cli/commands/ls_command.go
View File

@ -8,6 +8,7 @@ import (
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models"
s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
)
var ls_getPlugins func(path string) []m.InstalledPlugin = s.GetLocalPlugins
@ -31,7 +32,7 @@ var validateLsCommand = func(pluginDir string) error {
return nil
}
func lsCommand(c CommandLine) error {
func lsCommand(c utils.CommandLine) error {
pluginDir := c.PluginDirectory()
if err := validateLsCommand(pluginDir); err != nil {
return err

5
pkg/cmd/grafana-cli/commands/remove_command.go
View File

@ -5,12 +5,13 @@ import (
"fmt"
"strings"
services "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
)
var removePlugin func(pluginPath, id string) error = services.RemoveInstalledPlugin
func removeCommand(c CommandLine) error {
func removeCommand(c utils.CommandLine) error {
pluginPath := c.PluginDirectory()
plugin := c.Args().First()

4
pkg/cmd/grafana-cli/commands/reset_password_command.go
View File

@ -6,13 +6,15 @@ import (
"github.com/fatih/color"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/util"
)
const AdminUserId = 1
func resetPasswordCommand(c CommandLine) error {
func resetPasswordCommand(c utils.CommandLine, sqlStore *sqlstore.SqlStore) error {
newPassword := c.Args().First()
password := models.Password(newPassword)

3
pkg/cmd/grafana-cli/commands/upgrade_all_command.go
View File

@ -4,6 +4,7 @@ import (
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models"
s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
"github.com/hashicorp/go-version"
)
@ -27,7 +28,7 @@ func ShouldUpgrade(installed string, remote m.Plugin) bool {
return false
}
func upgradeAllCommand(c CommandLine) error {
func upgradeAllCommand(c utils.CommandLine) error {
pluginsDir := c.PluginDirectory()
localPlugins := s.GetLocalPlugins(pluginsDir)

3
pkg/cmd/grafana-cli/commands/upgrade_command.go
View File

@ -4,9 +4,10 @@ import (
"github.com/fatih/color"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
)
func upgradeCommand(c CommandLine) error {
func upgradeCommand(c utils.CommandLine) error {
pluginsDir := c.PluginDirectory()
pluginName := c.Args().First()

16
pkg/cmd/grafana-cli/commands/command_line.go → pkg/cmd/grafana-cli/utils/command_line.go
View File

@ -1,4 +1,4 @@
package commands
package utils
import (
"github.com/codegangsta/cli"
@ -22,30 +22,30 @@ type CommandLine interface {
PluginURL() string
}
type contextCommandLine struct {
type ContextCommandLine struct {
*cli.Context
}
func (c *contextCommandLine) ShowHelp() {
func (c *ContextCommandLine) ShowHelp() {
cli.ShowCommandHelp(c.Context, c.Command.Name)
}
func (c *contextCommandLine) ShowVersion() {
func (c *ContextCommandLine) ShowVersion() {
cli.ShowVersion(c.Context)
}
func (c *contextCommandLine) Application() *cli.App {
func (c *ContextCommandLine) Application() *cli.App {
return c.App
}
func (c *contextCommandLine) PluginDirectory() string {
func (c *ContextCommandLine) PluginDirectory() string {
return c.GlobalString("pluginsDir")
}
func (c *contextCommandLine) RepoDirectory() string {
func (c *ContextCommandLine) RepoDirectory() string {
return c.GlobalString("repo")
}
func (c *contextCommandLine) PluginURL() string {
func (c *ContextCommandLine) PluginURL() string {
return c.GlobalString("pluginUrl")
}

17
pkg/util/strings.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"math"
"regexp"
"strings"
"time"
)
@ -66,3 +67,19 @@ func GetAgeString(t time.Time) string {
return "< 1m"
}
// ToCamelCase changes kebab case, snake case or mixed strings to camel case. See unit test for examples.
func ToCamelCase(str string) string {
var finalParts []string
parts := strings.Split(str, "_")
for _, part := range parts {
finalParts = append(finalParts, strings.Split(part, "-")...)
}
for index, part := range finalParts[1:] {
finalParts[index+1] = strings.Title(part)
}
return strings.Join(finalParts, "")
}

9
pkg/util/strings_test.go
View File

@ -37,3 +37,12 @@ func TestDateAge(t *testing.T) {
So(GetAgeString(time.Now().Add(-time.Hour*24*409)), ShouldEqual, "1y")
})
}
func TestToCamelCase(t *testing.T) {
Convey("ToCamelCase", t, func() {
So(ToCamelCase("kebab-case-string"), ShouldEqual, "kebabCaseString")
So(ToCamelCase("snake_case_string"), ShouldEqual, "snakeCaseString")
So(ToCamelCase("mixed-case_string"), ShouldEqual, "mixedCaseString")
So(ToCamelCase("alreadyCamelCase"), ShouldEqual, "alreadyCamelCase")
})
}