diff --git a/devenv/dashboards/bulk-testing/bulkdash.jsonnet b/devenv/dashboards/bulk-testing/bulkdash.jsonnet index 17b3f8983af..4c82fd36f69 100644 --- a/devenv/dashboards/bulk-testing/bulkdash.jsonnet +++ b/devenv/dashboards/bulk-testing/bulkdash.jsonnet @@ -1137,4 +1137,4 @@ "title": "Big Dashboard", "uid": "000000003", "version": 16 -} \ No newline at end of file +} diff --git a/devenv/setup.sh b/devenv/setup.sh index d6f8f969e75..0a8958131fb 100755 --- a/devenv/setup.sh +++ b/devenv/setup.sh @@ -5,10 +5,10 @@ bulkDashboard() { requiresJsonnet COUNTER=0 - MAX=400 + MAX=4 while [ $COUNTER -lt $MAX ]; do jsonnet -o "dashboards/bulk-testing/dashboard${COUNTER}.json" -e "local bulkDash = import 'dashboards/bulk-testing/bulkdash.jsonnet'; bulkDash + { uid: 'uid-${COUNTER}', title: 'title-${COUNTER}' }" - let COUNTER=COUNTER+1 + let COUNTER=COUNTER+1 done ln -s -f -r ./dashboards/bulk-testing/bulk-dashboards.yaml ../conf/provisioning/dashboards/custom.yaml @@ -58,4 +58,4 @@ main() { fi } -main "$@" \ No newline at end of file +main "$@" diff --git a/pkg/models/dashboards.go b/pkg/models/dashboards.go index eb44c1bc582..4b84d840113 100644 --- a/pkg/models/dashboards.go +++ b/pkg/models/dashboards.go @@ -254,6 +254,7 @@ type DashboardProvisioning struct { DashboardId int64 Name string ExternalId string + CheckSum string Updated int64 } diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index cd4598794bc..e1cb92abc83 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -4,12 +4,14 @@ import ( "context" "errors" "fmt" + "io/ioutil" "os" "path/filepath" "strings" "time" "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/bus" @@ -161,13 +163,18 @@ func (fr *fileReader) saveDashboard(path string, folderId int64, fileInfo os.Fil provisionedData, alreadyProvisioned := provisionedDashboardRefs[path] upToDate := alreadyProvisioned && provisionedData.Updated >= resolvedFileInfo.ModTime().Unix() - dash, err := fr.readDashboardFromFile(path, resolvedFileInfo.ModTime(), folderId) + jsonFile, err := fr.readDashboardFromFile(path, resolvedFileInfo.ModTime(), folderId) if err != nil { fr.log.Error("failed to load dashboard from ", "file", path, "error", err) return provisioningMetadata, nil } + if provisionedData != nil && jsonFile.checkSum == provisionedData.CheckSum { + upToDate = true + } + // keeps track of what uid's and title's we have already provisioned + dash := jsonFile.dashboard provisioningMetadata.uid = dash.Dashboard.Uid provisioningMetadata.title = dash.Dashboard.Title @@ -185,7 +192,13 @@ func (fr *fileReader) saveDashboard(path string, folderId int64, fileInfo os.Fil } fr.log.Debug("saving new dashboard", "file", path) - dp := &models.DashboardProvisioning{ExternalId: path, Name: fr.Cfg.Name, Updated: resolvedFileInfo.ModTime().Unix()} + dp := &models.DashboardProvisioning{ + ExternalId: path, + Name: fr.Cfg.Name, + Updated: resolvedFileInfo.ModTime().Unix(), + CheckSum: jsonFile.checkSum, + } + _, err = fr.dashboardService.SaveProvisionedDashboard(dash, dp) return provisioningMetadata, err } @@ -283,14 +296,30 @@ func validateWalkablePath(fileInfo os.FileInfo) (bool, error) { return true, nil } -func (fr *fileReader) readDashboardFromFile(path string, lastModified time.Time, folderId int64) (*dashboards.SaveDashboardDTO, error) { +type dashboardJsonFile struct { + dashboard *dashboards.SaveDashboardDTO + checkSum string + lastModified time.Time +} + +func (fr *fileReader) readDashboardFromFile(path string, lastModified time.Time, folderId int64) (*dashboardJsonFile, error) { reader, err := os.Open(path) if err != nil { return nil, err } defer reader.Close() - data, err := simplejson.NewFromReader(reader) + all, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + + checkSum, err := util.Md5SumString(string(all)) + if err != nil { + return nil, err + } + + data, err := simplejson.NewJson(all) if err != nil { return nil, err } @@ -300,7 +329,11 @@ func (fr *fileReader) readDashboardFromFile(path string, lastModified time.Time, return nil, err } - return dash, nil + return &dashboardJsonFile{ + dashboard: dash, + checkSum: checkSum, + lastModified: lastModified, + }, nil } type provisioningMetadata struct { @@ -328,7 +361,6 @@ func (checker provisioningSanityChecker) track(pm provisioningMetadata) { if len(pm.title) > 0 { checker.titleUsage[pm.title] += 1 } - } func (checker provisioningSanityChecker) logWarnings(log log.Logger) { @@ -343,5 +375,4 @@ func (checker provisioningSanityChecker) logWarnings(log log.Logger) { log.Error("the same 'title' is used more than once", "title", title, "provider", checker.provisioningProvider) } } - } diff --git a/pkg/services/sqlstore/migrations/dashboard_mig.go b/pkg/services/sqlstore/migrations/dashboard_mig.go index 170498c4bd9..b770afb1b4e 100644 --- a/pkg/services/sqlstore/migrations/dashboard_mig.go +++ b/pkg/services/sqlstore/migrations/dashboard_mig.go @@ -211,4 +211,8 @@ func addDashboardMigration(mg *Migrator) { "name": "name", "external_id": "external_id", }) + + mg.AddMigration("Add check_sum column", NewAddColumnMigration(dashboardExtrasTableV2, &Column{ + Name: "check_sum", Type: DB_NVarchar, Length: 32, Nullable: true, + })) } diff --git a/pkg/util/md5.go b/pkg/util/md5.go new file mode 100644 index 00000000000..2473a1a406c --- /dev/null +++ b/pkg/util/md5.go @@ -0,0 +1,26 @@ +package util + +import ( + "crypto/md5" + "encoding/hex" + "io" + "strings" +) + +// Md5Sum calculates the md5sum of a stream +func Md5Sum(reader io.Reader) (string, error) { + var returnMD5String string + hash := md5.New() + if _, err := io.Copy(hash, reader); err != nil { + return returnMD5String, err + } + hashInBytes := hash.Sum(nil)[:16] + returnMD5String = hex.EncodeToString(hashInBytes) + return returnMD5String, nil +} + +// Md5Sum calculates the md5sum of a string +func Md5SumString(input string) (string, error) { + buffer := strings.NewReader(input) + return Md5Sum(buffer) +} diff --git a/pkg/util/md5_test.go b/pkg/util/md5_test.go new file mode 100644 index 00000000000..1338d42bb51 --- /dev/null +++ b/pkg/util/md5_test.go @@ -0,0 +1,17 @@ +package util + +import "testing" + +func TestMd5Sum(t *testing.T) { + input := "dont hash passwords with md5" + + have, err := Md5SumString(input) + if err != nil { + t.Fatal("expected err to be nil") + } + + want := "2d6a56c82d09d374643b926d3417afba" + if have != want { + t.Fatalf("expected: %s got: %s", want, have) + } +}