History and Version Control for Dashboard Updates

A simple version control system for dashboards. Closes #1504.

Goals

1. To create a new dashboard version every time a dashboard is saved.
2. To allow users to view all versions of a given dashboard.
3. To allow users to rollback to a previous version of a dashboard.
4. To allow users to compare two versions of a dashboard.

Usage

Navigate to a dashboard, and click the settings cog. From there, click
the "Changelog" button to be brought to the Changelog view. In this
view, a table containing each version of a dashboard can be seen. Each
entry in the table represents a dashboard version. A selectable
checkbox, the version number, date created, name of the user who created
that version, and commit message is shown in the table, along with a
button that allows a user to restore to a previous version of that
dashboard. If a user wants to restore to a previous version of their
dashboard, they can do so by clicking the previously mentioned button.
If a user wants to compare two different versions of a dashboard, they
can do so by clicking the checkbox of two different dashboard versions,
then clicking the "Compare versions" button located below the dashboard.
From there, the user is brought to a view showing a summary of the
dashboard differences. Each summarized change contains a link that can
be clicked to take the user a JSON diff highlighting the changes line by
line.

Overview of Changes

Backend Changes

- A `dashboard_version` table was created to store each dashboard
  version, along with a dashboard version model and structs to represent
  the queries and commands necessary for the dashboard version API
  methods.
- API endpoints were created to support working with dashboard
  versions.
- Methods were added to create, update, read, and destroy dashboard
  versions in the database.
  - Logic was added to compute the diff between two versions, and
  display it to the user.
  - The dashboard migration logic was updated to save a "Version
  1" of each existing dashboard in the database.

Frontend Changes

- New views
- Methods to pull JSON and HTML from endpoints

New API Endpoints

Each endpoint requires the authorization header to be sent in
the format,

```
Authorization: Bearer <jwt>
```

where `<jwt>` is a JSON web token obtained from the Grafana
admin panel.

`GET "/api/dashboards/db/:dashboardId/versions?orderBy=<string>&limit=<int>&start=<int>"`

Get all dashboard versions for the given dashboard ID. Accepts
three URL parameters:

- `orderBy` String to order the results by. Possible values
  are `version`, `created`, `created_by`, `message`. Default
  is `versions`. Ordering is always in descending order.
- `limit` Maximum number of results to return
- `start` Position in results to start from

`GET "/api/dashboards/db/:dashboardId/versions/:id"`

Get an individual dashboard version by ID, for the given
dashboard ID.

`POST "/api/dashboards/db/:dashboardId/restore"`

Restore to the given dashboard version. Post body is of
content-type `application/json`, and must contain.

```json
{
  "dashboardId": <int>,
  "version": <int>
}
```

`GET "/api/dashboards/db/:dashboardId/compare/:versionA...:versionB"`

Compare two dashboard versions by ID for the given
dashboard ID, returning a JSON delta formatted
representation of the diff. The URL format follows
what GitHub does. For example, visiting
[/api/dashboards/db/18/compare/22...33](http://ec2-54-80-139-44.compute-1.amazonaws.com:3000/api/dashboards/db/18/compare/22...33)
will return the diff between versions 22 and 33 for
the dashboard ID 18.

Dependencies Added

- The Go package [gojsondiff](https://github.com/yudai/gojsondiff)
  was added and vendored.

pkg/services/sqlstore/dashboard.go

@@ -3,6 +3,7 @@ package sqlstore
 import (
 	"bytes"
 	"fmt"
+	"time"
 
 	"github.com/go-xorm/xorm"
 	"github.com/grafana/grafana/pkg/bus"
@@ -69,17 +70,43 @@ func SaveDashboard(cmd *m.SaveDashboardCommand) error {
 			}
 		}
 
-		affectedRows := int64(0)
+		parentVersion := dash.Version
+		version, err := getMaxVersion(sess, dash.Id)
+		if err != nil {
+			return err
+		}
+		dash.Version = version
 
+		affectedRows := int64(0)
 		if dash.Id == 0 {
 			metrics.M_Models_Dashboard_Insert.Inc(1)
+			dash.Data.Set("version", dash.Version)
 			affectedRows, err = sess.Insert(dash)
 		} else {
-			dash.Version += 1
 			dash.Data.Set("version", dash.Version)
 			affectedRows, err = sess.Id(dash.Id).Update(dash)
 		}
+		if err != nil {
+			return err
+		}
+		if affectedRows == 0 {
+			return m.ErrDashboardNotFound
+		}
 
+		dashVersion := &m.DashboardVersion{
+			DashboardId:   dash.Id,
+			ParentVersion: parentVersion,
+			RestoredFrom:  -1,
+			Version:       dash.Version,
+			Created:       time.Now(),
+			CreatedBy:     dash.UpdatedBy,
+			Message:       cmd.Message,
+			Data:          dash.Data,
+		}
+		affectedRows, err = sess.Insert(dashVersion)
+		if err != nil {
+			return err
+		}
 		if affectedRows == 0 {
 			return m.ErrDashboardNotFound
 		}
@@ -234,6 +261,7 @@ func DeleteDashboard(cmd *m.DeleteDashboardCommand) error {
 			"DELETE FROM star WHERE dashboard_id = ? ",
 			"DELETE FROM dashboard WHERE id = ?",
 			"DELETE FROM playlist_item WHERE type = 'dashboard_by_id' AND value = ?",
+			"DELETE FROM dashboard_version WHERE dashboard_id = ?",
 		}
 
 		for _, sql := range deletes {

pkg/services/sqlstore/dashboard_version.go

@@ -0,0 +1,274 @@
+package sqlstore
+
+import (
+	"encoding/json"
+	"errors"
+	"time"
+
+	"github.com/go-xorm/xorm"
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/components/formatter"
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	m "github.com/grafana/grafana/pkg/models"
+
+	diff "github.com/yudai/gojsondiff"
+	deltaFormatter "github.com/yudai/gojsondiff/formatter"
+)
+
+var (
+	// ErrUnsupportedDiffType occurs when an invalid diff type is used.
+	ErrUnsupportedDiffType = errors.New("sqlstore: unsupported diff type")
+
+	// ErrNilDiff occurs when two compared interfaces are identical.
+	ErrNilDiff = errors.New("sqlstore: diff is nil")
+)
+
+func init() {
+	bus.AddHandler("sql", CompareDashboardVersionsCommand)
+	bus.AddHandler("sql", GetDashboardVersion)
+	bus.AddHandler("sql", GetDashboardVersions)
+	bus.AddHandler("sql", RestoreDashboardVersion)
+}
+
+// CompareDashboardVersionsCommand computes the JSON diff of two versions,
+// assigning the delta of the diff to the `Delta` field.
+func CompareDashboardVersionsCommand(cmd *m.CompareDashboardVersionsCommand) error {
+	original, err := getDashboardVersion(cmd.DashboardId, cmd.Original)
+	if err != nil {
+		return err
+	}
+
+	newDashboard, err := getDashboardVersion(cmd.DashboardId, cmd.New)
+	if err != nil {
+		return err
+	}
+
+	left, jsonDiff, err := getDiff(original, newDashboard)
+	if err != nil {
+		return err
+	}
+
+	switch cmd.DiffType {
+	case m.DiffDelta:
+
+		deltaOutput, err := deltaFormatter.NewDeltaFormatter().Format(jsonDiff)
+		if err != nil {
+			return err
+		}
+		cmd.Delta = []byte(deltaOutput)
+
+	case m.DiffJSON:
+		jsonOutput, err := formatter.NewJSONFormatter(left).Format(jsonDiff)
+		if err != nil {
+			return err
+		}
+		cmd.Delta = []byte(jsonOutput)
+
+	case m.DiffBasic:
+		basicOutput, err := formatter.NewBasicFormatter(left).Format(jsonDiff)
+		if err != nil {
+			return err
+		}
+		cmd.Delta = basicOutput
+
+	default:
+		return ErrUnsupportedDiffType
+	}
+
+	return nil
+}
+
+// GetDashboardVersion gets the dashboard version for the given dashboard ID
+// and version number.
+func GetDashboardVersion(query *m.GetDashboardVersionCommand) error {
+	result, err := getDashboardVersion(query.DashboardId, query.Version)
+	if err != nil {
+		return err
+	}
+
+	query.Result = result
+	return nil
+}
+
+// GetDashboardVersions gets all dashboard versions for the given dashboard ID.
+func GetDashboardVersions(query *m.GetDashboardVersionsCommand) error {
+	order := ""
+
+	// the query builder in xorm doesn't provide a way to set
+	// a default order, so we perform this check
+	if query.OrderBy != "" {
+		order = " desc"
+	}
+	err := x.In("dashboard_id", query.DashboardId).
+		OrderBy(query.OrderBy+order).
+		Limit(query.Limit, query.Start).
+		Find(&query.Result)
+	if err != nil {
+		return err
+	}
+
+	if len(query.Result) < 1 {
+		return m.ErrNoVersionsForDashboardId
+	}
+	return nil
+}
+
+// RestoreDashboardVersion restores the dashboard data to the given version.
+func RestoreDashboardVersion(cmd *m.RestoreDashboardVersionCommand) error {
+	return inTransaction(func(sess *xorm.Session) error {
+		// check if dashboard version exists in dashboard_version table
+		//
+		// normally we could use the getDashboardVersion func here, but since
+		// we're in a transaction, we need to run the queries using the
+		// session instead of using the global `x`, so we copy those functions
+		// here, replacing `x` with `sess`
+		dashboardVersion := m.DashboardVersion{}
+		has, err := sess.Where(
+			"dashboard_id=? AND version=?",
+			cmd.DashboardId,
+			cmd.Version,
+		).Get(&dashboardVersion)
+		if err != nil {
+			return err
+		}
+		if !has {
+			return m.ErrDashboardVersionNotFound
+		}
+		dashboardVersion.Data.Set("id", dashboardVersion.DashboardId)
+
+		// get the dashboard version
+		dashboard := m.Dashboard{Id: cmd.DashboardId}
+		has, err = sess.Get(&dashboard)
+		if err != nil {
+			return err
+		}
+		if has == false {
+			return m.ErrDashboardNotFound
+		}
+
+		version, err := getMaxVersion(sess, dashboard.Id)
+		if err != nil {
+			return err
+		}
+
+		// revert and save to a new dashboard version
+		dashboard.Data = dashboardVersion.Data
+		dashboard.Updated = time.Now()
+		dashboard.UpdatedBy = cmd.UserId
+		dashboard.Version = version
+		dashboard.Data.Set("version", dashboard.Version)
+		affectedRows, err := sess.Id(dashboard.Id).Update(dashboard)
+		if err != nil {
+			return err
+		}
+		if affectedRows == 0 {
+			return m.ErrDashboardNotFound
+		}
+
+		// save that version a new version
+		dashVersion := &m.DashboardVersion{
+			DashboardId:   dashboard.Id,
+			ParentVersion: cmd.Version,
+			RestoredFrom:  cmd.Version,
+			Version:       dashboard.Version,
+			Created:       time.Now(),
+			CreatedBy:     dashboard.UpdatedBy,
+			Message:       "",
+			Data:          dashboard.Data,
+		}
+		affectedRows, err = sess.Insert(dashVersion)
+		if err != nil {
+			return err
+		}
+		if affectedRows == 0 {
+			return m.ErrDashboardNotFound
+		}
+
+		cmd.Result = &dashboard
+		return nil
+	})
+}
+
+// getDashboardVersion is a helper function that gets the dashboard version for
+// the given dashboard ID and version ID.
+func getDashboardVersion(dashboardId int64, version int) (*m.DashboardVersion, error) {
+	dashboardVersion := m.DashboardVersion{}
+	has, err := x.Where("dashboard_id=? AND version=?", dashboardId, version).Get(&dashboardVersion)
+	if err != nil {
+		return nil, err
+	}
+	if !has {
+		return nil, m.ErrDashboardVersionNotFound
+	}
+
+	dashboardVersion.Data.Set("id", dashboardVersion.DashboardId)
+	return &dashboardVersion, nil
+}
+
+// getDashboard gets a dashboard by ID. Used for retrieving the dashboard
+// associated with dashboard versions.
+func getDashboard(dashboardId int64) (*m.Dashboard, error) {
+	dashboard := m.Dashboard{Id: dashboardId}
+	has, err := x.Get(&dashboard)
+	if err != nil {
+		return nil, err
+	}
+	if has == false {
+		return nil, m.ErrDashboardNotFound
+	}
+	return &dashboard, nil
+}
+
+// getDiff computes the diff of two dashboard versions.
+func getDiff(originalDash, newDash *m.DashboardVersion) (interface{}, diff.Diff, error) {
+	leftBytes, err := simplejson.NewFromAny(originalDash).Encode()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	rightBytes, err := simplejson.NewFromAny(newDash).Encode()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	jsonDiff, err := diff.New().Compare(leftBytes, rightBytes)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if !jsonDiff.Modified() {
+		return nil, nil, ErrNilDiff
+	}
+
+	left := make(map[string]interface{})
+	err = json.Unmarshal(leftBytes, &left)
+	return left, jsonDiff, nil
+}
+
+type version struct {
+	Max int
+}
+
+// getMaxVersion returns the highest version number in the `dashboard_version`
+// table.
+//
+// This is necessary because sqlite3 doesn't support autoincrement in the same
+// way that Postgres or MySQL do, so we use this to get around that. Since it's
+// impossible to delete a version in Grafana, this is believed to be a
+// safe-enough alternative.
+func getMaxVersion(sess *xorm.Session, dashboardId int64) (int, error) {
+	v := version{}
+	has, err := sess.Table("dashboard_version").
+		Select("MAX(version) AS max").
+		Where("dashboard_id = ?", dashboardId).
+		Get(&v)
+	if !has {
+		return 0, m.ErrDashboardNotFound
+	}
+	if err != nil {
+		return 0, err
+	}
+
+	v.Max++
+	return v.Max, nil
+}

pkg/services/sqlstore/dashboard_version_test.go

@@ -0,0 +1,188 @@
+package sqlstore
+
+import (
+	"reflect"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+
+	"github.com/grafana/grafana/pkg/components/simplejson"
+	m "github.com/grafana/grafana/pkg/models"
+)
+
+func updateTestDashboard(dashboard *m.Dashboard, data map[string]interface{}) {
+	data["title"] = dashboard.Title
+
+	saveCmd := m.SaveDashboardCommand{
+		OrgId:     dashboard.OrgId,
+		Overwrite: true,
+		Dashboard: simplejson.NewFromAny(data),
+	}
+
+	err := SaveDashboard(&saveCmd)
+	So(err, ShouldBeNil)
+}
+
+func TestGetDashboardVersion(t *testing.T) {
+	Convey("Testing dashboard version retrieval", t, func() {
+		InitTestDB(t)
+
+		Convey("Get a Dashboard ID and version ID", func() {
+			savedDash := insertTestDashboard("test dash 26", 1, "diff")
+
+			cmd := m.GetDashboardVersionCommand{
+				DashboardId: savedDash.Id,
+				Version:     savedDash.Version,
+			}
+
+			err := GetDashboardVersion(&cmd)
+			So(err, ShouldBeNil)
+			So(savedDash.Id, ShouldEqual, cmd.DashboardId)
+			So(savedDash.Version, ShouldEqual, cmd.Version)
+
+			dashCmd := m.GetDashboardQuery{
+				OrgId: savedDash.OrgId,
+				Slug:  savedDash.Slug,
+			}
+			err = GetDashboard(&dashCmd)
+			So(err, ShouldBeNil)
+			eq := reflect.DeepEqual(dashCmd.Result.Data, cmd.Result.Data)
+			So(eq, ShouldEqual, true)
+		})
+
+		Convey("Attempt to get a version that doesn't exist", func() {
+			cmd := m.GetDashboardVersionCommand{
+				DashboardId: int64(999),
+				Version:     123,
+			}
+
+			err := GetDashboardVersion(&cmd)
+			So(err, ShouldNotBeNil)
+			So(err, ShouldEqual, m.ErrDashboardVersionNotFound)
+		})
+	})
+}
+
+func TestGetDashboardVersions(t *testing.T) {
+	Convey("Testing dashboard versions retrieval", t, func() {
+		InitTestDB(t)
+		savedDash := insertTestDashboard("test dash 43", 1, "diff-all")
+
+		Convey("Get all versions for a given Dashboard ID", func() {
+			cmd := m.GetDashboardVersionsCommand{
+				DashboardId: savedDash.Id,
+			}
+
+			err := GetDashboardVersions(&cmd)
+			So(err, ShouldBeNil)
+			So(len(cmd.Result), ShouldEqual, 1)
+		})
+
+		Convey("Attempt to get the versions for a non-existent Dashboard ID", func() {
+			cmd := m.GetDashboardVersionsCommand{
+				DashboardId: int64(999),
+			}
+
+			err := GetDashboardVersions(&cmd)
+			So(err, ShouldNotBeNil)
+			So(err, ShouldEqual, m.ErrNoVersionsForDashboardId)
+			So(len(cmd.Result), ShouldEqual, 0)
+		})
+
+		Convey("Get all versions for an updated dashboard", func() {
+			updateTestDashboard(savedDash, map[string]interface{}{
+				"tags": "different-tag",
+			})
+
+			cmd := m.GetDashboardVersionsCommand{
+				DashboardId: savedDash.Id,
+			}
+			err := GetDashboardVersions(&cmd)
+			So(err, ShouldBeNil)
+			So(len(cmd.Result), ShouldEqual, 2)
+		})
+	})
+}
+
+func TestCompareDashboardVersions(t *testing.T) {
+	Convey("Testing dashboard version comparison", t, func() {
+		InitTestDB(t)
+
+		savedDash := insertTestDashboard("test dash 43", 1, "x")
+		updateTestDashboard(savedDash, map[string]interface{}{
+			"tags": "y",
+		})
+
+		Convey("Compare two versions that are different", func() {
+			getVersionCmd := m.GetDashboardVersionsCommand{
+				DashboardId: savedDash.Id,
+			}
+			err := GetDashboardVersions(&getVersionCmd)
+			So(err, ShouldBeNil)
+			So(len(getVersionCmd.Result), ShouldEqual, 2)
+
+			cmd := m.CompareDashboardVersionsCommand{
+				DashboardId: savedDash.Id,
+				Original:    getVersionCmd.Result[0].Version,
+				New:         getVersionCmd.Result[1].Version,
+				DiffType:    m.DiffDelta,
+			}
+			err = CompareDashboardVersionsCommand(&cmd)
+			So(err, ShouldBeNil)
+			So(cmd.Delta, ShouldNotBeNil)
+		})
+
+		Convey("Compare two versions that are the same", func() {
+			cmd := m.CompareDashboardVersionsCommand{
+				DashboardId: savedDash.Id,
+				Original:    savedDash.Version,
+				New:         savedDash.Version,
+				DiffType:    m.DiffDelta,
+			}
+
+			err := CompareDashboardVersionsCommand(&cmd)
+			So(err, ShouldNotBeNil)
+			So(cmd.Delta, ShouldBeNil)
+		})
+
+		Convey("Compare two versions that don't exist", func() {
+			cmd := m.CompareDashboardVersionsCommand{
+				DashboardId: savedDash.Id,
+				Original:    123,
+				New:         456,
+				DiffType:    m.DiffDelta,
+			}
+
+			err := CompareDashboardVersionsCommand(&cmd)
+			So(err, ShouldNotBeNil)
+			So(cmd.Delta, ShouldBeNil)
+		})
+	})
+}
+
+func TestRestoreDashboardVersion(t *testing.T) {
+	Convey("Testing dashboard version restoration", t, func() {
+		InitTestDB(t)
+		savedDash := insertTestDashboard("test dash 26", 1, "restore")
+		updateTestDashboard(savedDash, map[string]interface{}{
+			"tags": "not restore",
+		})
+
+		Convey("Restore dashboard to a previous version", func() {
+			versionsCmd := m.GetDashboardVersionsCommand{
+				DashboardId: savedDash.Id,
+			}
+			err := GetDashboardVersions(&versionsCmd)
+			So(err, ShouldBeNil)
+
+			cmd := m.RestoreDashboardVersionCommand{
+				DashboardId: savedDash.Id,
+				Version:     savedDash.Version,
+				UserId:      0,
+			}
+
+			err = RestoreDashboardVersion(&cmd)
+			So(err, ShouldBeNil)
+		})
+	})
+}

pkg/services/sqlstore/migrations/dashboard_version_mig.go

@@ -0,0 +1,54 @@
+package migrations
+
+import . "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
+
+func addDashboardVersionMigration(mg *Migrator) {
+	dashboardVersionV1 := Table{
+		Name: "dashboard_version",
+		Columns: []*Column{
+			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
+			{Name: "dashboard_id", Type: DB_BigInt},
+			{Name: "parent_version", Type: DB_Int, Nullable: false},
+			{Name: "restored_from", Type: DB_Int, Nullable: false},
+			{Name: "version", Type: DB_Int, Nullable: false},
+			{Name: "created", Type: DB_DateTime, Nullable: false},
+			{Name: "created_by", Type: DB_BigInt, Nullable: false},
+			{Name: "message", Type: DB_Text, Nullable: false},
+			{Name: "data", Type: DB_Text, Nullable: false},
+		},
+		Indices: []*Index{
+			{Cols: []string{"dashboard_id"}},
+			{Cols: []string{"dashboard_id", "version"}, Type: UniqueIndex},
+		},
+	}
+
+	mg.AddMigration("create dashboard_version table v1", NewAddTableMigration(dashboardVersionV1))
+	mg.AddMigration("add index dashboard_version.dashboard_id", NewAddIndexMigration(dashboardVersionV1, dashboardVersionV1.Indices[0]))
+	mg.AddMigration("add unique index dashboard_version.dashboard_id and dashboard_version.version", NewAddIndexMigration(dashboardVersionV1, dashboardVersionV1.Indices[1]))
+
+	const rawSQL = `INSERT INTO dashboard_version
+(
+	dashboard_id,
+	version,
+	parent_version,
+	restored_from,
+	created,
+	created_by,
+	message,
+	data
+)
+SELECT
+	dashboard.id,
+	dashboard.version + 1,
+	dashboard.version,
+	dashboard.version,
+	dashboard.updated,
+	dashboard.updated_by,
+	'',
+	dashboard.data
+FROM dashboard;`
+	mg.AddMigration("save existing dashboard data in dashboard_version table v1", new(RawSqlMigration).
+		Sqlite(rawSQL).
+		Postgres(rawSQL).
+		Mysql(rawSQL))
+}

pkg/services/sqlstore/migrations/migrations.go

@@ -25,6 +25,7 @@ func AddMigrations(mg *Migrator) {
 	addAlertMigrations(mg)
 	addAnnotationMig(mg)
 	addTestDataMigrations(mg)
+	addDashboardVersionMigration(mg)
 }
 
 func addMigrationLogMigrations(mg *Migrator) {