history from SQL query

pkg/registry/apis/dashboard/access/README.md

@@ -0,0 +1,15 @@
+This implements a ResourceServer backed by the existing dashboard SQL tables.
+
+There are a few oddities worth noting.  This is not a totally accurate implementation,
+but it is good enough to drive the UI needs and let kubectl list work!
+
+1. The resourceVersion is the dashboard version
+  - each resource starts at 1 and increases
+  - there are duplicate resourceVersions!
+  - the resourceVersion is never set on the list commands
+
+1. Results are always sorted by internal id ascending
+  - this ensures everything is returned
+
+1. The history objects have createdTimestamp == updatedTimestamp
+  - not real, but good enough

pkg/registry/apis/dashboard/access/sql_dashboards.go

@@ -19,7 +19,6 @@ import (
 	"github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
 	gapiutil "github.com/grafana/grafana/pkg/services/apiserver/utils"
 	"github.com/grafana/grafana/pkg/services/dashboards"
-	dashver "github.com/grafana/grafana/pkg/services/dashboardversion"
 	"github.com/grafana/grafana/pkg/services/provisioning"
 	"github.com/grafana/grafana/pkg/services/sqlstore/session"
 	"github.com/grafana/grafana/pkg/storage/unified/resource"
@@ -30,15 +29,12 @@ var (
 )
 
 type dashboardRow struct {
-	// The numeric resource version for this dashboard
-	ResourceVersion int64
+	// The numeric version for this dashboard
+	Version int64
 
 	// Dashboard resource
 	Dash *dashboardsV0.Dashboard
 
-	// Title -- this may come from saved metadata rather than the body
-	Title string
-
 	// The folder UID (needed for access control checks)
 	FolderUID string
 
@@ -56,7 +52,6 @@ type dashboardSqlAccess struct {
 	namespacer   request.NamespaceMapper
 	dashStore    dashboards.Store
 	provisioning provisioning.ProvisioningService
-	versions     dashver.Service
 
 	// Typically one... the server wrapper
 	subscribers []chan *resource.WrittenEvent
@@ -67,21 +62,19 @@ func NewDashboardAccess(sql db.DB,
 	namespacer request.NamespaceMapper,
 	dashStore dashboards.Store,
 	provisioning provisioning.ProvisioningService,
-	versions dashver.Service) DashboardAccess {
+) DashboardAccess {
 	return &dashboardSqlAccess{
 		sql:          sql,
 		sess:         sql.GetSqlxSession(),
 		namespacer:   namespacer,
 		dashStore:    dashStore,
 		provisioning: provisioning,
-		versions:     versions,
 	}
 }
 
 const selector = `SELECT
 	dashboard.org_id, dashboard.id,
-	dashboard.uid,slug,
-	dashboard.folder_uid,
+	dashboard.uid, dashboard.folder_uid,
 	dashboard.created,dashboard.created_by,CreatedUSER.uid as created_uid,
 	dashboard.updated,dashboard.updated_by,UpdatedUSER.uid as updated_uid,
 	plugin_id,
@@ -89,14 +82,29 @@ const selector = `SELECT
 	dashboard_provisioning.external_id as origin_path,
 	dashboard_provisioning.check_sum as origin_key,
 	dashboard_provisioning.updated as origin_ts,
-	dashboard.version,
-	title,
-	dashboard.data
+	dashboard.version, '', dashboard.data
   FROM dashboard
   LEFT OUTER JOIN dashboard_provisioning ON dashboard.id = dashboard_provisioning.dashboard_id
   LEFT OUTER JOIN user AS CreatedUSER ON dashboard.created_by = CreatedUSER.id
   LEFT OUTER JOIN user AS UpdatedUSER ON dashboard.created_by = UpdatedUSER.id
-  WHERE is_folder = false`
+  WHERE dashboard.is_folder = false`
+
+const history = `SELECT
+	dashboard.org_id, dashboard.id,
+	dashboard.uid, dashboard.folder_uid,
+	dashboard_version.created,dashboard_version.created_by,CreatedUSER.uid as created_uid,
+	dashboard_version.created,dashboard_version.created_by,CreatedUSER.uid as updated_uid,
+	plugin_id,
+	dashboard_provisioning.name as origin_name,
+	dashboard_provisioning.external_id as origin_path,
+	dashboard_provisioning.check_sum as origin_key,
+	dashboard_provisioning.updated as origin_ts,
+	dashboard_version.version, dashboard_version.message, dashboard_version.data
+  FROM dashboard
+  LEFT OUTER JOIN dashboard_provisioning ON dashboard.id = dashboard_provisioning.dashboard_id
+  LEFT OUTER JOIN dashboard_version  ON dashboard.id = dashboard_version.dashboard_id
+  LEFT OUTER JOIN user AS CreatedUSER ON dashboard_version.created_by = CreatedUSER.id
+  WHERE dashboard.is_folder = false`
 
 func (a *dashboardSqlAccess) getRows(ctx context.Context, query *DashboardQuery) (*rowsWrapper, int, error) {
 	if len(query.Labels) > 0 {
@@ -107,24 +115,50 @@ func (a *dashboardSqlAccess) getRows(ctx context.Context, query *DashboardQuery)
 		// }
 	}
 
+	sqlcmd := selector
 	args := []any{query.OrgID}
-	sqlcmd := fmt.Sprintf("%s AND dashboard.org_id=$%d", selector, len(args))
 
 	limit := query.Limit
 	if limit < 1 {
 		limit = 15 //
 	}
 
-	if query.UID != "" {
+	if query.FromHistory {
+		sqlcmd = fmt.Sprintf("%s AND dashboard.org_id=$%d\n  ", history, len(args))
+
+		if query.UID == "" {
+			return nil, 0, fmt.Errorf("history query must have a UID")
+		}
+
 		args = append(args, query.UID)
 		sqlcmd = fmt.Sprintf("%s AND dashboard.uid=$%d", sqlcmd, len(args))
-	} else if query.MinID > 0 {
-		args = append(args, query.MinID)
-		sqlcmd = fmt.Sprintf("%s AND dashboard.id>=$%d", sqlcmd, len(args))
+
+		if query.Version > 0 {
+			args = append(args, query.Version)
+			sqlcmd = fmt.Sprintf("%s AND dashboard_version.version=$%d", sqlcmd, len(args))
+		} else if query.MinID > 0 {
+			args = append(args, query.MinID)
+			sqlcmd = fmt.Sprintf("%s AND dashboard_version.version<$%d", sqlcmd, len(args))
+		}
+
+		args = append(args, (limit + 2)) // add more so we can include a next token
+		sqlcmd = fmt.Sprintf("%s\n   ORDER BY dashboard_version.version desc LIMIT $%d", sqlcmd, len(args))
+	} else {
+		sqlcmd = fmt.Sprintf("%s AND dashboard.org_id=$%d\n  ", selector, len(args))
+
+		if query.UID != "" {
+			args = append(args, query.UID)
+			sqlcmd = fmt.Sprintf("%s AND dashboard.uid=$%d", sqlcmd, len(args))
+		} else if query.MinID > 0 {
+			args = append(args, query.MinID)
+			sqlcmd = fmt.Sprintf("%s AND dashboard.id>=$%d", sqlcmd, len(args))
+		}
+
+		args = append(args, (limit + 2)) // add more so we can include a next token
+		sqlcmd = fmt.Sprintf("%s\n   ORDER BY dashboard.id asc LIMIT $%d", sqlcmd, len(args))
 	}
 
-	args = append(args, (limit + 2)) // add more so we can include a next token
-	sqlcmd = fmt.Sprintf("%s ORDER BY dashboard.id asc LIMIT $%d", sqlcmd, len(args))
+	fmt.Printf("%s // %v\n", sqlcmd, args)
 
 	rows, err := a.doQuery(ctx, sqlcmd, args...)
 	if err != nil {
@@ -199,7 +233,6 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows) (*dashboardRow, error) {
 
 	var dashboard_id int64
 	var orgId int64
-	var slug string
 	var folder_uid sql.NullString
 	var updated time.Time
 	var updatedByID int64
@@ -208,6 +241,7 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows) (*dashboardRow, error) {
 	var created time.Time
 	var createdByID int64
 	var createdByUID sql.NullString
+	var message sql.NullString
 
 	var plugin_id string
 	var origin_name sql.NullString
@@ -217,20 +251,18 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows) (*dashboardRow, error) {
 	var data []byte // the dashboard JSON
 	var version int64
 
-	err := rows.Scan(&orgId, &dashboard_id, &dash.Name,
-		&slug, &folder_uid,
+	err := rows.Scan(&orgId, &dashboard_id, &dash.Name, &folder_uid,
 		&created, &createdByID, &createdByUID,
 		&updated, &updatedByID, &updatedByUID,
 		&plugin_id,
 		&origin_name, &origin_path, &origin_hash, &origin_ts,
-		&version,
-		&row.Title, &data,
+		&version, &message, &data,
 	)
 
 	row.token = &continueToken{orgId: orgId, id: dashboard_id}
 	if err == nil {
-		row.ResourceVersion = updated.UnixNano() + version
-		dash.ResourceVersion = fmt.Sprintf("%d", row.ResourceVersion)
+		row.Version = version
+		dash.ResourceVersion = fmt.Sprintf("%d", version)
 		dash.Namespace = a.namespacer(orgId)
 		dash.UID = gapiutil.CalculateClusterWideUID(dash)
 		dash.SetCreationTimestamp(metav1.NewTime(created))
@@ -239,7 +271,6 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows) (*dashboardRow, error) {
 			return nil, err
 		}
 		meta.SetUpdatedTimestamp(&updated)
-		meta.SetSlug(slug)
 		if createdByID > 0 {
 			meta.SetCreatedBy(identity.NewNamespaceID(identity.NamespaceUser, createdByID).String())
 		} else if createdByID < 0 {
@@ -250,6 +281,9 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows) (*dashboardRow, error) {
 		} else if updatedByID < 0 {
 			meta.SetCreatedBy(identity.NewNamespaceID(identity.NamespaceProvisioning, 0).String())
 		}
+		if message.Valid && message.String != "" {
+			meta.SetMessage(message.String)
+		}
 		if folder_uid.Valid {
 			meta.SetFolder(folder_uid.String)
 			row.FolderUID = folder_uid.String
@@ -287,7 +321,6 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows) (*dashboardRow, error) {
 				return row, err
 			}
 			dash.Spec.Set("id", dashboard_id) // add it so we can get it from the body later
-			row.Title = dash.Spec.GetNestedString("title")
 		}
 	}
 	return row, err

pkg/registry/apis/dashboard/access/storage.go

@@ -11,11 +11,10 @@ import (
 	"github.com/grafana/grafana/pkg/apimachinery/utils"
 	dashboard "github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1"
 	"github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
-	dashver "github.com/grafana/grafana/pkg/services/dashboardversion"
 	"github.com/grafana/grafana/pkg/storage/unified/resource"
 )
 
-func getDashbaordFromEvent(event resource.WriteEvent) (*dashboard.Dashboard, error) {
+func getDashboardFromEvent(event resource.WriteEvent) (*dashboard.Dashboard, error) {
 	obj, ok := event.Object.GetRuntimeObject()
 	if ok && obj != nil {
 		dash, ok := obj.(*dashboard.Dashboard)
@@ -60,7 +59,7 @@ func (a *dashboardSqlAccess) WriteEvent(ctx context.Context, event resource.Writ
 	// The difference depends on embedded internal ID
 	case resource.WatchEvent_ADDED, resource.WatchEvent_MODIFIED:
 		{
-			dash, err := getDashbaordFromEvent(event)
+			dash, err := getDashboardFromEvent(event)
 			if err != nil {
 				return 0, err
 			}
@@ -118,7 +117,7 @@ func (a *dashboardSqlAccess) GetDashboard(ctx context.Context, orgId int64, uid
 		return nil, 0, err
 	}
 
-	return row.Dash, row.ResourceVersion, nil
+	return row.Dash, row.Version, nil
 }
 
 // Read implements ResourceStoreServer.
@@ -201,7 +200,7 @@ func (a *dashboardSqlAccess) List(ctx context.Context, req *resource.ListRequest
 			return list, err
 		}
 		list.Items = append(list.Items, &resource.ResourceWrapper{
-			ResourceVersion: row.ResourceVersion,
+			ResourceVersion: row.Version,
 			Value:           val,
 		})
 	}
@@ -267,46 +266,75 @@ func (a *dashboardSqlAccess) GetBlob(ctx context.Context, key *resource.Resource
 }
 
 func (a *dashboardSqlAccess) History(ctx context.Context, req *resource.HistoryRequest) (*resource.HistoryResponse, error) {
-	ns, err := request.ParseNamespace(req.Key.Namespace)
+	info, err := request.ParseNamespace(req.Key.Namespace)
 	if err == nil {
-		err = isDashboardKey(req.Key, true)
+		err = isDashboardKey(req.Key, false)
 	}
 	if err != nil {
 		return nil, err
 	}
-	versions, err := a.versions.List(ctx, &dashver.ListDashboardVersionsQuery{
-		OrgID:        ns.OrgID,
-		DashboardUID: req.Key.Name,
-		Limit:        100,
-	})
+
+	token, err := readContinueToken(req.NextPageToken)
 	if err != nil {
 		return nil, err
 	}
+	if token.orgId > 0 && token.orgId != info.OrgID {
+		return nil, fmt.Errorf("token and orgID mismatch")
+	}
 
-	rsp := &resource.HistoryResponse{}
-	for _, version := range versions {
-		partial := &metav1.PartialObjectMetadata{}
-		meta, err := utils.MetaAccessor(partial)
-		if err != nil {
-			return nil, err
-		}
-		meta.SetName(version.DashboardUID)
-		meta.SetCreationTimestamp(metav1.NewTime(version.Created)) // ???
-		meta.SetUpdatedTimestampMillis(version.Created.UnixMilli())
-		meta.SetMessage(version.Message)
-		meta.SetResourceVersionInt64(version.Created.UnixMilli())
+	query := &DashboardQuery{
+		OrgID:       info.OrgID,
+		Limit:       int(req.Limit),
+		MaxBytes:    2 * 1024 * 1024, // 2MB,
+		MinID:       token.id,
+		FromHistory: true,
+		UID:         req.Key.Name,
+	}
 
-		bytes, err := json.Marshal(partial)
-		if err != nil {
-			return nil, err
+	rows, limit, err := a.getRows(ctx, query)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = rows.Close() }()
+
+	totalSize := 0
+	list := &resource.HistoryResponse{}
+	for {
+		row, err := rows.Next()
+		if err != nil || row == nil {
+			return list, err
 		}
-		rsp.Items = append(rsp.Items, &resource.ResourceMeta{
-			ResourceVersion:   version.Created.UnixMilli(),
-			PartialObjectMeta: bytes,
+
+		totalSize += row.Bytes
+		if len(list.Items) > 0 && (totalSize > query.MaxBytes || len(list.Items) >= limit) {
+			// if query.Requirements.Folder != nil {
+			// 	row.token.folder = *query.Requirements.Folder
+			// }
+			row.token.id = row.Version              // Use the version as the increment
+			list.NextPageToken = row.token.String() // will skip this one but start here next time
+			return list, err
+		}
+
+		partial := &metav1.PartialObjectMetadata{
+			ObjectMeta: row.Dash.ObjectMeta,
+		}
+		partial.UID = "" // it is not useful/helpful/accurate and just confusing now
+
+		val, err := json.Marshal(partial)
+		if err != nil {
+			return list, err
+		}
+		full, err := json.Marshal(row.Dash.Spec)
+		if err != nil {
+			return list, err
+		}
+		list.Items = append(list.Items, &resource.ResourceMeta{
+			ResourceVersion:   row.Version,
+			PartialObjectMeta: val,
+			Size:              int32(len(full)),
+			Hash:              "??", // hash the full?
 		})
 	}
-	return rsp, err
-
 }
 
 // Used for efficient provisioning

pkg/registry/apis/dashboard/access/types.go

@@ -16,6 +16,9 @@ type DashboardQuery struct {
 	MaxBytes int
 	MinID    int64 // from continue token
 
+	FromHistory bool
+	Version     int64
+
 	// The label requirements
 	Labels []*resource.Requirement
 }

pkg/registry/apis/dashboard/register.go

@@ -15,7 +15,6 @@ import (
 	common "k8s.io/kube-openapi/pkg/common"
 	"k8s.io/kube-openapi/pkg/spec3"
 
-	"github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1"
 	dashboard "github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1"
 	"github.com/grafana/grafana/pkg/apiserver/builder"
 	grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
@@ -27,7 +26,6 @@ import (
 	"github.com/grafana/grafana/pkg/services/apiserver/endpoints/request"
 	gapiutil "github.com/grafana/grafana/pkg/services/apiserver/utils"
 	"github.com/grafana/grafana/pkg/services/dashboards"
-	dashver "github.com/grafana/grafana/pkg/services/dashboardversion"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/provisioning"
 	"github.com/grafana/grafana/pkg/setting"
@@ -48,7 +46,6 @@ type DashboardsAPIBuilder struct {
 func RegisterAPIService(cfg *setting.Cfg, features featuremgmt.FeatureToggles,
 	apiregistration builder.APIRegistrar,
 	dashboardService dashboards.DashboardService,
-	dashboardVersionService dashver.Service,
 	accessControl accesscontrol.AccessControl,
 	provisioning provisioning.ProvisioningService,
 	dashStore dashboards.Store,
@@ -69,7 +66,7 @@ func RegisterAPIService(cfg *setting.Cfg, features featuremgmt.FeatureToggles,
 
 		store: &dashboardStorage{
 			resource: dashboard.DashboardResourceInfo,
-			access:   access.NewDashboardAccess(sql, namespacer, dashStore, provisioning, dashboardVersionService),
+			access:   access.NewDashboardAccess(sql, namespacer, dashStore, provisioning),
 			tableConverter: gapiutil.NewTableConverter(
 				dashboard.DashboardResourceInfo.GroupResource(),
 				[]metav1.TableColumnDefinition{
@@ -78,7 +75,7 @@ func RegisterAPIService(cfg *setting.Cfg, features featuremgmt.FeatureToggles,
 					{Name: "Created At", Type: "date"},
 				},
 				func(obj any) ([]interface{}, error) {
-					dash, ok := obj.(*v0alpha1.Dashboard)
+					dash, ok := obj.(*dashboard.Dashboard)
 					if ok {
 						if dash != nil {
 							return []interface{}{
@@ -97,7 +94,7 @@ func RegisterAPIService(cfg *setting.Cfg, features featuremgmt.FeatureToggles,
 }
 
 func (b *DashboardsAPIBuilder) GetGroupVersion() schema.GroupVersion {
-	return v0alpha1.DashboardResourceInfo.GroupVersion()
+	return dashboard.DashboardResourceInfo.GroupVersion()
 }
 
 func (b *DashboardsAPIBuilder) GetDesiredDualWriterMode(dualWrite bool, modeMap map[string]grafanarest.DualWriterMode) grafanarest.DualWriterMode {
@@ -107,18 +104,18 @@ func (b *DashboardsAPIBuilder) GetDesiredDualWriterMode(dualWrite bool, modeMap
 
 func addKnownTypes(scheme *runtime.Scheme, gv schema.GroupVersion) {
 	scheme.AddKnownTypes(gv,
-		&v0alpha1.Dashboard{},
-		&v0alpha1.DashboardList{},
-		&v0alpha1.DashboardWithAccessInfo{},
-		&v0alpha1.DashboardVersionList{},
-		&v0alpha1.VersionsQueryOptions{},
+		&dashboard.Dashboard{},
+		&dashboard.DashboardList{},
+		&dashboard.DashboardWithAccessInfo{},
+		&dashboard.DashboardVersionList{},
+		&dashboard.VersionsQueryOptions{},
 		&metav1.PartialObjectMetadata{},
 		&metav1.PartialObjectMetadataList{},
 	)
 }
 
 func (b *DashboardsAPIBuilder) InstallSchema(scheme *runtime.Scheme) error {
-	resourceInfo := v0alpha1.DashboardResourceInfo
+	resourceInfo := dashboard.DashboardResourceInfo
 	addKnownTypes(scheme, resourceInfo.GroupVersion())
 
 	// Link this version to the internal representation.
@@ -144,7 +141,7 @@ func (b *DashboardsAPIBuilder) GetAPIGroupInfo(
 	desiredMode grafanarest.DualWriterMode,
 	reg prometheus.Registerer,
 ) (*genericapiserver.APIGroupInfo, error) {
-	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(v0alpha1.GROUP, scheme, metav1.ParameterCodec, codecs)
+	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(dashboard.GROUP, scheme, metav1.ParameterCodec, codecs)
 
 	dash := b.store.resource
 	legacyStore, err := b.store.newStore(scheme, optsGetter)
@@ -174,12 +171,12 @@ func (b *DashboardsAPIBuilder) GetAPIGroupInfo(
 	// 		reg)
 	// }
 
-	apiGroupInfo.VersionedResourcesStorageMap[v0alpha1.VERSION] = storage
+	apiGroupInfo.VersionedResourcesStorageMap[dashboard.VERSION] = storage
 	return &apiGroupInfo, nil
 }
 
 func (b *DashboardsAPIBuilder) GetOpenAPIDefinitions() common.GetOpenAPIDefinitions {
-	return v0alpha1.GetOpenAPIDefinitions
+	return dashboard.GetOpenAPIDefinitions
 }
 
 func (b *DashboardsAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.OpenAPI, error) {
@@ -190,8 +187,8 @@ func (b *DashboardsAPIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.Op
 	root := "/apis/" + b.GetGroupVersion().String() + "/"
 
 	// Hide the ability to list or watch across all tenants
-	delete(oas.Paths.Paths, root+v0alpha1.DashboardResourceInfo.GroupResource().Resource)
-	delete(oas.Paths.Paths, root+"watch/"+v0alpha1.DashboardResourceInfo.GroupResource().Resource)
+	delete(oas.Paths.Paths, root+dashboard.DashboardResourceInfo.GroupResource().Resource)
+	delete(oas.Paths.Paths, root+"watch/"+dashboard.DashboardResourceInfo.GroupResource().Resource)
 
 	// The root API discovery list
 	sub := oas.Paths.Paths[root]