EntityAPI: Save nested summary info in the SQL database (#61732)

pkg/models/entity.go

@@ -94,10 +94,6 @@ type EntitySummary struct {
 	// URL safe version of the name.  It will be unique within the folder
 	Slug string `json:"slug,omitempty"`
 
-	// URL should only be set if the value is not derived directly from kind+uid
-	// NOTE: this may go away with a more robust GRN solution /!\
-	URL string `json:"URL,omitempty"`
-
 	// When errors exist
 	Error *EntityErrorInfo `json:"error,omitempty"`
 

pkg/services/searchV2/bluge.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strconv"
 	"strings"
 	"time"
 
@@ -14,6 +15,7 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/slugify"
 	"github.com/grafana/grafana/pkg/models"
 )
 
@@ -188,13 +190,21 @@ func getNonFolderDashboardDoc(dash dashboard, location string) *bluge.Document {
 }
 
 func getDashboardPanelDocs(dash dashboard, location string) []*bluge.Document {
+	dashURL := fmt.Sprintf("/d/%s/%s", dash.uid, slugify.Slugify(dash.summary.Name))
+
 	var docs []*bluge.Document
 	for _, panel := range dash.summary.Nested {
 		if panel.Kind == "panel-row" {
 			continue // for now, we are excluding rows from the search index
 		}
+		idx := strings.LastIndex(panel.UID, "#")
+		panelId, err := strconv.Atoi(panel.UID[idx+1:])
+		if err != nil {
+			continue
+		}
 
-		doc := newSearchDocument(panel.UID, panel.Name, panel.Description, panel.URL).
+		url := fmt.Sprintf("%s?viewPanel=%d", dashURL, panelId)
+		doc := newSearchDocument(panel.UID, panel.Name, panel.Description, url).
 			AddField(bluge.NewKeywordField(documentFieldLocation, location).Aggregatable().StoreValue()).
 			AddField(bluge.NewKeywordField(documentFieldKind, string(entityKindPanel)).Aggregatable().StoreValue()) // likely want independent index for this
 

pkg/services/sqlstore/migrations/entity_store_mig.go

@@ -36,7 +36,7 @@ func addEntityStoreMigrations(mg *migrator.Migrator) {
 			{Name: "slug", Type: migrator.DB_NVarchar, Length: 189, Nullable: false}, // from title
 
 			// The raw entity body (any byte array)
-			{Name: "body", Type: migrator.DB_LongBlob, Nullable: false},
+			{Name: "body", Type: migrator.DB_LongBlob, Nullable: true}, // null when nested or remote
 			{Name: "size", Type: migrator.DB_BigInt, Nullable: false},
 			{Name: "etag", Type: migrator.DB_NVarchar, Length: 32, Nullable: false, IsLatin: true}, // md5(body)
 			{Name: "version", Type: migrator.DB_NVarchar, Length: 128, Nullable: false},
@@ -79,6 +79,8 @@ func addEntityStoreMigrations(mg *migrator.Migrator) {
 			getLatinPathColumn("slug_path"),                             ///slug/slug/slug/
 			{Name: "tree", Type: migrator.DB_Text, Nullable: false},     // JSON []{uid, title}
 			{Name: "depth", Type: migrator.DB_Int, Nullable: false},     // starts at 1
+			{Name: "left", Type: migrator.DB_Int, Nullable: false},      // MPTT
+			{Name: "right", Type: migrator.DB_Int, Nullable: false},     // MPTT
 			{Name: "detached", Type: migrator.DB_Bool, Nullable: false}, // a parent folder was not found
 		},
 		Indices: []*migrator.Index{
@@ -93,9 +95,11 @@ func addEntityStoreMigrations(mg *migrator.Migrator) {
 			{Name: "grn", Type: migrator.DB_NVarchar, Length: grnLength, Nullable: false},
 			{Name: "label", Type: migrator.DB_NVarchar, Length: 191, Nullable: false},
 			{Name: "value", Type: migrator.DB_NVarchar, Length: 1024, Nullable: false},
+			{Name: "parent_grn", Type: migrator.DB_NVarchar, Length: grnLength, Nullable: true},
 		},
 		Indices: []*migrator.Index{
 			{Cols: []string{"grn", "label"}, Type: migrator.UniqueIndex},
+			{Cols: []string{"parent_grn"}, Type: migrator.IndexType},
 		},
 	})
 
@@ -104,6 +108,7 @@ func addEntityStoreMigrations(mg *migrator.Migrator) {
 		Columns: []*migrator.Column{
 			// Source:
 			{Name: "grn", Type: migrator.DB_NVarchar, Length: grnLength, Nullable: false},
+			{Name: "parent_grn", Type: migrator.DB_NVarchar, Length: grnLength, Nullable: true},
 
 			// Address (defined in the body, not resolved, may be invalid and change)
 			{Name: "kind", Type: migrator.DB_NVarchar, Length: 255, Nullable: false},
@@ -120,6 +125,7 @@ func addEntityStoreMigrations(mg *migrator.Migrator) {
 			{Cols: []string{"grn"}, Type: migrator.IndexType},
 			{Cols: []string{"kind"}, Type: migrator.IndexType},
 			{Cols: []string{"resolved_to"}, Type: migrator.IndexType},
+			{Cols: []string{"parent_grn"}, Type: migrator.IndexType},
 		},
 	})
 
@@ -147,6 +153,34 @@ func addEntityStoreMigrations(mg *migrator.Migrator) {
 		},
 	})
 
+	tables = append(tables, migrator.Table{
+		Name: "entity_nested",
+		Columns: []*migrator.Column{
+			{Name: "grn", Type: migrator.DB_NVarchar, Length: grnLength, Nullable: false, IsPrimaryKey: true},
+			{Name: "parent_grn", Type: migrator.DB_NVarchar, Length: grnLength, Nullable: false},
+
+			// The entity identifier
+			{Name: "tenant_id", Type: migrator.DB_BigInt, Nullable: false},
+			{Name: "kind", Type: migrator.DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "uid", Type: migrator.DB_NVarchar, Length: 40, Nullable: false},
+			{Name: "folder", Type: migrator.DB_NVarchar, Length: 40, Nullable: false},
+
+			// Summary data (always extracted from the `body` column)
+			{Name: "name", Type: migrator.DB_NVarchar, Length: 255, Nullable: false},
+			{Name: "description", Type: migrator.DB_NVarchar, Length: 255, Nullable: true},
+			{Name: "labels", Type: migrator.DB_Text, Nullable: true}, // JSON object
+			{Name: "fields", Type: migrator.DB_Text, Nullable: true}, // JSON object
+			{Name: "errors", Type: migrator.DB_Text, Nullable: true}, // JSON object
+		},
+		Indices: []*migrator.Index{
+			{Cols: []string{"parent_grn"}},
+			{Cols: []string{"kind"}},
+			{Cols: []string{"folder"}},
+			{Cols: []string{"uid"}},
+			{Cols: []string{"tenant_id", "kind", "uid"}, Type: migrator.UniqueIndex},
+		},
+	})
+
 	// !!! This should not run in production!
 	// The object store SQL schema is still in active development and this
 	// will only be called when the feature toggle is enabled
@@ -158,7 +192,7 @@ func addEntityStoreMigrations(mg *migrator.Migrator) {
 	// Migration cleanups: given that this is a complex setup
 	// that requires a lot of testing before we are ready to push out of dev
 	// this script lets us easy wipe previous changes and initialize clean tables
-	suffix := " (v12)" // change this when we want to wipe and reset the object tables
+	suffix := " (v31)" // change this when we want to wipe and reset the object tables
 	mg.AddMigration("EntityStore init: cleanup"+suffix, migrator.NewRawSQLMigration(strings.TrimSpace(`
 		DELETE FROM migration_log WHERE migration_id LIKE 'EntityStore init%';
 	`)))

pkg/services/store/entity/sqlstash/folder_support.go

@@ -11,15 +11,23 @@ import (
 
 type folderInfo struct {
 	UID  string `json:"uid"`
-	Name string `json:"name"`
+	Name string `json:"name"` // original display name
-	Slug string `json:"slug"`
+	Slug string `json:"slug"` // full slug
+
+	// original slug
+	originalSlug string
+
+	depth int32
+	left  int32
+	right int32
 
 	// Build the tree
-	ParentUID string `json:"-"`
+	parentUID string
 
-	// Added after query
+	// Calculated after query
-	children []*folderInfo
 	parent   *folderInfo
+	children []*folderInfo
+	stack    []*folderInfo
 }
 
 // This will replace all entries in `entity_folder`
@@ -32,7 +40,6 @@ func updateFolderTree(ctx context.Context, tx *session.SessionTx, tenant int64)
 	}
 
 	all := []*folderInfo{}
-	lookup := make(map[string]*folderInfo)
 	rows, err := tx.Query(ctx, "SELECT uid,folder,name,slug FROM entity WHERE kind=? AND tenant_id=? ORDER BY slug asc;",
 		models.StandardKindFolder, tenant)
 	if err != nil {
@@ -42,11 +49,10 @@ func updateFolderTree(ctx context.Context, tx *session.SessionTx, tenant int64)
 		folder := folderInfo{
 			children: []*folderInfo{},
 		}
-		err = rows.Scan(&folder.UID, &folder.ParentUID, &folder.Name, &folder.Slug)
+		err = rows.Scan(&folder.UID, &folder.parentUID, &folder.Name, &folder.originalSlug)
 		if err != nil {
 			return err
 		}
-		lookup[folder.UID] = &folder
 		all = append(all, &folder)
 	}
 	err = rows.Close()
@@ -54,16 +60,43 @@ func updateFolderTree(ctx context.Context, tx *session.SessionTx, tenant int64)
 		return err
 	}
 
+	root, lost, err := buildFolderTree(all)
+	if err != nil {
+		return err
+	}
+
+	err = insertFolderInfo(ctx, tx, tenant, root, false)
+	if err != nil {
+		return err
+	}
+
+	for _, folder := range lost {
+		err = insertFolderInfo(ctx, tx, tenant, folder, true)
+		if err != nil {
+			return err
+		}
+	}
+	return err
+}
+
+func buildFolderTree(all []*folderInfo) (*folderInfo, []*folderInfo, error) {
+	lost := []*folderInfo{}
+	lookup := make(map[string]*folderInfo)
+	for _, folder := range all {
+		lookup[folder.UID] = folder
+	}
+
 	root := &folderInfo{
 		Name:     "Root",
+		UID:      "",
 		children: []*folderInfo{},
+		left:     1,
 	}
 	lookup[""] = root
-	lost := []*folderInfo{}
 
 	// already sorted by slug
 	for _, folder := range all {
-		parent, ok := lookup[folder.ParentUID]
+		parent, ok := lookup[folder.parentUID]
 		if ok {
 			folder.parent = parent
 			parent.children = append(parent.children, folder)
@@ -72,40 +105,49 @@ func updateFolderTree(ctx context.Context, tx *session.SessionTx, tenant int64)
 		}
 	}
 
-	for _, folder := range root.children {
+	_, err := setMPTTOrder(root, []*folderInfo{}, int32(1))
-		err = addFolderInfo(ctx, tx, tenant, []*folderInfo{folder}, false)
+	return root, lost, err
-		if err != nil {
-			return err
-		}
-	}
-	for _, folder := range lost {
-		err = addFolderInfo(ctx, tx, tenant, []*folderInfo{folder}, true)
-		if err != nil {
-			return err
-		}
-	}
-	return err
 }
 
-func addFolderInfo(ctx context.Context, tx *session.SessionTx, tenant int64, tree []*folderInfo, isDetached bool) error {
+// https://imrannazar.com/Modified-Preorder-Tree-Traversal
-	folder := tree[len(tree)-1] // last item in the tree
+func setMPTTOrder(folder *folderInfo, stack []*folderInfo, idx int32) (int32, error) {
+	var err error
+	folder.depth = int32(len(stack))
+	folder.left = idx
+	folder.stack = stack
 
-	js, _ := json.Marshal(tree)
+	if folder.depth > 0 {
-	slugPath := "/"
+		folder.Slug = "/"
-	for _, f := range tree {
+		for _, f := range stack {
-		slugPath += f.Slug + "/"
+			folder.Slug += f.originalSlug + "/"
+		}
 	}
+
+	for _, child := range folder.children {
+		idx, err = setMPTTOrder(child, append(stack, child), idx+1)
+		if err != nil {
+			return idx, err
+		}
+	}
+	folder.right = idx + 1
+	return folder.right, nil
+}
+
+func insertFolderInfo(ctx context.Context, tx *session.SessionTx, tenant int64, folder *folderInfo, isDetached bool) error {
+	js, _ := json.Marshal(folder.stack)
 	grn := entity.GRN{TenantId: tenant, Kind: models.StandardKindFolder, UID: folder.UID}
 	_, err := tx.Exec(ctx,
 		`INSERT INTO entity_folder `+
-			"(grn, tenant_id, uid, slug_path, tree, depth, detached) "+
+			"(grn, tenant_id, uid, slug_path, tree, depth, left, right, detached) "+
-			`VALUES (?, ?, ?, ?, ?, ?, ?)`,
+			`VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
 		grn.ToGRNString(),
 		tenant,
 		folder.UID,
-		slugPath,
+		folder.Slug,
 		string(js),
-		len(tree),
+		folder.depth,
+		folder.left,
+		folder.right,
 		isDetached,
 	)
 	if err != nil {
@@ -113,7 +155,7 @@ func addFolderInfo(ctx context.Context, tx *session.SessionTx, tenant int64, tre
 	}
 
 	for _, sub := range folder.children {
-		err := addFolderInfo(ctx, tx, tenant, append(tree, sub), isDetached)
+		err := insertFolderInfo(ctx, tx, tenant, sub, isDetached)
 		if err != nil {
 			return err
 		}

pkg/services/store/entity/sqlstash/folder_support_test.go

@@ -0,0 +1,63 @@
+package sqlstash
+
+import (
+	_ "embed"
+	"encoding/json"
+	"testing"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana-plugin-sdk-go/experimental"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFolderSupport(t *testing.T) {
+	root, lost, err := buildFolderTree([]*folderInfo{
+		{UID: "A", parentUID: "", Name: "A", originalSlug: "a"},
+		{UID: "AA", parentUID: "A", Name: "AA", originalSlug: "aa"},
+		{UID: "B", parentUID: "", Name: "B", originalSlug: "b"},
+	})
+	require.NoError(t, err)
+	require.NotNil(t, root)
+	require.NotNil(t, lost)
+	require.Empty(t, lost)
+
+	frame := treeToFrame(root)
+	experimental.CheckGoldenJSONFrame(t, "testdata", "simple", frame, true)
+}
+
+func treeToFrame(root *folderInfo) *data.Frame {
+	frame := data.NewFrame("",
+		data.NewFieldFromFieldType(data.FieldTypeString, 0), // UID
+		data.NewFieldFromFieldType(data.FieldTypeString, 0), // Name
+		data.NewFieldFromFieldType(data.FieldTypeString, 0), // Slug
+		data.NewFieldFromFieldType(data.FieldTypeInt32, 0),  // Depth
+		data.NewFieldFromFieldType(data.FieldTypeInt32, 0),  // Left
+		data.NewFieldFromFieldType(data.FieldTypeInt32, 0),  // Right
+		data.NewFieldFromFieldType(data.FieldTypeJSON, 0),   // Tree
+	)
+	frame.Fields[0].Name = "UID"
+	frame.Fields[1].Name = "name"
+	frame.Fields[2].Name = "slug"
+	frame.Fields[3].Name = "depth"
+	frame.Fields[4].Name = "left"
+	frame.Fields[5].Name = "right"
+	frame.Fields[6].Name = "tree"
+	appendFolder(root, frame)
+	return frame
+}
+
+func appendFolder(folder *folderInfo, frame *data.Frame) {
+	b, _ := json.Marshal(folder.stack)
+	frame.AppendRow(
+		folder.UID,
+		folder.Name,
+		folder.Slug,
+		folder.depth,
+		folder.left,
+		folder.right,
+		json.RawMessage(b),
+	)
+	for _, sub := range folder.children {
+		appendFolder(sub, frame)
+	}
+}

pkg/services/store/entity/sqlstash/sql_storage_server.go

@@ -304,7 +304,6 @@ func (s *sqlEntityServer) AdminWrite(ctx context.Context, r *entity.AdminWriteEn
 		return nil, err
 	}
 
-	isFolder := models.StandardKindFolder == r.GRN.Kind
 	etag := createContentsHash(body)
 	rsp := &entity.WriteEntityResponse{
 		GRN:    grn,
@@ -372,10 +371,13 @@ func (s *sqlEntityServer) AdminWrite(ctx context.Context, r *entity.AdminWriteEn
 
 		if isUpdate {
 			// Clear the labels+refs
-			if _, err := tx.Exec(ctx, "DELETE FROM entity_labels WHERE grn=?", oid); err != nil {
+			if _, err := tx.Exec(ctx, "DELETE FROM entity_labels WHERE grn=? OR parent_grn=?", oid, oid); err != nil {
 				return err
 			}
-			if _, err := tx.Exec(ctx, "DELETE FROM entity_ref WHERE grn=?", oid); err != nil {
+			if _, err := tx.Exec(ctx, "DELETE FROM entity_ref WHERE grn=? OR parent_grn=?", oid, oid); err != nil {
+				return err
+			}
+			if _, err := tx.Exec(ctx, "DELETE FROM entity_nested WHERE parent_grn=?", oid); err != nil {
 				return err
 			}
 		}
@@ -398,37 +400,6 @@ func (s *sqlEntityServer) AdminWrite(ctx context.Context, r *entity.AdminWriteEn
 			return err
 		}
 
-		// 2. Add the labels rows
-		for k, v := range summary.model.Labels {
-			_, err = tx.Exec(ctx,
-				`INSERT INTO entity_labels `+
-					"(grn, label, value) "+
-					`VALUES (?, ?, ?)`,
-				oid, k, v,
-			)
-			if err != nil {
-				return err
-			}
-		}
-
-		// 3. Add the references rows
-		for _, ref := range summary.model.References {
-			resolved, err := s.resolver.Resolve(ctx, ref)
-			if err != nil {
-				return err
-			}
-			_, err = tx.Exec(ctx, `INSERT INTO entity_ref (`+
-				"grn, kind, type, uid, "+
-				"resolved_ok, resolved_to, resolved_warning, resolved_time) "+
-				`VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
-				oid, ref.Kind, ref.Type, ref.UID,
-				resolved.OK, resolved.Key, resolved.Warning, resolved.Timestamp,
-			)
-			if err != nil {
-				return err
-			}
-		}
-
 		// 5. Add/update the main `entity` table
 		rsp.Entity = versionInfo
 		if isUpdate {
@@ -447,43 +418,43 @@ func (s *sqlEntityServer) AdminWrite(ctx context.Context, r *entity.AdminWriteEn
 				origin.Source, origin.Key, timestamp,
 				oid,
 			)
-
+		} else {
-			if isFolder && err == nil {
+			if createdAt < 1000 {
-				err = updateFolderTree(ctx, tx, grn.TenantId)
+				createdAt = updatedAt
+			}
+			if createdBy == "" {
+				createdBy = updatedBy
 			}
-			return err
-		}
 
-		if createdAt < 1000 {
+			_, err = tx.Exec(ctx, "INSERT INTO entity ("+
-			createdAt = updatedAt
+				"grn, tenant_id, kind, uid, folder, "+
+				"size, body, etag, version, "+
+				"updated_at, updated_by, created_at, created_by, "+
+				"name, description, slug, "+
+				"labels, fields, errors, "+
+				"origin, origin_key, origin_ts) "+
+				"VALUES (?, ?, ?, ?, ?, "+
+				" ?, ?, ?, ?, "+
+				" ?, ?, ?, ?, "+
+				" ?, ?, ?, "+
+				" ?, ?, ?, "+
+				" ?, ?, ?)",
+				oid, grn.TenantId, grn.Kind, grn.UID, r.Folder,
+				versionInfo.Size, body, etag, versionInfo.Version,
+				updatedAt, createdBy, createdAt, createdBy,
+				summary.model.Name, summary.model.Description, summary.model.Slug,
+				summary.labels, summary.fields, summary.errors,
+				origin.Source, origin.Key, origin.Time,
+			)
 		}
-		if createdBy == "" {
+		if err == nil && models.StandardKindFolder == r.GRN.Kind {
-			createdBy = updatedBy
-		}
-
-		_, err = tx.Exec(ctx, "INSERT INTO entity ("+
-			"grn, tenant_id, kind, uid, folder, "+
-			"size, body, etag, version, "+
-			"updated_at, updated_by, created_at, created_by, "+
-			"name, description, slug, "+
-			"labels, fields, errors, "+
-			"origin, origin_key, origin_ts) "+
-			"VALUES (?, ?, ?, ?, ?, "+
-			" ?, ?, ?, ?, "+
-			" ?, ?, ?, ?, "+
-			" ?, ?, ?, "+
-			" ?, ?, ?, "+
-			" ?, ?, ?)",
-			oid, grn.TenantId, grn.Kind, grn.UID, r.Folder,
-			versionInfo.Size, body, etag, versionInfo.Version,
-			updatedAt, createdBy, createdAt, createdBy,
-			summary.model.Name, summary.model.Description, summary.model.Slug,
-			summary.labels, summary.fields, summary.errors,
-			origin.Source, origin.Key, origin.Time,
-		)
-		if isFolder && err == nil {
 			err = updateFolderTree(ctx, tx, grn.TenantId)
 		}
+		if err == nil {
+			summary.folder = r.Folder
+			summary.parent_grn = grn
+			return s.writeSearchInfo(ctx, tx, oid, summary)
+		}
 		return err
 	})
 	rsp.SummaryJson = summary.marshaled
@@ -534,6 +505,92 @@ func (s *sqlEntityServer) selectForUpdate(ctx context.Context, tx *session.Sessi
 	return current, err
 }
 
+func (s *sqlEntityServer) writeSearchInfo(
+	ctx context.Context,
+	tx *session.SessionTx,
+	grn string,
+	summary *summarySupport,
+) error {
+	parent_grn := summary.getParentGRN()
+
+	// Add the labels rows
+	for k, v := range summary.model.Labels {
+		_, err := tx.Exec(ctx,
+			`INSERT INTO entity_labels `+
+				"(grn, label, value, parent_grn) "+
+				`VALUES (?, ?, ?, ?)`,
+			grn, k, v, parent_grn,
+		)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Resolve references
+	for _, ref := range summary.model.References {
+		resolved, err := s.resolver.Resolve(ctx, ref)
+		if err != nil {
+			return err
+		}
+		_, err = tx.Exec(ctx, `INSERT INTO entity_ref (`+
+			"grn, parent_grn, kind, type, uid, "+
+			"resolved_ok, resolved_to, resolved_warning, resolved_time) "+
+			`VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+			grn, parent_grn, ref.Kind, ref.Type, ref.UID,
+			resolved.OK, resolved.Key, resolved.Warning, resolved.Timestamp,
+		)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Traverse entities and insert refs
+	if summary.model.Nested != nil {
+		for _, childModel := range summary.model.Nested {
+			grn = (&entity.GRN{
+				TenantId: summary.parent_grn.TenantId,
+				Kind:     childModel.Kind,
+				UID:      childModel.UID, // append???
+			}).ToGRNString()
+
+			child, err := newSummarySupport(childModel)
+			if err != nil {
+				return err
+			}
+			child.isNested = true
+			child.folder = summary.folder
+			child.parent_grn = summary.parent_grn
+			parent_grn := child.getParentGRN()
+
+			_, err = tx.Exec(ctx, "INSERT INTO entity_nested ("+
+				"parent_grn, grn, "+
+				"tenant_id, kind, uid, folder, "+
+				"name, description, "+
+				"labels, fields, errors) "+
+				"VALUES (?, ?,"+
+				" ?, ?, ?, ?,"+
+				" ?, ?,"+
+				" ?, ?, ?)",
+				*parent_grn, grn,
+				summary.parent_grn.TenantId, childModel.Kind, childModel.UID, summary.folder,
+				child.name, child.description,
+				child.labels, child.fields, child.errors,
+			)
+
+			if err != nil {
+				return err
+			}
+
+			err = s.writeSearchInfo(ctx, tx, grn, child)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
 func (s *sqlEntityServer) prepare(ctx context.Context, r *entity.AdminWriteEntityRequest) (*summarySupport, []byte, error) {
 	grn := r.GRN
 	builder := s.kinds.GetSummaryBuilder(grn.Kind)
@@ -589,14 +646,26 @@ func doDelete(ctx context.Context, tx *session.SessionTx, grn *entity.GRN) (bool
 	}
 
 	// TODO: keep history? would need current version bump, and the "write" would have to get from history
-	_, _ = tx.Exec(ctx, "DELETE FROM entity_history WHERE grn=?", str)
+	_, err = tx.Exec(ctx, "DELETE FROM entity_history WHERE grn=?", str)
-	_, _ = tx.Exec(ctx, "DELETE FROM entity_labels WHERE grn=?", str)
+	if err != nil {
-	_, _ = tx.Exec(ctx, "DELETE FROM entity_ref WHERE grn=?", str)
+		return false, err
+	}
+	_, err = tx.Exec(ctx, "DELETE FROM entity_labels WHERE grn=? OR parent_grn=?", str, str)
+	if err != nil {
+		return false, err
+	}
+	_, err = tx.Exec(ctx, "DELETE FROM entity_ref WHERE grn=? OR parent_grn=?", str, str)
+	if err != nil {
+		return false, err
+	}
+	_, err = tx.Exec(ctx, "DELETE FROM entity_nested WHERE parent_grn=?", str)
+	if err != nil {
+		return false, err
+	}
 
 	if grn.Kind == models.StandardKindFolder {
 		err = updateFolderTree(ctx, tx, grn.TenantId)
 	}
-
 	return rows > 0, err
 }
 

pkg/services/store/entity/sqlstash/summary_handler.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 
 	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/store/entity"
 )
 
 type summarySupport struct {
@@ -15,6 +16,11 @@ type summarySupport struct {
 	fields      *string
 	errors      *string // should not allow saving with this!
 	marshaled   []byte
+
+	// metadata for nested objects
+	parent_grn *entity.GRN
+	folder     string
+	isNested   bool // set when this is for a nested item
 }
 
 func newSummarySupport(summary *models.EntitySummary) (*summarySupport, error) {
@@ -100,3 +106,11 @@ func (s summarySupport) toEntitySummary() (*models.EntitySummary, error) {
 	}
 	return summary, err
 }
+
+func (s *summarySupport) getParentGRN() *string {
+	if s.isNested {
+		t := s.parent_grn.ToGRNString()
+		return &t
+	}
+	return nil
+}

pkg/services/store/entity/sqlstash/testdata/simple.jsonc

@@ -0,0 +1,147 @@
+//  🌟 This was machine generated.  Do not edit. 🌟
+//  
+//  Frame[0] 
+//  Name: 
+//  Dimensions: 7 Fields by 4 Rows
+//  +----------------+----------------+----------------+---------------+---------------+---------------+--------------------------------------------------------------------------------+
+//  | Name: UID      | Name: name     | Name: slug     | Name: depth   | Name: left    | Name: right   | Name: tree                                                                     |
+//  | Labels:        | Labels:        | Labels:        | Labels:       | Labels:       | Labels:       | Labels:                                                                        |
+//  | Type: []string | Type: []string | Type: []string | Type: []int32 | Type: []int32 | Type: []int32 | Type: []json.RawMessage                                                        |
+//  +----------------+----------------+----------------+---------------+---------------+---------------+--------------------------------------------------------------------------------+
+//  |                | Root           |                | 0             | 1             | 8             | []                                                                             |
+//  | A              | A              | /a/            | 1             | 2             | 5             | [{"uid":"A","name":"A","slug":"/a/"}]                                          |
+//  | AA             | AA             | /a/aa/         | 2             | 3             | 4             | [{"uid":"A","name":"A","slug":"/a/"},{"uid":"AA","name":"AA","slug":"/a/aa/"}] |
+//  | B              | B              | /b/            | 1             | 6             | 7             | [{"uid":"B","name":"B","slug":"/b/"}]                                          |
+//  +----------------+----------------+----------------+---------------+---------------+---------------+--------------------------------------------------------------------------------+
+//  
+//  
+//  🌟 This was machine generated.  Do not edit. 🌟
+{
+  "status": 200,
+  "frames": [
+    {
+      "schema": {
+        "fields": [
+          {
+            "name": "UID",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          },
+          {
+            "name": "name",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          },
+          {
+            "name": "slug",
+            "type": "string",
+            "typeInfo": {
+              "frame": "string"
+            }
+          },
+          {
+            "name": "depth",
+            "type": "number",
+            "typeInfo": {
+              "frame": "int32"
+            }
+          },
+          {
+            "name": "left",
+            "type": "number",
+            "typeInfo": {
+              "frame": "int32"
+            }
+          },
+          {
+            "name": "right",
+            "type": "number",
+            "typeInfo": {
+              "frame": "int32"
+            }
+          },
+          {
+            "name": "tree",
+            "type": "other",
+            "typeInfo": {
+              "frame": "json.RawMessage"
+            }
+          }
+        ]
+      },
+      "data": {
+        "values": [
+          [
+            "",
+            "A",
+            "AA",
+            "B"
+          ],
+          [
+            "Root",
+            "A",
+            "AA",
+            "B"
+          ],
+          [
+            "",
+            "/a/",
+            "/a/aa/",
+            "/b/"
+          ],
+          [
+            0,
+            1,
+            2,
+            1
+          ],
+          [
+            1,
+            2,
+            3,
+            6
+          ],
+          [
+            8,
+            5,
+            4,
+            7
+          ],
+          [
+            [],
+            [
+              {
+                "uid": "A",
+                "name": "A",
+                "slug": "/a/"
+              }
+            ],
+            [
+              {
+                "uid": "A",
+                "name": "A",
+                "slug": "/a/"
+              },
+              {
+                "uid": "AA",
+                "name": "AA",
+                "slug": "/a/aa/"
+              }
+            ],
+            [
+              {
+                "uid": "B",
+                "name": "B",
+                "slug": "/b/"
+              }
+            ]
+          ]
+        ]
+      }
+    }
+  ]
+}

pkg/services/store/kind/dashboard/summary.go

@@ -4,10 +4,8 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"fmt"
 	"strconv"
 
-	"github.com/grafana/grafana/pkg/infra/slugify"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
 )
@@ -58,10 +56,8 @@ func NewStaticDashboardSummaryBuilder(lookup DatasourceLookup, sanitize bool) mo
 		}
 
 		dashboardRefs := NewReferenceAccumulator()
-		url := fmt.Sprintf("/d/%s/%s", uid, slugify.Slugify(dash.Title))
 		summary.Name = dash.Title
 		summary.Description = dash.Description
-		summary.URL = url
 		for _, v := range dash.Tags {
 			summary.Labels[v] = ""
 		}
@@ -78,7 +74,6 @@ func NewStaticDashboardSummaryBuilder(lookup DatasourceLookup, sanitize bool) mo
 			}
 			p.Name = panel.Title
 			p.Description = panel.Description
-			p.URL = fmt.Sprintf("%s?viewPanel=%d", url, panel.ID)
 			p.Fields = make(map[string]interface{}, 0)
 			p.Fields["type"] = panel.Type
 

pkg/services/store/kind/dashboard/testdata/gdev-walk-graph-gradient-area-fills.json

@@ -5,7 +5,6 @@
     "graph": "",
     "panel-tests": ""
   },
-  "URL": "/d/graph-gradient-area-fills.json/panel-tests-graph-gradient-area-fills",
   "fields": {
     "schemaVersion": 18
   },
@@ -14,7 +13,6 @@
       "uid": "graph-gradient-area-fills.json#2",
       "kind": "panel",
       "name": "Req/s",
-      "URL": "/d/graph-gradient-area-fills.json/panel-tests-graph-gradient-area-fills?viewPanel=2",
       "fields": {
         "type": "graph"
       },
@@ -33,7 +31,6 @@
       "uid": "graph-gradient-area-fills.json#11",
       "kind": "panel",
       "name": "Req/s",
-      "URL": "/d/graph-gradient-area-fills.json/panel-tests-graph-gradient-area-fills?viewPanel=11",
       "fields": {
         "type": "graph"
       },
@@ -52,7 +49,6 @@
       "uid": "graph-gradient-area-fills.json#7",
       "kind": "panel",
       "name": "Memory",
-      "URL": "/d/graph-gradient-area-fills.json/panel-tests-graph-gradient-area-fills?viewPanel=7",
       "fields": {
         "type": "graph"
       },
@@ -71,7 +67,6 @@
       "uid": "graph-gradient-area-fills.json#10",
       "kind": "panel",
       "name": "Req/s",
-      "URL": "/d/graph-gradient-area-fills.json/panel-tests-graph-gradient-area-fills?viewPanel=10",
       "fields": {
         "type": "graph"
       },

pkg/services/store/kind/dashboard/testdata/gdev-walk-graph-shared-tooltips.json

@@ -5,7 +5,6 @@
     "graph-ng": "",
     "panel-tests": ""
   },
-  "URL": "/d/graph-shared-tooltips.json/panel-tests-shared-tooltips",
   "fields": {
     "schemaVersion": 28
   },
@@ -14,7 +13,6 @@
       "uid": "graph-shared-tooltips.json#4",
       "kind": "panel",
       "name": "two units",
-      "URL": "/d/graph-shared-tooltips.json/panel-tests-shared-tooltips?viewPanel=4",
       "fields": {
         "type": "timeseries"
       },
@@ -33,7 +31,6 @@
       "uid": "graph-shared-tooltips.json#13",
       "kind": "panel",
       "name": "Speed vs Temperature (XY)",
-      "URL": "/d/graph-shared-tooltips.json/panel-tests-shared-tooltips?viewPanel=13",
       "fields": {
         "type": "xychart"
       },
@@ -62,7 +59,6 @@
       "uid": "graph-shared-tooltips.json#2",
       "kind": "panel",
       "name": "Cursor info",
-      "URL": "/d/graph-shared-tooltips.json/panel-tests-shared-tooltips?viewPanel=2",
       "fields": {
         "type": "debug"
       },
@@ -81,7 +77,6 @@
       "uid": "graph-shared-tooltips.json#5",
       "kind": "panel",
       "name": "Only temperature",
-      "URL": "/d/graph-shared-tooltips.json/panel-tests-shared-tooltips?viewPanel=5",
       "fields": {
         "type": "timeseries"
       },
@@ -100,7 +95,6 @@
       "uid": "graph-shared-tooltips.json#9",
       "kind": "panel",
       "name": "Only Speed",
-      "URL": "/d/graph-shared-tooltips.json/panel-tests-shared-tooltips?viewPanel=9",
       "fields": {
         "type": "timeseries"
       },
@@ -119,7 +113,6 @@
       "uid": "graph-shared-tooltips.json#11",
       "kind": "panel",
       "name": "Panel Title",
-      "URL": "/d/graph-shared-tooltips.json/panel-tests-shared-tooltips?viewPanel=11",
       "fields": {
         "type": "timeseries"
       },
@@ -138,7 +131,6 @@
       "uid": "graph-shared-tooltips.json#8",
       "kind": "panel",
       "name": "flot panel (temperature)",
-      "URL": "/d/graph-shared-tooltips.json/panel-tests-shared-tooltips?viewPanel=8",
       "fields": {
         "type": "graph"
       },
@@ -157,7 +149,6 @@
       "uid": "graph-shared-tooltips.json#10",
       "kind": "panel",
       "name": "flot panel (no units)",
-      "URL": "/d/graph-shared-tooltips.json/panel-tests-shared-tooltips?viewPanel=10",
       "fields": {
         "type": "graph"
       },

pkg/services/store/kind/dashboard/testdata/gdev-walk-graph-time-regions.json

@@ -5,7 +5,6 @@
     "graph": "",
     "panel-tests": ""
   },
-  "URL": "/d/graph-time-regions.json/panel-tests-graph-time-regions",
   "fields": {
     "schemaVersion": 18
   },
@@ -14,7 +13,6 @@
       "uid": "graph-time-regions.json#2",
       "kind": "panel",
       "name": "Business Hours",
-      "URL": "/d/graph-time-regions.json/panel-tests-graph-time-regions?viewPanel=2",
       "fields": {
         "type": "graph"
       },
@@ -34,7 +32,6 @@
       "uid": "graph-time-regions.json#4",
       "kind": "panel",
       "name": "Sunday's 20-23",
-      "URL": "/d/graph-time-regions.json/panel-tests-graph-time-regions?viewPanel=4",
       "fields": {
         "type": "graph"
       },
@@ -54,7 +51,6 @@
       "uid": "graph-time-regions.json#3",
       "kind": "panel",
       "name": "Each day of week",
-      "URL": "/d/graph-time-regions.json/panel-tests-graph-time-regions?viewPanel=3",
       "fields": {
         "type": "graph"
       },
@@ -74,7 +70,6 @@
       "uid": "graph-time-regions.json#5",
       "kind": "panel",
       "name": "05:00",
-      "URL": "/d/graph-time-regions.json/panel-tests-graph-time-regions?viewPanel=5",
       "fields": {
         "type": "graph"
       },
@@ -94,7 +89,6 @@
       "uid": "graph-time-regions.json#7",
       "kind": "panel",
       "name": "From 22:00 to 00:30 (crossing midnight)",
-      "URL": "/d/graph-time-regions.json/panel-tests-graph-time-regions?viewPanel=7",
       "fields": {
         "type": "graph"
       },

pkg/services/store/kind/dashboard/testdata/gdev-walk-graph_tests.json

@@ -5,7 +5,6 @@
     "graph": "",
     "panel-tests": ""
   },
-  "URL": "/d/graph_tests.json/panel-tests-graph",
   "fields": {
     "schemaVersion": 16
   },
@@ -14,7 +13,6 @@
       "uid": "graph_tests.json#1",
       "kind": "panel",
       "name": "No Data Points Warning",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=1",
       "fields": {
         "type": "graph"
       },
@@ -34,7 +32,6 @@
       "uid": "graph_tests.json#2",
       "kind": "panel",
       "name": "Datapoints Outside Range Warning",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=2",
       "fields": {
         "type": "graph"
       },
@@ -54,7 +51,6 @@
       "uid": "graph_tests.json#3",
       "kind": "panel",
       "name": "Random walk series",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=3",
       "fields": {
         "type": "graph"
       },
@@ -74,7 +70,6 @@
       "uid": "graph_tests.json#4",
       "kind": "panel",
       "name": "Millisecond res x-axis and tooltip",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=4",
       "fields": {
         "type": "graph"
       },
@@ -93,7 +88,6 @@
     {
       "uid": "graph_tests.json#6",
       "kind": "panel",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=6",
       "fields": {
         "type": "text"
       },
@@ -112,7 +106,6 @@
       "uid": "graph_tests.json#5",
       "kind": "panel",
       "name": "2 yaxis and axis labels",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=5",
       "fields": {
         "type": "graph"
       },
@@ -131,7 +124,6 @@
     {
       "uid": "graph_tests.json#7",
       "kind": "panel",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=7",
       "fields": {
         "type": "text"
       },
@@ -150,7 +142,6 @@
       "uid": "graph_tests.json#8",
       "kind": "panel",
       "name": "null value connected",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=8",
       "fields": {
         "type": "graph"
       },
@@ -170,7 +161,6 @@
       "uid": "graph_tests.json#10",
       "kind": "panel",
       "name": "null value null as zero",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=10",
       "fields": {
         "type": "graph"
       },
@@ -189,7 +179,6 @@
     {
       "uid": "graph_tests.json#13",
       "kind": "panel",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=13",
       "fields": {
         "type": "text"
       },
@@ -208,7 +197,6 @@
       "uid": "graph_tests.json#9",
       "kind": "panel",
       "name": "Stacking value ontop of nulls",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=9",
       "fields": {
         "type": "graph"
       },
@@ -227,7 +215,6 @@
     {
       "uid": "graph_tests.json#14",
       "kind": "panel",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=14",
       "fields": {
         "type": "text"
       },
@@ -246,7 +233,6 @@
       "uid": "graph_tests.json#12",
       "kind": "panel",
       "name": "Stacking all series null segment",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=12",
       "fields": {
         "type": "graph"
       },
@@ -265,7 +251,6 @@
     {
       "uid": "graph_tests.json#15",
       "kind": "panel",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=15",
       "fields": {
         "type": "text"
       },
@@ -284,7 +269,6 @@
       "uid": "graph_tests.json#21",
       "kind": "panel",
       "name": "Null between points",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=21",
       "fields": {
         "type": "graph"
       },
@@ -303,7 +287,6 @@
     {
       "uid": "graph_tests.json#22",
       "kind": "panel",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=22",
       "fields": {
         "type": "text"
       },
@@ -322,7 +305,6 @@
       "uid": "graph_tests.json#20",
       "kind": "panel",
       "name": "Legend Table Single Series Should Take Minimum Height",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=20",
       "fields": {
         "type": "graph"
       },
@@ -342,7 +324,6 @@
       "uid": "graph_tests.json#16",
       "kind": "panel",
       "name": "Legend Table No Scroll Visible",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=16",
       "fields": {
         "type": "graph"
       },
@@ -362,7 +343,6 @@
       "uid": "graph_tests.json#17",
       "kind": "panel",
       "name": "Legend Table Should Scroll",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=17",
       "fields": {
         "type": "graph"
       },
@@ -382,7 +362,6 @@
       "uid": "graph_tests.json#18",
       "kind": "panel",
       "name": "Legend Table No Scroll Visible",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=18",
       "fields": {
         "type": "graph"
       },
@@ -402,7 +381,6 @@
       "uid": "graph_tests.json#19",
       "kind": "panel",
       "name": "Legend Table No Scroll Visible",
-      "URL": "/d/graph_tests.json/panel-tests-graph?viewPanel=19",
       "fields": {
         "type": "graph"
       },

pkg/services/store/kind/dashboard/testdata/gdev-walk-graph_y_axis.json

@@ -4,7 +4,6 @@
     "gdev": "",
     "panel-tests": ""
   },
-  "URL": "/d/graph_y_axis.json/panel-tests-graph-y-axis-ticks",
   "fields": {
     "schemaVersion": 19
   },
@@ -13,7 +12,6 @@
       "uid": "graph_y_axis.json#7",
       "kind": "panel",
       "name": "Data from 0 - 10K (unit short)",
-      "URL": "/d/graph_y_axis.json/panel-tests-graph-y-axis-ticks?viewPanel=7",
       "fields": {
         "type": "graph"
       },
@@ -32,7 +30,6 @@
       "uid": "graph_y_axis.json#5",
       "kind": "panel",
       "name": "Data from 0 - 10K (unit bytes metric)",
-      "URL": "/d/graph_y_axis.json/panel-tests-graph-y-axis-ticks?viewPanel=5",
       "fields": {
         "type": "graph"
       },
@@ -51,7 +48,6 @@
       "uid": "graph_y_axis.json#4",
       "kind": "panel",
       "name": "Data from 0 - 10K (unit bytes IEC)",
-      "URL": "/d/graph_y_axis.json/panel-tests-graph-y-axis-ticks?viewPanel=4",
       "fields": {
         "type": "graph"
       },
@@ -70,7 +66,6 @@
       "uid": "graph_y_axis.json#2",
       "kind": "panel",
       "name": "Data from 0 - 10K (unit short)",
-      "URL": "/d/graph_y_axis.json/panel-tests-graph-y-axis-ticks?viewPanel=2",
       "fields": {
         "type": "graph"
       },
@@ -89,7 +84,6 @@
       "uid": "graph_y_axis.json#3",
       "kind": "panel",
       "name": "Data from 0.0002 - 0.001 (unit short)",
-      "URL": "/d/graph_y_axis.json/panel-tests-graph-y-axis-ticks?viewPanel=3",
       "fields": {
         "type": "graph"
       },
@@ -108,7 +102,6 @@
       "uid": "graph_y_axis.json#6",
       "kind": "panel",
       "name": "Data from 12000 - 30000 (unit ms)",
-      "URL": "/d/graph_y_axis.json/panel-tests-graph-y-axis-ticks?viewPanel=6",
       "fields": {
         "type": "graph"
       },
@@ -127,7 +120,6 @@
       "uid": "graph_y_axis.json#9",
       "kind": "panel",
       "name": "Data from 0 - 1B (unit short)",
-      "URL": "/d/graph_y_axis.json/panel-tests-graph-y-axis-ticks?viewPanel=9",
       "fields": {
         "type": "graph"
       },
@@ -146,7 +138,6 @@
       "uid": "graph_y_axis.json#10",
       "kind": "panel",
       "name": "Data from 0 - 1B (unit bytes)",
-      "URL": "/d/graph_y_axis.json/panel-tests-graph-y-axis-ticks?viewPanel=10",
       "fields": {
         "type": "graph"
       },
@@ -165,7 +156,6 @@
       "uid": "graph_y_axis.json#8",
       "kind": "panel",
       "name": "Data from 12000 - 30000 (unit ms)",
-      "URL": "/d/graph_y_axis.json/panel-tests-graph-y-axis-ticks?viewPanel=8",
       "fields": {
         "type": "graph"
       },