Unified storage Indexer: Add kind to hardcoded resource types (#96149)

* add kind to hardcoded resource types for indexer

* call ListOptions in Watch

* Use GaugeVec instead of CounterVec

* index folders before dashboards

pkg/storage/unified/resource/index.go

@@ -195,24 +195,24 @@ func (i *Index) InitForTenant(ctx context.Context, namespace string) (int, error
 	resourceTypes := fetchResourceTypes()
 	totalObjectsFetched := 0
 	for _, rt := range resourceTypes {
-		logger.Debug("indexing resource", "kind", rt.Key.Resource, "list_limit", i.opts.ListLimit, "batch_size", i.opts.BatchSize, "workers", i.opts.Workers, "namespace", namespace)
-		r := &ListRequest{Options: rt, Limit: int64(i.opts.ListLimit)}
+		logger.Debug("indexing resource", "kind", rt.Kind, "list_limit", i.opts.ListLimit, "batch_size", i.opts.BatchSize, "workers", i.opts.Workers, "namespace", namespace)
+		r := &ListRequest{Options: rt.ListOptions, Limit: int64(i.opts.ListLimit)}
 		r.Options.Key.Namespace = namespace // scope the list to a tenant or this will take forever when US has 1M+ resources
 
 		// Paginate through the list of resources and index each page
 		for {
-			logger.Debug("fetching resource list", "kind", rt.Key.Resource, "namespace", namespace)
+			logger.Debug("fetching resource list", "kind", rt.Kind, "namespace", namespace)
 			list, err := i.s.List(ctx, r)
 			if err != nil {
 				return totalObjectsFetched, err
 			}
 
 			// Record the number of objects indexed for the kind
-			IndexServerMetrics.IndexedKinds.WithLabelValues(rt.Key.Resource).Add(float64(len(list.Items)))
+			IndexServerMetrics.IndexedKinds.WithLabelValues(rt.Kind).Add(float64(len(list.Items)))
 
 			totalObjectsFetched += len(list.Items)
 
-			logger.Debug("indexing batch", "kind", rt.Key.Resource, "count", len(list.Items), "namespace", namespace)
+			logger.Debug("indexing batch", "kind", rt.Kind, "count", len(list.Items), "namespace", namespace)
 			//add changes to batches for shards with changes in the List
 			err = i.writeBatch(ctx, list)
 			if err != nil {
@@ -309,6 +309,9 @@ func (i *Index) Delete(ctx context.Context, uid string, key *ResourceKey) error
 	if err != nil {
 		return err
 	}
+
+	IndexServerMetrics.IndexedKinds.WithLabelValues(key.Resource).Dec()
+
 	return nil
 }
 
@@ -474,29 +477,43 @@ func createInMemoryIndex() (bleve.Index, string, error) {
 	return index, "", err
 }
 
+type IndexerListOptions struct {
+	*ListOptions
+	Kind string
+}
+
 // TODO - fetch from api
-func fetchResourceTypes() []*ListOptions {
-	items := []*ListOptions{}
-	items = append(items,
-		&ListOptions{
-			Key: &ResourceKey{
-				Group:    "playlist.grafana.app",
-				Resource: "playlists",
+// Folders need to be indexed first as dashboards depend on them to be indexed already.
+func fetchResourceTypes() []*IndexerListOptions {
+	return []*IndexerListOptions{
+		{
+			ListOptions: &ListOptions{
+				Key: &ResourceKey{
+					Group:    "folder.grafana.app",
+					Resource: "folders",
+				},
 			},
+			Kind: "Folder",
 		},
-		&ListOptions{
-			Key: &ResourceKey{
-				Group:    "folder.grafana.app",
-				Resource: "folders",
+		{
+			ListOptions: &ListOptions{
+				Key: &ResourceKey{
+					Group:    "playlist.grafana.app",
+					Resource: "playlists",
+				},
 			},
+			Kind: "Playlist",
 		},
-		&ListOptions{
-			Key: &ResourceKey{
-				Group:    "dashboard.grafana.app",
-				Resource: "dashboards",
+		{
+			ListOptions: &ListOptions{
+				Key: &ResourceKey{
+					Group:    "dashboard.grafana.app",
+					Resource: "dashboards",
+				},
 			},
-		})
-	return items
+			Kind: "Dashboard",
+		},
+	}
 }
 
 func getSortFields(request *SearchRequest) []string {

pkg/storage/unified/resource/index_metrics.go

@@ -23,7 +23,7 @@ type IndexMetrics struct {
 	IndexLatency      *prometheus.HistogramVec
 	IndexSize         prometheus.Gauge
 	IndexedDocs       prometheus.Gauge
-	IndexedKinds      *prometheus.CounterVec
+	IndexedKinds      *prometheus.GaugeVec
 	IndexCreationTime *prometheus.HistogramVec
 }
 
@@ -53,7 +53,7 @@ func NewIndexMetrics(indexDir string, indexServer *IndexServer) *IndexMetrics {
 				Name:      "indexed_docs",
 				Help:      "Number of indexed documents by resource",
 			}),
-			IndexedKinds: prometheus.NewCounterVec(prometheus.CounterOpts{
+			IndexedKinds: prometheus.NewGaugeVec(prometheus.GaugeOpts{
 				Namespace: "index_server",
 				Name:      "indexed_kinds",
 				Help:      "Number of indexed documents by kind",

pkg/storage/unified/resource/index_server.go

@@ -76,7 +76,7 @@ func (is *IndexServer) Watch(ctx context.Context) error {
 	rtList := fetchResourceTypes()
 	for _, rt := range rtList {
 		wr := &WatchRequest{
-			Options: rt,
+			Options: rt.ListOptions,
 		}
 
 		go func() {
@@ -229,7 +229,7 @@ func getData(wr *WatchEvent_Resource) (*Data, error) {
 
 	key := &ResourceKey{
 		Group:     r.Group,
-		Resource:  r.Kind,
+		Resource:  r.Kind, // We use Kind as resource key since watch events don't have a resource name on them
 		Namespace: r.Namespace,
 		Name:      r.Name,
 	}