Usage: add gauge for panel/datasource/transformer types (#48991)

pkg/services/searchV2/bluge.go

@@ -25,6 +25,7 @@ const (
 	documentFieldDescription = "description"
 	documentFieldLocation    = "location" // parent path
 	documentFieldPanelType   = "panel_type"
+	documentFieldTransformer = "transformer"
 	documentFieldDSUID       = "ds_uid"
 	documentFieldDSType      = "ds_type"
 	documentFieldInternalID  = "__internal_id" // only for migrations! (indexed as a string)
@@ -171,6 +172,25 @@ func getDashboardPanelDocs(dash dashboard, location string) []*bluge.Document {
 			AddField(bluge.NewKeywordField(documentFieldLocation, location).Aggregatable().StoreValue()).
 			AddField(bluge.NewKeywordField(documentFieldKind, string(entityKindPanel)).Aggregatable().StoreValue()) // likely want independent index for this
 
+		for _, xform := range panel.Transformer {
+			doc.AddField(bluge.NewKeywordField(documentFieldTransformer, xform).Aggregatable())
+		}
+
+		for _, ds := range panel.Datasource {
+			if ds.UID != "" {
+				doc.AddField(bluge.NewKeywordField(documentFieldDSUID, ds.UID).
+					StoreValue().
+					Aggregatable().
+					SearchTermPositions())
+			}
+			if ds.Type != "" {
+				doc.AddField(bluge.NewKeywordField(documentFieldDSType, ds.Type).
+					StoreValue().
+					Aggregatable().
+					SearchTermPositions())
+			}
+		}
+
 		docs = append(docs, doc)
 	}
 	return docs

pkg/services/searchV2/extract/dashboard.go

@@ -203,7 +203,7 @@ func readPanelInfo(iter *jsoniter.Iterator, lookup DatasourceLookup) PanelInfo {
 			for iter.ReadArray() {
 				for sub := iter.ReadObject(); sub != ""; sub = iter.ReadObject() {
 					if sub == "id" {
-						panel.Transformations = append(panel.Transformations, iter.ReadString())
+						panel.Transformer = append(panel.Transformer, iter.ReadString())
 					} else {
 						iter.Skip()
 					}

pkg/services/searchV2/extract/testdata/devdash-graph-shared-tooltips-info.json

@@ -35,7 +35,7 @@
           "type": "default.type"
         }
       ],
-      "transformations": [
+      "transformer": [
         "seriesToColumns",
         "organize"
       ]

pkg/services/searchV2/extract/types.go

@@ -9,13 +9,13 @@ type DataSourceRef struct {
 }
 
 type PanelInfo struct {
-	ID              int64           `json:"id"`
-	Title           string          `json:"title"`
-	Description     string          `json:"description,omitempty"`
-	Type            string          `json:"type,omitempty"` // PluginID
-	PluginVersion   string          `json:"pluginVersion,omitempty"`
-	Datasource      []DataSourceRef `json:"datasource,omitempty"`      // UIDs
-	Transformations []string        `json:"transformations,omitempty"` // ids of the transformation steps
+	ID            int64           `json:"id"`
+	Title         string          `json:"title"`
+	Description   string          `json:"description,omitempty"`
+	Type          string          `json:"type,omitempty"` // PluginID
+	PluginVersion string          `json:"pluginVersion,omitempty"`
+	Datasource    []DataSourceRef `json:"datasource,omitempty"`  // UIDs
+	Transformer   []string        `json:"transformer,omitempty"` // ids of the transformation steps
 
 	// Rows define panels as sub objects
 	Collapsed []PanelInfo `json:"collapsed,omitempty"`

pkg/services/searchV2/index.go

@@ -135,12 +135,17 @@ func (i *dashboardIndex) buildOrgIndex(ctx context.Context, orgID int64) (int, e
 		"orgId", orgID,
 		"orgSearchIndexLoadTime", orgSearchIndexLoadTime,
 		"orgSearchIndexBuildTime", orgSearchIndexBuildTime,
-		"orgSearchIndexTotalTime", orgSearchIndexTotalTime)
+		"orgSearchIndexTotalTime", orgSearchIndexTotalTime,
+		"orgSearchDashboardCount", len(dashboards))
 
 	i.mu.Lock()
 	i.perOrgReader[orgID] = reader
 	i.perOrgWriter[orgID] = writer
 	i.mu.Unlock()
+
+	if orgID == 1 {
+		go updateUsageStats(context.Background(), reader, i.logger)
+	}
 	return len(dashboards), nil
 }
 

pkg/services/searchV2/usage.go

@@ -0,0 +1,74 @@
+package searchV2
+
+import (
+	"context"
+
+	"github.com/blugelabs/bluge"
+	"github.com/blugelabs/bluge/search"
+	"github.com/blugelabs/bluge/search/aggregations"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+type usageGauge struct {
+	field string
+	gauge *prometheus.GaugeVec
+}
+
+var (
+	infoPanelUsage = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name:      "panel_type_usage",
+		Help:      "a metric indicating how many panels across all dashboards use each plugin panel type",
+		Namespace: "grafana",
+	}, []string{"name"})
+
+	infoDatasourceUsage = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name:      "panel_datasource_usage",
+		Help:      "indicates how many panels across all dashboards reference each datasource type",
+		Namespace: "grafana",
+	}, []string{"name"})
+
+	infoTransformerUsage = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Name:      "panel_transformer_usage",
+		Help:      "indicates how many panels use each transformer type",
+		Namespace: "grafana",
+	}, []string{"name"})
+
+	panelUsage = []usageGauge{
+		{field: documentFieldDSType, gauge: infoDatasourceUsage},
+		{field: documentFieldPanelType, gauge: infoPanelUsage},
+		{field: documentFieldTransformer, gauge: infoTransformerUsage},
+	}
+)
+
+func updateUsageStats(ctx context.Context, reader *bluge.Reader, logger log.Logger) {
+	req := bluge.NewAllMatches(bluge.NewTermQuery("panel").SetField(documentFieldKind))
+	for _, usage := range panelUsage {
+		req.AddAggregation(usage.field, aggregations.NewTermsAggregation(search.Field(usage.field), 50))
+	}
+
+	// execute this search on the reader
+	documentMatchIterator, err := reader.Search(ctx, req)
+	if err != nil {
+		logger.Error("error executing search: %v", err)
+		return
+	}
+
+	// need to iterate through the document matches, otherwise the aggregations are empty?
+	match, err := documentMatchIterator.Next()
+	for err == nil && match != nil {
+		match, err = documentMatchIterator.Next()
+	}
+
+	aggs := documentMatchIterator.Aggregations()
+	for _, usage := range panelUsage {
+		bucket := aggs.Buckets(usage.field)
+		for _, v := range bucket {
+			if v.Name() == "" {
+				continue
+			}
+			usage.gauge.WithLabelValues(v.Name()).Set(float64(v.Count()))
+		}
+	}
+}