Indexing PoC: Add search/browse (#94126)

* adds Filter gRPC and make protobuf

* adds route for querying the filter gRPC

* wires up Filter gRPC call

* [WIP] index from start

* renames gRPC endpoint to "Search"

* adds /apis/search route into k8s routes. Hacky for now.

* updates readme - wrong casing

* adds feature toggle for unified storage search

* hides US search behind feature flag. Clean up print statements.

* removes indexer - will be added in another PR

* Search: Add API Builder

* adds required method

* implementing UpdateAPIGroupInfo (WIP)

* adds groupversion

* commenting out for now

* remove unneeded code from experimenting and update register.go to match interface required

* list resources and load into index

* pass context

* namespaces search route

* lint

* watch

* add todo

* add todo

* merge

* cleanup

* add todo

* gen protobuf

* lint; fix migration issue

* Updates index mapping function to map unified storage object Value

* Changes Index() to pointer receiver - fixes panic

* add delete

* cleanup

* gets search/browse functioning. Results show up as base64 encoded. Still a WIP.

* Doesnt json re-encode gRPC response in search handler

* add kind to SearchRequest proto

* Updates query interface to be more generic. Make proto. Parses query params in api server.

* make protobuf

* removes unused method and imports

* Returns all indexed fields in search results. Adds pagination support (limit + offset).

* remove comment

* remove unused struct

* gets tenant in search k8s api handler

* adds hardcoded spec field mappings - starting with playlists

* adds all spec fields to search results

* moved helper function for field mappings into index

* only includes allowed spec fields in search results

* cleans up error handling

* removes debug log

---------

Co-authored-by: leonorfmartins <leonorfmartins@gmail.com>
Co-authored-by: Todd Treece <todd.treece@grafana.com>
Co-authored-by: Scott Lepper <scott.lepper@gmail.com>

pkg/storage/unified/resource/index.go

@@ -6,11 +6,13 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"strings"
 
 	"github.com/blevesearch/bleve/v2"
 	"github.com/blevesearch/bleve/v2/analysis/lang/en"
 	"github.com/blevesearch/bleve/v2/mapping"
 	"github.com/google/uuid"
+	"golang.org/x/exp/slices"
 )
 
 type Shard struct {
@@ -111,7 +113,7 @@ func (i *Index) Delete(ctx context.Context, uid string, key *ResourceKey) error
 	return nil
 }
 
-func (i *Index) Search(ctx context.Context, tenant string, query string) ([]string, error) {
+func (i *Index) Search(ctx context.Context, tenant string, query string, limit int, offset int) ([]SearchSummary, error) {
 	if tenant == "" {
 		tenant = "default"
 	}
@@ -119,20 +121,50 @@ func (i *Index) Search(ctx context.Context, tenant string, query string) ([]stri
 	if err != nil {
 		return nil, err
 	}
+
+	// use 10 as a default limit for now
+	if limit <= 0 {
+		limit = 10
+	}
+
 	req := bleve.NewSearchRequest(bleve.NewQueryStringQuery(query))
-	req.Fields = []string{"kind", "spec.title"}
+	req.From = offset
+	req.Size = limit
+
+	req.Fields = []string{"*"} // return all indexed fields in search results
 
 	res, err := shard.index.Search(req)
 	if err != nil {
 		return nil, err
 	}
-
 	hits := res.Hits
-	results := []string{}
-	for _, hit := range hits {
-		val := fmt.Sprintf("%s:%s", hit.Fields["kind"], hit.Fields["spec.title"])
-		results = append(results, val)
+
+	results := make([]SearchSummary, len(hits))
+	for resKey, hit := range hits {
+		searchSummary := SearchSummary{}
+
+		// add common fields to search results
+		searchSummary.Kind = hit.Fields["kind"].(string)
+		searchSummary.Metadata.CreationTimestamp = hit.Fields["metadata.creationTimestamp"].(string)
+		searchSummary.Metadata.Uid = hit.Fields["metadata.uid"].(string)
+
+		// add allowed indexed spec fields to search results
+		specResult := map[string]interface{}{}
+		for k, v := range hit.Fields {
+			if strings.HasPrefix(k, "spec.") {
+				mappedFields := specFieldMappings(searchSummary.Kind)
+				// should only include spec fields we care about in search results
+				if slices.Contains(mappedFields, k) {
+					specKey := strings.TrimPrefix(k, "spec.")
+					specResult[specKey] = v
+				}
+			}
+			searchSummary.Spec = specResult
+		}
+
+		results[resKey] = searchSummary
 	}
+
 	return results, nil
 }
 
@@ -140,11 +172,17 @@ func tenant(res *Resource) string {
 	return res.Metadata.Namespace
 }
 
+type SearchSummary struct {
+	Kind     string `json:"kind"`
+	Metadata `json:"metadata"`
+	Spec     map[string]interface{} `json:"spec"`
+}
+
 type Metadata struct {
 	Name              string
 	Namespace         string
-	Uid               string
-	CreationTimestamp string
+	Uid               string `json:"uid"`
+	CreationTimestamp string `json:"creationTimestamp"`
 	Labels            map[string]string
 	Annotations       map[string]string
 }
@@ -170,27 +208,26 @@ func createFileIndex() (bleve.Index, string, error) {
 	return index, indexPath, err
 }
 
-// TODO: clean this up.  it was copied from owens performance test
 func createIndexMappings() *mapping.IndexMappingImpl {
-	//Create mapping for the name and creationTimestamp fields in the metadata
-	nameFieldMapping := bleve.NewTextFieldMapping()
+	//Create mapping for the creationTimestamp field in the metadata
 	creationTimestampFieldMapping := bleve.NewDateTimeFieldMapping()
+	uidMapping := bleve.NewTextFieldMapping()
 	metaMapping := bleve.NewDocumentMapping()
-	metaMapping.AddFieldMappingsAt("name", nameFieldMapping)
 	metaMapping.AddFieldMappingsAt("creationTimestamp", creationTimestampFieldMapping)
+	metaMapping.AddFieldMappingsAt("uid", uidMapping)
 	metaMapping.Dynamic = false
 	metaMapping.Enabled = true
 
+	// Spec is different for all resources, so we create a dynamic mapping for it to index all fields (for now)
 	specMapping := bleve.NewDocumentMapping()
-	specMapping.AddFieldMappingsAt("title", nameFieldMapping)
-	specMapping.Dynamic = false
+	specMapping.Dynamic = true
 	specMapping.Enabled = true
 
 	//Create a sub-document mapping for the metadata field
 	objectMapping := bleve.NewDocumentMapping()
 	objectMapping.AddSubDocumentMapping("metadata", metaMapping)
 	objectMapping.AddSubDocumentMapping("spec", specMapping)
-	objectMapping.Dynamic = false
+	objectMapping.Dynamic = true
 	objectMapping.Enabled = true
 
 	// a generic reusable mapping for english text
@@ -248,3 +285,14 @@ func fetchResourceTypes() []*ListOptions {
 	})
 	return items
 }
+
+func specFieldMappings(kind string) []string {
+	mappedFields := map[string][]string{
+		"Playlist": {
+			"spec.title",
+			"spec.interval",
+		},
+	}
+
+	return mappedFields[kind]
+}