Search: remove internal ID from bluge index (#49205)

Co-authored-by: Alexander Emelin <frvzmb@gmail.com>

pkg/services/searchV2/bluge.go

@@ -25,14 +25,13 @@ const (
 	documentFieldTag         = "tag"
 	documentFieldURL         = "url"
 	documentFieldName        = "name"
-	documentFieldName_ngram  = "name_ngram"
+	documentFieldNameNgram   = "name_ngram"
 	documentFieldDescription = "description"
 	documentFieldLocation    = "location" // parent path
 	documentFieldPanelType   = "panel_type"
 	documentFieldTransformer = "transformer"
 	documentFieldDSUID       = "ds_uid"
 	documentFieldDSType      = "ds_type"
-	documentFieldInternalID  = "__internal_id" // only for migrations! (indexed as a string)
 )
 
 func initIndex(dashboards []dashboard, logger log.Logger, extendDoc ExtendDashboardFunc) (*bluge.Reader, *bluge.Writer, error) {
@@ -149,10 +148,9 @@ func getFolderDashboardDoc(dash dashboard) *bluge.Document {
 		AddField(bluge.NewKeywordField(documentFieldKind, string(entityKindFolder)).Aggregatable().StoreValue()).
 		AddField(bluge.NewKeywordField(documentFieldURL, url).StoreValue()).
 		AddField(bluge.NewTextField(documentFieldName, dash.info.Title).StoreValue().SearchTermPositions()).
-		AddField(getNameNGramField(dash.info.Title)).
 		AddField(bluge.NewTextField(documentFieldDescription, dash.info.Description).SearchTermPositions()).
-		// Add legacy ID (for lookup by internal ID)
+		AddField(getNameNGramField(dash.info.Title)).
-		AddField(bluge.NewKeywordField(documentFieldInternalID, fmt.Sprintf("%d", dash.id)).Aggregatable().StoreValue())
+		AddField(bluge.NewTextField(documentFieldDescription, dash.info.Description).SearchTermPositions())
 }
 
 func getNonFolderDashboardDoc(dash dashboard, location string) *bluge.Document {
@@ -167,9 +165,6 @@ func getNonFolderDashboardDoc(dash dashboard, location string) *bluge.Document {
 		AddField(getNameNGramField(dash.info.Title)).
 		AddField(bluge.NewTextField(documentFieldDescription, dash.info.Description).SearchTermPositions())
 
-	// Add legacy ID (for lookup by internal ID)
-	doc.AddField(bluge.NewKeywordField(documentFieldInternalID, fmt.Sprintf("%d", dash.id)))
-
 	for _, tag := range dash.info.Tags {
 		doc.AddField(bluge.NewKeywordField(documentFieldTag, tag).
 			StoreValue().
@@ -255,36 +250,7 @@ var ngramQueryAnalyzer = &analysis.Analyzer{
 }
 
 func getNameNGramField(name string) bluge.Field {
-	return bluge.NewTextField(documentFieldName_ngram, name).WithAnalyzer(ngramIndexAnalyzer)
+	return bluge.NewTextField(documentFieldNameNgram, name).WithAnalyzer(ngramIndexAnalyzer)
-}
-
-func getDashboardFolderUID(reader *bluge.Reader, folderID int64) (string, error) {
-	fullQuery := bluge.NewBooleanQuery()
-	fullQuery.AddMust(bluge.NewTermQuery(strconv.FormatInt(folderID, 10)).SetField(documentFieldInternalID))
-	fullQuery.AddMust(bluge.NewTermQuery(string(entityKindFolder)).SetField(documentFieldKind))
-	req := bluge.NewAllMatches(fullQuery)
-	req.WithStandardAggregations()
-	documentMatchIterator, err := reader.Search(context.Background(), req)
-	if err != nil {
-		return "", err
-	}
-	var uid string
-	match, err := documentMatchIterator.Next()
-	for err == nil && match != nil {
-		// load the identifier for this match
-		err = match.VisitStoredFields(func(field string, value []byte) bool {
-			if field == documentFieldUID {
-				uid = string(value)
-			}
-			return true
-		})
-		if err != nil {
-			return "", err
-		}
-		// load the next document match
-		match, err = documentMatchIterator.Next()
-	}
-	return uid, err
 }
 
 func getDashboardPanelIDs(reader *bluge.Reader, dashboardUID string) ([]string, error) {
@@ -293,7 +259,6 @@ func getDashboardPanelIDs(reader *bluge.Reader, dashboardUID string) ([]string,
 	fullQuery.AddMust(bluge.NewTermQuery(dashboardUID).SetField(documentFieldDSUID))
 	fullQuery.AddMust(bluge.NewTermQuery(string(entityKindPanel)).SetField(documentFieldKind))
 	req := bluge.NewAllMatches(fullQuery)
-	req.WithStandardAggregations()
 	documentMatchIterator, err := reader.Search(context.Background(), req)
 	if err != nil {
 		return nil, err
@@ -320,7 +285,7 @@ func getDashboardPanelIDs(reader *bluge.Reader, dashboardUID string) ([]string,
 func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader, filter ResourceFilter, q DashboardQuery, extender QueryExtender) *backend.DataResponse {
 	response := &backend.DataResponse{}
 
-	// Folder listing structure
+	// Folder listing structure.
 	idx := strings.Index(q.Query, ":")
 	if idx > 0 {
 		key := q.Query[0:idx]
@@ -345,7 +310,7 @@ func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader,
 	fullQuery := bluge.NewBooleanQuery()
 	fullQuery.AddMust(newPermissionFilter(filter, logger))
 
-	// Only show dashboard / folders
+	// Only show dashboard / folders / panels.
 	if len(q.Kind) > 0 {
 		bq := bluge.NewBooleanQuery()
 		for _, k := range q.Kind {
@@ -400,7 +365,7 @@ func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader,
 			AddShould(bluge.NewMatchPhraseQuery(q.Query).SetField(documentFieldName).SetBoost(6)).
 			AddShould(bluge.NewMatchPhraseQuery(q.Query).SetField(documentFieldDescription).SetBoost(3)).
 			AddShould(bluge.NewMatchQuery(q.Query).
-				SetField(documentFieldName_ngram).
+				SetField(documentFieldNameNgram).
 				SetAnalyzer(ngramQueryAnalyzer).SetBoost(1))
 
 		if len(q.Query) > 4 {
@@ -447,8 +412,6 @@ func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader,
 	dvfieldNames := []string{"type"}
 	sctx := search.NewSearchContext(0, 0)
 
-	// numericFields := map[string]bool{"schemaVersion": true, "panelCount": true}
-
 	fScore := data.NewFieldFromFieldType(data.FieldTypeFloat64, 0)
 	fUID := data.NewFieldFromFieldType(data.FieldTypeString, 0)
 	fKind := data.NewFieldFromFieldType(data.FieldTypeString, 0)
@@ -500,19 +463,10 @@ func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader,
 		name := ""
 		url := ""
 		loc := ""
-		var ds_uids []string
+		var dsUIDs []string
 		var tags []string
 
 		err = match.VisitStoredFields(func(field string, value []byte) bool {
-			// if numericFields[field] {
-			// 	num, err2 := bluge.DecodeNumericFloat64(value)
-			// 	if err2 != nil {
-			// 		vals[field] = num
-			// 	}
-			// } else {
-			// 	vals[field] = string(value)
-			// }
-
 			switch field {
 			case documentFieldUID:
 				uid = string(value)
@@ -527,7 +481,7 @@ func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader,
 			case documentFieldLocation:
 				loc = string(value)
 			case documentFieldDSUID:
-				ds_uids = append(ds_uids, string(value))
+				dsUIDs = append(dsUIDs, string(value))
 			case documentFieldTag:
 				tags = append(tags, string(value))
 			default:
@@ -563,8 +517,8 @@ func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader,
 			fTags.Append(nil)
 		}
 
-		if len(ds_uids) > 0 {
+		if len(dsUIDs) > 0 {
-			js, _ := json.Marshal(ds_uids)
+			js, _ := json.Marshal(dsUIDs)
 			jsb := json.RawMessage(js)
 			fDSUIDs.Append(&jsb)
 		} else {
@@ -598,7 +552,7 @@ func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader,
 	aggs := documentMatchIterator.Aggregations()
 
 	header := &customMeta{
-		Count: aggs.Count(), // Total cound
+		Count: aggs.Count(), // Total count.
 	}
 	if q.Explain {
 		header.MaxScore = aggs.Metric("max_score")

pkg/services/searchV2/index.go

@@ -34,6 +34,10 @@ type eventStore interface {
 	GetAllEventsAfter(ctx context.Context, id int64) ([]*store.EntityEvent, error)
 }
 
+// While we migrate away from internal IDs... this lets us lookup values in SQL
+// NOTE: folderId is unique across all orgs
+type folderUIDLookup = func(ctx context.Context, folderId int64) (string, error)
+
 type dashboard struct {
 	id       int64
 	uid      string
@@ -46,25 +50,27 @@ type dashboard struct {
 }
 
 type dashboardIndex struct {
-	mu           sync.RWMutex
+	mu             sync.RWMutex
-	loader       dashboardLoader
+	loader         dashboardLoader
-	perOrgReader map[int64]*bluge.Reader // orgId -> bluge reader
+	perOrgReader   map[int64]*bluge.Reader // orgId -> bluge reader
-	perOrgWriter map[int64]*bluge.Writer // orgId -> bluge writer
+	perOrgWriter   map[int64]*bluge.Writer // orgId -> bluge writer
-	eventStore   eventStore
+	eventStore     eventStore
-	logger       log.Logger
+	logger         log.Logger
-	buildSignals chan int64
+	buildSignals   chan int64
-	extender     DocumentExtender
+	extender       DocumentExtender
+	folderIdLookup folderUIDLookup
 }
 
-func newDashboardIndex(dashLoader dashboardLoader, evStore eventStore, extender DocumentExtender) *dashboardIndex {
+func newDashboardIndex(dashLoader dashboardLoader, evStore eventStore, extender DocumentExtender, folderIDs folderUIDLookup) *dashboardIndex {
 	return &dashboardIndex{
-		loader:       dashLoader,
+		loader:         dashLoader,
-		eventStore:   evStore,
+		eventStore:     evStore,
-		perOrgReader: map[int64]*bluge.Reader{},
+		perOrgReader:   map[int64]*bluge.Reader{},
-		perOrgWriter: map[int64]*bluge.Writer{},
+		perOrgWriter:   map[int64]*bluge.Writer{},
-		logger:       log.New("dashboardIndex"),
+		logger:         log.New("dashboardIndex"),
-		buildSignals: make(chan int64),
+		buildSignals:   make(chan int64),
-		extender:     extender,
+		extender:       extender,
+		folderIdLookup: folderIDs,
 	}
 }
 
@@ -351,9 +357,9 @@ func (i *dashboardIndex) applyDashboardEvent(ctx context.Context, orgID int64, d
 
 	// In the future we can rely on operation types to reduce work here.
 	if len(dbDashboards) == 0 {
-		newReader, err = i.removeDashboard(writer, reader, dashboardUID)
+		newReader, err = i.removeDashboard(ctx, writer, reader, dashboardUID)
 	} else {
-		newReader, err = i.updateDashboard(orgID, writer, reader, dbDashboards[0])
+		newReader, err = i.updateDashboard(ctx, orgID, writer, reader, dbDashboards[0])
 	}
 	if err != nil {
 		return err
@@ -362,7 +368,7 @@ func (i *dashboardIndex) applyDashboardEvent(ctx context.Context, orgID int64, d
 	return nil
 }
 
-func (i *dashboardIndex) removeDashboard(writer *bluge.Writer, reader *bluge.Reader, dashboardUID string) (*bluge.Reader, error) {
+func (i *dashboardIndex) removeDashboard(_ context.Context, writer *bluge.Writer, reader *bluge.Reader, dashboardUID string) (*bluge.Reader, error) {
 	// Find all panel docs to remove with dashboard.
 	panelIDs, err := getDashboardPanelIDs(reader, dashboardUID)
 	if err != nil {
@@ -392,7 +398,7 @@ func stringInSlice(str string, slice []string) bool {
 	return false
 }
 
-func (i *dashboardIndex) updateDashboard(orgID int64, writer *bluge.Writer, reader *bluge.Reader, dash dashboard) (*bluge.Reader, error) {
+func (i *dashboardIndex) updateDashboard(ctx context.Context, orgID int64, writer *bluge.Writer, reader *bluge.Reader, dash dashboard) (*bluge.Reader, error) {
 	batch := bluge.NewBatch()
 
 	extendDoc := i.extender.GetDashboardExtender(orgID, dash.uid)
@@ -409,7 +415,7 @@ func (i *dashboardIndex) updateDashboard(orgID int64, writer *bluge.Writer, read
 			folderUID = "general"
 		} else {
 			var err error
-			folderUID, err = getDashboardFolderUID(reader, dash.folderID)
+			folderUID, err = i.folderIdLookup(ctx, dash.folderID)
 			if err != nil {
 				return nil, err
 			}
@@ -547,6 +553,27 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
 	return dashboards, err
 }
 
+func newFolderIDLookup(sql *sqlstore.SQLStore) folderUIDLookup {
+	return func(ctx context.Context, folderID int64) (string, error) {
+		uid := ""
+		err := sql.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
+			sess.Table("dashboard").
+				Where("id = ?", folderID).
+				Cols("uid")
+
+			res, err := sess.Query("SELECT uid FROM dashboard WHERE id=?", folderID)
+			if err != nil {
+				return err
+			}
+			if len(res) > 0 {
+				uid = string(res[0]["uid"])
+			}
+			return nil
+		})
+		return uid, err
+	}
+}
+
 type dashboardQueryResult struct {
 	Id       int64
 	Uid      string

pkg/services/searchV2/index_test.go

@@ -48,7 +48,11 @@ func initTestIndexFromDashesExtended(t *testing.T, dashboards []dashboard, exten
 	dashboardLoader := &testDashboardLoader{
 		dashboards: dashboards,
 	}
-	index := newDashboardIndex(dashboardLoader, &store.MockEntityEventsService{}, extender)
+	index := newDashboardIndex(
+		dashboardLoader,
+		&store.MockEntityEventsService{},
+		extender,
+		func(ctx context.Context, folderId int64) (string, error) { return "x", nil })
 	require.NotNil(t, index)
 	numDashboards, err := index.buildOrgIndex(context.Background(), testOrgID)
 	require.NoError(t, err)
@@ -110,7 +114,7 @@ func TestDashboardIndexUpdates(t *testing.T) {
 	t.Run("dashboard-delete", func(t *testing.T) {
 		index, reader, writer := initTestIndexFromDashes(t, testDashboards)
 
-		newReader, err := index.removeDashboard(writer, reader, "2")
+		newReader, err := index.removeDashboard(context.Background(), writer, reader, "2")
 		require.NoError(t, err)
 
 		checkSearchResponse(t, filepath.Base(t.Name())+".txt", newReader, testAllowAllFilter,
@@ -121,7 +125,7 @@ func TestDashboardIndexUpdates(t *testing.T) {
 	t.Run("dashboard-create", func(t *testing.T) {
 		index, reader, writer := initTestIndexFromDashes(t, testDashboards)
 
-		newReader, err := index.updateDashboard(testOrgID, writer, reader, dashboard{
+		newReader, err := index.updateDashboard(context.Background(), testOrgID, writer, reader, dashboard{
 			id:  3,
 			uid: "3",
 			info: &extract.DashboardInfo{
@@ -138,7 +142,7 @@ func TestDashboardIndexUpdates(t *testing.T) {
 	t.Run("dashboard-update", func(t *testing.T) {
 		index, reader, writer := initTestIndexFromDashes(t, testDashboards)
 
-		newReader, err := index.updateDashboard(testOrgID, writer, reader, dashboard{
+		newReader, err := index.updateDashboard(context.Background(), testOrgID, writer, reader, dashboard{
 			id:  2,
 			uid: "2",
 			info: &extract.DashboardInfo{

pkg/services/searchV2/service.go

@@ -39,9 +39,14 @@ func ProvideService(cfg *setting.Cfg, sql *sqlstore.SQLStore, entityEventStore s
 			sql: sql,
 			ac:  ac,
 		},
-		dashboardIndex: newDashboardIndex(newSQLDashboardLoader(sql), entityEventStore, extender.GetDocumentExtender()),
+		dashboardIndex: newDashboardIndex(
-		logger:         log.New("searchV2"),
+			newSQLDashboardLoader(sql),
-		extender:       extender,
+			entityEventStore,
+			extender.GetDocumentExtender(),
+			newFolderIDLookup(sql),
+		),
+		logger:   log.New("searchV2"),
+		extender: extender,
 	}
 	return s
 }