Search: Downgrade Bluge, refactor code (#51560)

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
2025-02-25 18:55:37 -06:00 · 2022-07-04 11:33:07 +03:00
parent 16f9830fc3
commit f233a74b70
6 changed files with 287 additions and 235 deletions
--- a/pkg/services/searchV2/index_test.go
+++ b/pkg/services/searchV2/index_test.go
@@ -37,17 +37,29 @@ var testDisallowAllFilter = func(uid string) bool {

 var testOrgID int64 = 1

-func initTestIndexFromDashes(t *testing.T, dashboards []dashboard) (*dashboardIndex, *bluge.Reader, *bluge.Writer) {
+func initTestOrgIndexFromDashes(t *testing.T, dashboards []dashboard) *orgIndex {
+	t.Helper()
+	searchIdx := initTestIndexFromDashesExtended(t, dashboards, &NoopDocumentExtender{})
+	return searchIdx.perOrgIndex[testOrgID]
+}
+
+func initTestOrgIndexFromDashesExtended(t *testing.T, dashboards []dashboard, extender DocumentExtender) *orgIndex {
+	t.Helper()
+	searchIdx := initTestIndexFromDashesExtended(t, dashboards, extender)
+	return searchIdx.perOrgIndex[testOrgID]
+}
+
+func initTestIndexFromDashes(t *testing.T, dashboards []dashboard) *searchIndex {
 	t.Helper()
 	return initTestIndexFromDashesExtended(t, dashboards, &NoopDocumentExtender{})
 }

-func initTestIndexFromDashesExtended(t *testing.T, dashboards []dashboard, extender DocumentExtender) (*dashboardIndex, *bluge.Reader, *bluge.Writer) {
+func initTestIndexFromDashesExtended(t *testing.T, dashboards []dashboard, extender DocumentExtender) *searchIndex {
 	t.Helper()
 	dashboardLoader := &testDashboardLoader{
 		dashboards: dashboards,
 	}
-	index := newDashboardIndex(
+	index := newSearchIndex(
 		dashboardLoader,
 		&store.MockEntityEventsService{},
 		extender,
@@ -56,21 +68,17 @@ func initTestIndexFromDashesExtended(t *testing.T, dashboards []dashboard, exten
 	numDashboards, err := index.buildOrgIndex(context.Background(), testOrgID)
 	require.NoError(t, err)
 	require.Equal(t, len(dashboardLoader.dashboards), numDashboards)
-	reader, ok := index.getOrgReader(testOrgID)
-	require.True(t, ok)
-	writer, ok := index.getOrgWriter(testOrgID)
-	require.True(t, ok)
-	return index, reader, writer
+	return index
 }

-func checkSearchResponse(t *testing.T, fileName string, reader *bluge.Reader, filter ResourceFilter, query DashboardQuery) {
+func checkSearchResponse(t *testing.T, fileName string, index *orgIndex, filter ResourceFilter, query DashboardQuery) {
 	t.Helper()
-	checkSearchResponseExtended(t, fileName, reader, filter, query, &NoopQueryExtender{})
+	checkSearchResponseExtended(t, fileName, index, filter, query, &NoopQueryExtender{})
 }

-func checkSearchResponseExtended(t *testing.T, fileName string, reader *bluge.Reader, filter ResourceFilter, query DashboardQuery, extender QueryExtender) {
+func checkSearchResponseExtended(t *testing.T, fileName string, index *orgIndex, filter ResourceFilter, query DashboardQuery, extender QueryExtender) {
 	t.Helper()
-	resp := doSearchQuery(context.Background(), testLogger, reader, filter, query, extender, "/pfix")
+	resp := doSearchQuery(context.Background(), testLogger, index, filter, query, extender, "/pfix")
 	experimental.CheckGoldenJSONResponse(t, "testdata", fileName, resp, true)
 }

@@ -93,15 +101,15 @@ var testDashboards = []dashboard{

 func TestDashboardIndex(t *testing.T) {
 	t.Run("basic-search", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, testDashboards)
-		checkSearchResponse(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashes(t, testDashboards)
+		checkSearchResponse(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "boom"},
 		)
 	})

 	t.Run("basic-filter", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, testDashboards)
-		checkSearchResponse(t, filepath.Base(t.Name()), reader, testDisallowAllFilter,
+		index := initTestOrgIndexFromDashes(t, testDashboards)
+		checkSearchResponse(t, filepath.Base(t.Name()), index, testDisallowAllFilter,
 			DashboardQuery{Query: "boom"},
 		)
 	})
@@ -109,20 +117,22 @@ func TestDashboardIndex(t *testing.T) {

 func TestDashboardIndexUpdates(t *testing.T) {
 	t.Run("dashboard-delete", func(t *testing.T) {
-		index, reader, writer := initTestIndexFromDashes(t, testDashboards)
-
-		newReader, err := index.removeDashboard(context.Background(), writer, reader, "2")
+		index := initTestIndexFromDashes(t, testDashboards)
+		orgIdx, ok := index.getOrgIndex(testOrgID)
+		require.True(t, ok)
+		err := index.removeDashboard(context.Background(), orgIdx, "2")
 		require.NoError(t, err)
-
-		checkSearchResponse(t, filepath.Base(t.Name()), newReader, testAllowAllFilter,
+		checkSearchResponse(t, filepath.Base(t.Name()), orgIdx, testAllowAllFilter,
 			DashboardQuery{Query: "boom"},
 		)
 	})

 	t.Run("dashboard-create", func(t *testing.T) {
-		index, reader, writer := initTestIndexFromDashes(t, testDashboards)
+		index := initTestIndexFromDashes(t, testDashboards)
+		orgIdx, ok := index.getOrgIndex(testOrgID)
+		require.True(t, ok)

-		newReader, err := index.updateDashboard(context.Background(), testOrgID, writer, reader, dashboard{
+		err := index.updateDashboard(context.Background(), testOrgID, orgIdx, dashboard{
 			id:  3,
 			uid: "3",
 			info: &extract.DashboardInfo{
@@ -131,15 +141,17 @@ func TestDashboardIndexUpdates(t *testing.T) {
 		})
 		require.NoError(t, err)

-		checkSearchResponse(t, filepath.Base(t.Name()), newReader, testAllowAllFilter,
+		checkSearchResponse(t, filepath.Base(t.Name()), orgIdx, testAllowAllFilter,
 			DashboardQuery{Query: "created"},
 		)
 	})

 	t.Run("dashboard-update", func(t *testing.T) {
-		index, reader, writer := initTestIndexFromDashes(t, testDashboards)
+		index := initTestIndexFromDashes(t, testDashboards)
+		orgIdx, ok := index.getOrgIndex(testOrgID)
+		require.True(t, ok)

-		newReader, err := index.updateDashboard(context.Background(), testOrgID, writer, reader, dashboard{
+		err := index.updateDashboard(context.Background(), testOrgID, orgIdx, dashboard{
 			id:  2,
 			uid: "2",
 			info: &extract.DashboardInfo{
@@ -148,7 +160,7 @@ func TestDashboardIndexUpdates(t *testing.T) {
 		})
 		require.NoError(t, err)

-		checkSearchResponse(t, filepath.Base(t.Name()), newReader, testAllowAllFilter,
+		checkSearchResponse(t, filepath.Base(t.Name()), orgIdx, testAllowAllFilter,
 			DashboardQuery{Query: "nginx"},
 		)
 	})
@@ -231,15 +243,15 @@ func TestDashboardIndexSort(t *testing.T) {
 	}

 	t.Run("sort-asc", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashesExtended(t, testSortDashboards, extender.GetDocumentExtender())
-		checkSearchResponseExtended(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashesExtended(t, testSortDashboards, extender.GetDocumentExtender())
+		checkSearchResponseExtended(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "*", Sort: "test"}, extender.GetQueryExtender(),
 		)
 	})

 	t.Run("sort-desc", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashesExtended(t, testSortDashboards, extender.GetDocumentExtender())
-		checkSearchResponseExtended(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashesExtended(t, testSortDashboards, extender.GetDocumentExtender())
+		checkSearchResponseExtended(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "*", Sort: "-test"}, extender.GetQueryExtender(),
 		)
 	})
@@ -264,29 +276,29 @@ var testPrefixDashboards = []dashboard{

 func TestDashboardIndex_PrefixSearch(t *testing.T) {
 	t.Run("prefix-search-beginning", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, testPrefixDashboards)
-		checkSearchResponse(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashes(t, testPrefixDashboards)
+		checkSearchResponse(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "Arch"},
 		)
 	})

 	t.Run("prefix-search-middle", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, testPrefixDashboards)
-		checkSearchResponse(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashes(t, testPrefixDashboards)
+		checkSearchResponse(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "Syn"},
 		)
 	})

 	t.Run("prefix-search-beginning-lower", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, testPrefixDashboards)
-		checkSearchResponse(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashes(t, testPrefixDashboards)
+		checkSearchResponse(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "arch"},
 		)
 	})

 	t.Run("prefix-search-middle-lower", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, testPrefixDashboards)
-		checkSearchResponse(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashes(t, testPrefixDashboards)
+		checkSearchResponse(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "syn"},
 		)
 	})
@@ -294,30 +306,30 @@ func TestDashboardIndex_PrefixSearch(t *testing.T) {

 func TestDashboardIndex_MultipleTokensInRow(t *testing.T) {
 	t.Run("multiple-tokens-beginning", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, testPrefixDashboards)
-		checkSearchResponse(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashes(t, testPrefixDashboards)
+		checkSearchResponse(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "Archer da"},
 		)
 	})

 	t.Run("multiple-tokens-beginning-lower", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, testPrefixDashboards)
-		checkSearchResponse(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashes(t, testPrefixDashboards)
+		checkSearchResponse(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "da archer"},
 		)
 	})

 	// Not sure it is great this matches, but
 	t.Run("multiple-tokens-middle", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, testPrefixDashboards)
-		checkSearchResponse(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashes(t, testPrefixDashboards)
+		checkSearchResponse(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "ar Da"},
 		)
 	})

 	t.Run("multiple-tokens-middle-lower", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, testPrefixDashboards)
-		checkSearchResponse(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashes(t, testPrefixDashboards)
+		checkSearchResponse(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "doc sy"},
 		)
 	})
@@ -335,8 +347,8 @@ var longPrefixDashboards = []dashboard{

 func TestDashboardIndex_PrefixNgramExceeded(t *testing.T) {
 	t.Run("prefix-search-ngram-exceeded", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, longPrefixDashboards)
-		checkSearchResponse(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashes(t, longPrefixDashboards)
+		checkSearchResponse(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "Eyjafjallajöku"},
 		)
 	})
@@ -361,14 +373,14 @@ var scatteredTokensDashboards = []dashboard{

 func TestDashboardIndex_MultipleTokensScattered(t *testing.T) {
 	t.Run("scattered-tokens-match", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, scatteredTokensDashboards)
-		checkSearchResponse(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashes(t, scatteredTokensDashboards)
+		checkSearchResponse(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "dead secret"},
 		)
 	})
 	t.Run("scattered-tokens-match-reversed", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, scatteredTokensDashboards)
-		checkSearchResponse(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashes(t, scatteredTokensDashboards)
+		checkSearchResponse(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "powerful secret"},
 		)
 	})
@@ -432,15 +444,15 @@ var dashboardsWithFolders = []dashboard{

 func TestDashboardIndex_Folders(t *testing.T) {
 	t.Run("folders-indexed", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, dashboardsWithFolders)
-		checkSearchResponse(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashes(t, dashboardsWithFolders)
+		checkSearchResponse(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "My folder", Kind: []string{string(entityKindFolder)}},
 		)
 	})
 	t.Run("folders-dashboard-has-folder", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, dashboardsWithFolders)
+		index := initTestOrgIndexFromDashes(t, dashboardsWithFolders)
 		// TODO: golden file compare does not work here.
-		resp := doSearchQuery(context.Background(), testLogger, reader, testAllowAllFilter,
+		resp := doSearchQuery(context.Background(), testLogger, index, testAllowAllFilter,
 			DashboardQuery{Query: "Dashboard in folder", Kind: []string{string(entityKindDashboard)}},
 			&NoopQueryExtender{}, "")
 		custom, ok := resp.Frames[0].Meta.Custom.(*customMeta)
@@ -449,19 +461,23 @@ func TestDashboardIndex_Folders(t *testing.T) {
 		require.Equal(t, "/dashboards/f/1/", custom.Locations["1"].URL)
 	})
 	t.Run("folders-dashboard-removed-on-folder-removed", func(t *testing.T) {
-		index, reader, writer := initTestIndexFromDashes(t, dashboardsWithFolders)
-		newReader, err := index.removeFolder(context.Background(), writer, reader, "1")
+		index := initTestIndexFromDashes(t, dashboardsWithFolders)
+		orgIdx, ok := index.getOrgIndex(testOrgID)
+		require.True(t, ok)
+		err := index.removeFolder(context.Background(), orgIdx, "1")
 		require.NoError(t, err)
 		// In response we expect one dashboard which does not belong to removed folder.
-		checkSearchResponse(t, filepath.Base(t.Name()), newReader, testAllowAllFilter,
+		checkSearchResponse(t, filepath.Base(t.Name()), orgIdx, testAllowAllFilter,
 			DashboardQuery{Query: "dash", Kind: []string{string(entityKindDashboard)}},
 		)
 	})
 	t.Run("folders-panels-removed-on-folder-removed", func(t *testing.T) {
-		index, reader, writer := initTestIndexFromDashes(t, dashboardsWithFolders)
-		newReader, err := index.removeFolder(context.Background(), writer, reader, "1")
+		index := initTestIndexFromDashes(t, dashboardsWithFolders)
+		orgIdx, ok := index.getOrgIndex(testOrgID)
+		require.True(t, ok)
+		err := index.removeFolder(context.Background(), orgIdx, "1")
 		require.NoError(t, err)
-		resp := doSearchQuery(context.Background(), testLogger, newReader, testAllowAllFilter,
+		resp := doSearchQuery(context.Background(), testLogger, orgIdx, testAllowAllFilter,
 			DashboardQuery{Query: "Panel", Kind: []string{string(entityKindPanel)}},
 			&NoopQueryExtender{}, "")
 		custom, ok := resp.Frames[0].Meta.Custom.(*customMeta)
@@ -492,10 +508,10 @@ var dashboardsWithPanels = []dashboard{

 func TestDashboardIndex_Panels(t *testing.T) {
 	t.Run("panels-indexed", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, dashboardsWithPanels)
+		index := initTestOrgIndexFromDashes(t, dashboardsWithPanels)
 		// TODO: golden file compare does not work here.
 		resp := doSearchQuery(
-			context.Background(), testLogger, reader, testAllowAllFilter,
+			context.Background(), testLogger, index, testAllowAllFilter,
 			DashboardQuery{Query: "Panel", Kind: []string{string(entityKindPanel)}},
 			&NoopQueryExtender{}, "")
 		custom, ok := resp.Frames[0].Meta.Custom.(*customMeta)
@@ -504,10 +520,12 @@ func TestDashboardIndex_Panels(t *testing.T) {
 		require.Equal(t, "/d/1/", custom.Locations["1"].URL)
 	})
 	t.Run("panels-panel-removed-on-dashboard-removed", func(t *testing.T) {
-		index, reader, writer := initTestIndexFromDashes(t, dashboardsWithPanels)
-		newReader, err := index.removeDashboard(context.Background(), writer, reader, "1")
+		index := initTestIndexFromDashes(t, dashboardsWithPanels)
+		orgIdx, ok := index.getOrgIndex(testOrgID)
+		require.True(t, ok)
+		err := index.removeDashboard(context.Background(), orgIdx, "1")
 		require.NoError(t, err)
-		checkSearchResponse(t, filepath.Base(t.Name()), newReader, testAllowAllFilter,
+		checkSearchResponse(t, filepath.Base(t.Name()), orgIdx, testAllowAllFilter,
 			DashboardQuery{Query: "Panel", Kind: []string{string(entityKindPanel)}},
 		)
 	})
@@ -532,15 +550,15 @@ var punctuationSplitNgramDashboards = []dashboard{

 func TestDashboardIndex_PunctuationNgram(t *testing.T) {
 	t.Run("ngram-punctuation-split", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, punctuationSplitNgramDashboards)
-		checkSearchResponse(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashes(t, punctuationSplitNgramDashboards)
+		checkSearchResponse(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "tork he"},
 		)
 	})

 	t.Run("ngram-simple", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, punctuationSplitNgramDashboards)
-		checkSearchResponse(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashes(t, punctuationSplitNgramDashboards)
+		checkSearchResponse(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "hea"},
 		)
 	})
@@ -558,8 +576,8 @@ var camelCaseNgramDashboards = []dashboard{

 func TestDashboardIndex_CamelCaseNgram(t *testing.T) {
 	t.Run("ngram-camel-case-split", func(t *testing.T) {
-		_, reader, _ := initTestIndexFromDashes(t, camelCaseNgramDashboards)
-		checkSearchResponse(t, filepath.Base(t.Name()), reader, testAllowAllFilter,
+		index := initTestOrgIndexFromDashes(t, camelCaseNgramDashboards)
+		checkSearchResponse(t, filepath.Base(t.Name()), index, testAllowAllFilter,
 			DashboardQuery{Query: "tork"},
 		)
 	})