SearchV2: Support soft deletion (#90217)

* soft delete
* Fix bench test
 Co-authored-by:  Bruno Abrantes <bruno@brunoabrantes.com>
* Add integration test for soft deletion

---------

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>

pkg/services/searchV2/index.go

@@ -849,7 +849,8 @@ func (l sqlDashboardLoader) loadAllDashboards(ctx context.Context, limit int, or
 			rows := make([]*dashboardQueryResult, 0)
 			err := l.sql.WithDbSession(dashboardQueryCtx, func(sess *db.Session) error {
 				sess.Table("dashboard").
-					Where("org_id = ?", orgID)
+					Where("org_id = ?", orgID).
+					Where("deleted IS NULL") // don't index soft delete files
 
 				if lastID > 0 {
 					sess.Where("id > ?", lastID)

pkg/services/searchV2/index_test.go

@@ -12,11 +12,21 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/experimental"
 	"github.com/stretchr/testify/require"
 
+	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/tracing"
+	"github.com/grafana/grafana/pkg/services/accesscontrol/actest"
+	"github.com/grafana/grafana/pkg/services/dashboards/database"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/services/folder/foldertest"
+	"github.com/grafana/grafana/pkg/services/org"
+	"github.com/grafana/grafana/pkg/services/org/orgtest"
+	"github.com/grafana/grafana/pkg/services/quota/quotatest"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/services/store"
 	"github.com/grafana/grafana/pkg/services/store/entity"
+	"github.com/grafana/grafana/pkg/services/tag/tagimpl"
+	"github.com/grafana/grafana/pkg/services/user"
 	"github.com/grafana/grafana/pkg/setting"
 )
 
@@ -731,3 +741,75 @@ func TestDashboardIndex_MultiTermPrefixMatch(t *testing.T) {
 		})
 	}
 }
+
+func setupIntegrationEnv(t *testing.T, folderCount, dashboardsPerFolder int, sqlStore *sqlstore.SQLStore) (*StandardSearchService, *user.SignedInUser, error) {
+	err := populateDB(folderCount, dashboardsPerFolder, sqlStore)
+	require.NoError(t, err, "error when populating the database for integration test")
+
+	// load all dashboards and folders
+	dbLoadingBatchSize := (dashboardsPerFolder + 1) * folderCount
+	cfg := &setting.Cfg{Search: setting.SearchSettings{DashboardLoadingBatchSize: dbLoadingBatchSize}}
+	features := featuremgmt.WithFeatures()
+	orgSvc := &orgtest.FakeOrgService{
+		ExpectedOrgs: []*org.OrgDTO{{ID: 1}},
+	}
+	searchService, ok := ProvideService(cfg, sqlStore, store.NewDummyEntityEventsService(), actest.FakeService{},
+		tracing.InitializeTracerForTest(), features, orgSvc, nil, foldertest.NewFakeService()).(*StandardSearchService)
+	require.True(t, ok)
+
+	err = runSearchService(searchService)
+	require.NoError(t, err, "error when running search service for integration test")
+
+	user := getSignedInUser(folderCount, dashboardsPerFolder)
+
+	return searchService, user, nil
+}
+
+func TestIntegrationSoftDeletion(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	// Set up search v2.
+	folderCount := 1
+	dashboardsPerFolder := 1
+	sqlStore, cfg := db.InitTestDBWithCfg(t)
+	searchService, testUser, err := setupIntegrationEnv(t, folderCount, dashboardsPerFolder, sqlStore)
+	require.NoError(t, err)
+
+	// Query search v2 to ensure "dashboard2" is present.
+	result := searchService.doDashboardQuery(context.Background(), testUser, 1, DashboardQuery{Kind: []string{string(entityKindDashboard)}})
+	require.NoError(t, result.Error)
+	require.NotZero(t, len(result.Frames))
+	for _, field := range result.Frames[0].Fields {
+		if field.Name == "uid" {
+			require.Equal(t, dashboardsPerFolder, field.Len())
+			break
+		}
+	}
+
+	// Set up dashboard store.
+	quotaService := quotatest.New(false, nil)
+	featureToggles := featuremgmt.WithFeatures(featuremgmt.FlagPanelTitleSearch, featuremgmt.FlagDashboardRestore)
+	dashboardStore, err := database.ProvideDashboardStore(sqlStore, cfg, featureToggles, tagimpl.ProvideService(sqlStore), quotaService)
+	require.NoError(t, err)
+
+	// Soft delete "dashboard2".
+	err = dashboardStore.SoftDeleteDashboard(context.Background(), 1, "dashboard2")
+	require.NoError(t, err)
+
+	// Reindex to ensure "dashboard2" is excluded from the index.
+	searchService.dashboardIndex.reIndexFromScratch(context.Background())
+
+	// Query search v2 to ensure "dashboard2" is no longer present.
+	expectedResultCount := dashboardsPerFolder - 1
+	result2 := searchService.doDashboardQuery(context.Background(), testUser, 1, DashboardQuery{Kind: []string{string(entityKindDashboard)}})
+	require.NoError(t, result2.Error)
+	require.NotZero(t, len(result2.Frames))
+	for _, field := range result2.Frames[0].Fields {
+		if field.Name == "uid" {
+			require.Equal(t, expectedResultCount, field.Len())
+			break
+		}
+	}
+}

pkg/services/searchV2/service_bench_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/accesscontrol/actest"
 	"github.com/grafana/grafana/pkg/services/dashboards"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/services/folder/foldertest"
 	"github.com/grafana/grafana/pkg/services/org"
 	"github.com/grafana/grafana/pkg/services/org/orgtest"
 	"github.com/grafana/grafana/pkg/services/store"
@@ -41,7 +42,7 @@ func setupBenchEnv(b *testing.B, folderCount, dashboardsPerFolder int) (*Standar
 		ExpectedOrgs: []*org.OrgDTO{{ID: 1}},
 	}
 	searchService, ok := ProvideService(cfg, sqlStore, store.NewDummyEntityEventsService(), actest.FakeService{},
-		tracing.InitializeTracerForTest(), features, orgSvc, nil, nil).(*StandardSearchService)
+		tracing.InitializeTracerForTest(), features, orgSvc, nil, foldertest.NewFakeService()).(*StandardSearchService)
 	require.True(b, ok)
 
 	err = runSearchService(searchService)