diff --git a/pkg/storage/unified/resource/search.go b/pkg/storage/unified/resource/search.go
index 9bbf25e1b8a..8498b252bfa 100644
--- a/pkg/storage/unified/resource/search.go
+++ b/pkg/storage/unified/resource/search.go
@@ -490,7 +490,7 @@ func (s *searchSupport) build(ctx context.Context, nsr NamespacedResource, size
 	}
 	fields := s.builders.GetFields(nsr)
 
-	s.log.Debug(fmt.Sprintf("TODO, build %+v (size:%d, rv:%d) // builder:%+v\n", nsr, size, rv, builder))
+	s.log.Debug("Building index", "resource", nsr.Resource, "size", size, "rv", rv)
 
 	key := &ResourceKey{
 		Group:     nsr.Group,
diff --git a/pkg/storage/unified/search/bleve.go b/pkg/storage/unified/search/bleve.go
index d0e0c376872..101ee49497d 100644
--- a/pkg/storage/unified/search/bleve.go
+++ b/pkg/storage/unified/search/bleve.go
@@ -111,14 +111,14 @@ func (b *bleveBackend) BuildIndex(ctx context.Context,
 	mapper := getBleveMappings(fields)
 
 	if size > b.opts.FileThreshold {
+		resourceDir := filepath.Join(b.opts.Root, key.Namespace,
+			fmt.Sprintf("%s.%s", key.Resource, key.Group),
+		)
 		fname := fmt.Sprintf("rv%d", resourceVersion)
 		if resourceVersion == 0 {
 			fname = b.start.Format("tmp-20060102-150405")
 		}
-		dir := filepath.Join(b.opts.Root, key.Namespace,
-			fmt.Sprintf("%s.%s", key.Resource, key.Group),
-			fname,
-		)
+		dir := filepath.Join(resourceDir, fname)
 		if resourceVersion > 0 {
 			info, _ := os.Stat(dir)
 			if info != nil && info.IsDir() {
@@ -128,6 +128,10 @@ func (b *bleveBackend) BuildIndex(ctx context.Context,
 					if err != nil || int64(found) != size {
 						b.log.Info("this size changed since the last time the index opened")
 						_ = index.Close()
+
+						// Pick a new file name
+						fname = b.start.Format("tmp-20060102-150405-changed")
+						dir = filepath.Join(resourceDir, fname)
 						index = nil
 					} else {
 						build = false // no need to build the index
@@ -138,8 +142,15 @@ func (b *bleveBackend) BuildIndex(ctx context.Context,
 
 		if index == nil {
 			index, err = bleve.New(dir, mapper)
+			if err != nil {
+				err = fmt.Errorf("error creating new bleve index: %s %w", dir, err)
+			}
 		}
 
+		// Start a background task to cleanup the old index directories
+		if index != nil && err == nil {
+			go b.cleanOldIndexes(resourceDir, fname)
+		}
 		resource.IndexMetrics.IndexTenants.WithLabelValues("file").Inc()
 	} else {
 		index, err = bleve.NewMemOnly(mapper)
@@ -183,6 +194,25 @@ func (b *bleveBackend) BuildIndex(ctx context.Context,
 	return idx, nil
 }
 
+func (b *bleveBackend) cleanOldIndexes(dir string, skip string) {
+	files, err := os.ReadDir(dir)
+	if err != nil {
+		b.log.Warn("error cleaning folders from", "directory", dir, "error", err)
+		return
+	}
+	for _, file := range files {
+		if file.IsDir() && file.Name() != skip {
+			fpath := filepath.Join(dir, file.Name())
+			err = os.RemoveAll(fpath)
+			if err != nil {
+				b.log.Error("Unable to remove old index folder", "directory", fpath, "error", err)
+			} else {
+				b.log.Error("Removed old index folder", "directory", fpath)
+			}
+		}
+	}
+}
+
 // TotalDocs returns the total number of documents across all indices
 func (b *bleveBackend) TotalDocs() int64 {
 	var totalDocs int64