Search: limit max size of batches during indexing (#49187)

pkg/services/searchV2/index.go

@@ -4,6 +4,9 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io/ioutil"
+	"os"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -79,13 +82,10 @@ func (i *dashboardIndex) run(ctx context.Context) error {
 		lastEventID = lastEvent.Id
 	}
 
-	// Build on start for orgID 1 but keep lazy for others.
-	started := time.Now()
-	numDashboards, err := i.buildOrgIndex(ctx, 1)
+	err = i.buildInitialIndex(ctx)
 	if err != nil {
-		return fmt.Errorf("can't build dashboard search index for org ID 1: %w", err)
+		return fmt.Errorf("can't build initial dashboard search index: %w", err)
 	}
-	i.logger.Info("Indexing for main org finished", "mainOrgIndexElapsed", time.Since(started), "numDashboards", numDashboards)
 
 	for {
 		select {
@@ -111,6 +111,93 @@ func (i *dashboardIndex) run(ctx context.Context) error {
 	}
 }
 
+func (i *dashboardIndex) buildInitialIndex(ctx context.Context) error {
+	memCtx, memCancel := context.WithCancel(ctx)
+	if os.Getenv("GF_SEARCH_DEBUG") != "" {
+		go i.debugMemStats(memCtx, 200*time.Millisecond)
+	}
+
+	// Build on start for orgID 1 but keep lazy for others.
+	started := time.Now()
+	numDashboards, err := i.buildOrgIndex(ctx, 1)
+	if err != nil {
+		memCancel()
+		return fmt.Errorf("can't build dashboard search index for org ID 1: %w", err)
+	}
+	i.logger.Info("Indexing for main org finished", "mainOrgIndexElapsed", time.Since(started), "numDashboards", numDashboards)
+	memCancel()
+
+	if os.Getenv("GF_SEARCH_DEBUG") != "" {
+		// May help to estimate size of index when introducing changes. Though it's not a direct
+		// match to a memory consumption, but at least make give some relative difference understanding.
+		// Moreover, changes in indexing can cause additional memory consumption upon initial index build
+		// which is not reflected here.
+		i.reportSizeOfIndexDiskBackup(1)
+	}
+	return nil
+}
+
+func (i *dashboardIndex) debugMemStats(ctx context.Context, frequency time.Duration) {
+	var maxHeapInuse uint64
+	var maxSys uint64
+
+	captureMemStats := func() {
+		var m runtime.MemStats
+		runtime.ReadMemStats(&m)
+		if m.HeapInuse > maxHeapInuse {
+			maxHeapInuse = m.HeapInuse
+		}
+		if m.Sys > maxSys {
+			maxSys = m.Sys
+		}
+	}
+
+	captureMemStats()
+
+	for {
+		select {
+		case <-ctx.Done():
+			i.logger.Warn("Memory stats during indexing", "maxHeapInUse", formatBytes(maxHeapInuse), "maxSys", formatBytes(maxSys))
+			return
+		case <-time.After(frequency):
+			captureMemStats()
+		}
+	}
+}
+
+func (i *dashboardIndex) reportSizeOfIndexDiskBackup(orgID int64) {
+	reader, _ := i.getOrgReader(orgID)
+
+	// create a temp directory to store the index
+	tmpDir, err := ioutil.TempDir("", "grafana.dashboard_index")
+	if err != nil {
+		i.logger.Error("can't create temp dir", "error", err)
+		return
+	}
+	defer func() {
+		err := os.RemoveAll(tmpDir)
+		if err != nil {
+			i.logger.Error("can't remove temp dir", "error", err, "tmpDir", tmpDir)
+			return
+		}
+	}()
+
+	cancel := make(chan struct{})
+	err = reader.Backup(tmpDir, cancel)
+	if err != nil {
+		i.logger.Error("can't create index disk backup", "error", err)
+		return
+	}
+
+	size, err := dirSize(tmpDir)
+	if err != nil {
+		i.logger.Error("can't calculate dir size", "error", err)
+		return
+	}
+
+	i.logger.Warn("Size of index disk backup", "size", formatBytes(uint64(size)))
+}
+
 func (i *dashboardIndex) buildOrgIndex(ctx context.Context, orgID int64) (int, error) {
 	started := time.Now()
 	ctx, cancel := context.WithTimeout(ctx, time.Minute)