Search: Update bluge version (#50377)

pkg/services/searchV2/bluge.go

@@ -50,7 +50,7 @@ func initIndex(dashboards []dashboard, logger log.Logger, extendDoc ExtendDashbo
 	// In order to reduce memory usage while initial indexing we are limiting
 	// the size of batch here.
 	docsInBatch := 0
-	maxBatchSize := 100
+	maxBatchSize := 300
 
 	flushIfRequired := func(force bool) error {
 		docsInBatch++

pkg/services/searchV2/index.go

@@ -3,9 +3,11 @@ package searchV2
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
+	"os/exec"
 	"runtime"
 	"strconv"
 	"strings"
@@ -195,19 +197,19 @@ func (i *dashboardIndex) buildInitialIndexes(ctx context.Context, orgIDs []int64
 }
 
 func (i *dashboardIndex) buildInitialIndex(ctx context.Context, orgID int64) error {
-	memCtx, memCancel := context.WithCancel(ctx)
+	debugCtx, debugCtxCancel := context.WithCancel(ctx)
 	if os.Getenv("GF_SEARCH_DEBUG") != "" {
-		go i.debugMemStats(memCtx, 200*time.Millisecond)
+		go i.debugResourceUsage(debugCtx, 200*time.Millisecond)
 	}
 
 	started := time.Now()
 	numDashboards, err := i.buildOrgIndex(ctx, orgID)
 	if err != nil {
-		memCancel()
-		return fmt.Errorf("can't build dashboard search index for org ID %d: %w", orgID, err)
+		debugCtxCancel()
+		return fmt.Errorf("can't build dashboard search index for org ID 1: %w", err)
 	}
 	i.logger.Info("Indexing for org finished", "orgIndexElapsed", time.Since(started), "orgId", orgID, "numDashboards", numDashboards)
-	memCancel()
+	debugCtxCancel()
 
 	if os.Getenv("GF_SEARCH_DEBUG") != "" {
 		// May help to estimate size of index when introducing changes. Though it's not a direct
@@ -219,7 +221,47 @@ func (i *dashboardIndex) buildInitialIndex(ctx context.Context, orgID int64) err
 	return nil
 }
 
-func (i *dashboardIndex) debugMemStats(ctx context.Context, frequency time.Duration) {
+// This is a naive implementation of process CPU getting (credits to
+// https://stackoverflow.com/a/11357813/1288429). Should work on both Linux and Darwin.
+// Since we only use this during development – seems simple and cheap solution to get
+// process CPU usage in cross-platform way.
+func getProcessCPU(currentPid int) (float64, error) {
+	cmd := exec.Command("ps", "aux")
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	err := cmd.Run()
+	if err != nil {
+		return 0, err
+	}
+	for {
+		line, err := out.ReadString('\n')
+		if err != nil {
+			break
+		}
+		tokens := strings.Split(line, " ")
+		ft := make([]string, 0)
+		for _, t := range tokens {
+			if t != "" && t != "\t" {
+				ft = append(ft, t)
+			}
+		}
+		pid, err := strconv.Atoi(ft[1])
+		if err != nil {
+			continue
+		}
+		if pid != currentPid {
+			continue
+		}
+		cpu, err := strconv.ParseFloat(ft[2], 64)
+		if err != nil {
+			return 0, err
+		}
+		return cpu, nil
+	}
+	return 0, errors.New("process not found")
+}
+
+func (i *dashboardIndex) debugResourceUsage(ctx context.Context, frequency time.Duration) {
 	var maxHeapInuse uint64
 	var maxSys uint64
 
@@ -234,15 +276,29 @@ func (i *dashboardIndex) debugMemStats(ctx context.Context, frequency time.Durat
 		}
 	}
 
+	var cpuUtilization []float64
+
+	captureCPUStats := func() {
+		cpu, err := getProcessCPU(os.Getpid())
+		if err != nil {
+			i.logger.Error("CPU stats error", "error", err)
+			return
+		}
+		// Just collect CPU utilization to a slice and show in the of index build.
+		cpuUtilization = append(cpuUtilization, cpu)
+	}
+
 	captureMemStats()
+	captureCPUStats()
 
 	for {
 		select {
 		case <-ctx.Done():
-			i.logger.Warn("Memory stats during indexing", "maxHeapInUse", formatBytes(maxHeapInuse), "maxSys", formatBytes(maxSys))
+			i.logger.Warn("Resource usage during indexing", "maxHeapInUse", formatBytes(maxHeapInuse), "maxSys", formatBytes(maxSys), "cpuPercent", cpuUtilization)
 			return
 		case <-time.After(frequency):
 			captureMemStats()
+			captureCPUStats()
 		}
 	}
 }