Search PoC: Improves initial indexing speed. Makes params configurable. (#95439)

* Improves initial indexing speed. Makes params configurable.

* fix linter errors

* removes kind param

* updates index test

* remove println from test

* removes error check in test

* adds log for high index latency ands updates max goroutine var with workers config var

* fix test timing out - set worker limit

* set the batch size

---------

Co-authored-by: Scott Lepper <scott.lepper@gmail.com>

pkg/setting/setting.go

@@ -532,8 +532,11 @@ type Cfg struct {
 	ShortLinkExpiration int
 
 	// Unified Storage
-	UnifiedStorage map[string]UnifiedStorageConfig
-	IndexPath      string
+	UnifiedStorage    map[string]UnifiedStorageConfig
+	IndexPath         string
+	IndexWorkers      int
+	IndexMaxBatchSize int
+	IndexListLimit    int
 }
 
 type UnifiedStorageConfig struct {
@@ -1343,7 +1346,6 @@ func (cfg *Cfg) parseINIFile(iniFile *ini.File) error {
 
 	// unified storage config
 	cfg.setUnifiedStorageConfig()
-	cfg.setIndexPath()
 
 	return nil
 }

pkg/setting/setting_unified_storage.go

@@ -35,8 +35,11 @@ func (cfg *Cfg) setUnifiedStorageConfig() {
 		}
 	}
 	cfg.UnifiedStorage = storageConfig
-}
 
-func (cfg *Cfg) setIndexPath() {
-	cfg.IndexPath = cfg.Raw.Section("unified_storage").Key("index_path").String()
+	// Set indexer config for unified storaae
+	section := cfg.Raw.Section("unified_storage")
+	cfg.IndexPath = section.Key("index_path").String()
+	cfg.IndexWorkers = section.Key("index_workers").MustInt(10)
+	cfg.IndexMaxBatchSize = section.Key("index_max_batch_size").MustInt(100)
+	cfg.IndexListLimit = section.Key("index_list_limit").MustInt(1000)
 }