Alerting: Support concurrent queries for saving alert instances (#70525)

This commit adds support for concurrent queries when saving alert
instances to the database. This is an experimental feature in
response to some customers experiencing delays between rule evaluation
and sending alerts to Alertmanager, resulting in flapping. It is
disabled by default.

pkg/services/ngalert/backtesting/engine.go

@@ -45,12 +45,13 @@ func NewEngine(appUrl *url.URL, evalFactory eval.EvaluatorFactory) *Engine {
 		evalFactory: evalFactory,
 		createStateManager: func() stateManager {
 			cfg := state.ManagerCfg{
-				Metrics:       nil,
-				ExternalURL:   appUrl,
-				InstanceStore: nil,
-				Images:        &NoopImageService{},
-				Clock:         clock.New(),
-				Historian:     nil,
+				Metrics:                 nil,
+				ExternalURL:             appUrl,
+				InstanceStore:           nil,
+				Images:                  &NoopImageService{},
+				Clock:                   clock.New(),
+				Historian:               nil,
+				MaxStateSaveConcurrency: 1,
 			}
 			return state.NewManager(cfg)
 		},