Alerting: Create alertingQueryOptimization feature flag for alert query optimization (#78932)

* Alerting: Create feature flag for alert query optimization

Adds a feature flag alertingQueryOptimization for an already existing 
functionality: alert query optimization. This feature flag will now be disabled 
by default.
This commit is contained in:
Matthew Jacobson 2024-01-10 15:52:58 -05:00 committed by GitHub
parent bba691777a
commit afa33f12b2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 40 additions and 14 deletions

View File

@ -58,6 +58,7 @@ Some features are enabled by default. You can disable these feature by setting t
| `logRowsPopoverMenu` | Enable filtering menu displayed when text of a log line is selected | Yes |
| `displayAnonymousStats` | Enables anonymous stats to be shown in the UI for Grafana | Yes |
| `lokiQueryHints` | Enables query hints for Loki | Yes |
| `alertingQueryOptimization` | Optimizes eligible queries in order to reduce load on datasources | |
## Preview feature toggles

View File

@ -173,4 +173,5 @@ export interface FeatureToggles {
alertingPreviewUpgrade?: boolean;
enablePluginsTracingByDefault?: boolean;
cloudRBACRoles?: boolean;
alertingQueryOptimization?: boolean;
}

View File

@ -1315,5 +1315,13 @@ var (
RequiresRestart: true,
Created: time.Date(2024, time.January, 10, 12, 0, 0, 0, time.UTC),
},
{
Name: "alertingQueryOptimization",
Description: "Optimizes eligible queries in order to reduce load on datasources",
Stage: FeatureStageGeneralAvailability,
Owner: grafanaAlertingSquad,
AllowSelfServe: false,
Created: time.Date(2024, time.January, 10, 12, 0, 0, 0, time.UTC),
},
}
)

View File

@ -154,3 +154,4 @@ lokiQueryHints,GA,@grafana/observability-logs,2023-12-18,false,false,false,true
alertingPreviewUpgrade,experimental,@grafana/alerting-squad,2024-01-03,false,false,true,false
enablePluginsTracingByDefault,experimental,@grafana/plugins-platform-backend,2024-01-09,false,false,true,false
cloudRBACRoles,experimental,@grafana/identity-access-team,2024-01-10,false,false,true,false
alertingQueryOptimization,GA,@grafana/alerting-squad,2024-01-10,false,false,false,false

1 Name Stage Owner Created requiresDevMode RequiresLicense RequiresRestart FrontendOnly
154 alertingPreviewUpgrade experimental @grafana/alerting-squad 2024-01-03 false false true false
155 enablePluginsTracingByDefault experimental @grafana/plugins-platform-backend 2024-01-09 false false true false
156 cloudRBACRoles experimental @grafana/identity-access-team 2024-01-10 false false true false
157 alertingQueryOptimization GA @grafana/alerting-squad 2024-01-10 false false false false

View File

@ -626,4 +626,8 @@ const (
// FlagCloudRBACRoles
// Enabled grafana cloud specific RBAC roles
FlagCloudRBACRoles = "cloudRBACRoles"
// FlagAlertingQueryOptimization
// Optimizes eligible queries in order to reduce load on datasources
FlagAlertingQueryOptimization = "alertingQueryOptimization"
)

View File

@ -69,8 +69,10 @@ func (srv TestingApiSrv) RouteTestGrafanaRuleConfig(c *contextmodel.ReqContext,
return response.ErrOrFallback(http.StatusInternalServerError, "failed to authorize access to rule group", err)
}
if _, err := store.OptimizeAlertQueries(rule.Data); err != nil {
return ErrResp(http.StatusInternalServerError, err, "Failed to optimize query")
if srv.featureManager.IsEnabled(c.Req.Context(), featuremgmt.FlagAlertingQueryOptimization) {
if _, err := store.OptimizeAlertQueries(rule.Data); err != nil {
return ErrResp(http.StatusInternalServerError, err, "Failed to optimize query")
}
}
evaluator, err := srv.evaluator.Create(eval.NewContext(c.Req.Context(), c.SignedInUser), rule.GetEvalCondition())
@ -165,9 +167,13 @@ func (srv TestingApiSrv) RouteEvalQueries(c *contextmodel.ReqContext, cmd apimod
cond.Condition = cond.Data[len(cond.Data)-1].RefID
}
optimizations, err := store.OptimizeAlertQueries(cond.Data)
if err != nil {
return ErrResp(http.StatusInternalServerError, err, "Failed to optimize query")
var optimizations []store.Optimization
if srv.featureManager.IsEnabled(c.Req.Context(), featuremgmt.FlagAlertingQueryOptimization) {
var err error
optimizations, err = store.OptimizeAlertQueries(cond.Data)
if err != nil {
return ErrResp(http.StatusInternalServerError, err, "Failed to optimize query")
}
}
evaluator, err := srv.evaluator.Create(eval.NewContext(c.Req.Context(), c.SignedInUser), cond)

View File

@ -17,6 +17,7 @@ import (
contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
"github.com/grafana/grafana/pkg/services/datasources"
fakes "github.com/grafana/grafana/pkg/services/datasources/fakes"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/ngalert/accesscontrol"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
@ -146,7 +147,7 @@ func TestRouteTestGrafanaRuleConfig(t *testing.T) {
{Action: datasources.ActionQuery, Scope: datasources.ScopeProvider.GetResourceScopeUID(data1.DatasourceUID)},
})
srv := createTestingApiSrv(t, nil, ac, eval_mocks.NewEvaluatorFactory(&eval_mocks.ConditionEvaluatorMock{}))
srv := createTestingApiSrv(t, nil, ac, eval_mocks.NewEvaluatorFactory(&eval_mocks.ConditionEvaluatorMock{}), &featuremgmt.FeatureManager{})
rule := validRule()
rule.GrafanaManagedAlert.Data = ApiAlertQueriesFromAlertQueries([]models.AlertQuery{data1, data2})
@ -180,7 +181,7 @@ func TestRouteTestGrafanaRuleConfig(t *testing.T) {
evalFactory := eval_mocks.NewEvaluatorFactory(evaluator)
srv := createTestingApiSrv(t, ds, ac, evalFactory)
srv := createTestingApiSrv(t, ds, ac, evalFactory, &featuremgmt.FeatureManager{})
rule := validRule()
rule.GrafanaManagedAlert.Data = ApiAlertQueriesFromAlertQueries([]models.AlertQuery{data1, data2})
@ -255,7 +256,7 @@ func TestRouteEvalQueries(t *testing.T) {
}
evaluator.EXPECT().EvaluateRaw(mock.Anything, mock.Anything).Return(result, nil)
srv := createTestingApiSrv(t, ds, ac, eval_mocks.NewEvaluatorFactory(evaluator))
srv := createTestingApiSrv(t, ds, ac, eval_mocks.NewEvaluatorFactory(evaluator), &featuremgmt.FeatureManager{})
response := srv.RouteEvalQueries(rc, definitions.EvalQueriesPayload{
Data: ApiAlertQueriesFromAlertQueries([]models.AlertQuery{data1, data2}),
@ -315,7 +316,7 @@ func TestRouteEvalQueries(t *testing.T) {
}
evaluator.EXPECT().EvaluateRaw(mock.Anything, mock.Anything).Return(result, nil)
srv := createTestingApiSrv(t, ds, ac, eval_mocks.NewEvaluatorFactory(evaluator))
srv := createTestingApiSrv(t, ds, ac, eval_mocks.NewEvaluatorFactory(evaluator), featuremgmt.WithManager(featuremgmt.FlagAlertingQueryOptimization))
response := srv.RouteEvalQueries(rc, definitions.EvalQueriesPayload{
Data: ApiAlertQueriesFromAlertQueries(queries),
@ -341,7 +342,7 @@ func TestRouteEvalQueries(t *testing.T) {
})
}
func createTestingApiSrv(t *testing.T, ds *fakes.FakeCacheService, ac *acMock.Mock, evaluator eval.EvaluatorFactory) *TestingApiSrv {
func createTestingApiSrv(t *testing.T, ds *fakes.FakeCacheService, ac *acMock.Mock, evaluator eval.EvaluatorFactory, featureManager *featuremgmt.FeatureManager) *TestingApiSrv {
if ac == nil {
ac = acMock.New()
}
@ -352,5 +353,6 @@ func createTestingApiSrv(t *testing.T, ds *fakes.FakeCacheService, ac *acMock.Mo
evaluator: evaluator,
cfg: config(t),
tracer: tracing.InitializeTracerForTest(),
featureManager: featureManager,
}
}

View File

@ -13,6 +13,7 @@ import (
"github.com/grafana/grafana/pkg/services/auth/identity"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/dashboards/dashboardaccess"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/folder"
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/search/model"
@ -552,10 +553,12 @@ func (st DBstore) GetAlertRulesForScheduling(ctx context.Context, query *ngmodel
st.Logger.Error("Invalid rule found in DB store, ignoring it", "func", "GetAlertRulesForScheduling", "error", err)
continue
}
if optimizations, err := OptimizeAlertQueries(rule.Data); err != nil {
st.Logger.Error("Could not migrate rule from range to instant query", "rule", rule.UID, "err", err)
} else if len(optimizations) > 0 {
st.Logger.Info("Migrated rule from range to instant query", "rule", rule.UID, "migrated_queries", len(optimizations))
if st.FeatureToggles.IsEnabled(ctx, featuremgmt.FlagAlertingQueryOptimization) {
if optimizations, err := OptimizeAlertQueries(rule.Data); err != nil {
st.Logger.Error("Could not migrate rule from range to instant query", "rule", rule.UID, "err", err)
} else if len(optimizations) > 0 {
st.Logger.Info("Migrated rule from range to instant query", "rule", rule.UID, "migrated_queries", len(optimizations))
}
}
rules = append(rules, rule)
}