Plugins: Add enablePluginsTracingByDefault feature flag (#80195)

* Add enablePluginsTracingByDefault feature flag

* Enable tracing for all plugins if enablePluginsTracingByDefault is set

* fix docstrings for IsEnabled and IsEnabledGlobally

* fix tests

* do not use separate feature manager

* add test case

* Revert "fix tests"

This reverts commit 46a2420ed1.

* cleanup

* fix plugin tracing disabled if wrong plugin setting is present

* add test case for enabled on plugin with wrong plugin setting but with enablePluginsTracingByDefault feature flag

* Add RequiresRestart = true to enablePluginsTracingByDefault

* re-generate feature flags

* pr review feedback

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -169,6 +169,7 @@ Experimental features might be changed or removed without prior notice.
 | `flameGraphItemCollapsing`                  | Allow collapsing of flame graph items                                                                                                                                                                                                                                             |
 | `pluginsSkipHostEnvVars`                    | Disables passing host environment variable to plugin processes                                                                                                                                                                                                                    |
 | `tableSharedCrosshair`                      | Enables shared crosshair in table panel                                                                                                                                                                                                                                           |
+| `enablePluginsTracingByDefault`             | Enable plugin tracing for all external plugins                                                                                                                                                                                                                                    |
 
 ## Development feature toggles
 

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -171,4 +171,5 @@ export interface FeatureToggles {
   alertStateHistoryAnnotationsFromLoki?: boolean;
   lokiQueryHints?: boolean;
   alertingPreviewUpgrade?: boolean;
+  enablePluginsTracingByDefault?: boolean;
 }

pkg/plugins/envvars/envvars.go

@@ -14,10 +14,10 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/backend/proxy"
 	"github.com/grafana/grafana-plugin-sdk-go/experimental/featuretoggles"
-
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/plugins/auth"
 	"github.com/grafana/grafana/pkg/plugins/config"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
 )
 
 const (
@@ -207,8 +207,8 @@ func (s *Service) GetConfigMap(ctx context.Context, pluginID string, _ *auth.Ext
 }
 
 func (s *Service) tracingEnvVars(plugin *plugins.Plugin) []string {
-	var pluginTracingEnabled bool
-	if v, exists := s.cfg.PluginSettings[plugin.ID]["tracing"]; exists {
+	pluginTracingEnabled := s.cfg.Features != nil && s.cfg.Features.IsEnabledGlobally(featuremgmt.FlagEnablePluginsTracingByDefault)
+	if v, exists := s.cfg.PluginSettings[plugin.ID]["tracing"]; exists && !pluginTracingEnabled {
 		pluginTracingEnabled = v == "true"
 	}
 	if !s.cfg.Tracing.IsEnabled() || !pluginTracingEnabled {

pkg/plugins/envvars/envvars_test.go

@@ -354,6 +354,30 @@ func TestInitializer_tracingEnvironmentVariables(t *testing.T) {
 			plugin: defaultPlugin,
 			exp:    expDefaultOtlp,
 		},
+		{
+			name: `enabled on plugin with no "tracing" plugin setting but with enablePluginsTracingByDefault feature flag`,
+			cfg: &config.Cfg{
+				Tracing: config.Tracing{
+					OpenTelemetry: defaultOTelCfg,
+				},
+				PluginSettings: map[string]map[string]string{pluginID: {}},
+				Features:       featuremgmt.WithFeatures(featuremgmt.FlagEnablePluginsTracingByDefault),
+			},
+			plugin: defaultPlugin,
+			exp:    expDefaultOtlp,
+		},
+		{
+			name: `enabled on plugin with plugin setting "tracing=false" but with enablePluginsTracingByDefault feature flag`,
+			cfg: &config.Cfg{
+				Tracing: config.Tracing{
+					OpenTelemetry: defaultOTelCfg,
+				},
+				PluginSettings: map[string]map[string]string{pluginID: {"tracing": "false"}},
+				Features:       featuremgmt.WithFeatures(featuremgmt.FlagEnablePluginsTracingByDefault),
+			},
+			plugin: defaultPlugin,
+			exp:    expDefaultOtlp,
+		},
 		{
 			name: "GF_PLUGIN_VERSION is not present if tracing is disabled",
 			cfg: &config.Cfg{

pkg/services/featuremgmt/models.go

@@ -8,11 +8,11 @@ import (
 )
 
 type FeatureToggles interface {
-	// Check if a feature is enabled for a given context.
+	// IsEnabled checks if a feature is enabled for a given context.
 	// The settings may be per user, tenant, or globally set in the cloud
 	IsEnabled(ctx context.Context, flag string) bool
 
-	// Check if a flag is configured globally.  For now, this is the same
+	// IsEnabledGlobally checks if a flag is configured globally.  For now, this is the same
 	// as the function above, however it will move to only checking flags that
 	// are configured by the operator and shared across all tenants.
 	// Use of global feature flags should be limited and careful as they require

pkg/services/featuremgmt/registry.go

@@ -1297,5 +1297,14 @@ var (
 			RequiresRestart: true,
 			Created:         time.Date(2024, time.January, 3, 12, 0, 0, 0, time.UTC),
 		},
+		{
+			Name:            "enablePluginsTracingByDefault",
+			Description:     "Enable plugin tracing for all external plugins",
+			FrontendOnly:    false,
+			Stage:           FeatureStageExperimental,
+			Owner:           grafanaPluginsPlatformSquad,
+			RequiresRestart: true,
+			Created:         time.Date(2024, time.January, 9, 12, 0, 0, 0, time.UTC),
+		},
 	}
 )

pkg/services/featuremgmt/toggles_gen.csv

@@ -152,3 +152,4 @@ displayAnonymousStats,GA,@grafana/identity-access-team,2023-11-29,false,false,fa
 alertStateHistoryAnnotationsFromLoki,experimental,@grafana/alerting-squad,2023-11-30,false,false,true,false
 lokiQueryHints,GA,@grafana/observability-logs,2023-12-18,false,false,false,true
 alertingPreviewUpgrade,experimental,@grafana/alerting-squad,2024-01-03,false,false,true,false
+enablePluginsTracingByDefault,experimental,@grafana/plugins-platform-backend,2024-01-09,false,false,true,false

pkg/services/featuremgmt/toggles_gen.go

@@ -618,4 +618,8 @@ const (
 	// FlagAlertingPreviewUpgrade
 	// Show Unified Alerting preview and upgrade page in legacy alerting
 	FlagAlertingPreviewUpgrade = "alertingPreviewUpgrade"
+
+	// FlagEnablePluginsTracingByDefault
+	// Enable plugin tracing for all external plugins
+	FlagEnablePluginsTracingByDefault = "enablePluginsTracingByDefault"
 )