Plugins: Automatic service account (and token) setup (#76473)

* Update cue to have an AuthProvider entry

* Cable the new auth provider

* Add feature flag check to the accesscontrol service

* Fix test

* Change the structure of externalServiceRegistration (#76673)

pkg/plugins/envvars/envvars.go

@@ -59,8 +59,10 @@ func (s *Service) Get(ctx context.Context, p *plugins.Plugin) []string {
 			fmt.Sprintf("GF_APP_URL=%s", s.cfg.GrafanaAppURL),
 			fmt.Sprintf("GF_PLUGIN_APP_CLIENT_ID=%s", p.ExternalService.ClientID),
 			fmt.Sprintf("GF_PLUGIN_APP_CLIENT_SECRET=%s", p.ExternalService.ClientSecret),
-			fmt.Sprintf("GF_PLUGIN_APP_PRIVATE_KEY=%s", p.ExternalService.PrivateKey),
 		)
+		if p.ExternalService.PrivateKey != "" {
+			hostEnv = append(hostEnv, fmt.Sprintf("GF_PLUGIN_APP_PRIVATE_KEY=%s", p.ExternalService.PrivateKey))
+		}
 	}
 
 	hostEnv = append(hostEnv, s.featureToggleEnableVar(ctx)...)

pkg/plugins/envvars/envvars_test.go

@@ -306,8 +306,8 @@ func TestInitializer_tracingEnvironmentVariables(t *testing.T) {
 	}
 }
 
-func TestInitializer_oauthEnvVars(t *testing.T) {
-	t.Run("backend datasource with oauth registration", func(t *testing.T) {
+func TestInitializer_authEnvVars(t *testing.T) {
+	t.Run("backend datasource with auth registration", func(t *testing.T) {
 		p := &plugins.Plugin{
 			JSONData: plugins.JSONData{
 				ID:                          "test",
@@ -322,7 +322,6 @@ func TestInitializer_oauthEnvVars(t *testing.T) {
 
 		envVarsProvider := NewProvider(&config.Cfg{
 			GrafanaAppURL: "https://myorg.com/",
-			Features:      featuremgmt.WithFeatures(featuremgmt.FlagExternalServiceAuth),
 		}, nil)
 		envVars := envVarsProvider.Get(context.Background(), p)
 		assert.Equal(t, "GF_VERSION=", envVars[0])

pkg/plugins/manager/testdata/external-registration/plugin.json

@@ -18,24 +18,11 @@
     "version": "1.0.0"
   },
   "externalServiceRegistration": {
-    "impersonation": {
-      "enabled" : true,
-      "groups" : true,
-      "permissions" :  [
-        {
-          "action": "read",
-          "scope": "datasource"
-        }
-      ]
-    },
-    "self": {
-      "enabled" : true,
-      "permissions" :  [
-        {
-          "action": "read",
-          "scope": "datasource"
-        }
-      ]
-    }
+    "permissions" :  [
+      {
+        "action": "read",
+        "scope": "datasource"
+      }
+    ]
   }
 }

pkg/plugins/manager/testdata/oauth-external-registration/plugin.json

@@ -0,0 +1,38 @@
+{
+  "id": "grafana-test-datasource",
+  "type": "datasource",
+  "name": "Test",
+  "backend": true,
+  "executable": "gpx_test_datasource",
+  "info": {
+    "author": {
+      "name": "Grafana Labs",
+      "url": "https://grafana.com"
+    },
+    "logos": {
+      "large": "img/ds.svg",
+      "small": "img/ds.svg"
+    },
+    "screenshots": [],
+    "updated": "2023-08-03",
+    "version": "1.0.0"
+  },
+  "externalServiceRegistration": {
+    "impersonation": {
+      "enabled" : true,
+      "groups" : true,
+      "permissions" :  [
+        {
+          "action": "read",
+          "scope": "datasource"
+        }
+      ]
+    },
+    "permissions" :  [
+      {
+        "action": "read",
+        "scope": "datasource"
+      }
+    ]
+  }
+}

pkg/plugins/pfs/pfs_test.go

@@ -134,6 +134,9 @@ func TestParsePluginTestdata(t *testing.T) {
 		"external-registration": {
 			rootid: "grafana-test-datasource",
 		},
+		"oauth-external-registration": {
+			rootid: "grafana-test-datasource",
+		},
 	}
 
 	staticRootPath, err := filepath.Abs("../manager/testdata")

pkg/plugins/plugindef/plugindef.cue

@@ -413,11 +413,15 @@ schemas: [{
 		// External service registration information
 		externalServiceRegistration: #ExternalServiceRegistration
 
+		// ExternalServiceRegistration allows the service to get a service account token
+		// (or to use the client_credentials grant if the token provider is the OAuth2 Server)
 		#ExternalServiceRegistration: {
+			// Permissions are the permissions that the external service needs its associated service account to have.
+			permissions?: [...#Permission]
+
 			// Impersonation describes the permissions that the external service will have on behalf of the user
+			// This is only available with the OAuth2 Server
 			impersonation?: #Impersonation
-			// Self describes the permissions that the external service will have on behalf of itself
-			self?: 		#Self
 		}
 
 		#Impersonation: {
@@ -432,14 +436,6 @@ schemas: [{
 			// gain more privileges than the impersonated user has.
 			permissions?: [...#Permission]
 		}
-
-		#Self: {
-			// Enabled allows the service to request access tokens for itself using the client_credentials grant
-			// Defaults to true.
-			enabled?: bool
-			// Permissions are the permissions that the external service needs its associated service account to have.
-			permissions?: [...#Permission]
-		}
 	}
 }]
 lenses: []

pkg/plugins/plugindef/plugindef_types_gen.go

@@ -122,10 +122,13 @@ type Dependency struct {
 // DependencyType defines model for Dependency.Type.
 type DependencyType string
 
-// ExternalServiceRegistration defines model for ExternalServiceRegistration.
+// ExternalServiceRegistration allows the service to get a service account token
+// (or to use the client_credentials grant if the token provider is the OAuth2 Server)
 type ExternalServiceRegistration struct {
 	Impersonation *Impersonation `json:"impersonation,omitempty"`
-	Self          *Self          `json:"self,omitempty"`
+
+	// Permissions are the permissions that the external service needs its associated service account to have.
+	Permissions []Permission `json:"permissions,omitempty"`
 }
 
 // Header describes an HTTP header that is forwarded with a proxied request for
@@ -314,7 +317,10 @@ type PluginDef struct {
 	// $GOARCH><.exe for Windows>`, e.g. `plugin_linux_amd64`.
 	// Combination of $GOOS and $GOARCH can be found here:
 	// https://golang.org/doc/install/source#environment.
-	Executable                  *string                     `json:"executable,omitempty"`
+	Executable *string `json:"executable,omitempty"`
+
+	// ExternalServiceRegistration allows the service to get a service account token
+	// (or to use the client_credentials grant if the token provider is the OAuth2 Server)
 	ExternalServiceRegistration ExternalServiceRegistration `json:"externalServiceRegistration"`
 
 	// [internal only] Excludes the plugin from listings in Grafana's UI. Only
@@ -472,16 +478,6 @@ type Route struct {
 	UrlParams []URLParam `json:"urlParams,omitempty"`
 }
 
-// Self defines model for Self.
-type Self struct {
-	// Enabled allows the service to request access tokens for itself using the client_credentials grant
-	// Defaults to true.
-	Enabled *bool `json:"enabled,omitempty"`
-
-	// Permissions are the permissions that the external service needs its associated service account to have.
-	Permissions []Permission `json:"permissions,omitempty"`
-}
-
 // TODO docs
 type TokenAuth struct {
 	// Parameters for the token authentication request.

pkg/services/accesscontrol/acimpl/service.go

@@ -397,7 +397,7 @@ func PermissionMatchesSearchOptions(permission accesscontrol.Permission, searchO
 }
 
 func (s *Service) SaveExternalServiceRole(ctx context.Context, cmd accesscontrol.SaveExternalServiceRoleCommand) error {
-	if !s.features.IsEnabled(featuremgmt.FlagExternalServiceAuth) {
+	if !(s.features.IsEnabled(featuremgmt.FlagExternalServiceAuth) || s.features.IsEnabled(featuremgmt.FlagExternalServiceAccounts)) {
 		s.log.Debug("Registering an external service role is behind a feature flag, enable it to use this feature.")
 		return nil
 	}
@@ -410,7 +410,7 @@ func (s *Service) SaveExternalServiceRole(ctx context.Context, cmd accesscontrol
 }
 
 func (s *Service) DeleteExternalServiceRole(ctx context.Context, externalServiceID string) error {
-	if !s.features.IsEnabled(featuremgmt.FlagExternalServiceAuth) {
+	if !(s.features.IsEnabled(featuremgmt.FlagExternalServiceAuth) || s.features.IsEnabled(featuremgmt.FlagExternalServiceAccounts)) {
 		s.log.Debug("Deleting an external service role is behind a feature flag, enable it to use this feature.")
 		return nil
 	}

pkg/services/accesscontrol/acimpl/service_test.go

@@ -815,7 +815,7 @@ func TestService_SaveExternalServiceRole(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			ctx := context.Background()
 			ac := setupTestEnv(t)
-			ac.features = featuremgmt.WithFeatures(featuremgmt.FlagExternalServiceAuth)
+			ac.features = featuremgmt.WithFeatures(featuremgmt.FlagExternalServiceAuth, featuremgmt.FlagExternalServiceAccounts)
 			for _, r := range tt.runs {
 				err := ac.SaveExternalServiceRole(ctx, r.cmd)
 				if r.wantErr {
@@ -861,7 +861,7 @@ func TestService_DeleteExternalServiceRole(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			ctx := context.Background()
 			ac := setupTestEnv(t)
-			ac.features = featuremgmt.WithFeatures(featuremgmt.FlagExternalServiceAuth)
+			ac.features = featuremgmt.WithFeatures(featuremgmt.FlagExternalServiceAuth, featuremgmt.FlagExternalServiceAccounts)
 
 			if tt.initCmd != nil {
 				err := ac.SaveExternalServiceRole(ctx, *tt.initCmd)

pkg/services/extsvcauth/extsvcaccounts/service_test.go

@@ -32,7 +32,7 @@ func setupTestEnv(t *testing.T) *TestEnv {
 	t.Helper()
 
 	cfg := setting.NewCfg()
-	fmgt := featuremgmt.WithFeatures(featuremgmt.FlagExternalServiceAuth)
+	fmgt := featuremgmt.WithFeatures(featuremgmt.FlagExternalServiceAccounts)
 
 	env := &TestEnv{
 		AcStore:  &actest.MockStore{},

pkg/services/extsvcauth/models.go

@@ -8,7 +8,8 @@ import (
 )
 
 const (
-	OAuth2Server AuthProvider = "OAuth2Server"
+	OAuth2Server    AuthProvider = "OAuth2Server"
+	ServiceAccounts AuthProvider = "ServiceAccounts"
 
 	// TmpOrgID is the orgID we use while global service accounts are not supported.
 	TmpOrgID int64 = 1

pkg/services/extsvcauth/registry/service.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/services/extsvcauth"
+	"github.com/grafana/grafana/pkg/services/extsvcauth/extsvcaccounts"
 	"github.com/grafana/grafana/pkg/services/extsvcauth/oauthserver"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 )
@@ -15,13 +16,15 @@ type Registry struct {
 	features    featuremgmt.FeatureToggles
 	logger      log.Logger
 	oauthServer oauthserver.OAuth2Server
+	saSvc       *extsvcaccounts.ExtSvcAccountsService
 }
 
-func ProvideExtSvcRegistry(oauthServer oauthserver.OAuth2Server, features featuremgmt.FeatureToggles) *Registry {
+func ProvideExtSvcRegistry(oauthServer oauthserver.OAuth2Server, saSvc *extsvcaccounts.ExtSvcAccountsService, features featuremgmt.FeatureToggles) *Registry {
 	return &Registry{
 		features:    features,
 		logger:      log.New("extsvcauth.registry"),
 		oauthServer: oauthServer,
+		saSvc:       saSvc,
 	}
 }
 
@@ -30,6 +33,13 @@ func ProvideExtSvcRegistry(oauthServer oauthserver.OAuth2Server, features featur
 // associated service account has the correct permissions.
 func (r *Registry) SaveExternalService(ctx context.Context, cmd *extsvcauth.ExternalServiceRegistration) (*extsvcauth.ExternalService, error) {
 	switch cmd.AuthProvider {
+	case extsvcauth.ServiceAccounts:
+		if !r.features.IsEnabled(featuremgmt.FlagExternalServiceAccounts) {
+			r.logger.Warn("Skipping external service authentication, flag disabled", "service", cmd.Name, "flag", featuremgmt.FlagExternalServiceAccounts)
+			return nil, nil
+		}
+		r.logger.Debug("Routing the External Service registration to the External Service Account service", "service", cmd.Name)
+		return r.saSvc.SaveExternalService(ctx, cmd)
 	case extsvcauth.OAuth2Server:
 		if !r.features.IsEnabled(featuremgmt.FlagExternalServiceAuth) {
 			r.logger.Warn("Skipping external service authentication, flag disabled", "service", cmd.Name, "flag", featuremgmt.FlagExternalServiceAuth)

pkg/services/pluginsintegration/loader/loader_test.go

@@ -498,12 +498,12 @@ func TestLoader_Load_ExternalRegistration(t *testing.T) {
 	boolPtr := func(b bool) *bool { return &b }
 	stringPtr := func(s string) *string { return &s }
 
-	t.Run("Load a plugin with external registration", func(t *testing.T) {
+	t.Run("Load a plugin with oauth client registration", func(t *testing.T) {
 		cfg := &config.Cfg{
 			Features:             fakes.NewFakeFeatureToggles(featuremgmt.FlagExternalServiceAuth),
 			PluginsAllowUnsigned: []string{"grafana-test-datasource"},
 		}
-		pluginPaths := []string{filepath.Join(testDataDir(t), "external-registration")}
+		pluginPaths := []string{filepath.Join(testDataDir(t), "oauth-external-registration")}
 		expected := []*plugins.Plugin{
 			{JSONData: plugins.JSONData{
 				ID:         "grafana-test-datasource",
@@ -539,13 +539,10 @@ func TestLoader_Load_ExternalRegistration(t *testing.T) {
 							},
 						},
 					},
-					Self: &plugindef.Self{
-						Enabled: boolPtr(true),
-						Permissions: []plugindef.Permission{
-							{
-								Action: "read",
-								Scope:  stringPtr("datasource"),
-							},
+					Permissions: []plugindef.Permission{
+						{
+							Action: "read",
+							Scope:  stringPtr("datasource"),
 						},
 					},
 				},
@@ -602,6 +599,96 @@ func TestLoader_Load_ExternalRegistration(t *testing.T) {
 			t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts...))
 		}
 	})
+
+	t.Run("Load a plugin with service account registration", func(t *testing.T) {
+		cfg := &config.Cfg{
+			Features:             fakes.NewFakeFeatureToggles(featuremgmt.FlagExternalServiceAuth),
+			PluginsAllowUnsigned: []string{"grafana-test-datasource"},
+		}
+		pluginPaths := []string{filepath.Join(testDataDir(t), "external-registration")}
+		expected := []*plugins.Plugin{
+			{JSONData: plugins.JSONData{
+				ID:         "grafana-test-datasource",
+				Type:       plugins.TypeDataSource,
+				Name:       "Test",
+				Backend:    true,
+				Executable: "gpx_test_datasource",
+				Info: plugins.Info{
+					Author: plugins.InfoLink{
+						Name: "Grafana Labs",
+						URL:  "https://grafana.com",
+					},
+					Version: "1.0.0",
+					Logos: plugins.Logos{
+						Small: "/public/plugins/grafana-test-datasource/img/ds.svg",
+						Large: "/public/plugins/grafana-test-datasource/img/ds.svg",
+					},
+					Updated:     "2023-08-03",
+					Screenshots: []plugins.Screenshots{},
+				},
+				Dependencies: plugins.Dependencies{
+					GrafanaVersion: "*",
+					Plugins:        []plugins.Dependency{},
+				},
+				ExternalServiceRegistration: &plugindef.ExternalServiceRegistration{
+					Permissions: []plugindef.Permission{
+						{
+							Action: "read",
+							Scope:  stringPtr("datasource"),
+						},
+					},
+				},
+			},
+				FS:        mustNewStaticFSForTests(t, pluginPaths[0]),
+				Class:     plugins.ClassExternal,
+				Signature: plugins.SignatureStatusUnsigned,
+				Module:    "/public/plugins/grafana-test-datasource/module.js",
+				BaseURL:   "/public/plugins/grafana-test-datasource",
+				ExternalService: &auth.ExternalService{
+					ClientID:     "client-id",
+					ClientSecret: "secretz",
+				},
+			},
+		}
+
+		backendFactoryProvider := fakes.NewFakeBackendProcessProvider()
+		backendFactoryProvider.BackendFactoryFunc = func(ctx context.Context, plugin *plugins.Plugin) backendplugin.PluginFactoryFunc {
+			return func(pluginID string, logger log.Logger, env func() []string) (backendplugin.Plugin, error) {
+				require.Equal(t, "grafana-test-datasource", pluginID)
+				require.Equal(t, []string{"GF_VERSION=", "GF_EDITION=", "GF_ENTERPRISE_LICENSE_PATH=",
+					"GF_ENTERPRISE_APP_URL=", "GF_ENTERPRISE_LICENSE_TEXT=", "GF_APP_URL=",
+					"GF_PLUGIN_APP_CLIENT_ID=client-id", "GF_PLUGIN_APP_CLIENT_SECRET=secretz",
+					"GF_INSTANCE_FEATURE_TOGGLES_ENABLE=externalServiceAuth"}, env())
+				return &fakes.FakeBackendPlugin{}, nil
+			}
+		}
+
+		l := newLoaderWithOpts(t, cfg, loaderDepOpts{
+			authServiceRegistry: &fakes.FakeAuthService{
+				Result: &auth.ExternalService{
+					ClientID:     "client-id",
+					ClientSecret: "secretz",
+				},
+			},
+			backendFactoryProvider: backendFactoryProvider,
+		})
+		got, err := l.Load(context.Background(), &fakes.FakePluginSource{
+			PluginClassFunc: func(ctx context.Context) plugins.Class {
+				return plugins.ClassExternal
+			},
+			PluginURIsFunc: func(ctx context.Context) []string {
+				return pluginPaths
+			},
+			DefaultSignatureFunc: func(ctx context.Context) (plugins.Signature, bool) {
+				return plugins.Signature{}, false
+			},
+		})
+
+		require.NoError(t, err)
+		if !cmp.Equal(got, expected, compareOpts...) {
+			t.Fatalf("Result mismatch (-want +got):\n%s", cmp.Diff(got, expected, compareOpts...))
+		}
+	})
 }
 
 func TestLoader_Load_CustomSource(t *testing.T) {

pkg/services/pluginsintegration/pipeline/steps.go

@@ -38,7 +38,8 @@ func newExternalServiceRegistration(cfg *config.Cfg, serviceRegistry auth.Extern
 
 // Register registers the external service with the external service registry, if the feature is enabled.
 func (r *ExternalServiceRegistration) Register(ctx context.Context, p *plugins.Plugin) (*plugins.Plugin, error) {
-	if p.ExternalServiceRegistration != nil && r.cfg.Features.IsEnabled(featuremgmt.FlagExternalServiceAuth) {
+	if p.ExternalServiceRegistration != nil &&
+		(r.cfg.Features.IsEnabled(featuremgmt.FlagExternalServiceAuth) || r.cfg.Features.IsEnabled(featuremgmt.FlagExternalServiceAccounts)) {
 		s, err := r.externalServiceRegistry.RegisterExternalService(ctx, p.ID, p.ExternalServiceRegistration)
 		if err != nil {
 			r.log.Error("Could not register an external service. Initialization skipped", "pluginId", p.ID, "error", err)

pkg/services/pluginsintegration/serviceregistration/serviceregistration.go

@@ -38,23 +38,26 @@ func (s *Service) RegisterExternalService(ctx context.Context, svcName string, s
 	}
 
 	self := extsvcauth.SelfCfg{}
-	if svc.Self != nil {
-		self.Permissions = toAccessControlPermissions(svc.Self.Permissions)
-		if svc.Self.Enabled != nil {
-			self.Enabled = *svc.Self.Enabled
-		} else {
-			self.Enabled = true
-		}
+	if len(svc.Permissions) > 0 {
+		self.Permissions = toAccessControlPermissions(svc.Permissions)
+		self.Enabled = true
 	}
 
-	extSvc, err := s.os.SaveExternalService(ctx, &extsvcauth.ExternalServiceRegistration{
-		Name:             svcName,
-		Impersonation:    impersonation,
-		Self:             self,
-		AuthProvider:     extsvcauth.OAuth2Server,
-		OAuthProviderCfg: &extsvcauth.OAuthProviderCfg{Key: &extsvcauth.KeyOption{Generate: true}},
-	})
-	if err != nil {
+	registration := &extsvcauth.ExternalServiceRegistration{
+		Name:          svcName,
+		Impersonation: impersonation,
+		Self:          self,
+	}
+
+	// Default authProvider now is ServiceAccounts
+	registration.AuthProvider = extsvcauth.ServiceAccounts
+	if svc.Impersonation != nil {
+		registration.AuthProvider = extsvcauth.OAuth2Server
+		registration.OAuthProviderCfg = &extsvcauth.OAuthProviderCfg{Key: &extsvcauth.KeyOption{Generate: true}}
+	}
+
+	extSvc, err := s.os.SaveExternalService(ctx, registration)
+	if err != nil || extSvc == nil {
 		return nil, err
 	}