AuthN: Support HA setups with External Service Account management (#78425)

* Lock when creating external service

* Add local lock back

* Improve function signature

* Define lockName separately to make it more explicit

* Update pkg/infra/serverlock/serverlock.go

Co-authored-by: Gabriel MABILLE <gamab@users.noreply.github.com>

* Update pkg/infra/serverlock/serverlock.go

Co-authored-by: Gabriel MABILLE <gamab@users.noreply.github.com>

---------

Co-authored-by: Gabriel MABILLE <gamab@users.noreply.github.com>

pkg/services/extsvcauth/registry/service.go

@@ -3,8 +3,10 @@ package registry
 import (
 	"context"
 	"sync"
+	"time"
 
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/serverlock"
 	"github.com/grafana/grafana/pkg/infra/slugify"
 	"github.com/grafana/grafana/pkg/services/extsvcauth"
 	"github.com/grafana/grafana/pkg/services/extsvcauth/oauthserver/oasimpl"
@@ -14,6 +16,12 @@ import (
 
 var _ extsvcauth.ExternalServiceRegistry = &Registry{}
 
+var lockTimeConfig = serverlock.LockTimeConfig{
+	MaxInterval: 2 * time.Minute,
+	MinWait:     1 * time.Second,
+	MaxWait:     5 * time.Second,
+}
+
 type Registry struct {
 	features featuremgmt.FeatureToggles
 	logger   log.Logger
@@ -22,9 +30,10 @@ type Registry struct {
 
 	extSvcProviders map[string]extsvcauth.AuthProvider
 	lock            sync.Mutex
+	serverLock      *serverlock.ServerLockService
 }
 
-func ProvideExtSvcRegistry(oauthServer *oasimpl.OAuth2ServiceImpl, saSvc *extsvcaccounts.ExtSvcAccountsService, features featuremgmt.FeatureToggles) *Registry {
+func ProvideExtSvcRegistry(oauthServer *oasimpl.OAuth2ServiceImpl, saSvc *extsvcaccounts.ExtSvcAccountsService, serverLock *serverlock.ServerLockService, features featuremgmt.FeatureToggles) *Registry {
 	return &Registry{
 		extSvcProviders: map[string]extsvcauth.AuthProvider{},
 		features:        features,
@@ -32,6 +41,7 @@ func ProvideExtSvcRegistry(oauthServer *oasimpl.OAuth2ServiceImpl, saSvc *extsvc
 		logger:          log.New("extsvcauth.registry"),
 		oauthReg:        oauthServer,
 		saReg:           saSvc,
+		serverLock:      serverLock,
 	}
 }
 
@@ -104,7 +114,7 @@ func (r *Registry) RemoveExternalService(ctx context.Context, name string) error
 		r.logger.Debug("Routing External Service removal to the OAuth2Server", "service", name)
 		return r.oauthReg.RemoveExternalService(ctx, name)
 	default:
-		return extsvcauth.ErrUnknownProvider.Errorf("unknow provider '%v'", provider)
+		return extsvcauth.ErrUnknownProvider.Errorf("unknown provider '%v'", provider)
 	}
 }
 
@@ -112,29 +122,42 @@ func (r *Registry) RemoveExternalService(ctx context.Context, name string) error
 // it generates client_id, secrets and any additional provider specificities (ex: rsa keys). It also ensures that the
 // associated service account has the correct permissions.
 func (r *Registry) SaveExternalService(ctx context.Context, cmd *extsvcauth.ExternalServiceRegistration) (*extsvcauth.ExternalService, error) {
-	// Record provider in case of removal
-	r.lock.Lock()
-	r.extSvcProviders[slugify.Slugify(cmd.Name)] = cmd.AuthProvider
-	r.lock.Unlock()
+	var (
+		errSave  error
+		extSvc   *extsvcauth.ExternalService
+		lockName = "ext-svc-save-" + cmd.Name
+	)
 
-	switch cmd.AuthProvider {
-	case extsvcauth.ServiceAccounts:
-		if !r.features.IsEnabled(ctx, featuremgmt.FlagExternalServiceAccounts) {
-			r.logger.Warn("Skipping External Service authentication, flag disabled", "service", cmd.Name, "flag", featuremgmt.FlagExternalServiceAccounts)
-			return nil, nil
+	err := r.serverLock.LockExecuteAndReleaseWithRetries(ctx, lockName, lockTimeConfig, func(ctx context.Context) {
+		// Record provider in case of removal
+		r.lock.Lock()
+		r.extSvcProviders[slugify.Slugify(cmd.Name)] = cmd.AuthProvider
+		r.lock.Unlock()
+
+		switch cmd.AuthProvider {
+		case extsvcauth.ServiceAccounts:
+			if !r.features.IsEnabled(ctx, featuremgmt.FlagExternalServiceAccounts) {
+				r.logger.Warn("Skipping External Service authentication, flag disabled", "service", cmd.Name, "flag", featuremgmt.FlagExternalServiceAccounts)
+				return
+			}
+			r.logger.Debug("Routing the External Service registration to the External Service Account service", "service", cmd.Name)
+			extSvc, errSave = r.saReg.SaveExternalService(ctx, cmd)
+		case extsvcauth.OAuth2Server:
+			if !r.features.IsEnabled(ctx, featuremgmt.FlagExternalServiceAuth) {
+				r.logger.Warn("Skipping External Service authentication, flag disabled", "service", cmd.Name, "flag", featuremgmt.FlagExternalServiceAuth)
+				return
+			}
+			r.logger.Debug("Routing the External Service registration to the OAuth2Server", "service", cmd.Name)
+			extSvc, errSave = r.oauthReg.SaveExternalService(ctx, cmd)
+		default:
+			errSave = extsvcauth.ErrUnknownProvider.Errorf("unknown provider '%v'", cmd.AuthProvider)
 		}
-		r.logger.Debug("Routing the External Service registration to the External Service Account service", "service", cmd.Name)
-		return r.saReg.SaveExternalService(ctx, cmd)
-	case extsvcauth.OAuth2Server:
-		if !r.features.IsEnabled(ctx, featuremgmt.FlagExternalServiceAuth) {
-			r.logger.Warn("Skipping External Service authentication, flag disabled", "service", cmd.Name, "flag", featuremgmt.FlagExternalServiceAuth)
-			return nil, nil
-		}
-		r.logger.Debug("Routing the External Service registration to the OAuth2Server", "service", cmd.Name)
-		return r.oauthReg.SaveExternalService(ctx, cmd)
-	default:
-		return nil, extsvcauth.ErrUnknownProvider.Errorf("unknow provider '%v'", cmd.AuthProvider)
+	})
+	if err != nil {
+		return nil, err
 	}
+
+	return extSvc, errSave
 }
 
 // retrieveExtSvcProviders fetches external services from store and map their associated provider

pkg/services/extsvcauth/registry/service_test.go

@@ -2,7 +2,6 @@ package registry
 
 import (
 	"context"
-	"sync"
 	"testing"
 
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -29,7 +28,6 @@ func setupTestEnv(t *testing.T) *TestEnv {
 		oauthReg:        env.oauthReg,
 		saReg:           env.saReg,
 		extSvcProviders: map[string]extsvcauth.AuthProvider{},
-		lock:            sync.Mutex{},
 	}
 	return &env
 }