Cloud Migrations: Create new service for cloud migrations (#80949)

* introduce feature toggle

* create base service structure

* fix sample metric

* register metrics

* add to codeowners

* separate api dtos from service models

* remove leading newline

.github/CODEOWNERS

@@ -604,6 +604,7 @@ cypress.config.js @grafana/grafana-frontend-platform
 /pkg/infra/httpclient/httpclientprovider/sigv4_middleware_test.go @grafana/grafana-operator-experience-squad
 /pkg/services/caching/ @grafana/grafana-operator-experience-squad
 /pkg/services/featuremgmt/ @grafana/grafana-operator-experience-squad
+/pkg/services/cloudmigrations/ @grafana/grafana-operator-experience-squad
 
 # Kind definitions
 /kinds/dashboard @grafana/dashboards-squad

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -173,6 +173,7 @@ Experimental features might be changed or removed without prior notice.
 | `enablePluginsTracingByDefault`             | Enable plugin tracing for all external plugins                                                                                                                                                                                                                                    |
 | `newFolderPicker`                           | Enables the nested folder picker without having nested folders enabled                                                                                                                                                                                                            |
 | `jitterAlertRules`                          | Distributes alert rule evaluations more evenly over time, by rule group                                                                                                                                                                                                           |
+| `onPremToCloudMigrations`                   | In-development feature that will allow users to easily migrate their on-prem Grafana instances to Grafana Cloud.                                                                                                                                                                  |
 
 ## Development feature toggles
 

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -176,4 +176,5 @@ export interface FeatureToggles {
   newFolderPicker?: boolean;
   jitterAlertRules?: boolean;
   jitterAlertRulesWithinGroups?: boolean;
+  onPremToCloudMigrations?: boolean;
 }

pkg/server/wire.go

@@ -49,6 +49,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/auth/jwt"
 	"github.com/grafana/grafana/pkg/services/authn/authnimpl"
 	"github.com/grafana/grafana/pkg/services/cleanup"
+	cloudmigrations "github.com/grafana/grafana/pkg/services/cloudmigrations/service"
 	"github.com/grafana/grafana/pkg/services/contexthandler"
 	"github.com/grafana/grafana/pkg/services/correlations"
 	"github.com/grafana/grafana/pkg/services/dashboardimport"
@@ -382,6 +383,8 @@ var wireBasicSet = wire.NewSet(
 	wire.Bind(new(ssosettings.Service), new(*ssoSettingsImpl.SSOSettingsService)),
 	idimpl.ProvideService,
 	wire.Bind(new(auth.IDService), new(*idimpl.Service)),
+	cloudmigrations.ProvideService,
+	// Kubernetes API server
 	grafanaapiserver.WireSet,
 	apiregistry.WireSet,
 )

pkg/services/cloudmigrations/api/api.go

@@ -0,0 +1,57 @@
+package api
+
+import (
+	"net/http"
+
+	"github.com/grafana/grafana/pkg/api/response"
+	"github.com/grafana/grafana/pkg/api/routing"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/middleware"
+	"github.com/grafana/grafana/pkg/services/cloudmigrations"
+	"github.com/grafana/grafana/pkg/services/cloudmigrations/models"
+	contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
+	"github.com/grafana/grafana/pkg/web"
+)
+
+type MigrationAPI struct {
+	cloudMigrationsService cloudmigrations.CloudMigrationService
+	routeRegister          routing.RouteRegister
+	log                    log.Logger
+}
+
+func RegisterApi(
+	rr routing.RouteRegister,
+	cms cloudmigrations.CloudMigrationService,
+) *MigrationAPI {
+	api := &MigrationAPI{
+		log:                    log.New("cloudmigrations.api"),
+		routeRegister:          rr,
+		cloudMigrationsService: cms,
+	}
+	api.registerEndpoints()
+	return api
+}
+
+// RegisterAPIEndpoints Registers Endpoints on Grafana Router
+func (api *MigrationAPI) registerEndpoints() {
+	api.routeRegister.Group("/api/cloudmigrations", func(apiRoute routing.RouteRegister) {
+		apiRoute.Post(
+			"/migrate_datasources",
+			routing.Wrap(api.MigrateDatasources),
+		)
+	}, middleware.ReqGrafanaAdmin)
+}
+
+func (api *MigrationAPI) MigrateDatasources(c *contextmodel.ReqContext) response.Response {
+	var req migrateDatasourcesRequestDTO
+	if err := web.Bind(c.Req, &req); err != nil {
+		return response.Error(http.StatusBadRequest, "bad request data", err)
+	}
+
+	resp, err := api.cloudMigrationsService.MigrateDatasources(c.Req.Context(), &models.MigrateDatasourcesRequest{MigrateToPDC: req.MigrateToPDC, MigrateCredentials: req.MigrateCredentials})
+	if err != nil {
+		return response.Error(http.StatusInternalServerError, "data source migrations error", err)
+	}
+
+	return response.JSON(http.StatusOK, migrateDatasourcesResponseDTO{DatasourcesMigrated: resp.DatasourcesMigrated})
+}

pkg/services/cloudmigrations/api/api_test.go

@@ -0,0 +1,12 @@
+package api
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_OnlyEnabledForGrafanaAdmi(t *testing.T) {
+	// TODO: implement
+	assert.True(t, true)
+}

pkg/services/cloudmigrations/api/models.go

@@ -0,0 +1,10 @@
+package api
+
+type migrateDatasourcesRequestDTO struct {
+	MigrateToPDC       bool `json:"migrateToPDC"`
+	MigrateCredentials bool `json:"migrateCredentials"`
+}
+
+type migrateDatasourcesResponseDTO struct {
+	DatasourcesMigrated int `json:"datasourcesMigrated"`
+}

pkg/services/cloudmigrations/cloudmigrations.go

@@ -0,0 +1,11 @@
+package cloudmigrations
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/services/cloudmigrations/models"
+)
+
+type CloudMigrationService interface {
+	MigrateDatasources(ctx context.Context, request *models.MigrateDatasourcesRequest) (*models.MigrateDatasourcesResponse, error)
+}

pkg/services/cloudmigrations/metrics/metric.go

@@ -0,0 +1,40 @@
+package metrics
+
+import (
+	"errors"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type Metrics struct {
+	log log.Logger
+}
+
+func RegisterMetrics(
+	prom prometheus.Registerer,
+) (*Metrics, error) {
+	s := &Metrics{
+		log: log.New("cloudmigrations.metrics"),
+	}
+
+	if err := s.registerMetrics(prom); err != nil {
+		return nil, err
+	}
+
+	return s, nil
+}
+
+func (s *Metrics) registerMetrics(prom prometheus.Registerer) error {
+	for _, m := range promMetrics {
+		if err := prom.Register(m); err != nil {
+			var alreadyRegisterErr prometheus.AlreadyRegisteredError
+			if errors.As(err, &alreadyRegisterErr) {
+				s.log.Warn("metric already registered", "metric", m)
+				continue
+			}
+			return err
+		}
+	}
+	return nil
+}

pkg/services/cloudmigrations/metrics/metrics.go

@@ -0,0 +1,19 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	namespace = "grafana"
+	subsystem = "cloudmigrations"
+)
+
+var promMetrics = []prometheus.Collector{
+	prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: subsystem,
+		Name:      "datasources_migrated",
+		Help:      "Total amount of data sources migrated",
+	}, []string{"pdc_converted"}),
+}

pkg/services/cloudmigrations/models/errors.go

@@ -0,0 +1,8 @@
+package models
+
+import "github.com/grafana/grafana/pkg/util/errutil"
+
+var (
+	ErrInternalNotImplementedError = errutil.Internal("cloudmigrations.notImplemented", errutil.WithPublicMessage("Internal server error"))
+	ErrFeatureDisabledError        = errutil.Internal("cloudmigrations.disabled", errutil.WithPublicMessage("Cloud migrations are disabled on this instance"))
+)

pkg/services/cloudmigrations/models/models.go

@@ -0,0 +1,10 @@
+package models
+
+type MigrateDatasourcesRequest struct {
+	MigrateToPDC       bool
+	MigrateCredentials bool
+}
+
+type MigrateDatasourcesResponse struct {
+	DatasourcesMigrated int
+}

pkg/services/cloudmigrations/service/noop_service.go

@@ -0,0 +1,17 @@
+package service
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/services/cloudmigrations"
+	"github.com/grafana/grafana/pkg/services/cloudmigrations/models"
+)
+
+// CloudMigrationsServiceImpl Define the Service Implementation.
+type NoopServiceImpl struct{}
+
+var _ cloudmigrations.CloudMigrationService = (*NoopServiceImpl)(nil)
+
+func (cm *NoopServiceImpl) MigrateDatasources(ctx context.Context, request *models.MigrateDatasourcesRequest) (*models.MigrateDatasourcesResponse, error) {
+	return nil, models.ErrFeatureDisabledError
+}

pkg/services/cloudmigrations/service/service.go

@@ -0,0 +1,70 @@
+package service
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/api/routing"
+	"github.com/grafana/grafana/pkg/infra/db"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/services/cloudmigrations"
+	"github.com/grafana/grafana/pkg/services/cloudmigrations/api"
+	"github.com/grafana/grafana/pkg/services/cloudmigrations/metrics"
+	"github.com/grafana/grafana/pkg/services/cloudmigrations/models"
+	"github.com/grafana/grafana/pkg/services/datasources"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// CloudMigrationsServiceImpl Define the Service Implementation.
+type CloudMigrationsServiceImpl struct {
+	log log.Logger
+	cfg *setting.Cfg
+
+	sqlStore  db.DB
+	features  featuremgmt.FeatureToggles
+	dsService datasources.DataSourceService
+
+	api     *api.MigrationAPI
+	metrics *metrics.Metrics
+}
+
+var LogPrefix = "cloudmigrations.service"
+
+var _ cloudmigrations.CloudMigrationService = (*CloudMigrationsServiceImpl)(nil)
+
+// ProvideService Factory for method used by wire to inject dependencies.
+// builds the service, and api, and configures routes
+func ProvideService(
+	cfg *setting.Cfg,
+	features featuremgmt.FeatureToggles,
+	sqlStore db.DB,
+	dsService datasources.DataSourceService,
+	routeRegister routing.RouteRegister,
+	prom prometheus.Registerer,
+) cloudmigrations.CloudMigrationService {
+	if !features.IsEnabledGlobally(featuremgmt.FlagOnPremToCloudMigrations) {
+		return &NoopServiceImpl{}
+	}
+
+	s := &CloudMigrationsServiceImpl{
+		log:       log.New(LogPrefix),
+		cfg:       cfg,
+		sqlStore:  sqlStore,
+		features:  features,
+		dsService: dsService,
+	}
+	s.api = api.RegisterApi(routeRegister, s)
+
+	if m, err := metrics.RegisterMetrics(prom); err != nil {
+		s.log.Warn("error registering prom metrics", "error", err.Error())
+	} else {
+		s.metrics = m
+	}
+
+	return s
+}
+
+func (cm *CloudMigrationsServiceImpl) MigrateDatasources(ctx context.Context, request *models.MigrateDatasourcesRequest) (*models.MigrateDatasourcesResponse, error) {
+	return nil, models.ErrInternalNotImplementedError
+}

pkg/services/cloudmigrations/service/service_test.go

@@ -0,0 +1,15 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	"github.com/grafana/grafana/pkg/services/cloudmigrations/models"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_NoopServiceDoesNothing(t *testing.T) {
+	s := &NoopServiceImpl{}
+	_, e := s.MigrateDatasources(context.Background(), &models.MigrateDatasourcesRequest{})
+	assert.ErrorIs(t, e, models.ErrFeatureDisabledError)
+}

pkg/services/featuremgmt/registry.go

@@ -1348,5 +1348,12 @@ var (
 			RequiresRestart:   true,
 			Created:           time.Date(2024, time.January, 17, 12, 0, 0, 0, time.UTC),
 		},
+		{
+			Name:        "onPremToCloudMigrations",
+			Description: "In-development feature that will allow users to easily migrate their on-prem Grafana instances to Grafana Cloud.",
+			Stage:       FeatureStageExperimental,
+			Owner:       grafanaOperatorExperienceSquad,
+			Created:     time.Date(2024, time.January, 22, 3, 30, 00, 00, time.UTC),
+		},
 	}
 )

pkg/services/featuremgmt/toggles_gen.csv

@@ -157,3 +157,4 @@ alertingQueryOptimization,GA,@grafana/alerting-squad,2024-01-10,false,false,fals
 newFolderPicker,experimental,@grafana/grafana-frontend-platform,2024-01-12,false,false,false,true
 jitterAlertRules,experimental,@grafana/alerting-squad,2024-01-17,false,false,true,false
 jitterAlertRulesWithinGroups,experimental,@grafana/alerting-squad,2024-01-17,false,false,true,false
+onPremToCloudMigrations,experimental,@grafana/grafana-operator-experience-squad,2024-01-22,false,false,false,false

pkg/services/featuremgmt/toggles_gen.go

@@ -638,4 +638,8 @@ const (
 	// FlagJitterAlertRulesWithinGroups
 	// Distributes alert rule evaluations more evenly over time, including spreading out rules within the same group
 	FlagJitterAlertRulesWithinGroups = "jitterAlertRulesWithinGroups"
+
+	// FlagOnPremToCloudMigrations
+	// In-development feature that will allow users to easily migrate their on-prem Grafana instances to Grafana Cloud.
+	FlagOnPremToCloudMigrations = "onPremToCloudMigrations"
 )