Export: introduce export plumbing (behind dev feature flag) (#48091)

.github/CODEOWNERS

@@ -58,6 +58,7 @@ go.sum @grafana/backend-platform
 /pkg/services/live/ @grafana/grafana-edge-squad
 /pkg/services/searchV2/ @grafana/grafana-edge-squad
 /pkg/services/store/ @grafana/grafana-edge-squad
+/pkg/services/export/ @grafana/grafana-edge-squad
 /pkg/infra/filestore/ @grafana/grafana-edge-squad
 pkg/tsdb/testdatasource/sims/ @grafana/grafana-edge-squad
 

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -50,6 +50,7 @@ export interface FeatureToggles {
   saveDashboardDrawer?: boolean;
   storage?: boolean;
   alertProvisioning?: boolean;
+  export?: boolean;
   storageLocalUpload?: boolean;
   azureMonitorResourcePickerForMetrics?: boolean;
   explore2Dashboard?: boolean;

pkg/api/api.go

@@ -523,6 +523,11 @@ func (hs *HTTPServer) registerRoutes() {
 			adminRoute.Get("/crawler/status", reqGrafanaAdmin, routing.Wrap(hs.ThumbService.CrawlerStatus))
 		}
 
+		if hs.Features.IsEnabled(featuremgmt.FlagExport) {
+			adminRoute.Get("/export", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleGetStatus))
+			adminRoute.Post("/export", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleRequestExport))
+		}
+
 		adminRoute.Post("/provisioning/dashboards/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersDashboards)), routing.Wrap(hs.AdminProvisioningReloadDashboards))
 		adminRoute.Post("/provisioning/plugins/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersPlugins)), routing.Wrap(hs.AdminProvisioningReloadPlugins))
 		adminRoute.Post("/provisioning/datasources/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersDatasources)), routing.Wrap(hs.AdminProvisioningReloadDatasources))

pkg/api/http_server.go

@@ -39,6 +39,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/datasources"
 	"github.com/grafana/grafana/pkg/services/datasources/permissions"
 	"github.com/grafana/grafana/pkg/services/encryption"
+	"github.com/grafana/grafana/pkg/services/export"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/hooks"
 	"github.com/grafana/grafana/pkg/services/ldap"
@@ -112,6 +113,7 @@ type HTTPServer struct {
 	Live                         *live.GrafanaLive
 	LivePushGateway              *pushhttp.Gateway
 	ThumbService                 thumbs.Service
+	ExportService                export.ExportService
 	StorageService               store.HTTPStorageService
 	ContextHandler               *contexthandler.ContextHandler
 	SQLStore                     sqlstore.Store
@@ -170,7 +172,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 	contextHandler *contexthandler.ContextHandler, features *featuremgmt.FeatureManager,
 	schemaService *schemaloader.SchemaLoaderService, alertNG *ngalert.AlertNG,
 	libraryPanelService librarypanels.Service, libraryElementService libraryelements.Service,
-	quotaService *quota.QuotaService, socialService social.Service, tracer tracing.Tracer,
+	quotaService *quota.QuotaService, socialService social.Service, tracer tracing.Tracer, exportService export.ExportService,
 	encryptionService encryption.Internal, grafanaUpdateChecker *updatechecker.GrafanaService,
 	pluginsUpdateChecker *updatechecker.PluginsService, searchUsersService searchusers.Service,
 	dataSourcesService datasources.DataSourceService, secretsService secrets.Service, queryDataService *query.Service,
@@ -217,6 +219,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 		AccessControl:                accessControl,
 		DataProxy:                    dataSourceProxy,
 		SearchService:                searchService,
+		ExportService:                exportService,
 		Live:                         live,
 		LivePushGateway:              livePushGateway,
 		PluginContextProvider:        plugCtxProvider,

pkg/server/wire.go

@@ -50,6 +50,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/datasourceproxy"
 	"github.com/grafana/grafana/pkg/services/datasources"
 	datasourceservice "github.com/grafana/grafana/pkg/services/datasources/service"
+	"github.com/grafana/grafana/pkg/services/export"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/guardian"
 	"github.com/grafana/grafana/pkg/services/hooks"
@@ -175,6 +176,7 @@ var wireBasicSet = wire.NewSet(
 	searchV2.ProvideService,
 	store.ProvideService,
 	store.ProvideHTTPService,
+	export.ProvideService,
 	live.ProvideService,
 	pushhttp.ProvideService,
 	plugincontext.ProvideService,

pkg/services/export/dummy_job.go

@@ -0,0 +1,103 @@
+package export
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"math/rand"
+	"sync"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+)
+
+var _ Job = new(dummyExportJob)
+
+type dummyExportJob struct {
+	logger log.Logger
+
+	statusMu    sync.Mutex
+	status      ExportStatus
+	cfg         ExportConfig
+	broadcaster statusBroadcaster
+}
+
+func startDummyExportJob(cfg ExportConfig, broadcaster statusBroadcaster) (Job, error) {
+	if cfg.Format != "git" {
+		return nil, errors.New("only git format is supported")
+	}
+
+	job := &dummyExportJob{
+		logger:      log.New("dummy_export_job"),
+		cfg:         cfg,
+		broadcaster: broadcaster,
+		status: ExportStatus{
+			Running: true,
+			Target:  "git export",
+			Started: time.Now().UnixMilli(),
+			Count:   int64(math.Round(10 + rand.Float64()*20)),
+			Current: 0,
+		},
+	}
+
+	broadcaster(job.status)
+	go job.start()
+	return job, nil
+}
+
+func (e *dummyExportJob) start() {
+	defer func() {
+		e.logger.Info("Finished dummy export job")
+
+		e.statusMu.Lock()
+		defer e.statusMu.Unlock()
+		s := e.status
+		if err := recover(); err != nil {
+			e.logger.Error("export panic", "error", err)
+			s.Status = fmt.Sprintf("ERROR: %v", err)
+		}
+		// Make sure it finishes OK
+		if s.Finished < 10 {
+			s.Finished = time.Now().UnixMilli()
+		}
+		s.Running = false
+		if s.Status == "" {
+			s.Status = "done"
+		}
+		e.status = s
+		e.broadcaster(s)
+	}()
+
+	e.logger.Info("Starting dummy export job")
+
+	ticker := time.NewTicker(1 * time.Second)
+	for t := range ticker.C {
+		e.statusMu.Lock()
+		e.status.Changed = t.UnixMilli()
+		e.status.Current++
+		e.status.Last = fmt.Sprintf("ITEM: %d", e.status.Current)
+		e.statusMu.Unlock()
+
+		// Wait till we are done
+		shouldStop := e.status.Current >= e.status.Count
+		e.broadcaster(e.status)
+
+		if shouldStop {
+			break
+		}
+	}
+}
+
+func (e *dummyExportJob) getStatus() ExportStatus {
+	e.statusMu.Lock()
+	defer e.statusMu.Unlock()
+
+	return e.status
+}
+
+func (e *dummyExportJob) getConfig() ExportConfig {
+	e.statusMu.Lock()
+	defer e.statusMu.Unlock()
+
+	return e.cfg
+}

pkg/services/export/service.go

@@ -0,0 +1,93 @@
+package export
+
+import (
+	"encoding/json"
+	"net/http"
+	"sync"
+
+	"github.com/grafana/grafana/pkg/api/response"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/grafana/grafana/pkg/services/live"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+)
+
+type ExportService interface {
+	// List folder contents
+	HandleGetStatus(c *models.ReqContext) response.Response
+
+	// Read raw file contents out of the store
+	HandleRequestExport(c *models.ReqContext) response.Response
+}
+
+type StandardExport struct {
+	logger log.Logger
+	sql    *sqlstore.SQLStore
+	glive  *live.GrafanaLive
+	mutex  sync.Mutex
+
+	// updated with mutex
+	exportJob Job
+}
+
+func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, gl *live.GrafanaLive) ExportService {
+	if !features.IsEnabled(featuremgmt.FlagExport) {
+		return &StubExport{}
+	}
+
+	return &StandardExport{
+		sql:       sql,
+		glive:     gl,
+		logger:    log.New("export_service"),
+		exportJob: &stoppedJob{},
+	}
+}
+
+func (ex *StandardExport) HandleGetStatus(c *models.ReqContext) response.Response {
+	ex.mutex.Lock()
+	defer ex.mutex.Unlock()
+
+	return response.JSON(http.StatusOK, ex.exportJob.getStatus())
+}
+
+func (ex *StandardExport) HandleRequestExport(c *models.ReqContext) response.Response {
+	var cfg ExportConfig
+	err := json.NewDecoder(c.Req.Body).Decode(&cfg)
+	if err != nil {
+		return response.Error(http.StatusBadRequest, "unable to read config", err)
+	}
+
+	ex.mutex.Lock()
+	defer ex.mutex.Unlock()
+
+	status := ex.exportJob.getStatus()
+	if status.Running {
+		ex.logger.Error("export already running")
+		return response.Error(http.StatusLocked, "export already running", nil)
+	}
+
+	job, err := startDummyExportJob(cfg, func(s ExportStatus) {
+		ex.broadcastStatus(c.OrgId, s)
+	})
+	if err != nil {
+		ex.logger.Error("failed to start export job", "err", err)
+		return response.Error(http.StatusBadRequest, "failed to start export job", err)
+	}
+
+	ex.exportJob = job
+	return response.JSON(http.StatusOK, ex.exportJob.getStatus())
+}
+
+func (ex *StandardExport) broadcastStatus(orgID int64, s ExportStatus) {
+	msg, err := json.Marshal(s)
+	if err != nil {
+		ex.logger.Warn("Error making message", "err", err)
+		return
+	}
+	err = ex.glive.Publish(orgID, "grafana/broadcast/export", msg)
+	if err != nil {
+		ex.logger.Warn("Error Publish message", "err", err)
+		return
+	}
+}

pkg/services/export/stopped_job.go

@@ -0,0 +1,19 @@
+package export
+
+import "time"
+
+var _ Job = new(stoppedJob)
+
+type stoppedJob struct {
+}
+
+func (e *stoppedJob) getStatus() ExportStatus {
+	return ExportStatus{
+		Running: false,
+		Changed: time.Now().UnixMilli(),
+	}
+}
+
+func (e *stoppedJob) getConfig() ExportConfig {
+	return ExportConfig{}
+}

pkg/services/export/stub.go

@@ -0,0 +1,20 @@
+package export
+
+import (
+	"net/http"
+
+	"github.com/grafana/grafana/pkg/api/response"
+	"github.com/grafana/grafana/pkg/models"
+)
+
+var _ ExportService = new(StubExport)
+
+type StubExport struct{}
+
+func (ex *StubExport) HandleGetStatus(c *models.ReqContext) response.Response {
+	return response.Error(http.StatusForbidden, "feature not enabled", nil)
+}
+
+func (ex *StubExport) HandleRequestExport(c *models.ReqContext) response.Response {
+	return response.Error(http.StatusForbidden, "feature not enabled", nil)
+}

pkg/services/export/types.go

@@ -0,0 +1,36 @@
+package export
+
+// Export status.  Only one running at a time
+type ExportStatus struct {
+	Running  bool   `json:"running"`
+	Target   string `json:"target"` // description of where it is going (no secrets)
+	Started  int64  `json:"started,omitempty"`
+	Finished int64  `json:"finished,omitempty"`
+	Changed  int64  `json:"update,omitempty"`
+	Count    int64  `json:"count,omitempty"`
+	Current  int64  `json:"current,omitempty"`
+	Last     string `json:"last,omitempty"`
+	Status   string `json:"status"` // ERROR, SUCCESS, ETC
+}
+
+// Basic export config (for now)
+type ExportConfig struct {
+	Format string          `json:"format"`
+	Git    GitExportConfig `json:"git"`
+}
+
+type GitExportConfig struct {
+	// General folder is either at the root or as a subfolder
+	GeneralAtRoot bool `json:"generalAtRoot"`
+
+	// Keeping all history is nice, but much slower
+	ExcludeHistory bool `json:"excludeHistory"`
+}
+
+type Job interface {
+	getStatus() ExportStatus
+	getConfig() ExportConfig
+}
+
+// Will broadcast the live status
+type statusBroadcaster func(s ExportStatus)

pkg/services/featuremgmt/registry.go

@@ -190,6 +190,12 @@ var (
 			Description: "Provisioning-friendly routes for alerting",
 			State:       FeatureStateAlpha,
 		},
+		{
+			Name:            "export",
+			Description:     "Export grafana instance (to git, etc)",
+			State:           FeatureStateAlpha,
+			RequiresDevMode: true,
+		},
 		{
 			Name:            "storageLocalUpload",
 			Description:     "allow uploads to local storage",

pkg/services/featuremgmt/toggles_gen.go

@@ -143,6 +143,10 @@ const (
 	// Provisioning-friendly routes for alerting
 	FlagAlertProvisioning = "alertProvisioning"
 
+	// FlagExport
+	// Export grafana instance (to git, etc)
+	FlagExport = "export"
+
 	// FlagStorageLocalUpload
 	// allow uploads to local storage
 	FlagStorageLocalUpload = "storageLocalUpload"

public/app/features/admin/ExportStartButton.tsx

@@ -0,0 +1,62 @@
+import { css } from '@emotion/css';
+import React, { useState } from 'react';
+
+import { GrafanaTheme2 } from '@grafana/data';
+import { getBackendSrv } from '@grafana/runtime';
+import { Button, CodeEditor, Modal, useTheme2 } from '@grafana/ui';
+
+export const ExportStartButton = () => {
+  const styles = getStyles(useTheme2());
+  const [open, setOpen] = useState(false);
+  const [body, setBody] = useState({
+    format: 'git',
+    git: {},
+  });
+  const onDismiss = () => setOpen(false);
+  const doStart = () => {
+    getBackendSrv()
+      .post('/api/admin/export', body)
+      .then((v) => {
+        console.log('GOT', v);
+        onDismiss();
+      });
+  };
+
+  return (
+    <>
+      <Modal title={'Export grafana instance'} isOpen={open} onDismiss={onDismiss}>
+        <div className={styles.wrap}>
+          <CodeEditor
+            height={200}
+            value={JSON.stringify(body, null, 2) ?? ''}
+            showLineNumbers={false}
+            readOnly={false}
+            language="json"
+            showMiniMap={false}
+            onBlur={(text: string) => {
+              setBody(JSON.parse(text)); // force JSON?
+            }}
+          />
+        </div>
+        <Modal.ButtonRow>
+          <Button onClick={doStart}>Start</Button>
+          <Button variant="secondary" onClick={onDismiss}>
+            Cancel
+          </Button>
+        </Modal.ButtonRow>
+      </Modal>
+
+      <Button onClick={() => setOpen(true)} variant="primary">
+        Export
+      </Button>
+    </>
+  );
+};
+
+const getStyles = (theme: GrafanaTheme2) => {
+  return {
+    wrap: css`
+      border: 2px solid #111;
+    `,
+  };
+};

public/app/features/admin/ExportStatus.tsx

@@ -0,0 +1,82 @@
+import { css } from '@emotion/css';
+import React, { useEffect, useState } from 'react';
+
+import { GrafanaTheme2, isLiveChannelMessageEvent, isLiveChannelStatusEvent, LiveChannelScope } from '@grafana/data';
+import { getBackendSrv, getGrafanaLiveSrv } from '@grafana/runtime';
+import { Button, useTheme2 } from '@grafana/ui';
+
+import { ExportStartButton } from './ExportStartButton';
+
+interface ExportStatusMessage {
+  running: boolean;
+  target: string;
+  started: number;
+  finished: number;
+  update: number;
+  count: number;
+  current: number;
+  last: string;
+  status: string;
+}
+
+export const ExportStatus = () => {
+  const styles = getStyles(useTheme2());
+  const [status, setStatus] = useState<ExportStatusMessage>();
+
+  useEffect(() => {
+    const subscription = getGrafanaLiveSrv()
+      .getStream<ExportStatusMessage>({
+        scope: LiveChannelScope.Grafana,
+        namespace: 'broadcast',
+        path: 'export',
+      })
+      .subscribe({
+        next: (evt) => {
+          if (isLiveChannelMessageEvent(evt)) {
+            setStatus(evt.message);
+          } else if (isLiveChannelStatusEvent(evt)) {
+            setStatus(evt.message);
+          }
+        },
+      });
+    return () => {
+      subscription.unsubscribe();
+    };
+  }, []);
+
+  if (!status) {
+    return (
+      <div className={styles.wrap}>
+        <ExportStartButton />
+      </div>
+    );
+  }
+
+  return (
+    <div className={styles.wrap}>
+      <pre>{JSON.stringify(status, null, 2)}</pre>
+      {Boolean(!status.running) && <ExportStartButton />}
+      {Boolean(status.running) && (
+        <Button
+          variant="secondary"
+          onClick={() => {
+            getBackendSrv().post('/api/admin/export/stop');
+          }}
+        >
+          Stop
+        </Button>
+      )}
+    </div>
+  );
+};
+
+const getStyles = (theme: GrafanaTheme2) => {
+  return {
+    wrap: css`
+      border: 4px solid red;
+    `,
+    running: css`
+      border: 4px solid green;
+    `,
+  };
+};

public/app/features/admin/ServerStats.tsx

@@ -10,6 +10,7 @@ import { contextSrv } from '../../core/services/context_srv';
 import { Loader } from '../plugins/admin/components/Loader';
 
 import { CrawlerStatus } from './CrawlerStatus';
+import { ExportStatus } from './ExportStatus';
 import { getServerStats, ServerStat } from './state/apis';
 
 export const ServerStats = () => {
@@ -98,6 +99,7 @@ export const ServerStats = () => {
       )}
 
       {config.featureToggles.dashboardPreviews && config.featureToggles.dashboardPreviewsAdmin && <CrawlerStatus />}
+      {config.featureToggles.export && <ExportStatus />}
     </>
   );
 };