diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b421e6f3663..8f364c8f8bc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -58,6 +58,7 @@ go.sum @grafana/backend-platform /pkg/services/live/ @grafana/grafana-edge-squad /pkg/services/searchV2/ @grafana/grafana-edge-squad /pkg/services/store/ @grafana/grafana-edge-squad +/pkg/services/export/ @grafana/grafana-edge-squad /pkg/infra/filestore/ @grafana/grafana-edge-squad pkg/tsdb/testdatasource/sims/ @grafana/grafana-edge-squad diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index 080394dd1a9..e2bdfadde9d 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -50,6 +50,7 @@ export interface FeatureToggles { saveDashboardDrawer?: boolean; storage?: boolean; alertProvisioning?: boolean; + export?: boolean; storageLocalUpload?: boolean; azureMonitorResourcePickerForMetrics?: boolean; explore2Dashboard?: boolean; diff --git a/pkg/api/api.go b/pkg/api/api.go index 24883640e05..030c920ada2 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -523,6 +523,11 @@ func (hs *HTTPServer) registerRoutes() { adminRoute.Get("/crawler/status", reqGrafanaAdmin, routing.Wrap(hs.ThumbService.CrawlerStatus)) } + if hs.Features.IsEnabled(featuremgmt.FlagExport) { + adminRoute.Get("/export", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleGetStatus)) + adminRoute.Post("/export", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleRequestExport)) + } + adminRoute.Post("/provisioning/dashboards/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersDashboards)), routing.Wrap(hs.AdminProvisioningReloadDashboards)) adminRoute.Post("/provisioning/plugins/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersPlugins)), routing.Wrap(hs.AdminProvisioningReloadPlugins)) adminRoute.Post("/provisioning/datasources/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersDatasources)), routing.Wrap(hs.AdminProvisioningReloadDatasources)) diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index 5491b86d316..0d4e4241744 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -39,6 +39,7 @@ import ( "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/datasources/permissions" "github.com/grafana/grafana/pkg/services/encryption" + "github.com/grafana/grafana/pkg/services/export" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/hooks" "github.com/grafana/grafana/pkg/services/ldap" @@ -112,6 +113,7 @@ type HTTPServer struct { Live *live.GrafanaLive LivePushGateway *pushhttp.Gateway ThumbService thumbs.Service + ExportService export.ExportService StorageService store.HTTPStorageService ContextHandler *contexthandler.ContextHandler SQLStore sqlstore.Store @@ -170,7 +172,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi contextHandler *contexthandler.ContextHandler, features *featuremgmt.FeatureManager, schemaService *schemaloader.SchemaLoaderService, alertNG *ngalert.AlertNG, libraryPanelService librarypanels.Service, libraryElementService libraryelements.Service, - quotaService *quota.QuotaService, socialService social.Service, tracer tracing.Tracer, + quotaService *quota.QuotaService, socialService social.Service, tracer tracing.Tracer, exportService export.ExportService, encryptionService encryption.Internal, grafanaUpdateChecker *updatechecker.GrafanaService, pluginsUpdateChecker *updatechecker.PluginsService, searchUsersService searchusers.Service, dataSourcesService datasources.DataSourceService, secretsService secrets.Service, queryDataService *query.Service, @@ -217,6 +219,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi AccessControl: accessControl, DataProxy: dataSourceProxy, SearchService: searchService, + ExportService: exportService, Live: live, LivePushGateway: livePushGateway, PluginContextProvider: plugCtxProvider, diff --git a/pkg/server/wire.go b/pkg/server/wire.go index 47e2d1eee5d..4461854bd5b 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -50,6 +50,7 @@ import ( "github.com/grafana/grafana/pkg/services/datasourceproxy" "github.com/grafana/grafana/pkg/services/datasources" datasourceservice "github.com/grafana/grafana/pkg/services/datasources/service" + "github.com/grafana/grafana/pkg/services/export" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/guardian" "github.com/grafana/grafana/pkg/services/hooks" @@ -175,6 +176,7 @@ var wireBasicSet = wire.NewSet( searchV2.ProvideService, store.ProvideService, store.ProvideHTTPService, + export.ProvideService, live.ProvideService, pushhttp.ProvideService, plugincontext.ProvideService, diff --git a/pkg/services/export/dummy_job.go b/pkg/services/export/dummy_job.go new file mode 100644 index 00000000000..029bc8edabb --- /dev/null +++ b/pkg/services/export/dummy_job.go @@ -0,0 +1,103 @@ +package export + +import ( + "errors" + "fmt" + "math" + "math/rand" + "sync" + "time" + + "github.com/grafana/grafana/pkg/infra/log" +) + +var _ Job = new(dummyExportJob) + +type dummyExportJob struct { + logger log.Logger + + statusMu sync.Mutex + status ExportStatus + cfg ExportConfig + broadcaster statusBroadcaster +} + +func startDummyExportJob(cfg ExportConfig, broadcaster statusBroadcaster) (Job, error) { + if cfg.Format != "git" { + return nil, errors.New("only git format is supported") + } + + job := &dummyExportJob{ + logger: log.New("dummy_export_job"), + cfg: cfg, + broadcaster: broadcaster, + status: ExportStatus{ + Running: true, + Target: "git export", + Started: time.Now().UnixMilli(), + Count: int64(math.Round(10 + rand.Float64()*20)), + Current: 0, + }, + } + + broadcaster(job.status) + go job.start() + return job, nil +} + +func (e *dummyExportJob) start() { + defer func() { + e.logger.Info("Finished dummy export job") + + e.statusMu.Lock() + defer e.statusMu.Unlock() + s := e.status + if err := recover(); err != nil { + e.logger.Error("export panic", "error", err) + s.Status = fmt.Sprintf("ERROR: %v", err) + } + // Make sure it finishes OK + if s.Finished < 10 { + s.Finished = time.Now().UnixMilli() + } + s.Running = false + if s.Status == "" { + s.Status = "done" + } + e.status = s + e.broadcaster(s) + }() + + e.logger.Info("Starting dummy export job") + + ticker := time.NewTicker(1 * time.Second) + for t := range ticker.C { + e.statusMu.Lock() + e.status.Changed = t.UnixMilli() + e.status.Current++ + e.status.Last = fmt.Sprintf("ITEM: %d", e.status.Current) + e.statusMu.Unlock() + + // Wait till we are done + shouldStop := e.status.Current >= e.status.Count + e.broadcaster(e.status) + + if shouldStop { + break + } + } +} + +func (e *dummyExportJob) getStatus() ExportStatus { + e.statusMu.Lock() + defer e.statusMu.Unlock() + + return e.status +} + +func (e *dummyExportJob) getConfig() ExportConfig { + e.statusMu.Lock() + defer e.statusMu.Unlock() + + return e.cfg +} diff --git a/pkg/services/export/service.go b/pkg/services/export/service.go new file mode 100644 index 00000000000..ee4f1b040ea --- /dev/null +++ b/pkg/services/export/service.go @@ -0,0 +1,93 @@ +package export + +import ( + "encoding/json" + "net/http" + "sync" + + "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/services/live" + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +type ExportService interface { + // List folder contents + HandleGetStatus(c *models.ReqContext) response.Response + + // Read raw file contents out of the store + HandleRequestExport(c *models.ReqContext) response.Response +} + +type StandardExport struct { + logger log.Logger + sql *sqlstore.SQLStore + glive *live.GrafanaLive + mutex sync.Mutex + + // updated with mutex + exportJob Job +} + +func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, gl *live.GrafanaLive) ExportService { + if !features.IsEnabled(featuremgmt.FlagExport) { + return &StubExport{} + } + + return &StandardExport{ + sql: sql, + glive: gl, + logger: log.New("export_service"), + exportJob: &stoppedJob{}, + } +} + +func (ex *StandardExport) HandleGetStatus(c *models.ReqContext) response.Response { + ex.mutex.Lock() + defer ex.mutex.Unlock() + + return response.JSON(http.StatusOK, ex.exportJob.getStatus()) +} + +func (ex *StandardExport) HandleRequestExport(c *models.ReqContext) response.Response { + var cfg ExportConfig + err := json.NewDecoder(c.Req.Body).Decode(&cfg) + if err != nil { + return response.Error(http.StatusBadRequest, "unable to read config", err) + } + + ex.mutex.Lock() + defer ex.mutex.Unlock() + + status := ex.exportJob.getStatus() + if status.Running { + ex.logger.Error("export already running") + return response.Error(http.StatusLocked, "export already running", nil) + } + + job, err := startDummyExportJob(cfg, func(s ExportStatus) { + ex.broadcastStatus(c.OrgId, s) + }) + if err != nil { + ex.logger.Error("failed to start export job", "err", err) + return response.Error(http.StatusBadRequest, "failed to start export job", err) + } + + ex.exportJob = job + return response.JSON(http.StatusOK, ex.exportJob.getStatus()) +} + +func (ex *StandardExport) broadcastStatus(orgID int64, s ExportStatus) { + msg, err := json.Marshal(s) + if err != nil { + ex.logger.Warn("Error making message", "err", err) + return + } + err = ex.glive.Publish(orgID, "grafana/broadcast/export", msg) + if err != nil { + ex.logger.Warn("Error Publish message", "err", err) + return + } +} diff --git a/pkg/services/export/stopped_job.go b/pkg/services/export/stopped_job.go new file mode 100644 index 00000000000..b9756f9d72f --- /dev/null +++ b/pkg/services/export/stopped_job.go @@ -0,0 +1,19 @@ +package export + +import "time" + +var _ Job = new(stoppedJob) + +type stoppedJob struct { +} + +func (e *stoppedJob) getStatus() ExportStatus { + return ExportStatus{ + Running: false, + Changed: time.Now().UnixMilli(), + } +} + +func (e *stoppedJob) getConfig() ExportConfig { + return ExportConfig{} +} diff --git a/pkg/services/export/stub.go b/pkg/services/export/stub.go new file mode 100644 index 00000000000..551bb730f5c --- /dev/null +++ b/pkg/services/export/stub.go @@ -0,0 +1,20 @@ +package export + +import ( + "net/http" + + "github.com/grafana/grafana/pkg/api/response" + "github.com/grafana/grafana/pkg/models" +) + +var _ ExportService = new(StubExport) + +type StubExport struct{} + +func (ex *StubExport) HandleGetStatus(c *models.ReqContext) response.Response { + return response.Error(http.StatusForbidden, "feature not enabled", nil) +} + +func (ex *StubExport) HandleRequestExport(c *models.ReqContext) response.Response { + return response.Error(http.StatusForbidden, "feature not enabled", nil) +} diff --git a/pkg/services/export/types.go b/pkg/services/export/types.go new file mode 100644 index 00000000000..20a1ec49e3d --- /dev/null +++ b/pkg/services/export/types.go @@ -0,0 +1,36 @@ +package export + +// Export status. Only one running at a time +type ExportStatus struct { + Running bool `json:"running"` + Target string `json:"target"` // description of where it is going (no secrets) + Started int64 `json:"started,omitempty"` + Finished int64 `json:"finished,omitempty"` + Changed int64 `json:"update,omitempty"` + Count int64 `json:"count,omitempty"` + Current int64 `json:"current,omitempty"` + Last string `json:"last,omitempty"` + Status string `json:"status"` // ERROR, SUCCESS, ETC +} + +// Basic export config (for now) +type ExportConfig struct { + Format string `json:"format"` + Git GitExportConfig `json:"git"` +} + +type GitExportConfig struct { + // General folder is either at the root or as a subfolder + GeneralAtRoot bool `json:"generalAtRoot"` + + // Keeping all history is nice, but much slower + ExcludeHistory bool `json:"excludeHistory"` +} + +type Job interface { + getStatus() ExportStatus + getConfig() ExportConfig +} + +// Will broadcast the live status +type statusBroadcaster func(s ExportStatus) diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index a96c4249061..1266e6f75f1 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -190,6 +190,12 @@ var ( Description: "Provisioning-friendly routes for alerting", State: FeatureStateAlpha, }, + { + Name: "export", + Description: "Export grafana instance (to git, etc)", + State: FeatureStateAlpha, + RequiresDevMode: true, + }, { Name: "storageLocalUpload", Description: "allow uploads to local storage", diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 31978faca12..1966db6a137 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -143,6 +143,10 @@ const ( // Provisioning-friendly routes for alerting FlagAlertProvisioning = "alertProvisioning" + // FlagExport + // Export grafana instance (to git, etc) + FlagExport = "export" + // FlagStorageLocalUpload // allow uploads to local storage FlagStorageLocalUpload = "storageLocalUpload" diff --git a/public/app/features/admin/ExportStartButton.tsx b/public/app/features/admin/ExportStartButton.tsx new file mode 100644 index 00000000000..f94d4f5b081 --- /dev/null +++ b/public/app/features/admin/ExportStartButton.tsx @@ -0,0 +1,62 @@ +import { css } from '@emotion/css'; +import React, { useState } from 'react'; + +import { GrafanaTheme2 } from '@grafana/data'; +import { getBackendSrv } from '@grafana/runtime'; +import { Button, CodeEditor, Modal, useTheme2 } from '@grafana/ui'; + +export const ExportStartButton = () => { + const styles = getStyles(useTheme2()); + const [open, setOpen] = useState(false); + const [body, setBody] = useState({ + format: 'git', + git: {}, + }); + const onDismiss = () => setOpen(false); + const doStart = () => { + getBackendSrv() + .post('/api/admin/export', body) + .then((v) => { + console.log('GOT', v); + onDismiss(); + }); + }; + + return ( + <> + +
+ { + setBody(JSON.parse(text)); // force JSON? + }} + /> +
+ + + + +
+ + + + ); +}; + +const getStyles = (theme: GrafanaTheme2) => { + return { + wrap: css` + border: 2px solid #111; + `, + }; +}; diff --git a/public/app/features/admin/ExportStatus.tsx b/public/app/features/admin/ExportStatus.tsx new file mode 100644 index 00000000000..6b638b8f358 --- /dev/null +++ b/public/app/features/admin/ExportStatus.tsx @@ -0,0 +1,82 @@ +import { css } from '@emotion/css'; +import React, { useEffect, useState } from 'react'; + +import { GrafanaTheme2, isLiveChannelMessageEvent, isLiveChannelStatusEvent, LiveChannelScope } from '@grafana/data'; +import { getBackendSrv, getGrafanaLiveSrv } from '@grafana/runtime'; +import { Button, useTheme2 } from '@grafana/ui'; + +import { ExportStartButton } from './ExportStartButton'; + +interface ExportStatusMessage { + running: boolean; + target: string; + started: number; + finished: number; + update: number; + count: number; + current: number; + last: string; + status: string; +} + +export const ExportStatus = () => { + const styles = getStyles(useTheme2()); + const [status, setStatus] = useState(); + + useEffect(() => { + const subscription = getGrafanaLiveSrv() + .getStream({ + scope: LiveChannelScope.Grafana, + namespace: 'broadcast', + path: 'export', + }) + .subscribe({ + next: (evt) => { + if (isLiveChannelMessageEvent(evt)) { + setStatus(evt.message); + } else if (isLiveChannelStatusEvent(evt)) { + setStatus(evt.message); + } + }, + }); + return () => { + subscription.unsubscribe(); + }; + }, []); + + if (!status) { + return ( +
+ +
+ ); + } + + return ( +
+
{JSON.stringify(status, null, 2)}
+ {Boolean(!status.running) && } + {Boolean(status.running) && ( + + )} +
+ ); +}; + +const getStyles = (theme: GrafanaTheme2) => { + return { + wrap: css` + border: 4px solid red; + `, + running: css` + border: 4px solid green; + `, + }; +}; diff --git a/public/app/features/admin/ServerStats.tsx b/public/app/features/admin/ServerStats.tsx index 672d1ba2746..1b3cf7799d6 100644 --- a/public/app/features/admin/ServerStats.tsx +++ b/public/app/features/admin/ServerStats.tsx @@ -10,6 +10,7 @@ import { contextSrv } from '../../core/services/context_srv'; import { Loader } from '../plugins/admin/components/Loader'; import { CrawlerStatus } from './CrawlerStatus'; +import { ExportStatus } from './ExportStatus'; import { getServerStats, ServerStat } from './state/apis'; export const ServerStats = () => { @@ -98,6 +99,7 @@ export const ServerStats = () => { )} {config.featureToggles.dashboardPreviews && config.featureToggles.dashboardPreviewsAdmin && } + {config.featureToggles.export && } ); };