diff --git a/pkg/api/api.go b/pkg/api/api.go index 4364d24ce3a..5fbb25a1b04 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -567,6 +567,7 @@ func (hs *HTTPServer) registerRoutes() { adminRoute.Get("/export", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleGetStatus)) adminRoute.Post("/export", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleRequestExport)) adminRoute.Post("/export/stop", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleRequestStop)) + adminRoute.Get("/export/options", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleGetOptions)) } adminRoute.Post("/encryption/rotate-data-keys", reqGrafanaAdmin, routing.Wrap(hs.AdminRotateDataEncryptionKeys)) diff --git a/pkg/services/export/commit_helper.go b/pkg/services/export/commit_helper.go index ae5d05edfe2..5848db9dd08 100644 --- a/pkg/services/export/commit_helper.go +++ b/pkg/services/export/commit_helper.go @@ -26,6 +26,7 @@ type commitHelper struct { users map[int64]*userInfo stopRequested bool broadcast func(path string) + exporter string // key for the current exporter } type commitBody struct { diff --git a/pkg/services/export/export_plugins.go b/pkg/services/export/export_plugins.go new file mode 100644 index 00000000000..47a03910269 --- /dev/null +++ b/pkg/services/export/export_plugins.go @@ -0,0 +1,53 @@ +package export + +import ( + "encoding/json" + "fmt" + "path" + "strings" + "time" + + "github.com/grafana/grafana/pkg/services/sqlstore" +) + +func exportPlugins(helper *commitHelper, job *gitExportJob) error { + return job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error { + type pResult struct { + PluginID string `xorm:"plugin_id" json:"-"` + Enabled string `xorm:"enabled" json:"enabled"` + Pinned string `xorm:"pinned" json:"pinned"` + JSONData json.RawMessage `xorm:"json_data" json:"json_data,omitempty"` + // TODO: secure!!!! + PluginVersion string `xorm:"plugin_version" json:"version"` + Created time.Time `xorm:"created" json:"created"` + Updated time.Time `xorm:"updated" json:"updated"` + } + + rows := make([]*pResult, 0) + + sess.Table("plugin_setting").Where("org_id = ?", helper.orgID) + + err := sess.Find(&rows) + if err != nil { + if strings.HasPrefix(err.Error(), "no such table") { + return nil + } + return err + } + + for _, row := range rows { + err = helper.add(commitOptions{ + body: []commitBody{{ + body: prettyJSON(row), + fpath: path.Join(helper.orgDir, "plugins", row.PluginID, "settings.json"), + }}, + comment: fmt.Sprintf("Plugin: %s", row.PluginID), + when: row.Updated, + }) + if err != nil { + return err + } + } + return err + }) +} diff --git a/pkg/services/export/git_export_job.go b/pkg/services/export/git_export_job.go index 7a4f508fb8d..4b13b5e44c8 100644 --- a/pkg/services/export/git_export_job.go +++ b/pkg/services/export/git_export_job.go @@ -32,8 +32,6 @@ type gitExportJob struct { helper *commitHelper } -type simpleExporter = func(helper *commitHelper, job *gitExportJob) error - func startGitExportJob(cfg ExportConfig, sql *sqlstore.SQLStore, dashboardsnapshotsService dashboardsnapshots.Service, rootDir string, orgID int64, broadcaster statusBroadcaster) (Job, error) { job := &gitExportJob{ logger: log.New("git_export_job"), @@ -152,7 +150,7 @@ func (e *gitExportJob) doExportWithHistory() error { return err } - err = e.doOrgExportWithHistory(e.helper) + err := e.process(exporters) if err != nil { return err } @@ -169,56 +167,30 @@ func (e *gitExportJob) doExportWithHistory() error { return err } -func (e *gitExportJob) doOrgExportWithHistory(helper *commitHelper) error { - include := e.cfg.Include - - exporters := []simpleExporter{} - if include.Dash { - exporters = append(exporters, exportDashboards) - - if include.DashThumbs { - exporters = append(exporters, exportDashboardThumbnails) +func (e *gitExportJob) process(exporters []Exporter) error { + if false { // NEEDS a real user ID first + err := exportSnapshots(e.helper, e) + if err != nil { + return err } } - if include.Alerts { - exporters = append(exporters, exportAlerts) - } + for _, exp := range exporters { + if e.cfg.Exclude[exp.Key] { + continue + } - if include.DS { - exporters = append(exporters, exportDataSources) - } + if exp.process != nil { + e.status.Target = exp.Key + e.helper.exporter = exp.Key + err := exp.process(e.helper, e) + if err != nil { + return err + } + } - if include.Auth { - exporters = append(exporters, dumpAuthTables) - } - - if include.Usage { - exporters = append(exporters, exportUsage) - } - - if include.Services { - exporters = append(exporters, exportFiles, - exportSystemPreferences, - exportSystemStars, - exportSystemPlaylists, - exportKVStore, - exportSystemShortURL, - exportLive) - } - - if include.Anno { - exporters = append(exporters, exportAnnotations) - } - - if include.Snapshots { - exporters = append(exporters, exportSnapshots) - } - - for _, fn := range exporters { - err := fn(helper, e) - if err != nil { - return err + if exp.Exporters != nil { + return e.process(exp.Exporters) } } return nil diff --git a/pkg/services/export/service.go b/pkg/services/export/service.go index d39f5fc4fff..e8f5e185c7e 100644 --- a/pkg/services/export/service.go +++ b/pkg/services/export/service.go @@ -23,6 +23,9 @@ type ExportService interface { // List folder contents HandleGetStatus(c *models.ReqContext) response.Response + // List Get Options + HandleGetOptions(c *models.ReqContext) response.Response + // Read raw file contents out of the store HandleRequestExport(c *models.ReqContext) response.Response @@ -30,6 +33,108 @@ type ExportService interface { HandleRequestStop(c *models.ReqContext) response.Response } +var exporters = []Exporter{ + { + Key: "auth", + Name: "Authentication", + Description: "Saves raw SQL tables", + process: dumpAuthTables, + }, + { + Key: "dash", + Name: "Dashboards", + Description: "Save dashboard JSON", + process: exportDashboards, + Exporters: []Exporter{ + { + Key: "dash_thumbs", + Name: "Dashboard thumbnails", + Description: "Save current dashboard preview images", + process: exportDashboardThumbnails, + }, + }, + }, + { + Key: "alerts", + Name: "Alerts", + Description: "Archive alert rules and configuration", + process: exportAlerts, + }, + { + Key: "ds", + Name: "Data sources", + Description: "Data source configurations", + process: exportDataSources, + }, + { + Key: "services", + Name: "Services", + Description: "Save service settings", + Exporters: []Exporter{ + { + Name: "Preferences", + Description: "User and team preferences", + process: exportSystemPreferences, + }, + { + Name: "Stars", + Description: "User stars", + process: exportSystemStars, + }, + { + Name: "Playlists", + Description: "Playlists", + process: exportSystemPlaylists, + }, + { + Name: "Key Value store", + Description: "Internal KV store", + process: exportKVStore, + }, + { + Name: "Short URLs", + Description: "saved links", + process: exportSystemShortURL, + }, + { + Name: "Grafana live", + Description: "archived messages", + process: exportLive, + }, + }, + }, + { + Key: "files", + Name: "Files", + Description: "Export internal file system", + process: exportFiles, + }, + { + Key: "anno", + Name: "Annotations", + Description: "Write an DataFrame for all annotations on a dashboard", + process: exportAnnotations, + }, + { + Key: "plugins", + Name: "Plugins", + Description: "Save settings for all configured plugins", + process: exportPlugins, + }, + { + Key: "usage", + Name: "Usage", + Description: "archive current usage stats", + process: exportUsage, + }, + // { + // Key: "snapshots", + // Name: "Snapshots", + // Description: "write snapshots", + // process: exportSnapshots, + // }, +} + type StandardExport struct { logger log.Logger glive *live.GrafanaLive @@ -59,6 +164,13 @@ func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, } } +func (ex *StandardExport) HandleGetOptions(c *models.ReqContext) response.Response { + info := map[string]interface{}{ + "exporters": exporters, + } + return response.JSON(http.StatusOK, info) +} + func (ex *StandardExport) HandleGetStatus(c *models.ReqContext) response.Response { ex.mutex.Lock() defer ex.mutex.Unlock() @@ -114,7 +226,12 @@ func (ex *StandardExport) HandleRequestExport(c *models.ReqContext) response.Res } ex.exportJob = job - return response.JSON(http.StatusOK, ex.exportJob.getStatus()) + + info := map[string]interface{}{ + "cfg": cfg, // parsed job we are running + "status": ex.exportJob.getStatus(), + } + return response.JSON(http.StatusOK, info) } func (ex *StandardExport) broadcastStatus(orgID int64, s ExportStatus) { diff --git a/pkg/services/export/stub.go b/pkg/services/export/stub.go index 47e9267bb62..a0ad9a106f8 100644 --- a/pkg/services/export/stub.go +++ b/pkg/services/export/stub.go @@ -15,6 +15,10 @@ func (ex *StubExport) HandleGetStatus(c *models.ReqContext) response.Response { return response.Error(http.StatusForbidden, "feature not enabled", nil) } +func (ex *StubExport) HandleGetOptions(c *models.ReqContext) response.Response { + return response.Error(http.StatusForbidden, "feature not enabled", nil) +} + func (ex *StubExport) HandleRequestExport(c *models.ReqContext) response.Response { return response.Error(http.StatusForbidden, "feature not enabled", nil) } diff --git a/pkg/services/export/types.go b/pkg/services/export/types.go index d6247a2d89c..5deade2fd69 100644 --- a/pkg/services/export/types.go +++ b/pkg/services/export/types.go @@ -19,17 +19,7 @@ type ExportConfig struct { GeneralFolderPath string `json:"generalFolderPath"` KeepHistory bool `json:"history"` - Include struct { - Auth bool `json:"auth"` - DS bool `json:"ds"` - Dash bool `json:"dash"` - DashThumbs bool `json:"dash_thumbs"` - Alerts bool `json:"alerts"` - Services bool `json:"services"` - Usage bool `json:"usage"` - Anno bool `json:"anno"` - Snapshots bool `json:"snapshots"` - } `json:"include"` + Exclude map[string]bool `json:"exclude"` // Depends on the format Git GitExportConfig `json:"git"` @@ -45,3 +35,12 @@ type Job interface { // Will broadcast the live status type statusBroadcaster func(s ExportStatus) + +type Exporter struct { + Key string `json:"key"` + Name string `json:"name"` + Description string `json:"description"` + Exporters []Exporter `json:"exporters,omitempty"` + + process func(helper *commitHelper, job *gitExportJob) error +} diff --git a/public/app/features/storage/ExportView.tsx b/public/app/features/storage/ExportView.tsx index 5b26869927f..beefbc99f6f 100644 --- a/public/app/features/storage/ExportView.tsx +++ b/public/app/features/storage/ExportView.tsx @@ -1,9 +1,22 @@ -import React, { useEffect, useState } from 'react'; -import { useLocalStorage } from 'react-use'; +import React, { useEffect, useState, useCallback } from 'react'; +import { useAsync, useLocalStorage } from 'react-use'; -import { isLiveChannelMessageEvent, isLiveChannelStatusEvent, LiveChannelScope } from '@grafana/data'; +import { isLiveChannelMessageEvent, isLiveChannelStatusEvent, LiveChannelScope, SelectableValue } from '@grafana/data'; import { getBackendSrv, getGrafanaLiveSrv } from '@grafana/runtime'; -import { Button, CodeEditor, HorizontalGroup, LinkButton } from '@grafana/ui'; +import { + Button, + CodeEditor, + Collapse, + Field, + HorizontalGroup, + InlineField, + InlineFieldRow, + InlineSwitch, + Input, + LinkButton, + Select, + Switch, +} from '@grafana/ui'; import { StorageView } from './types'; @@ -21,63 +34,93 @@ interface ExportStatusMessage { status: string; } -interface ExportInclude { - auth: boolean; - ds: boolean; - dash: boolean; - dash_thumbs: boolean; - alerts: boolean; - services: boolean; - usage: boolean; - anno: boolean; - snapshots: boolean; -} - interface ExportJob { - format: 'git'; + format: string; // 'git'; generalFolderPath: string; history: boolean; - include: ExportInclude; + exclude: Record; git?: {}; } -const includAll: ExportInclude = { - auth: true, - ds: true, - dash: true, - dash_thumbs: true, - alerts: true, - services: true, - usage: true, - anno: true, - snapshots: false, // will fail until we have a real user -}; - const defaultJob: ExportJob = { format: 'git', generalFolderPath: 'general', history: true, - include: includAll, + exclude: {}, git: {}, }; +interface ExporterInfo { + key: string; + name: string; + description: string; + children?: ExporterInfo[]; +} + +const formats: Array> = [ + { label: 'GIT', value: 'git', description: 'Exports a fresh git repository' }, +]; + interface Props { onPathChange: (p: string, v?: StorageView) => void; } +const labelWith = 18; + export const ExportView = ({ onPathChange }: Props) => { const [status, setStatus] = useState(); - const [rawBody, setBody] = useLocalStorage(EXPORT_LOCAL_STORAGE_KEY, defaultJob); - const body = { ...defaultJob, ...rawBody, include: { ...includAll, ...rawBody?.include } }; + const [body, setBody] = useLocalStorage(EXPORT_LOCAL_STORAGE_KEY, defaultJob); + const [details, setDetails] = useState(false); + + const serverOptions = useAsync(() => { + return getBackendSrv().get<{ exporters: ExporterInfo[] }>('/api/admin/export/options'); + }, []); const doStart = () => { - getBackendSrv().post('/api/admin/export', body); + getBackendSrv() + .post('/api/admin/export', body) + .then((v) => { + if (v.cfg && v.status.running) { + setBody(v.cfg); // saves the valid parsed body + } + }); }; + const doStop = () => { getBackendSrv().post('/api/admin/export/stop'); }; + const setInclude = useCallback( + (k: string, v: boolean) => { + if (!serverOptions.value || !body) { + return; + } + const exclude: Record = {}; + if (k === '*') { + if (!v) { + for (let exp of serverOptions.value.exporters) { + exclude[exp.key] = true; + } + } + setBody({ ...body, exclude }); + return; + } + + for (let exp of serverOptions.value.exporters) { + let val = body.exclude?.[exp.key]; + if (k === exp.key) { + val = !v; + } + if (val) { + exclude[exp.key] = val; + } + } + setBody({ ...body, exclude }); + }, + [body, setBody, serverOptions] + ); + useEffect(() => { const subscription = getGrafanaLiveSrv() .getStream({ @@ -120,18 +163,53 @@ export const ExportView = ({ onPathChange }: Props) => { {!Boolean(status?.running) && (

Export grafana instance

- { - setBody(JSON.parse(text)); // force JSON? - }} - /> -
+ + setBody({ ...body!, generalFolderPath: v.currentTarget.value })} + placeholder="root folder path" + /> +
)} +
+
+ + + { + setBody(JSON.parse(text)); // force JSON? + }} + /> + ); };