Export: Export plugin settings (#52129)

pkg/api/api.go

@@ -567,6 +567,7 @@ func (hs *HTTPServer) registerRoutes() {
 			adminRoute.Get("/export", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleGetStatus))
 			adminRoute.Post("/export", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleRequestExport))
 			adminRoute.Post("/export/stop", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleRequestStop))
+			adminRoute.Get("/export/options", reqGrafanaAdmin, routing.Wrap(hs.ExportService.HandleGetOptions))
 		}
 
 		adminRoute.Post("/encryption/rotate-data-keys", reqGrafanaAdmin, routing.Wrap(hs.AdminRotateDataEncryptionKeys))

pkg/services/export/commit_helper.go

@@ -26,6 +26,7 @@ type commitHelper struct {
 	users         map[int64]*userInfo
 	stopRequested bool
 	broadcast     func(path string)
+	exporter      string // key for the current exporter
 }
 
 type commitBody struct {

pkg/services/export/export_plugins.go

@@ -0,0 +1,53 @@
+package export
+
+import (
+	"encoding/json"
+	"fmt"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+)
+
+func exportPlugins(helper *commitHelper, job *gitExportJob) error {
+	return job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error {
+		type pResult struct {
+			PluginID string          `xorm:"plugin_id" json:"-"`
+			Enabled  string          `xorm:"enabled" json:"enabled"`
+			Pinned   string          `xorm:"pinned" json:"pinned"`
+			JSONData json.RawMessage `xorm:"json_data" json:"json_data,omitempty"`
+			// TODO: secure!!!!
+			PluginVersion string    `xorm:"plugin_version" json:"version"`
+			Created       time.Time `xorm:"created" json:"created"`
+			Updated       time.Time `xorm:"updated" json:"updated"`
+		}
+
+		rows := make([]*pResult, 0)
+
+		sess.Table("plugin_setting").Where("org_id = ?", helper.orgID)
+
+		err := sess.Find(&rows)
+		if err != nil {
+			if strings.HasPrefix(err.Error(), "no such table") {
+				return nil
+			}
+			return err
+		}
+
+		for _, row := range rows {
+			err = helper.add(commitOptions{
+				body: []commitBody{{
+					body:  prettyJSON(row),
+					fpath: path.Join(helper.orgDir, "plugins", row.PluginID, "settings.json"),
+				}},
+				comment: fmt.Sprintf("Plugin: %s", row.PluginID),
+				when:    row.Updated,
+			})
+			if err != nil {
+				return err
+			}
+		}
+		return err
+	})
+}

pkg/services/export/git_export_job.go

@@ -32,8 +32,6 @@ type gitExportJob struct {
 	helper      *commitHelper
 }
 
-type simpleExporter = func(helper *commitHelper, job *gitExportJob) error
-
 func startGitExportJob(cfg ExportConfig, sql *sqlstore.SQLStore, dashboardsnapshotsService dashboardsnapshots.Service, rootDir string, orgID int64, broadcaster statusBroadcaster) (Job, error) {
 	job := &gitExportJob{
 		logger:                    log.New("git_export_job"),
@@ -152,7 +150,7 @@ func (e *gitExportJob) doExportWithHistory() error {
 			return err
 		}
 
-		err = e.doOrgExportWithHistory(e.helper)
+		err := e.process(exporters)
 		if err != nil {
 			return err
 		}
@@ -169,56 +167,30 @@ func (e *gitExportJob) doExportWithHistory() error {
 	return err
 }
 
-func (e *gitExportJob) doOrgExportWithHistory(helper *commitHelper) error {
+func (e *gitExportJob) process(exporters []Exporter) error {
-	include := e.cfg.Include
+	if false { // NEEDS a real user ID first
-
+		err := exportSnapshots(e.helper, e)
-	exporters := []simpleExporter{}
+		if err != nil {
-	if include.Dash {
+			return err
-		exporters = append(exporters, exportDashboards)
-
-		if include.DashThumbs {
-			exporters = append(exporters, exportDashboardThumbnails)
 		}
 	}
 
-	if include.Alerts {
+	for _, exp := range exporters {
-		exporters = append(exporters, exportAlerts)
+		if e.cfg.Exclude[exp.Key] {
-	}
+			continue
+		}
 
-	if include.DS {
+		if exp.process != nil {
-		exporters = append(exporters, exportDataSources)
+			e.status.Target = exp.Key
-	}
+			e.helper.exporter = exp.Key
+			err := exp.process(e.helper, e)
+			if err != nil {
+				return err
+			}
+		}
 
-	if include.Auth {
+		if exp.Exporters != nil {
-		exporters = append(exporters, dumpAuthTables)
+			return e.process(exp.Exporters)
-	}
-
-	if include.Usage {
-		exporters = append(exporters, exportUsage)
-	}
-
-	if include.Services {
-		exporters = append(exporters, exportFiles,
-			exportSystemPreferences,
-			exportSystemStars,
-			exportSystemPlaylists,
-			exportKVStore,
-			exportSystemShortURL,
-			exportLive)
-	}
-
-	if include.Anno {
-		exporters = append(exporters, exportAnnotations)
-	}
-
-	if include.Snapshots {
-		exporters = append(exporters, exportSnapshots)
-	}
-
-	for _, fn := range exporters {
-		err := fn(helper, e)
-		if err != nil {
-			return err
 		}
 	}
 	return nil

pkg/services/export/service.go

@@ -23,6 +23,9 @@ type ExportService interface {
 	// List folder contents
 	HandleGetStatus(c *models.ReqContext) response.Response
 
+	// List Get Options
+	HandleGetOptions(c *models.ReqContext) response.Response
+
 	// Read raw file contents out of the store
 	HandleRequestExport(c *models.ReqContext) response.Response
 
@@ -30,6 +33,108 @@ type ExportService interface {
 	HandleRequestStop(c *models.ReqContext) response.Response
 }
 
+var exporters = []Exporter{
+	{
+		Key:         "auth",
+		Name:        "Authentication",
+		Description: "Saves raw SQL tables",
+		process:     dumpAuthTables,
+	},
+	{
+		Key:         "dash",
+		Name:        "Dashboards",
+		Description: "Save dashboard JSON",
+		process:     exportDashboards,
+		Exporters: []Exporter{
+			{
+				Key:         "dash_thumbs",
+				Name:        "Dashboard thumbnails",
+				Description: "Save current dashboard preview images",
+				process:     exportDashboardThumbnails,
+			},
+		},
+	},
+	{
+		Key:         "alerts",
+		Name:        "Alerts",
+		Description: "Archive alert rules and configuration",
+		process:     exportAlerts,
+	},
+	{
+		Key:         "ds",
+		Name:        "Data sources",
+		Description: "Data source configurations",
+		process:     exportDataSources,
+	},
+	{
+		Key:         "services",
+		Name:        "Services",
+		Description: "Save service settings",
+		Exporters: []Exporter{
+			{
+				Name:        "Preferences",
+				Description: "User and team preferences",
+				process:     exportSystemPreferences,
+			},
+			{
+				Name:        "Stars",
+				Description: "User stars",
+				process:     exportSystemStars,
+			},
+			{
+				Name:        "Playlists",
+				Description: "Playlists",
+				process:     exportSystemPlaylists,
+			},
+			{
+				Name:        "Key Value store",
+				Description: "Internal KV store",
+				process:     exportKVStore,
+			},
+			{
+				Name:        "Short URLs",
+				Description: "saved links",
+				process:     exportSystemShortURL,
+			},
+			{
+				Name:        "Grafana live",
+				Description: "archived messages",
+				process:     exportLive,
+			},
+		},
+	},
+	{
+		Key:         "files",
+		Name:        "Files",
+		Description: "Export internal file system",
+		process:     exportFiles,
+	},
+	{
+		Key:         "anno",
+		Name:        "Annotations",
+		Description: "Write an DataFrame for all annotations on a dashboard",
+		process:     exportAnnotations,
+	},
+	{
+		Key:         "plugins",
+		Name:        "Plugins",
+		Description: "Save settings for all configured plugins",
+		process:     exportPlugins,
+	},
+	{
+		Key:         "usage",
+		Name:        "Usage",
+		Description: "archive current usage stats",
+		process:     exportUsage,
+	},
+	// {
+	// 	Key:         "snapshots",
+	// 	Name:        "Snapshots",
+	// 	Description: "write snapshots",
+	// 	process:     exportSnapshots,
+	// },
+}
+
 type StandardExport struct {
 	logger  log.Logger
 	glive   *live.GrafanaLive
@@ -59,6 +164,13 @@ func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles,
 	}
 }
 
+func (ex *StandardExport) HandleGetOptions(c *models.ReqContext) response.Response {
+	info := map[string]interface{}{
+		"exporters": exporters,
+	}
+	return response.JSON(http.StatusOK, info)
+}
+
 func (ex *StandardExport) HandleGetStatus(c *models.ReqContext) response.Response {
 	ex.mutex.Lock()
 	defer ex.mutex.Unlock()
@@ -114,7 +226,12 @@ func (ex *StandardExport) HandleRequestExport(c *models.ReqContext) response.Res
 	}
 
 	ex.exportJob = job
-	return response.JSON(http.StatusOK, ex.exportJob.getStatus())
+
+	info := map[string]interface{}{
+		"cfg":    cfg, // parsed job we are running
+		"status": ex.exportJob.getStatus(),
+	}
+	return response.JSON(http.StatusOK, info)
 }
 
 func (ex *StandardExport) broadcastStatus(orgID int64, s ExportStatus) {

pkg/services/export/stub.go

@@ -15,6 +15,10 @@ func (ex *StubExport) HandleGetStatus(c *models.ReqContext) response.Response {
 	return response.Error(http.StatusForbidden, "feature not enabled", nil)
 }
 
+func (ex *StubExport) HandleGetOptions(c *models.ReqContext) response.Response {
+	return response.Error(http.StatusForbidden, "feature not enabled", nil)
+}
+
 func (ex *StubExport) HandleRequestExport(c *models.ReqContext) response.Response {
 	return response.Error(http.StatusForbidden, "feature not enabled", nil)
 }

pkg/services/export/types.go

@@ -19,17 +19,7 @@ type ExportConfig struct {
 	GeneralFolderPath string `json:"generalFolderPath"`
 	KeepHistory       bool   `json:"history"`
 
-	Include struct {
+	Exclude map[string]bool `json:"exclude"`
-		Auth       bool `json:"auth"`
-		DS         bool `json:"ds"`
-		Dash       bool `json:"dash"`
-		DashThumbs bool `json:"dash_thumbs"`
-		Alerts     bool `json:"alerts"`
-		Services   bool `json:"services"`
-		Usage      bool `json:"usage"`
-		Anno       bool `json:"anno"`
-		Snapshots  bool `json:"snapshots"`
-	} `json:"include"`
 
 	// Depends on the format
 	Git GitExportConfig `json:"git"`
@@ -45,3 +35,12 @@ type Job interface {
 
 // Will broadcast the live status
 type statusBroadcaster func(s ExportStatus)
+
+type Exporter struct {
+	Key         string     `json:"key"`
+	Name        string     `json:"name"`
+	Description string     `json:"description"`
+	Exporters   []Exporter `json:"exporters,omitempty"`
+
+	process func(helper *commitHelper, job *gitExportJob) error
+}

public/app/features/storage/ExportView.tsx

@@ -1,9 +1,22 @@
-import React, { useEffect, useState } from 'react';
+import React, { useEffect, useState, useCallback } from 'react';
-import { useLocalStorage } from 'react-use';
+import { useAsync, useLocalStorage } from 'react-use';
 
-import { isLiveChannelMessageEvent, isLiveChannelStatusEvent, LiveChannelScope } from '@grafana/data';
+import { isLiveChannelMessageEvent, isLiveChannelStatusEvent, LiveChannelScope, SelectableValue } from '@grafana/data';
 import { getBackendSrv, getGrafanaLiveSrv } from '@grafana/runtime';
-import { Button, CodeEditor, HorizontalGroup, LinkButton } from '@grafana/ui';
+import {
+  Button,
+  CodeEditor,
+  Collapse,
+  Field,
+  HorizontalGroup,
+  InlineField,
+  InlineFieldRow,
+  InlineSwitch,
+  Input,
+  LinkButton,
+  Select,
+  Switch,
+} from '@grafana/ui';
 
 import { StorageView } from './types';
 
@@ -21,63 +34,93 @@ interface ExportStatusMessage {
   status: string;
 }
 
-interface ExportInclude {
-  auth: boolean;
-  ds: boolean;
-  dash: boolean;
-  dash_thumbs: boolean;
-  alerts: boolean;
-  services: boolean;
-  usage: boolean;
-  anno: boolean;
-  snapshots: boolean;
-}
-
 interface ExportJob {
-  format: 'git';
+  format: string; // 'git';
   generalFolderPath: string;
   history: boolean;
-  include: ExportInclude;
+  exclude: Record<string, boolean>;
 
   git?: {};
 }
 
-const includAll: ExportInclude = {
-  auth: true,
-  ds: true,
-  dash: true,
-  dash_thumbs: true,
-  alerts: true,
-  services: true,
-  usage: true,
-  anno: true,
-  snapshots: false, // will fail until we have a real user
-};
-
 const defaultJob: ExportJob = {
   format: 'git',
   generalFolderPath: 'general',
   history: true,
-  include: includAll,
+  exclude: {},
   git: {},
 };
 
+interface ExporterInfo {
+  key: string;
+  name: string;
+  description: string;
+  children?: ExporterInfo[];
+}
+
+const formats: Array<SelectableValue<string>> = [
+  { label: 'GIT', value: 'git', description: 'Exports a fresh git repository' },
+];
+
 interface Props {
   onPathChange: (p: string, v?: StorageView) => void;
 }
 
+const labelWith = 18;
+
 export const ExportView = ({ onPathChange }: Props) => {
   const [status, setStatus] = useState<ExportStatusMessage>();
-  const [rawBody, setBody] = useLocalStorage<ExportJob>(EXPORT_LOCAL_STORAGE_KEY, defaultJob);
+  const [body, setBody] = useLocalStorage<ExportJob>(EXPORT_LOCAL_STORAGE_KEY, defaultJob);
-  const body = { ...defaultJob, ...rawBody, include: { ...includAll, ...rawBody?.include } };
+  const [details, setDetails] = useState(false);
+
+  const serverOptions = useAsync(() => {
+    return getBackendSrv().get<{ exporters: ExporterInfo[] }>('/api/admin/export/options');
+  }, []);
 
   const doStart = () => {
-    getBackendSrv().post('/api/admin/export', body);
+    getBackendSrv()
+      .post('/api/admin/export', body)
+      .then((v) => {
+        if (v.cfg && v.status.running) {
+          setBody(v.cfg); // saves the valid parsed body
+        }
+      });
   };
+
   const doStop = () => {
     getBackendSrv().post('/api/admin/export/stop');
   };
 
+  const setInclude = useCallback(
+    (k: string, v: boolean) => {
+      if (!serverOptions.value || !body) {
+        return;
+      }
+      const exclude: Record<string, boolean> = {};
+      if (k === '*') {
+        if (!v) {
+          for (let exp of serverOptions.value.exporters) {
+            exclude[exp.key] = true;
+          }
+        }
+        setBody({ ...body, exclude });
+        return;
+      }
+
+      for (let exp of serverOptions.value.exporters) {
+        let val = body.exclude?.[exp.key];
+        if (k === exp.key) {
+          val = !v;
+        }
+        if (val) {
+          exclude[exp.key] = val;
+        }
+      }
+      setBody({ ...body, exclude });
+    },
+    [body, setBody, serverOptions]
+  );
+
   useEffect(() => {
     const subscription = getGrafanaLiveSrv()
       .getStream<ExportStatusMessage>({
@@ -120,18 +163,53 @@ export const ExportView = ({ onPathChange }: Props) => {
       {!Boolean(status?.running) && (
         <div>
           <h3>Export grafana instance</h3>
-          <CodeEditor
+          <Field label="Format">
-            height={275}
+            <Select
-            value={JSON.stringify(body, null, 2) ?? ''}
+              options={formats}
-            showLineNumbers={false}
+              width={40}
-            readOnly={false}
+              value={formats.find((v) => v.value === body?.format)}
-            language="json"
+              onChange={(v) => setBody({ ...body!, format: v.value! })}
-            showMiniMap={false}
+            />
-            onBlur={(text: string) => {
+          </Field>
-              setBody(JSON.parse(text)); // force JSON?
+          <Field label="Keep history">
-            }}
+            <Switch value={body?.history} onChange={(v) => setBody({ ...body!, history: v.currentTarget.checked })} />
-          />
+          </Field>
-          <br />
+
+          <Field label="Include">
+            <>
+              <InlineFieldRow>
+                <InlineField label="Toggle all" labelWidth={labelWith}>
+                  <InlineSwitch
+                    value={Object.keys(body?.exclude ?? {}).length === 0}
+                    onChange={(v) => setInclude('*', v.currentTarget.checked)}
+                  />
+                </InlineField>
+              </InlineFieldRow>
+              {serverOptions.value && (
+                <div>
+                  {serverOptions.value.exporters.map((ex) => (
+                    <InlineFieldRow key={ex.key}>
+                      <InlineField label={ex.name} labelWidth={labelWith} tooltip={ex.description}>
+                        <InlineSwitch
+                          value={body?.exclude?.[ex.key] !== true}
+                          onChange={(v) => setInclude(ex.key, v.currentTarget.checked)}
+                        />
+                      </InlineField>
+                    </InlineFieldRow>
+                  ))}
+                </div>
+              )}
+            </>
+          </Field>
+
+          <Field label="General folder" description="Set the folder name for items without a real folder">
+            <Input
+              width={40}
+              value={body?.generalFolderPath ?? ''}
+              onChange={(v) => setBody({ ...body!, generalFolderPath: v.currentTarget.value })}
+              placeholder="root folder path"
+            />
+          </Field>
 
           <HorizontalGroup>
             <Button onClick={doStart} variant="primary">
@@ -143,6 +221,22 @@ export const ExportView = ({ onPathChange }: Props) => {
           </HorizontalGroup>
         </div>
       )}
+      <br />
+      <br />
+
+      <Collapse label="Request details" isOpen={details} onToggle={setDetails} collapsible={true}>
+        <CodeEditor
+          height={275}
+          value={JSON.stringify(body, null, 2) ?? ''}
+          showLineNumbers={false}
+          readOnly={false}
+          language="json"
+          showMiniMap={false}
+          onBlur={(text: string) => {
+            setBody(JSON.parse(text)); // force JSON?
+          }}
+        />
+      </Collapse>
     </div>
   );
 };