Dashboards: load from storage (#51949)

.betterer.results

@@ -5843,12 +5843,15 @@ exports[`better eslint`] = {
       [0, 0, 0, "Do not use any type assertions.", "0"],
       [0, 0, 0, "Do not use any type assertions.", "1"]
     ],
-    "public/app/features/storage/helper.ts:5381": [
+    "public/app/features/storage/storage.ts:5381": [
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"],
       [0, 0, 0, "Unexpected any. Specify a different type.", "1"],
-      [0, 0, 0, "Do not use any type assertions.", "2"],
+      [0, 0, 0, "Unexpected any. Specify a different type.", "2"],
       [0, 0, 0, "Do not use any type assertions.", "3"],
-      [0, 0, 0, "Unexpected any. Specify a different type.", "4"]
+      [0, 0, 0, "Do not use any type assertions.", "4"],
+      [0, 0, 0, "Unexpected any. Specify a different type.", "5"],
+      [0, 0, 0, "Unexpected any. Specify a different type.", "6"],
+      [0, 0, 0, "Do not use any type assertions.", "7"]
     ],
     "public/app/features/teams/CreateTeam.test.tsx:5381": [
       [0, 0, 0, "Unexpected any. Specify a different type.", "0"]

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -41,6 +41,7 @@ export interface FeatureToggles {
   annotationComments?: boolean;
   migrationLocking?: boolean;
   storage?: boolean;
+  dashboardsFromStorage?: boolean;
   export?: boolean;
   storageLocalUpload?: boolean;
   azureMonitorResourcePickerForMetrics?: boolean;

pkg/api/api.go

@@ -108,6 +108,10 @@ func (hs *HTTPServer) registerRoutes() {
 	r.Get("/dashboards/*", reqSignedIn, hs.Index)
 	r.Get("/goto/:uid", reqSignedIn, hs.redirectFromShortURL, hs.Index)
 
+	if hs.Features.IsEnabled(featuremgmt.FlagDashboardsFromStorage) {
+		r.Get("/g/*", reqSignedIn, hs.Index)
+	}
+
 	if hs.Features.IsEnabled(featuremgmt.FlagPublicDashboards) {
 		r.Get("/public-dashboards/:accessToken", publicdashboardsapi.SetPublicDashboardFlag(), hs.Index)
 	}

pkg/services/featuremgmt/registry.go

@@ -141,6 +141,11 @@ var (
 			Description: "Configurable storage for dashboards, datasources, and resources",
 			State:       FeatureStateAlpha,
 		},
+		{
+			Name:        "dashboardsFromStorage",
+			Description: "Load dashboards from the generic storage interface",
+			State:       FeatureStateAlpha,
+		},
 		{
 			Name:            "export",
 			Description:     "Export grafana instance (to git, etc)",

pkg/services/featuremgmt/toggles_gen.go

@@ -107,6 +107,10 @@ const (
 	// Configurable storage for dashboards, datasources, and resources
 	FlagStorage = "storage"
 
+	// FlagDashboardsFromStorage
+	// Load dashboards from the generic storage interface
+	FlagDashboardsFromStorage = "dashboardsFromStorage"
+
 	// FlagExport
 	// Export grafana instance (to git, etc)
 	FlagExport = "export"

public/app/core/services/backend_srv.ts

@@ -14,10 +14,11 @@ import { catchError, filter, map, mergeMap, retryWhen, share, takeUntil, tap, th
 import { v4 as uuidv4 } from 'uuid';
 
 import { AppEvents, DataQueryErrorType } from '@grafana/data';
-import { BackendSrv as BackendService, BackendSrvRequest, FetchError, FetchResponse } from '@grafana/runtime';
+import { BackendSrv as BackendService, BackendSrvRequest, config, FetchError, FetchResponse } from '@grafana/runtime';
 import appEvents from 'app/core/app_events';
 import { getConfig } from 'app/core/config';
 import { DashboardSearchHit } from 'app/features/search/types';
+import { getGrafanaStorage } from 'app/features/storage/storage';
 import { TokenRevokedModal } from 'app/features/users/TokenRevokedModal';
 import { DashboardDTO, FolderDTO } from 'app/types';
 
@@ -430,7 +431,10 @@ export class BackendSrv implements BackendService {
     return this.get('/api/search', query);
   }
 
-  getDashboardByUid(uid: string) {
+  getDashboardByUid(uid: string): Promise<DashboardDTO> {
+    if (uid.indexOf('/') > 0 && config.featureToggles.dashboardsFromStorage) {
+      return getGrafanaStorage().getDashboard(uid);
+    }
     return this.get<DashboardDTO>(`/api/dashboards/uid/${uid}`);
   }
 

public/app/features/dashboard/components/SaveDashboard/SaveDashboardDrawer.tsx

@@ -1,7 +1,7 @@
 import React, { useMemo, useState } from 'react';
 import { useAsync } from 'react-use';
 
-import { Drawer, Tab, TabsBar } from '@grafana/ui';
+import { Drawer, Spinner, Tab, TabsBar } from '@grafana/ui';
 import { backendSrv } from 'app/core/services/backend_srv';
 
 import { jsonDiff } from '../VersionHistory/utils';
@@ -70,6 +70,14 @@ export const SaveDashboardDrawer = ({ dashboard, onDismiss, onSaveSuccess, isCop
       return <SaveDashboardDiff diff={data.diff} oldValue={previous.value} newValue={data.clone} />;
     }
 
+    if (state.loading) {
+      return (
+        <div>
+          <Spinner />
+        </div>
+      );
+    }
+
     if (isNew || isCopy) {
       return (
         <SaveDashboardAsForm

public/app/features/dashboard/components/SaveDashboard/types.ts

@@ -1,4 +1,5 @@
 import { CloneOptions, DashboardModel } from 'app/features/dashboard/state/DashboardModel';
+import { DashboardDataDTO } from 'app/types';
 
 import { Diffs } from '../VersionHistory/utils';
 
@@ -16,6 +17,13 @@ export interface SaveDashboardOptions extends CloneOptions {
   makeEditable?: boolean;
 }
 
+export interface SaveDashboardCommand {
+  dashboard: DashboardDataDTO;
+  message?: string;
+  folderId?: number;
+  overwrite?: boolean;
+}
+
 export interface SaveDashboardFormProps {
   dashboard: DashboardModel;
   onCancel: () => void;

public/app/features/dashboard/components/SaveDashboard/useDashboardSave.tsx

@@ -36,6 +36,9 @@ export const useDashboardSave = (dashboard: DashboardModel) => {
 
   const notifyApp = useAppNotification();
   useEffect(() => {
+    if (state.error) {
+      notifyApp.error(state.error.message ?? 'Error saving dashboard');
+    }
     if (state.value) {
       dashboard.version = state.value.version;
       dashboard.clearUnsavedChanges();

public/app/features/dashboard/services/DashboardLoaderSrv.ts

@@ -8,6 +8,8 @@ import { backendSrv } from 'app/core/services/backend_srv';
 import impressionSrv from 'app/core/services/impression_srv';
 import kbn from 'app/core/utils/kbn';
 import { getDatasourceSrv } from 'app/features/plugins/datasource_srv';
+import { getGrafanaStorage } from 'app/features/storage/storage';
+import { DashboardRoutes } from 'app/types';
 
 import { appEvents } from '../../../core/core';
 
@@ -39,6 +41,8 @@ export class DashboardLoaderSrv {
       promise = backendSrv.get('/api/snapshots/' + slug).catch(() => {
         return this._dashboardLoadFailed('Snapshot not found', true);
       });
+    } else if (type === DashboardRoutes.Path) {
+      promise = getGrafanaStorage().getDashboard(slug!);
     } else if (type === 'ds') {
       promise = this._loadFromDatasource(slug); // explore dashboards as code
     } else if (type === 'public') {

public/app/features/dashboard/state/initDashboard.ts

@@ -87,6 +87,10 @@ async function fetchDashboard(
       case DashboardRoutes.New: {
         return getNewDashboardModelData(args.urlFolderId, args.panelType);
       }
+      case DashboardRoutes.Path: {
+        const path = args.urlSlug ?? '';
+        return await dashboardLoaderSrv.loadDashboard(DashboardRoutes.Path, path, path);
+      }
       default:
         throw { message: 'Unknown route ' + args.routeName };
     }

public/app/features/manage-dashboards/state/actions.ts

@@ -2,8 +2,10 @@ import { DataSourceInstanceSettings, locationUtil } from '@grafana/data';
 import { getDataSourceSrv, locationService, getBackendSrv, isFetchError } from '@grafana/runtime';
 import { notifyApp } from 'app/core/actions';
 import { createErrorNotification } from 'app/core/copy/appNotification';
+import { SaveDashboardCommand } from 'app/features/dashboard/components/SaveDashboard/types';
 import { dashboardWatcher } from 'app/features/live/dashboard/dashboardWatcher';
-import { DashboardDataDTO, DashboardDTO, FolderInfo, PermissionLevelString, ThunkResult } from 'app/types';
+import { getGrafanaStorage } from 'app/features/storage/storage';
+import { DashboardDTO, FolderInfo, PermissionLevelString, ThunkResult } from 'app/types';
 
 import { LibraryElementExport } from '../../dashboard/components/DashExportModal/DashboardExporter';
 import { getLibraryPanel } from '../../library-panels/state/api';
@@ -262,16 +264,13 @@ export function deleteFoldersAndDashboards(folderUids: string[], dashboardUids:
   return executeInOrder(tasks);
 }
 
-export interface SaveDashboardOptions {
+export function saveDashboard(options: SaveDashboardCommand) {
-  dashboard: DashboardDataDTO;
-  message?: string;
-  folderId?: number;
-  overwrite?: boolean;
-}
-
-export function saveDashboard(options: SaveDashboardOptions) {
   dashboardWatcher.ignoreNextSave();
 
+  if (options.dashboard.uid.indexOf('/') > 0) {
+    return getGrafanaStorage().saveDashboard(options);
+  }
+
   return getBackendSrv().post('/api/dashboards/db/', {
     dashboard: options.dashboard,
     message: options.message ?? '',

public/app/features/storage/FileView.tsx

@@ -8,7 +8,7 @@ import AutoSizer from 'react-virtualized-auto-sizer';
 import { DataFrame, GrafanaTheme2 } from '@grafana/data';
 import { CodeEditor, useStyles2 } from '@grafana/ui';
 
-import { getGrafanaStorage } from './helper';
+import { getGrafanaStorage } from './storage';
 import { StorageView } from './types';
 
 interface FileDisplayInfo {

public/app/features/storage/StorageFolderPage.tsx

@@ -0,0 +1,79 @@
+import { css } from '@emotion/css';
+import React, { FC } from 'react';
+import { useAsync } from 'react-use';
+
+import { DataFrame, GrafanaTheme2 } from '@grafana/data';
+import { Card, Icon, Spinner, useStyles2 } from '@grafana/ui';
+import { GrafanaRouteComponentProps } from 'app/core/navigation/types';
+
+import { getGrafanaStorage } from './storage';
+
+export interface Props extends GrafanaRouteComponentProps<{ slug: string }> {}
+
+export const StorageFolderPage: FC<Props> = (props) => {
+  const slug = props.match.params.slug;
+
+  const styles = useStyles2(getStyles);
+  const listing = useAsync((): Promise<DataFrame | undefined> => {
+    return getGrafanaStorage().list(slug);
+  }, [slug]);
+
+  let base = document.location.pathname;
+  if (!base.endsWith('/')) {
+    base += '/';
+  }
+  let parent = '';
+  const idx = base.lastIndexOf('/', base.length - 2);
+  if (idx > 0) {
+    parent = base.substring(0, idx);
+  }
+
+  const renderListing = () => {
+    if (listing.value) {
+      const names = listing.value.fields[0].values.toArray();
+      return names.map((item: string) => {
+        let name = item;
+        const isFolder = name.indexOf('.') < 0;
+        const isDash = !isFolder && name.endsWith('.json');
+        return (
+          <Card key={name} href={isFolder || isDash ? base + name : undefined}>
+            <Card.Heading>{name}</Card.Heading>
+            <Card.Figure>
+              <Icon name={isFolder ? 'folder' : isDash ? 'gf-grid' : 'file-alt'} size="sm" />
+            </Card.Figure>
+          </Card>
+        );
+      });
+    }
+    if (listing.loading) {
+      return <Spinner />;
+    }
+    return <div>?</div>;
+  };
+
+  return (
+    <div className={styles.wrapper}>
+      {slug?.length > 0 && (
+        <>
+          <h1>{slug}</h1>
+          <Card href={parent}>
+            <Card.Heading>{parent}</Card.Heading>
+            <Card.Figure>
+              <Icon name="arrow-left" size="sm" />
+            </Card.Figure>
+          </Card>
+          <br />
+        </>
+      )}
+      {renderListing()}
+    </div>
+  );
+};
+
+const getStyles = (theme: GrafanaTheme2) => ({
+  wrapper: css`
+    margin: 50px;
+  `,
+});
+
+export default StorageFolderPage;

public/app/features/storage/StoragePage.tsx

@@ -3,8 +3,8 @@ import React, { useMemo, useState } from 'react';
 import { useAsync } from 'react-use';
 
 import { DataFrame, GrafanaTheme2, isDataFrame, ValueLinkConfig } from '@grafana/data';
-import { locationService } from '@grafana/runtime';
+import { config, locationService } from '@grafana/runtime';
-import { useStyles2, IconName, Spinner, TabsBar, Tab, Button, HorizontalGroup } from '@grafana/ui';
+import { useStyles2, IconName, Spinner, TabsBar, Tab, Button, HorizontalGroup, LinkButton } from '@grafana/ui';
 import appEvents from 'app/core/app_events';
 import { Page } from 'app/core/components/Page/Page';
 import { useNavModel } from 'app/core/hooks/useNavModel';
@@ -18,7 +18,7 @@ import { ExportView } from './ExportView';
 import { FileView } from './FileView';
 import { FolderView } from './FolderView';
 import { RootView } from './RootView';
-import { getGrafanaStorage, filenameAlreadyExists } from './helper';
+import { getGrafanaStorage, filenameAlreadyExists } from './storage';
 import { StorageView } from './types';
 
 interface RouteParams {
@@ -162,12 +162,19 @@ export default function StoragePage(props: Props) {
     }
     const canAddFolder = isFolder && path.startsWith('resources');
     const canDelete = path.startsWith('resources/');
+    const canViewDashboard =
+      path.startsWith('devenv/') && config.featureToggles.dashboardsFromStorage && (isFolder || path.endsWith('.json'));
 
     return (
       <div className={styles.wrapper}>
         <HorizontalGroup width="100%" justify="space-between" spacing={'md'} height={25}>
           <Breadcrumb pathName={path} onPathChange={setPath} rootIcon={navModel.node.icon as IconName} />
           <HorizontalGroup>
+            {canViewDashboard && (
+              <LinkButton icon="dashboard" href={`g/${path}`}>
+                Dashboard
+              </LinkButton>
+            )}
             {canAddFolder && <Button onClick={() => setIsAddingNewFolder(true)}>New Folder</Button>}
             {canDelete && (
               <Button

public/app/features/storage/UploadView.tsx

@@ -5,7 +5,7 @@ import SVG from 'react-inlinesvg';
 import { GrafanaTheme2 } from '@grafana/data';
 import { Alert, Button, ButtonGroup, Checkbox, Field, FileDropzone, useStyles2 } from '@grafana/ui';
 
-import { filenameAlreadyExists, getGrafanaStorage } from './helper';
+import { filenameAlreadyExists, getGrafanaStorage } from './storage';
 import { UploadReponse } from './types';
 
 interface Props {

public/app/features/storage/storage.ts

@@ -1,5 +1,8 @@
 import { DataFrame, dataFrameFromJSON, DataFrameJSON, getDisplayProcessor } from '@grafana/data';
 import { config, getBackendSrv } from '@grafana/runtime';
+import { backendSrv } from 'app/core/services/backend_srv';
+import { SaveDashboardCommand } from 'app/features/dashboard/components/SaveDashboard/types';
+import { DashboardDTO } from 'app/types';
 
 import { UploadReponse } from './types';
 
@@ -10,6 +13,13 @@ export interface GrafanaStorage {
   upload: (folder: string, file: File, overwriteExistingFile: boolean) => Promise<UploadReponse>;
   createFolder: (path: string) => Promise<{ error?: string }>;
   delete: (path: { isFolder: boolean; path: string }) => Promise<{ error?: string }>;
+
+  /**
+   * Temporary shim that will return a DashboardDTO shape for files in storage
+   * Longer term, this will call an "Entity API" that is eventually backed by storage
+   */
+  getDashboard: (path: string) => Promise<DashboardDTO>;
+  saveDashboard: (options: SaveDashboardCommand) => Promise<any>;
 }
 
 class SimpleStorage implements GrafanaStorage {
@@ -103,6 +113,75 @@ class SimpleStorage implements GrafanaStorage {
     }
     return body;
   }
+
+  // Temporary shim that can be loaded into the existing dashboard page structure
+  async getDashboard(path: string): Promise<DashboardDTO> {
+    if (!config.featureToggles.dashboardsFromStorage) {
+      return Promise.reject('Dashboards from storage is not enabled');
+    }
+
+    if (!path.endsWith('.json')) {
+      path += '.json';
+    }
+    const result = await backendSrv.get(`/api/storage/read/${path}`);
+    result.uid = path;
+    delete result.id; // Saved with the dev dashboards!
+
+    return {
+      meta: {
+        uid: path,
+        slug: path,
+        canEdit: true,
+        canSave: true,
+        canStar: false, // needs id
+      },
+      dashboard: result,
+    };
+  }
+
+  async saveDashboard(options: SaveDashboardCommand): Promise<any> {
+    if (!config.featureToggles.dashboardsFromStorage) {
+      return Promise.reject('Dashboards from storage is not enabled');
+    }
+
+    const blob = new Blob([JSON.stringify(options.dashboard)], {
+      type: 'application/json',
+    });
+
+    const uid = options.dashboard.uid;
+    const formData = new FormData();
+    if (options.message) {
+      formData.append('message', options.message);
+    }
+    formData.append('overwriteExistingFile', options.overwrite === false ? 'false' : 'true');
+    formData.append('file.path', uid);
+    formData.append('file', blob);
+    const res = await fetch('/api/storage/upload', {
+      method: 'POST',
+      body: formData,
+    });
+
+    let body = (await res.json()) as UploadReponse;
+    if (res.status !== 200 && !body?.err) {
+      console.log('SAVE', options, body);
+      return Promise.reject({ message: body?.message ?? res.statusText });
+    }
+
+    return {
+      uid,
+      url: `/g/${uid}`,
+      slug: uid,
+      status: 'success',
+    };
+  }
+}
+
+export function filenameAlreadyExists(folderName: string, fileNames: string[]) {
+  const lowerCase = folderName.toLowerCase();
+  const trimmedLowerCase = lowerCase.trim();
+  const existingTrimmedLowerCaseNames = fileNames.map((f) => f.trim().toLowerCase());
+
+  return existingTrimmedLowerCaseNames.includes(trimmedLowerCase);
 }
 
 let storage: GrafanaStorage | undefined;
@@ -113,11 +192,3 @@ export function getGrafanaStorage() {
   }
   return storage;
 }
-
-export function filenameAlreadyExists(folderName: string, fileNames: string[]) {
-  const lowerCase = folderName.toLowerCase();
-  const trimmedLowerCase = lowerCase.trim();
-  const existingTrimmedLowerCaseNames = fileNames.map((f) => f.trim().toLowerCase());
-
-  return existingTrimmedLowerCaseNames.includes(trimmedLowerCase);
-}

public/app/routes/routes.tsx

@@ -443,6 +443,7 @@ export function getAppRoutes(): RouteDescriptor[] {
         () => import(/* webpackChunkName: "NotificationsPage"*/ 'app/features/notifications/NotificationsPage')
       ),
     },
+    ...getBrowseStorageRoutes(),
     ...getDynamicDashboardRoutes(),
     ...getPluginCatalogRoutes(),
     ...getLiveRoutes(),
@@ -460,6 +461,28 @@ export function getAppRoutes(): RouteDescriptor[] {
   ];
 }
 
+export function getBrowseStorageRoutes(cfg = config): RouteDescriptor[] {
+  if (!cfg.featureToggles.dashboardsFromStorage) {
+    return [];
+  }
+  return [
+    {
+      path: '/g/:slug*.json', // suffix will eventually include dashboard
+      pageClass: 'page-dashboard',
+      routeName: DashboardRoutes.Path,
+      component: SafeDynamicImport(
+        () => import(/* webpackChunkName: "DashboardPage" */ '../features/dashboard/containers/DashboardPage')
+      ),
+    },
+    {
+      path: '/g/:slug*',
+      component: SafeDynamicImport(
+        () => import(/* webpackChunkName: "StorageFolderPage" */ '../features/storage/StorageFolderPage')
+      ),
+    },
+  ];
+}
+
 export function getDynamicDashboardRoutes(cfg = config): RouteDescriptor[] {
   if (!cfg.featureToggles.scenes) {
     return [];

public/app/types/dashboard.ts

@@ -11,6 +11,8 @@ export interface DashboardDTO {
 }
 
 export interface DashboardMeta {
+  slug?: string;
+  uid?: string;
   canSave?: boolean;
   canEdit?: boolean;
   canDelete?: boolean;
@@ -27,6 +29,7 @@ export interface DashboardMeta {
   isStarred?: boolean;
   showSettings?: boolean;
   expires?: string;
+  isFolder?: boolean;
   isSnapshot?: boolean;
   folderTitle?: string;
   folderUrl?: string;
@@ -59,12 +62,16 @@ export interface DashboardDataDTO {
     list: VariableModel[];
   };
   panels?: any[];
+
+  /** @deprecated -- components should key on uid rather than id */
+  id?: number;
 }
 
 export enum DashboardRoutes {
   Home = 'home-dashboard',
   New = 'new-dashboard',
   Normal = 'normal-dashboard',
+  Path = 'path-dashboard',
   Scripted = 'scripted-dashboard',
   Public = 'public-dashboard',
 }