CloudMigrations: Add support for migration of Library Elements (Panels) resources (#93898)

* CloudMigrations: create snapshots of Library Elements

* CloudMigrations: render library element resource in resources table

* CloudMigrations: create newtype with necessary fields for library element creation

pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration.go

@@ -26,6 +26,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/folder"
 	"github.com/grafana/grafana/pkg/services/gcom"
+	"github.com/grafana/grafana/pkg/services/libraryelements"
 	"github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore"
 	"github.com/grafana/grafana/pkg/services/secrets"
 	secretskv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
@@ -53,13 +54,14 @@ type Service struct {
 	gmsClient     gmsclient.Client
 	objectStorage objectstorage.ObjectStorage
 
-	dsService        datasources.DataSourceService
-	gcomService      gcom.Service
-	dashboardService dashboards.DashboardService
-	folderService    folder.Service
-	pluginStore      pluginstore.Store
-	secretsService   secrets.Service
-	kvStore          *kvstore.NamespacedKVStore
+	dsService              datasources.DataSourceService
+	gcomService            gcom.Service
+	dashboardService       dashboards.DashboardService
+	folderService          folder.Service
+	pluginStore            pluginstore.Store
+	secretsService         secrets.Service
+	kvStore                *kvstore.NamespacedKVStore
+	libraryElementsService libraryelements.Service
 
 	api     *api.CloudMigrationAPI
 	tracer  tracing.Tracer
@@ -93,24 +95,26 @@ func ProvideService(
 	folderService folder.Service,
 	pluginStore pluginstore.Store,
 	kvStore kvstore.KVStore,
+	libraryElementsService libraryelements.Service,
 ) (cloudmigration.Service, error) {
 	if !features.IsEnabledGlobally(featuremgmt.FlagOnPremToCloudMigrations) {
 		return &NoopServiceImpl{}, nil
 	}
 
 	s := &Service{
-		store:            &sqlStore{db: db, secretsStore: secretsStore, secretsService: secretsService},
-		log:              log.New(LogPrefix),
-		cfg:              cfg,
-		features:         features,
-		dsService:        dsService,
-		tracer:           tracer,
-		metrics:          newMetrics(),
-		secretsService:   secretsService,
-		dashboardService: dashboardService,
-		folderService:    folderService,
-		pluginStore:      pluginStore,
-		kvStore:          kvstore.WithNamespace(kvStore, 0, "cloudmigration"),
+		store:                  &sqlStore{db: db, secretsStore: secretsStore, secretsService: secretsService},
+		log:                    log.New(LogPrefix),
+		cfg:                    cfg,
+		features:               features,
+		dsService:              dsService,
+		tracer:                 tracer,
+		metrics:                newMetrics(),
+		secretsService:         secretsService,
+		dashboardService:       dashboardService,
+		folderService:          folderService,
+		pluginStore:            pluginStore,
+		kvStore:                kvstore.WithNamespace(kvStore, 0, "cloudmigration"),
+		libraryElementsService: libraryElementsService,
 	}
 	s.api = api.RegisterApi(routeRegister, s, tracer)
 

pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration_test.go

@@ -24,6 +24,8 @@ import (
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/folder"
 	"github.com/grafana/grafana/pkg/services/folder/foldertest"
+	libraryelementsfake "github.com/grafana/grafana/pkg/services/libraryelements/fake"
+	libraryelements "github.com/grafana/grafana/pkg/services/libraryelements/model"
 	"github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore"
 	secretsfakes "github.com/grafana/grafana/pkg/services/secrets/fakes"
 	secretskv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
@@ -562,6 +564,36 @@ func TestReportEvent(t *testing.T) {
 	})
 }
 
+func TestGetLibraryElementsCommands(t *testing.T) {
+	s := setUpServiceTest(t, false).(*Service)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+
+	libraryElementService, ok := s.libraryElementsService.(*libraryelementsfake.LibraryElementService)
+	require.True(t, ok)
+	require.NotNil(t, libraryElementService)
+
+	folderUID := "folder-uid"
+	createLibraryElementCmd := libraryelements.CreateLibraryElementCommand{
+		FolderUID: &folderUID,
+		Name:      "library-element-1",
+		Model:     []byte{},
+		Kind:      int64(libraryelements.PanelElement),
+		UID:       "library-element-uid-1",
+	}
+
+	user := &user.SignedInUser{OrgID: 1}
+
+	_, err := libraryElementService.CreateElement(ctx, user, createLibraryElementCmd)
+	require.NoError(t, err)
+
+	cmds, err := s.getLibraryElementsCommands(ctx, user)
+	require.NoError(t, err)
+	require.Len(t, cmds, 1)
+	require.Equal(t, createLibraryElementCmd.UID, cmds[0].UID)
+}
+
 func ctxWithSignedInUser() context.Context {
 	c := &contextmodel.ReqContext{
 		SignedInUser: &user.SignedInUser{OrgID: 1},
@@ -626,6 +658,7 @@ func setUpServiceTest(t *testing.T, withDashboardMock bool) cloudmigration.Servi
 		mockFolder,
 		&pluginstore.FakePluginStore{},
 		kvstore.ProvideService(sqlStore),
+		&libraryelementsfake.LibraryElementService{},
 	)
 	require.NoError(t, err)
 

pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt.go

@@ -3,6 +3,7 @@ package cloudmigrationimpl
 import (
 	"context"
 	cryptoRand "crypto/rand"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -18,6 +19,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/datasources"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/folder"
+	libraryelements "github.com/grafana/grafana/pkg/services/libraryelements/model"
 	"github.com/grafana/grafana/pkg/services/user"
 	"github.com/grafana/grafana/pkg/util/retryer"
 	"golang.org/x/crypto/nacl/box"
@@ -38,9 +40,15 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
 		return nil, err
 	}
 
+	libraryElements, err := s.getLibraryElementsCommands(ctx, signedInUser)
+	if err != nil {
+		s.log.Error("Failed to get library elements", "err", err)
+		return nil, err
+	}
+
 	migrationDataSlice := make(
 		[]cloudmigration.MigrateDataRequestItem, 0,
-		len(dataSources)+len(dashs)+len(folders),
+		len(dataSources)+len(dashs)+len(folders)+len(libraryElements),
 	)
 
 	for _, ds := range dataSources {
@@ -78,6 +86,15 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
 		})
 	}
 
+	for _, libraryElement := range libraryElements {
+		migrationDataSlice = append(migrationDataSlice, cloudmigration.MigrateDataRequestItem{
+			Type:  cloudmigration.LibraryElementDataType,
+			RefID: libraryElement.UID,
+			Name:  libraryElement.Name,
+			Data:  libraryElement,
+		})
+	}
+
 	migrationData := &cloudmigration.MigrateDataRequest{
 		Items: migrationDataSlice,
 	}
@@ -169,6 +186,60 @@ func (s *Service) getDashboardAndFolderCommands(ctx context.Context, signedInUse
 	return dashboardCmds, folderCmds, nil
 }
 
+type libraryElement struct {
+	FolderUID *string         `json:"folderUid"`
+	Name      string          `json:"name"`
+	UID       string          `json:"uid"`
+	Model     json.RawMessage `json:"model"`
+	Kind      int64           `json:"kind"`
+}
+
+// getLibraryElementsCommands returns the json payloads required by the library elements creation API
+func (s *Service) getLibraryElementsCommands(ctx context.Context, signedInUser *user.SignedInUser) ([]libraryElement, error) {
+	const perPage = 100
+
+	cmds := make([]libraryElement, 0)
+
+	page := 1
+	count := 0
+
+	for {
+		query := libraryelements.SearchLibraryElementsQuery{
+			PerPage: perPage,
+			Page:    page,
+		}
+
+		libraryElements, err := s.libraryElementsService.GetAllElements(ctx, signedInUser, query)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get all library elements: %w", err)
+		}
+
+		for _, element := range libraryElements.Elements {
+			var folderUID *string
+			if len(element.FolderUID) > 0 {
+				folderUID = &element.FolderUID
+			}
+
+			cmds = append(cmds, libraryElement{
+				FolderUID: folderUID,
+				Name:      element.Name,
+				Model:     element.Model,
+				Kind:      element.Kind,
+				UID:       element.UID,
+			})
+		}
+
+		page += 1
+		count += libraryElements.PerPage
+
+		if len(libraryElements.Elements) == 0 || count >= int(libraryElements.TotalCount) {
+			break
+		}
+	}
+
+	return cmds, nil
+}
+
 // asynchronous process for writing the snapshot to the filesystem and updating the snapshot status
 func (s *Service) buildSnapshot(ctx context.Context, signedInUser *user.SignedInUser, maxItemsPerPartition uint32, metadata []byte, snapshotMeta cloudmigration.CloudMigrationSnapshot) error {
 	// TODO -- make sure we can only build one snapshot at a time
@@ -229,6 +300,7 @@ func (s *Service) buildSnapshot(ctx context.Context, signedInUser *user.SignedIn
 	for _, resourceType := range []cloudmigration.MigrateDataType{
 		cloudmigration.DatasourceDataType,
 		cloudmigration.FolderDataType,
+		cloudmigration.LibraryElementDataType,
 		cloudmigration.DashboardDataType,
 	} {
 		for chunk := range slices.Chunk(resourcesGroupedByType[resourceType], int(maxItemsPerPartition)) {

public/app/features/migrate-to-cloud/api/endpoints.gen.ts

@@ -56,6 +56,9 @@ const injectedRtkApi = api.injectEndpoints({
     getDashboardByUid: build.query<GetDashboardByUidApiResponse, GetDashboardByUidApiArg>({
       query: (queryArg) => ({ url: `/dashboards/uid/${queryArg.uid}` }),
     }),
+    getLibraryElementByUid: build.query<GetLibraryElementByUidApiResponse, GetLibraryElementByUidApiArg>({
+      query: (queryArg) => ({ url: `/library-elements/${queryArg.libraryElementUid}` }),
+    }),
   }),
   overrideExisting: false,
 });
@@ -130,6 +133,11 @@ export type GetDashboardByUidApiResponse = /** status 200 (empty) */ DashboardFu
 export type GetDashboardByUidApiArg = {
   uid: string;
 };
+export type GetLibraryElementByUidApiResponse =
+  /** status 200 (empty) */ LibraryElementResponseIsAResponseStructForLibraryElementDto;
+export type GetLibraryElementByUidApiArg = {
+  libraryElementUid: string;
+};
 export type CloudMigrationSessionResponseDto = {
   created?: string;
   slug?: string;
@@ -264,6 +272,39 @@ export type DashboardFullWithMeta = {
   dashboard?: Json;
   meta?: DashboardMeta;
 };
+export type LibraryElementDtoMetaUserDefinesModelForLibraryElementDtoMetaUser = {
+  avatarUrl?: string;
+  id?: number;
+  name?: string;
+};
+export type LibraryElementDtoMetaIsTheMetaInformationForLibraryElementDto = {
+  connectedDashboards?: number;
+  created?: string;
+  createdBy?: LibraryElementDtoMetaUserDefinesModelForLibraryElementDtoMetaUser;
+  folderName?: string;
+  folderUid?: string;
+  updated?: string;
+  updatedBy?: LibraryElementDtoMetaUserDefinesModelForLibraryElementDtoMetaUser;
+};
+export type LibraryElementDtoIsTheFrontendDtoForEntities = {
+  description?: string;
+  /** Deprecated: use FolderUID instead */
+  folderId?: number;
+  folderUid?: string;
+  id?: number;
+  kind?: number;
+  meta?: LibraryElementDtoMetaIsTheMetaInformationForLibraryElementDto;
+  model?: object;
+  name?: string;
+  orgId?: number;
+  schemaVersion?: number;
+  type?: string;
+  uid?: string;
+  version?: number;
+};
+export type LibraryElementResponseIsAResponseStructForLibraryElementDto = {
+  result?: LibraryElementDtoIsTheFrontendDtoForEntities;
+};
 export const {
   useGetSessionListQuery,
   useCreateSessionMutation,
@@ -278,4 +319,5 @@ export const {
   useCreateCloudMigrationTokenMutation,
   useDeleteCloudMigrationTokenMutation,
   useGetDashboardByUidQuery,
+  useGetLibraryElementByUidQuery,
 } = injectedRtkApi;

public/app/features/migrate-to-cloud/api/index.ts

@@ -47,6 +47,7 @@ export const cloudMigrationAPI = generatedAPI.enhanceEndpoints({
     },
 
     getDashboardByUid: suppressErrorsOnQuery,
+    getLibraryElementByUid: suppressErrorsOnQuery,
   },
 });
 

public/app/features/migrate-to-cloud/fixtures/migrationItems.ts

@@ -29,3 +29,17 @@ export function wellFormedDashboardMigrationItem(
     ...partial,
   };
 }
+
+export function wellFormedLibraryElementMigrationItem(
+  seed = 1,
+  partial: Partial<MigrateDataResponseItemDto> = {}
+): MigrateDataResponseItemDto {
+  const random = Chance(seed);
+
+  return {
+    type: 'LIBRARY_ELEMENT',
+    refId: random.guid(),
+    status: random.pickone(['OK', 'ERROR']),
+    ...partial,
+  };
+}

public/app/features/migrate-to-cloud/fixtures/mswAPI.ts

@@ -27,6 +27,28 @@ function createMockAPI(): SetupServer {
       });
     }),
 
+    http.get('/api/library-elements/:uid', ({ request, params }) => {
+      if (params.uid === 'library-element-404') {
+        return HttpResponse.json(
+          {
+            message: 'Library element not found',
+          },
+          {
+            status: 404,
+          }
+        );
+      }
+
+      return HttpResponse.json({
+        result: {
+          name: 'My Library Element',
+          meta: {
+            folderName: 'FolderName',
+          },
+        },
+      });
+    }),
+
     http.post('/api/cloudmigration/migration', async ({ request }) => {
       const data = await request.json();
       const authToken = typeof data === 'object' && data && data.authToken;

public/app/features/migrate-to-cloud/onprem/NameCell.tsx

@@ -9,7 +9,7 @@ import { getSvgSize } from '@grafana/ui/src/components/Icon/utils';
 import { Trans } from 'app/core/internationalization';
 import { useGetFolderQuery } from 'app/features/browse-dashboards/api/browseDashboardsAPI';
 
-import { useGetDashboardByUidQuery } from '../api';
+import { useGetDashboardByUidQuery, useGetLibraryElementByUidQuery } from '../api';
 
 import { ResourceTableItem } from './types';
 
@@ -36,7 +36,7 @@ function ResourceInfo({ data }: { data: ResourceTableItem }) {
     case 'FOLDER':
       return <FolderInfo data={data} />;
     case 'LIBRARY_ELEMENT':
-      return null;
+      return <LibraryElementInfo data={data} />;
   }
 }
 
@@ -134,6 +134,44 @@ function FolderInfo({ data }: { data: ResourceTableItem }) {
   );
 }
 
+function LibraryElementInfo({ data }: { data: ResourceTableItem }) {
+  const uid = data.refId;
+  const { data: libraryElementData, isError, isLoading } = useGetLibraryElementByUidQuery({ libraryElementUid: uid });
+
+  const name = useMemo(() => {
+    return data?.name || (libraryElementData?.result?.name ?? uid);
+  }, [data, libraryElementData, uid]);
+
+  if (isError) {
+    return (
+      <>
+        <Text italic>
+          <Trans i18nKey="migrate-to-cloud.resource-table.error-library-element-title">
+            Unable to load library element
+          </Trans>
+        </Text>
+
+        <Text color="secondary">
+          <Trans i18nKey="migrate-to-cloud.resource-table.error-library-element-sub">Library Element {uid}</Trans>
+        </Text>
+      </>
+    );
+  }
+
+  if (isLoading || !libraryElementData) {
+    return <InfoSkeleton />;
+  }
+
+  const folderName = libraryElementData?.result?.meta?.folderName ?? 'General';
+
+  return (
+    <>
+      <span>{name}</span>
+      <Text color="secondary">{folderName}</Text>
+    </>
+  );
+}
+
 function InfoSkeleton() {
   return (
     <>
@@ -155,6 +193,8 @@ function ResourceIcon({ resource }: { resource: ResourceTableItem }) {
     return <img className={styles.icon} src={datasource.meta.info.logos.small} alt="" />;
   } else if (resource.type === 'DATASOURCE') {
     return <Icon size="xl" name="database" />;
+  } else if (resource.type === 'LIBRARY_ELEMENT') {
+    return <Icon size="xl" name="library-panel" />;
   }
 
   return undefined;

public/app/features/migrate-to-cloud/onprem/ResourcesTable.test.tsx

@@ -4,7 +4,11 @@ import { TestProvider } from 'test/helpers/TestProvider';
 import { setBackendSrv, config } from '@grafana/runtime';
 import { backendSrv } from 'app/core/services/backend_srv';
 
-import { wellFormedDashboardMigrationItem, wellFormedDatasourceMigrationItem } from '../fixtures/migrationItems';
+import {
+  wellFormedDashboardMigrationItem,
+  wellFormedDatasourceMigrationItem,
+  wellFormedLibraryElementMigrationItem,
+} from '../fixtures/migrationItems';
 import { registerMockAPI } from '../fixtures/mswAPI';
 import { wellFormedDatasource } from '../fixtures/others';
 
@@ -86,6 +90,28 @@ describe('ResourcesTable', () => {
     expect(await screen.findByText('Dashboard dashboard-404')).toBeInTheDocument();
   });
 
+  it('renders library elements', async () => {
+    const resources = [wellFormedLibraryElementMigrationItem(1)];
+
+    render({ resources });
+
+    expect(await screen.findByText('My Library Element')).toBeInTheDocument();
+    expect(await screen.findByText('FolderName')).toBeInTheDocument();
+  });
+
+  it('renders library elements when their data is missing', async () => {
+    const resources = [
+      wellFormedLibraryElementMigrationItem(2, {
+        refId: 'library-element-404',
+      }),
+    ];
+
+    render({ resources });
+
+    expect(await screen.findByText('Unable to load library element')).toBeInTheDocument();
+    expect(await screen.findByText('Library Element library-element-404')).toBeInTheDocument();
+  });
+
   it('renders the success status correctly', () => {
     const resources = [
       wellFormedDatasourceMigrationItem(1, {

public/app/features/migrate-to-cloud/onprem/TypeCell.tsx

@@ -11,6 +11,8 @@ export function prettyTypeName(type: ResourceTableItem['type']) {
       return t('migrate-to-cloud.resource-type.dashboard', 'Dashboard');
     case 'FOLDER':
       return t('migrate-to-cloud.resource-type.folder', 'Folder');
+    case 'LIBRARY_ELEMENT':
+      return t('migrate-to-cloud.resource-type.library_element', 'Library Element');
     default:
       return t('migrate-to-cloud.resource-type.unknown', 'Unknown');
   }

public/app/features/migrate-to-cloud/onprem/useNotifyOnSuccess.tsx

@@ -45,6 +45,8 @@ function getTranslatedMessage(snapshot: GetSnapshotResponseDto) {
       types.push(t('migrate-to-cloud.migrated-counts.datasources', 'data sources'));
     } else if (type === 'FOLDER') {
       types.push(t('migrate-to-cloud.migrated-counts.folders', 'folders'));
+    } else if (type === 'LIBRARY_ELEMENT') {
+      types.push(t('migrate-to-cloud.migrated-counts.library_elements', 'library elements'));
     }
   }
 

public/locales/en-US/grafana.json

@@ -1395,7 +1395,8 @@
     "migrated-counts": {
       "dashboards": "dashboards",
       "datasources": "data sources",
-      "folders": "folders"
+      "folders": "folders",
+      "library_elements": "library elements"
     },
     "migration-token": {
       "delete-button": "Delete token",
@@ -1467,6 +1468,8 @@
       "warning-details-button": "Details"
     },
     "resource-table": {
+      "error-library-element-sub": "Library Element {uid}",
+      "error-library-element-title": "Unable to load library element",
       "unknown-datasource-title": "Data source {{datasourceUID}}",
       "unknown-datasource-type": "Unknown data source"
     },
@@ -1474,6 +1477,7 @@
       "dashboard": "Dashboard",
       "datasource": "Data source",
       "folder": "Folder",
+      "library_element": "Library Element",
       "unknown": "Unknown"
     },
     "summary": {

public/locales/pseudo-LOCALE/grafana.json

@@ -1395,7 +1395,8 @@
     "migrated-counts": {
       "dashboards": "đäşĥþőäřđş",
       "datasources": "đäŧä şőūřčęş",
-      "folders": "ƒőľđęřş"
+      "folders": "ƒőľđęřş",
+      "library_elements": "ľįþřäřy ęľęmęʼnŧş"
     },
     "migration-token": {
       "delete-button": "Đęľęŧę ŧőĸęʼn",
@@ -1467,6 +1468,8 @@
       "warning-details-button": "Đęŧäįľş"
     },
     "resource-table": {
+      "error-library-element-sub": "Ŀįþřäřy Ēľęmęʼnŧ {ūįđ}",
+      "error-library-element-title": "Ůʼnäþľę ŧő ľőäđ ľįþřäřy ęľęmęʼnŧ",
       "unknown-datasource-title": "Đäŧä şőūřčę {{datasourceUID}}",
       "unknown-datasource-type": "Ůʼnĸʼnőŵʼn đäŧä şőūřčę"
     },
@@ -1474,6 +1477,7 @@
       "dashboard": "Đäşĥþőäřđ",
       "datasource": "Đäŧä şőūřčę",
       "folder": "Főľđęř",
+      "library_element": "Ŀįþřäřy Ēľęmęʼnŧ",
       "unknown": "Ůʼnĸʼnőŵʼn"
     },
     "summary": {

scripts/generate-rtk-apis.ts

@@ -28,6 +28,7 @@ const config: ConfigFile = {
         'getCloudMigrationToken',
 
         'getDashboardByUid',
+        'getLibraryElementByUid',
       ],
     },
     '../public/app/features/preferences/api/user/endpoints.gen.ts': {