IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

CloudMigrations: Store parent folder name in cloud_migration_resource table (#94009)

Browse Source 
* use name in fe

* store parent folder name in local db

* clean up

* tiny test

* trial react

* rename to parent name

* go lint

* generate api and ts

* go tests

* rearrange

* clean

* update with suggestions from josh

* make library elements work

* updates from comments

* global migration types

* parent name for alter table
This commit is contained in:
Dana Axinte
2024-10-07 06:35:08 -04:00
committed by GitHub
parent 5a9bd1d1cf
commit d88be2819d
16 changed files with 309 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.betterer.results
View File

@@ -4593,8 +4593,7 @@ exports[`better eslint`] = {
"public/app/features/migrate-to-cloud/onprem/NameCell.tsx:5381": [
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "0"],
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "1"],
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "2"],
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "3"]
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "2"]
],
"public/app/features/notifications/StoredNotifications.tsx:5381": [
[0, 0, 0, "No untranslated strings. Wrap text with <Trans />", "0"]

11
pkg/services/cloudmigration/api/api.go
View File

@@ -385,11 +385,12 @@ func (cma *CloudMigrationAPI) GetSnapshot(c *contextmodel.ReqContext) response.R
dtoResults := make([]MigrateDataResponseItemDTO, len(results))
for i := 0; i < len(results); i++ {
dtoResults[i] = MigrateDataResponseItemDTO{
Name: results[i].Name,
Type: MigrateDataType(results[i].Type),
RefID: results[i].RefID,
Status: ItemStatus(results[i].Status),
Message: results[i].Error,
Name: results[i].Name,
Type: MigrateDataType(results[i].Type),
RefID: results[i].RefID,
Status: ItemStatus(results[i].Status),
Message: results[i].Error,
ParentName: results[i].ParentName,
}
}

2
pkg/services/cloudmigration/api/api_test.go
View File

@@ -345,7 +345,7 @@ func TestCloudMigrationAPI_GetSnapshot(t *testing.T) {
requestUrl: "/api/cloudmigration/migration/1234/snapshot/1",
basicRole: org.RoleAdmin,
expectedHttpResult: http.StatusOK,
expectedBody: `{"uid":"fake_uid","status":"CREATING","sessionUid":"1234","created":"0001-01-01T00:00:00Z","finished":"0001-01-01T00:00:00Z","results":[],"stats":{"types":{},"statuses":{},"total":0}}`,
expectedBody: `{"uid":"fake_uid","status":"CREATING","sessionUid":"1234","created":"0001-01-01T00:00:00Z","finished":"0001-01-01T00:00:00Z","results":[{"name":"dashboard name","parentName":"dashboard parent name","type":"DASHBOARD","refId":"123","status":"PENDING"},{"name":"datasource name","parentName":"dashboard parent name","type":"DATASOURCE","refId":"456","status":"OK"}],"stats":{"types":{},"statuses":{},"total":0}}`,
},
{
desc: "should return 403 if no used is not admin",

3
pkg/services/cloudmigration/api/dtos.go
View File

@@ -106,7 +106,8 @@ type MigrateDataResponseDTO struct {
}
type MigrateDataResponseItemDTO struct {
Name string `json:"name"`
Name string `json:"name"`
ParentName string `json:"parentName"`
// required:true
Type MigrateDataType `json:"type"`
// required:true

121
pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration_test.go
View File

@@ -2,8 +2,10 @@ package cloudmigrationimpl
import (
"context"
"maps"
"os"
"path/filepath"
"slices"
"testing"
"time"
@@ -392,6 +394,7 @@ func Test_NonCoreDataSourcesHaveWarning(t *testing.T) {
Results: []cloudmigration.CloudMigrationResource{
{
Name: "1 name",
ParentName: "1 parent name",
Type: cloudmigration.DatasourceDataType,
RefID: "1", // this will be core
Status: cloudmigration.ItemStatusOK,
@@ -399,6 +402,7 @@ func Test_NonCoreDataSourcesHaveWarning(t *testing.T) {
},
{
Name: "2 name",
ParentName: "",
Type: cloudmigration.DatasourceDataType,
RefID: "2", // this will be non-core
Status: cloudmigration.ItemStatusOK,
@@ -406,6 +410,7 @@ func Test_NonCoreDataSourcesHaveWarning(t *testing.T) {
},
{
Name: "3 name",
ParentName: "3 parent name",
Type: cloudmigration.DatasourceDataType,
RefID: "3", // this will be non-core with an error
Status: cloudmigration.ItemStatusError,
@@ -414,6 +419,7 @@ func Test_NonCoreDataSourcesHaveWarning(t *testing.T) {
},
{
Name: "4 name",
ParentName: "4 folder name",
Type: cloudmigration.DatasourceDataType,
RefID: "4", // this will be deleted
Status: cloudmigration.ItemStatusOK,
@@ -564,6 +570,121 @@ func TestReportEvent(t *testing.T) {
require.Equal(t, 1, gmsMock.reportEventCalled)
})
}
func TestGetFolderNamesForFolderUIDs(t *testing.T) {
s := setUpServiceTest(t, false).(*Service)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
user := &user.SignedInUser{OrgID: 1}
testcases := []struct {
folders []*folder.Folder
folderUIDs []string
expectedFolderNames []string
}{
{
folders: []*folder.Folder{
{UID: "folderUID-A", Title: "Folder A", OrgID: 1},
{UID: "folderUID-B", Title: "Folder B", OrgID: 1},
},
folderUIDs: []string{"folderUID-A", "folderUID-B"},
expectedFolderNames: []string{"Folder A", "Folder B"},
},
{
folders: []*folder.Folder{
{UID: "folderUID-A", Title: "Folder A", OrgID: 1},
},
folderUIDs: []string{"folderUID-A"},
expectedFolderNames: []string{"Folder A"},
},
{
folders: []*folder.Folder{},
folderUIDs: []string{"folderUID-A"},
expectedFolderNames: []string{""},
},
{
folders: []*folder.Folder{
{UID: "folderUID-A", Title: "Folder A", OrgID: 1},
},
folderUIDs: []string{"folderUID-A", "folderUID-B"},
expectedFolderNames: []string{"Folder A", ""},
},
{
folders: []*folder.Folder{},
folderUIDs: []string{""},
expectedFolderNames: []string{""},
},
{
folders: []*folder.Folder{},
folderUIDs: []string{},
expectedFolderNames: []string{},
},
}
for _, tc := range testcases {
s.folderService = &foldertest.FakeService{ExpectedFolders: tc.folders}
folderUIDsToFolders, err := s.getFolderNamesForFolderUIDs(ctx, user, tc.folderUIDs)
require.NoError(t, err)
resFolderNames := slices.Collect(maps.Values(folderUIDsToFolders))
require.Len(t, resFolderNames, len(tc.expectedFolderNames))
require.ElementsMatch(t, resFolderNames, tc.expectedFolderNames)
}
}
func TestGetParentNames(t *testing.T) {
s := setUpServiceTest(t, false).(*Service)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
user := &user.SignedInUser{OrgID: 1}
libraryElementFolderUID := "folderUID-A"
testcases := []struct {
fakeFolders []*folder.Folder
folders []folder.CreateFolderCommand
dashboards []dashboards.Dashboard
libraryElements []libraryElement
expectedDashParentNames []string
expectedFoldParentNames []string
}{
{
fakeFolders: []*folder.Folder{
{UID: "folderUID-A", Title: "Folder A", OrgID: 1, ParentUID: ""},
{UID: "folderUID-B", Title: "Folder B", OrgID: 1, ParentUID: "folderUID-A"},
},
folders: []folder.CreateFolderCommand{
{UID: "folderUID-C", Title: "Folder A", OrgID: 1, ParentUID: "folderUID-A"},
},
dashboards: []dashboards.Dashboard{
{UID: "dashboardUID-0", OrgID: 1, FolderUID: ""},
{UID: "dashboardUID-1", OrgID: 1, FolderUID: "folderUID-A"},
{UID: "dashboardUID-2", OrgID: 1, FolderUID: "folderUID-B"},
},
libraryElements: []libraryElement{
{UID: "libraryElementUID-0", FolderUID: &libraryElementFolderUID},
},
expectedDashParentNames: []string{"", "Folder A", "Folder B"},
expectedFoldParentNames: []string{"Folder A"},
},
}
for _, tc := range testcases {
s.folderService = &foldertest.FakeService{ExpectedFolders: tc.fakeFolders}
dataUIDsToParentNamesByType, err := s.getParentNames(ctx, user, tc.dashboards, tc.folders, tc.libraryElements)
require.NoError(t, err)
resDashParentNames := slices.Collect(maps.Values(dataUIDsToParentNamesByType[cloudmigration.DashboardDataType]))
require.Len(t, resDashParentNames, len(tc.expectedDashParentNames))
require.ElementsMatch(t, resDashParentNames, tc.expectedDashParentNames)
resFoldParentNames := slices.Collect(maps.Values(dataUIDsToParentNamesByType[cloudmigration.FolderDataType]))
require.Len(t, resFoldParentNames, len(tc.expectedFoldParentNames))
require.ElementsMatch(t, resFoldParentNames, tc.expectedFoldParentNames)
}
}
func TestGetLibraryElementsCommands(t *testing.T) {
s := setUpServiceTest(t, false).(*Service)

18
pkg/services/cloudmigration/cloudmigrationimpl/fake/cloudmigration_fake.go
View File

@@ -98,10 +98,28 @@ func (m FakeServiceImpl) GetSnapshot(ctx context.Context, query cloudmigration.G
if m.ReturnError {
return nil, fmt.Errorf("mock error")
}
cloudMigrationResources := []cloudmigration.CloudMigrationResource{
{
Type: cloudmigration.DashboardDataType,
RefID: "123",
Status: cloudmigration.ItemStatusPending,
Name: "dashboard name",
ParentName: "dashboard parent name",
},
{
Type: cloudmigration.DatasourceDataType,
RefID: "456",
Status: cloudmigration.ItemStatusOK,
Name: "datasource name",
ParentName: "dashboard parent name",
},
}
return &cloudmigration.CloudMigrationSnapshot{
UID: "fake_uid",
SessionUID: "fake_uid",
Status: cloudmigration.SnapshotStatusCreating,
Resources: cloudMigrationResources,
}, nil
}

104
pkg/services/cloudmigration/cloudmigrationimpl/snapshot_mgmt.go
View File

@@ -27,6 +27,13 @@ import (
"go.opentelemetry.io/otel/codes"
)
var currentMigrationTypes = []cloudmigration.MigrateDataType{
cloudmigration.DatasourceDataType,
cloudmigration.FolderDataType,
cloudmigration.LibraryElementDataType,
cloudmigration.DashboardDataType,
}
func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.SignedInUser) (*cloudmigration.MigrateDataRequest, error) {
ctx, span := s.tracer.Start(ctx, "CloudMigrationService.getMigrationDataJSON")
defer span.End()
@@ -100,8 +107,15 @@ func (s *Service) getMigrationDataJSON(ctx context.Context, signedInUser *user.S
})
}
// Obtain the names of parent elements for Dashboard and Folders data types
parentNamesByType, err := s.getParentNames(ctx, signedInUser, dashs, folders, libraryElements)
if err != nil {
s.log.Error("Failed to get parent folder names", "err", err)
}
migrationData := &cloudmigration.MigrateDataRequest{
Items: migrationDataSlice,
Items: migrationDataSlice,
ItemParentNames: parentNamesByType,
}
return migrationData, nil
@@ -306,20 +320,21 @@ func (s *Service) buildSnapshot(ctx context.Context, signedInUser *user.SignedIn
Data: item.Data,
})
parentName := ""
if _, exists := migrationData.ItemParentNames[item.Type]; exists {
parentName = migrationData.ItemParentNames[item.Type][item.RefID]
}
localSnapshotResource[i] = cloudmigration.CloudMigrationResource{
Name: item.Name,
Type: item.Type,
RefID: item.RefID,
Status: cloudmigration.ItemStatusPending,
Name: item.Name,
Type: item.Type,
RefID: item.RefID,
Status: cloudmigration.ItemStatusPending,
ParentName: parentName,
}
}
for _, resourceType := range []cloudmigration.MigrateDataType{
cloudmigration.DatasourceDataType,
cloudmigration.FolderDataType,
cloudmigration.LibraryElementDataType,
cloudmigration.DashboardDataType,
} {
for _, resourceType := range currentMigrationTypes {
for chunk := range slices.Chunk(resourcesGroupedByType[resourceType], int(maxItemsPerPartition)) {
if err := snapshotWriter.Write(string(resourceType), chunk); err != nil {
return fmt.Errorf("writing resources to snapshot writer: resourceType=%s %w", resourceType, err)
@@ -533,3 +548,70 @@ func sortFolders(input []folder.CreateFolderCommand) []folder.CreateFolderComman
return input
}
// getFolderNamesForFolderUIDs queries the folders service to obtain folder names for a list of folderUIDs
func (s *Service) getFolderNamesForFolderUIDs(ctx context.Context, signedInUser *user.SignedInUser, folderUIDs []string) (map[string](string), error) {
folders, err := s.folderService.GetFolders(ctx, folder.GetFoldersQuery{
UIDs: folderUIDs,
SignedInUser: signedInUser,
WithFullpathUIDs: true,
})
if err != nil {
s.log.Error("Failed to obtain folders from folder UIDs", "err", err)
return nil, err
}
folderUIDsToNames := make(map[string](string), len(folderUIDs))
for _, folderUID := range folderUIDs {
folderUIDsToNames[folderUID] = ""
}
for _, f := range folders {
folderUIDsToNames[f.UID] = f.Title
}
return folderUIDsToNames, nil
}
// getParentNames finds the parent names for resources and returns a map of data type: {data UID : parentName}
// for dashboards, folders and library elements - the parent is the parent folder
func (s *Service) getParentNames(ctx context.Context, signedInUser *user.SignedInUser, dashboards []dashboards.Dashboard, folders []folder.CreateFolderCommand, libraryElements []libraryElement) (map[cloudmigration.MigrateDataType]map[string](string), error) {
parentNamesByType := make(map[cloudmigration.MigrateDataType]map[string](string))
for _, dataType := range currentMigrationTypes {
parentNamesByType[dataType] = make(map[string]string)
}
// Obtain list of unique folderUIDs
parentFolderUIDsSet := make(map[string]struct{}, len(dashboards)+len(folders)+len(libraryElements))
for _, dashboard := range dashboards {
parentFolderUIDsSet[dashboard.FolderUID] = struct{}{}
}
for _, f := range folders {
parentFolderUIDsSet[f.ParentUID] = struct{}{}
}
for _, libraryElement := range libraryElements {
parentFolderUIDsSet[*libraryElement.FolderUID] = struct{}{}
}
parentFolderUIDsSlice := make([]string, 0, len(parentFolderUIDsSet))
for parentFolderUID := range parentFolderUIDsSet {
parentFolderUIDsSlice = append(parentFolderUIDsSlice, parentFolderUID)
}
// Obtain folder names given a list of folderUIDs
foldersUIDsToFolderName, err := s.getFolderNamesForFolderUIDs(ctx, signedInUser, parentFolderUIDsSlice)
if err != nil {
s.log.Error("Failed to get parent folder names from folder UIDs", "err", err)
return parentNamesByType, err
}
// Prepare map of {data type: {data UID : parentName}}
for _, dashboard := range dashboards {
parentNamesByType[cloudmigration.DashboardDataType][dashboard.UID] = foldersUIDsToFolderName[dashboard.FolderUID]
}
for _, f := range folders {
parentNamesByType[cloudmigration.FolderDataType][f.UID] = foldersUIDsToFolderName[f.ParentUID]
}
for _, libraryElement := range libraryElements {
parentNamesByType[cloudmigration.LibraryElementDataType][libraryElement.UID] = foldersUIDsToFolderName[*libraryElement.FolderUID]
}
return parentNamesByType, err
}

4
pkg/services/cloudmigration/model.go
View File

@@ -75,6 +75,7 @@ type CloudMigrationResource struct {
Error string `xorm:"error_string" json:"error"`
SnapshotUID string `xorm:"snapshot_uid"`
ParentName string `xorm:"parent_name" json:"parentName"`
}
type MigrateDataType string
@@ -185,7 +186,8 @@ type Base64HGInstance struct {
// GMS domain structs
type MigrateDataRequest struct {
Items []MigrateDataRequestItem
Items []MigrateDataRequestItem
ItemParentNames map[MigrateDataType]map[string](string)
}
type MigrateDataRequestItem struct {

6
pkg/services/sqlstore/migrations/cloud_migrations.go
View File

@@ -164,4 +164,10 @@ func addCloudMigrationsMigrations(mg *Migrator) {
Type: DB_Text,
Nullable: true,
}))
mg.AddMigration("add cloud_migration_resource.parent_name column", NewAddColumnMigration(migrationResourceTable, &Column{
Name: "parent_name",
Type: DB_Text,
Nullable: true,
}))
}

4
public/api-enterprise-spec.json
View File

@@ -3382,6 +3382,7 @@
}
},
"CorrelationType": {
"description": "the type of correlation, either query for containing query information, or external for containing an external URL\n+enum",
"type": "string"
},
"CreateAccessTokenResponseDTO": {
@@ -5419,6 +5420,9 @@
"name": {
"type": "string"
},
"parentName": {
"type": "string"
},
"refId": {
"type": "string"
},

3
public/api-merged.json
View File

@@ -16909,6 +16909,9 @@
"name": {
"type": "string"
},
"parentName": {
"type": "string"
},
"refId": {
"type": "string"
},

1
public/app/features/migrate-to-cloud/api/endpoints.gen.ts
View File

@@ -166,6 +166,7 @@ export type CreateSnapshotResponseDto = {
export type MigrateDataResponseItemDto = {
message?: string;
name?: string;
parentName?: string;
refId: string;
status: 'OK' | 'WARNING' | 'ERROR' | 'PENDING' | 'UNKNOWN';
type: 'DASHBOARD' | 'DATASOURCE' | 'FOLDER' | 'LIBRARY_ELEMENT';

81
public/app/features/migrate-to-cloud/onprem/NameCell.tsx
View File

@@ -40,14 +40,6 @@ function ResourceInfo({ data }: { data: ResourceTableItem }) {
}
}
function getDashboardTitle(dashboardData: object) {
if ('title' in dashboardData && typeof dashboardData.title === 'string') {
return dashboardData.title;
}
return undefined;
}
function DatasourceInfo({ data }: { data: ResourceTableItem }) {
const datasourceUID = data.refId;
const datasource = useDatasource(datasourceUID);
@@ -75,72 +67,91 @@ function DatasourceInfo({ data }: { data: ResourceTableItem }) {
);
}
function getTitleFromDashboardJSON(dashboardData: object | undefined): string | null {
if (dashboardData && 'title' in dashboardData && typeof dashboardData.title === 'string') {
return dashboardData.title;
}
return null;
}
function DashboardInfo({ data }: { data: ResourceTableItem }) {
const dashboardUID = data.refId;
// TODO: really, the API should return this directly
const { data: dashboardData, isError } = useGetDashboardByUidQuery({
uid: dashboardUID,
});
const skipApiCall = !!data.name && !!data.parentName;
const {
data: dashboardData,
isLoading,
isError,
} = useGetDashboardByUidQuery({ uid: dashboardUID }, { skip: skipApiCall });
const dashboardName = useMemo(() => {
return (dashboardData?.dashboard && getDashboardTitle(dashboardData.dashboard)) ?? dashboardUID;
}, [dashboardData, dashboardUID]);
const dashboardName = data.name || getTitleFromDashboardJSON(dashboardData?.dashboard) || dashboardUID;
const dashboardParentName = data.parentName || dashboardData?.meta?.folderTitle || 'Dashboards';
if (isError) {
// Not translated because this is only temporary until the data comes through in the MigrationRun API
return (
<>
<Text italic>Unable to load dashboard</Text>
<Text italic>
<Trans i18nKey="migrate-to-cloud.resource-table.dashboard-load-error">Unable to load dashboard</Trans>
</Text>
<Text color="secondary">Dashboard {dashboardUID}</Text>
</>
);
}
if (!dashboardData) {
if (isLoading) {
return <InfoSkeleton />;
}
return (
<>
<span>{dashboardName}</span>
<Text color="secondary">{dashboardData.meta?.folderTitle ?? 'Dashboards'}</Text>
<Text color="secondary">{dashboardParentName}</Text>
</>
);
}
function FolderInfo({ data }: { data: ResourceTableItem }) {
const { data: folderData, isLoading, isError } = useGetFolderQuery(data.refId);
const folderUID = data.refId;
const skipApiCall = !!data.name && !!data.parentName;
if (isLoading || !folderData) {
return <InfoSkeleton />;
}
const { data: folderData, isLoading, isError } = useGetFolderQuery(folderUID, { skip: skipApiCall });
const folderName = data.name || folderData?.title;
const folderParentName = data.parentName || folderData?.parents?.[folderData.parents.length - 1]?.title;
if (isError) {
return (
<>
<Text italic>Unable to load dashboard</Text>
<Text color="secondary">Dashboard {data.refId}</Text>
<Text italic>Unable to load folder</Text>
<Text color="secondary">Folder {data.refId}</Text>
</>
);
}
const parentFolderName = folderData.parents?.[folderData.parents.length - 1]?.title;
if (isLoading) {
return <InfoSkeleton />;
}
return (
<>
<span>{folderData.title}</span>
<Text color="secondary">{parentFolderName ?? 'Dashboards'}</Text>
<span>{folderName}</span>
<Text color="secondary">{folderParentName ?? 'Dashboards'}</Text>
</>
);
}
function LibraryElementInfo({ data }: { data: ResourceTableItem }) {
const uid = data.refId;
const { data: libraryElementData, isError, isLoading } = useGetLibraryElementByUidQuery({ libraryElementUid: uid });
const skipApiCall = !!data.name && !!data.parentName;
const name = useMemo(() => {
return data?.name || (libraryElementData?.result?.name ?? uid);
}, [data, libraryElementData, uid]);
const {
data: libraryElementData,
isError,
isLoading,
} = useGetLibraryElementByUidQuery({ libraryElementUid: uid }, { skip: skipApiCall });
const name = data.name || libraryElementData?.result?.name || uid;
const parentName = data.parentName || libraryElementData?.result?.meta?.folderName || 'General';
if (isError) {
return (
@@ -158,16 +169,14 @@ function LibraryElementInfo({ data }: { data: ResourceTableItem }) {
);
}
if (isLoading || !libraryElementData) {
if (isLoading) {
return <InfoSkeleton />;
}
const folderName = libraryElementData?.result?.meta?.folderName ?? 'General';
return (
<>
<span>{name}</span>
<Text color="secondary">{folderName}</Text>
<Text color="secondary">{parentName}</Text>
</>
);
}

1
public/locales/en-US/grafana.json
View File

@@ -1478,6 +1478,7 @@
"warning-details-button": "Details"
},
"resource-table": {
"dashboard-load-error": "Unable to load dashboard",
"error-library-element-sub": "Library Element {uid}",
"error-library-element-title": "Unable to load library element",
"unknown-datasource-title": "Data source {{datasourceUID}}",

1
public/locales/pseudo-LOCALE/grafana.json
View File

@@ -1478,6 +1478,7 @@
"warning-details-button": "Đęŧäįľş"
},
"resource-table": {
"dashboard-load-error": "Ůʼnäþľę ŧő ľőäđ đäşĥþőäřđ",
"error-library-element-sub": "Ŀįþřäřy Ēľęmęʼnŧ {ūįđ}",
"error-library-element-title": "Ůʼnäþľę ŧő ľőäđ ľįþřäřy ęľęmęʼnŧ",
"unknown-datasource-title": "Đäŧä şőūřčę {{datasourceUID}}",

3
public/openapi3.json
View File

@@ -7130,6 +7130,9 @@
"name": {
"type": "string"
},
"parentName": {
"type": "string"
},
"refId": {
"type": "string"
},