mirror of
https://github.com/grafana/grafana.git
synced 2026-08-18 17:15:08 -05:00
Previews: crawler as a background service (#44891)
* add SQL migrations * dashboard previews from sql: poc * added todos * refactor: use the same enums where possible * use useEffect, always return json * added todo * refactor + delete files after use * refactor + fix manual thumbnail upload * refactor: move all interactions with sqlStore to thumbnail repo * refactor: remove file operations in thumb crawler/service * refactor: fix dashboard_thumbs sql store * refactor: extracted thumbnail fetching/updating to a hook * refactor: store thumbnails in redux store * refactor: store thumbnails in redux store * refactor: private'd repo methods * removed redux storage, saving images as blobs * allow for configurable rendering timeouts * added 1) query for dashboards with stale thumbnails, 2) command for marking thumbnails as stale * use sql-based queue in crawler * ui for marking thumbnails as stale * replaced `stale` boolean prop with `state` enum * introduce rendering session * compilation errors * fix crawler stop button * rename thumbnail state frozen to locked * #44449: fix merge conflicts * #44449: remove thumb methods from `Store` interface * #44449: clean filepath, defer file closing * #44449: fix rendering.Theme cyclic import * #44449: linting * #44449: linting * #44449: mutex'd crawlerStatus access * #44449: added integration tests for `sqlstore.dashboard_thumbs` * #44449: added comments to explain the `ThumbnailState` enum * #44449: use os.ReadFile rather then os.Open * #44449: always enable dashboardPreviews feature during integration tests * #44449: remove sleep time, adjust number of threads * #44449: review fix: add `orgId` to `DashboardThumbnailMeta` * #44449: review fix: automatic parsing of thumbnailState * #44449: lint fixes * #44449: crawler as a background service v0.1 * #44449: use ServerLockService * #44449: use ServerLockService * #44449: review fix: prefer `WithDbSession` over `WithTransactionalDbSession` * #44449: review fix: add a comment explaining source of the filepath * #44449: review fix: added filepath validation * #44449: fix FindDashboardsWithStaleThumbnails to include `theme` and `kind` in search params * #44449: fix FindDashboardsWithStaleThumbnails to include `theme` and `kind` in search params * #44449: create function for crawler on demand * #44449: improve crawler logging * #44449: fix wire * #44449: uncomment dummy thumb service, fix ticker interval * #44449: prevent race condition * #44449: improve logging * #44449: fix theme * #44449: review fixes https://github.com/grafana/grafana/pull/45063/files @fzambia * #44449: add missing unlock * #44449: merge * #44449: review fix - logger @fzambia https://github.com/grafana/grafana/pull/45063/files * #44449: formatting * #44449: merge conflict fix * #44449: merge conflict fix * #44449: merge conflict fix * #44449: naming fix * #44449: update authOpts * #44449: change authOpts.role back to admin * #44449: fix `walk` signature, move ctx to a first argument * #44449: add `dashboardPreviewsScheduler` feature flag Co-authored-by: Ryan McKinley <ryantxu@gmail.com> Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
This commit is contained in:
Artur Wierzbicki
co-authored by
Ryan McKinley
Alexander Emelin
Ryan McKinley
Alexander Emelin
parent
586b89f776
commit
0276b029fc
@@ -84,7 +84,7 @@ func (ss *SQLStore) UpdateThumbnailState(ctx context.Context, cmd *models.Update
|
||||
func (ss *SQLStore) FindDashboardsWithStaleThumbnails(ctx context.Context, cmd *models.FindDashboardsWithStaleThumbnailsCommand) ([]*models.DashboardWithStaleThumbnail, error) {
|
||||
err := ss.WithDbSession(ctx, func(sess *DBSession) error {
|
||||
sess.Table("dashboard")
|
||||
sess.Join("LEFT", "dashboard_thumbnail", "dashboard.id = dashboard_thumbnail.dashboard_id")
|
||||
sess.Join("LEFT", "dashboard_thumbnail", "dashboard.id = dashboard_thumbnail.dashboard_id AND dashboard_thumbnail.theme = ? AND dashboard_thumbnail.kind = ?", cmd.Theme, cmd.Kind)
|
||||
sess.Where("dashboard.is_folder = ?", dialect.BooleanStr(false))
|
||||
sess.Where("(dashboard.version != dashboard_thumbnail.dashboard_version "+
|
||||
"OR dashboard_thumbnail.state = ? "+
|
||||
|
||||
@@ -11,6 +11,9 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var theme = models.ThemeDark
|
||||
var kind = models.ThumbnailKindDefault
|
||||
|
||||
func TestSqlStorage(t *testing.T) {
|
||||
|
||||
var sqlStore *SQLStore
|
||||
@@ -52,7 +55,10 @@ func TestSqlStorage(t *testing.T) {
|
||||
|
||||
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
|
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
}
|
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res, 0)
|
||||
@@ -64,7 +70,10 @@ func TestSqlStorage(t *testing.T) {
|
||||
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
|
||||
updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateStale)
|
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
}
|
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res, 1)
|
||||
@@ -78,7 +87,10 @@ func TestSqlStorage(t *testing.T) {
|
||||
updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateStale)
|
||||
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
|
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
}
|
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res, 0)
|
||||
@@ -88,7 +100,10 @@ func TestSqlStorage(t *testing.T) {
|
||||
setup()
|
||||
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
|
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
}
|
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res, 1)
|
||||
@@ -104,7 +119,10 @@ func TestSqlStorage(t *testing.T) {
|
||||
"tags": "different-tag",
|
||||
})
|
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
}
|
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res, 1)
|
||||
@@ -121,7 +139,10 @@ func TestSqlStorage(t *testing.T) {
|
||||
"tags": "different-tag",
|
||||
})
|
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
}
|
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res, 0)
|
||||
@@ -136,7 +157,10 @@ func TestSqlStorage(t *testing.T) {
|
||||
"tags": "different-tag",
|
||||
})
|
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
}
|
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res, 0)
|
||||
@@ -152,6 +176,8 @@ func TestSqlStorage(t *testing.T) {
|
||||
})
|
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
IncludeManuallyUploadedThumbnails: true,
|
||||
}
|
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
|
||||
@@ -168,8 +194,8 @@ func getThumbnail(t *testing.T, sqlStore *SQLStore, dashboardUID string, orgId i
|
||||
DashboardUID: dashboardUID,
|
||||
OrgId: orgId,
|
||||
PanelID: 0,
|
||||
Kind: models.ThumbnailKindDefault,
|
||||
Theme: models.ThemeDark,
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -185,8 +211,8 @@ func upsertTestDashboardThumbnail(t *testing.T, sqlStore *SQLStore, dashboardUID
|
||||
DashboardUID: dashboardUID,
|
||||
OrgId: orgId,
|
||||
PanelID: 0,
|
||||
Kind: models.ThumbnailKindDefault,
|
||||
Theme: models.ThemeDark,
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
},
|
||||
DashboardVersion: dashboardVersion,
|
||||
Image: make([]byte, 0),
|
||||
@@ -206,8 +232,8 @@ func updateThumbnailState(t *testing.T, sqlStore *SQLStore, dashboardUID string,
|
||||
DashboardUID: dashboardUID,
|
||||
OrgId: orgId,
|
||||
PanelID: 0,
|
||||
Kind: models.ThumbnailKindDefault,
|
||||
Theme: models.ThemeDark,
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
},
|
||||
State: state,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user