mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Previews: crawler as a background service (#44891)
* add SQL migrations * dashboard previews from sql: poc * added todos * refactor: use the same enums where possible * use useEffect, always return json * added todo * refactor + delete files after use * refactor + fix manual thumbnail upload * refactor: move all interactions with sqlStore to thumbnail repo * refactor: remove file operations in thumb crawler/service * refactor: fix dashboard_thumbs sql store * refactor: extracted thumbnail fetching/updating to a hook * refactor: store thumbnails in redux store * refactor: store thumbnails in redux store * refactor: private'd repo methods * removed redux storage, saving images as blobs * allow for configurable rendering timeouts * added 1) query for dashboards with stale thumbnails, 2) command for marking thumbnails as stale * use sql-based queue in crawler * ui for marking thumbnails as stale * replaced `stale` boolean prop with `state` enum * introduce rendering session * compilation errors * fix crawler stop button * rename thumbnail state frozen to locked * #44449: fix merge conflicts * #44449: remove thumb methods from `Store` interface * #44449: clean filepath, defer file closing * #44449: fix rendering.Theme cyclic import * #44449: linting * #44449: linting * #44449: mutex'd crawlerStatus access * #44449: added integration tests for `sqlstore.dashboard_thumbs` * #44449: added comments to explain the `ThumbnailState` enum * #44449: use os.ReadFile rather then os.Open * #44449: always enable dashboardPreviews feature during integration tests * #44449: remove sleep time, adjust number of threads * #44449: review fix: add `orgId` to `DashboardThumbnailMeta` * #44449: review fix: automatic parsing of thumbnailState * #44449: lint fixes * #44449: crawler as a background service v0.1 * #44449: use ServerLockService * #44449: use ServerLockService * #44449: review fix: prefer `WithDbSession` over `WithTransactionalDbSession` * #44449: review fix: add a comment explaining source of the filepath * #44449: review fix: added filepath validation * #44449: fix FindDashboardsWithStaleThumbnails to include `theme` and `kind` in search params * #44449: fix FindDashboardsWithStaleThumbnails to include `theme` and `kind` in search params * #44449: create function for crawler on demand * #44449: improve crawler logging * #44449: fix wire * #44449: uncomment dummy thumb service, fix ticker interval * #44449: prevent race condition * #44449: improve logging * #44449: fix theme * #44449: review fixes https://github.com/grafana/grafana/pull/45063/files @fzambia * #44449: add missing unlock * #44449: merge * #44449: review fix - logger @fzambia https://github.com/grafana/grafana/pull/45063/files * #44449: formatting * #44449: merge conflict fix * #44449: merge conflict fix * #44449: merge conflict fix * #44449: naming fix * #44449: update authOpts * #44449: change authOpts.role back to admin * #44449: fix `walk` signature, move ctx to a first argument * #44449: add `dashboardPreviewsScheduler` feature flag Co-authored-by: Ryan McKinley <ryantxu@gmail.com> Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
This commit is contained in:
Artur Wierzbicki
GitHub
parent
586b89f776
commit
0276b029fc
@ -22,6 +22,7 @@ export interface FeatureToggles {
|
||||
['service-accounts']?: boolean;
|
||||
database_metrics?: boolean;
|
||||
dashboardPreviews?: boolean;
|
||||
dashboardPreviewsScheduler?: boolean;
|
||||
['live-config']?: boolean;
|
||||
['live-pipeline']?: boolean;
|
||||
['live-service-web-worker']?: boolean;
|
||||
|
||||
@ -119,6 +119,8 @@ type DashboardWithStaleThumbnail struct {
|
||||
|
||||
type FindDashboardsWithStaleThumbnailsCommand struct {
|
||||
IncludeManuallyUploadedThumbnails bool
|
||||
Theme Theme
|
||||
Kind ThumbnailKind
|
||||
Result []*DashboardWithStaleThumbnail
|
||||
}
|
||||
|
||||
|
||||
@ -323,8 +323,8 @@ func GetDashboardUrl(uid string, slug string) string {
|
||||
}
|
||||
|
||||
// GetKioskModeDashboardUrl returns the HTML url for a dashboard in kiosk mode.
|
||||
func GetKioskModeDashboardUrl(uid string, slug string) string {
|
||||
return fmt.Sprintf("%s?kiosk", GetDashboardUrl(uid, slug))
|
||||
func GetKioskModeDashboardUrl(uid string, slug string, theme Theme) string {
|
||||
return fmt.Sprintf("%s?kiosk&theme=%s", GetDashboardUrl(uid, slug), string(theme))
|
||||
}
|
||||
|
||||
// GetFullDashboardUrl returns the full URL for a dashboard.
|
||||
|
||||
@ -22,6 +22,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/rendering"
|
||||
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
|
||||
"github.com/grafana/grafana/pkg/services/serviceaccounts"
|
||||
"github.com/grafana/grafana/pkg/services/thumbs"
|
||||
"github.com/grafana/grafana/pkg/services/updatechecker"
|
||||
)
|
||||
|
||||
@ -32,7 +33,7 @@ func ProvideBackgroundServiceRegistry(
|
||||
provisioning *provisioning.ProvisioningServiceImpl, alerting *alerting.AlertEngine, usageStats *uss.UsageStats,
|
||||
grafanaUpdateChecker *updatechecker.GrafanaService, pluginsUpdateChecker *updatechecker.PluginsService,
|
||||
metrics *metrics.InternalMetricsService, secretsService *secretsManager.SecretsService,
|
||||
remoteCache *remotecache.RemoteCache,
|
||||
remoteCache *remotecache.RemoteCache, thumbnailsService thumbs.Service,
|
||||
// Need to make sure these are initialized, is there a better place to put them?
|
||||
_ *plugindashboards.Service, _ *dashboardsnapshots.Service, _ *pluginsettings.Service,
|
||||
_ *alerting.AlertNotificationService, _ serviceaccounts.Service,
|
||||
@ -55,7 +56,8 @@ func ProvideBackgroundServiceRegistry(
|
||||
usageStats,
|
||||
tracing,
|
||||
remoteCache,
|
||||
secretsService)
|
||||
secretsService,
|
||||
thumbnailsService)
|
||||
}
|
||||
|
||||
// BackgroundServiceRegistry provides background services.
|
||||
|
||||
@ -33,6 +33,11 @@ var (
|
||||
Description: "Create and show thumbnails for dashboard search results",
|
||||
State: FeatureStateAlpha,
|
||||
},
|
||||
{
|
||||
Name: "dashboardPreviewsScheduler",
|
||||
Description: "Schedule automatic updates to dashboard previews",
|
||||
State: FeatureStateAlpha,
|
||||
},
|
||||
{
|
||||
Name: "live-config",
|
||||
Description: "Save grafana live configuration in SQL tables",
|
||||
|
||||
@ -27,6 +27,10 @@ const (
|
||||
// Create and show thumbnails for dashboard search results
|
||||
FlagDashboardPreviews = "dashboardPreviews"
|
||||
|
||||
// FlagDashboardPreviewsScheduler
|
||||
// Schedule automatic updates to dashboard previews
|
||||
FlagDashboardPreviewsScheduler = "dashboardPreviewsScheduler"
|
||||
|
||||
// FlagLiveConfig
|
||||
// Save grafana live configuration in SQL tables
|
||||
FlagLiveConfig = "live-config"
|
||||
|
||||
@ -84,7 +84,7 @@ func (ss *SQLStore) UpdateThumbnailState(ctx context.Context, cmd *models.Update
|
||||
func (ss *SQLStore) FindDashboardsWithStaleThumbnails(ctx context.Context, cmd *models.FindDashboardsWithStaleThumbnailsCommand) ([]*models.DashboardWithStaleThumbnail, error) {
|
||||
err := ss.WithDbSession(ctx, func(sess *DBSession) error {
|
||||
sess.Table("dashboard")
|
||||
sess.Join("LEFT", "dashboard_thumbnail", "dashboard.id = dashboard_thumbnail.dashboard_id")
|
||||
sess.Join("LEFT", "dashboard_thumbnail", "dashboard.id = dashboard_thumbnail.dashboard_id AND dashboard_thumbnail.theme = ? AND dashboard_thumbnail.kind = ?", cmd.Theme, cmd.Kind)
|
||||
sess.Where("dashboard.is_folder = ?", dialect.BooleanStr(false))
|
||||
sess.Where("(dashboard.version != dashboard_thumbnail.dashboard_version "+
|
||||
"OR dashboard_thumbnail.state = ? "+
|
||||
|
||||
@ -11,6 +11,9 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var theme = models.ThemeDark
|
||||
var kind = models.ThumbnailKindDefault
|
||||
|
||||
func TestSqlStorage(t *testing.T) {
|
||||
|
||||
var sqlStore *SQLStore
|
||||
@ -52,7 +55,10 @@ func TestSqlStorage(t *testing.T) {
|
||||
|
||||
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
|
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
}
|
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res, 0)
|
||||
@ -64,7 +70,10 @@ func TestSqlStorage(t *testing.T) {
|
||||
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
|
||||
updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateStale)
|
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
}
|
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res, 1)
|
||||
@ -78,7 +87,10 @@ func TestSqlStorage(t *testing.T) {
|
||||
updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateStale)
|
||||
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
|
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
}
|
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res, 0)
|
||||
@ -88,7 +100,10 @@ func TestSqlStorage(t *testing.T) {
|
||||
setup()
|
||||
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
|
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
}
|
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res, 1)
|
||||
@ -104,7 +119,10 @@ func TestSqlStorage(t *testing.T) {
|
||||
"tags": "different-tag",
|
||||
})
|
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
}
|
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res, 1)
|
||||
@ -121,7 +139,10 @@ func TestSqlStorage(t *testing.T) {
|
||||
"tags": "different-tag",
|
||||
})
|
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
}
|
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res, 0)
|
||||
@ -136,7 +157,10 @@ func TestSqlStorage(t *testing.T) {
|
||||
"tags": "different-tag",
|
||||
})
|
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
}
|
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res, 0)
|
||||
@ -152,6 +176,8 @@ func TestSqlStorage(t *testing.T) {
|
||||
})
|
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
IncludeManuallyUploadedThumbnails: true,
|
||||
}
|
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
|
||||
@ -168,8 +194,8 @@ func getThumbnail(t *testing.T, sqlStore *SQLStore, dashboardUID string, orgId i
|
||||
DashboardUID: dashboardUID,
|
||||
OrgId: orgId,
|
||||
PanelID: 0,
|
||||
Kind: models.ThumbnailKindDefault,
|
||||
Theme: models.ThemeDark,
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
},
|
||||
}
|
||||
|
||||
@ -185,8 +211,8 @@ func upsertTestDashboardThumbnail(t *testing.T, sqlStore *SQLStore, dashboardUID
|
||||
DashboardUID: dashboardUID,
|
||||
OrgId: orgId,
|
||||
PanelID: 0,
|
||||
Kind: models.ThumbnailKindDefault,
|
||||
Theme: models.ThemeDark,
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
},
|
||||
DashboardVersion: dashboardVersion,
|
||||
Image: make([]byte, 0),
|
||||
@ -206,8 +232,8 @@ func updateThumbnailState(t *testing.T, sqlStore *SQLStore, dashboardUID string,
|
||||
DashboardUID: dashboardUID,
|
||||
OrgId: orgId,
|
||||
PanelID: 0,
|
||||
Kind: models.ThumbnailKindDefault,
|
||||
Theme: models.ThemeDark,
|
||||
Kind: kind,
|
||||
Theme: theme,
|
||||
},
|
||||
State: state,
|
||||
}
|
||||
|
||||
@ -8,6 +8,9 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
"github.com/grafana/grafana/pkg/services/live"
|
||||
"github.com/grafana/grafana/pkg/services/rendering"
|
||||
@ -26,6 +29,7 @@ type simpleCrawler struct {
|
||||
statusMutex sync.RWMutex
|
||||
queue []*models.DashboardWithStaleThumbnail
|
||||
queueMutex sync.Mutex
|
||||
log log.Logger
|
||||
renderingSession rendering.Session
|
||||
}
|
||||
|
||||
@ -35,6 +39,7 @@ func newSimpleCrawler(renderService rendering.Service, gl *live.GrafanaLive, rep
|
||||
threadCount: 6,
|
||||
glive: gl,
|
||||
thumbnailRepo: repo,
|
||||
log: log.New("thumbnails_crawler"),
|
||||
status: crawlStatus{
|
||||
State: initializing,
|
||||
Complete: 0,
|
||||
@ -62,63 +67,47 @@ func (r *simpleCrawler) next() *models.DashboardWithStaleThumbnail {
|
||||
func (r *simpleCrawler) broadcastStatus() {
|
||||
s, err := r.Status()
|
||||
if err != nil {
|
||||
tlog.Warn("error reading status")
|
||||
r.log.Warn("Error reading status", "err", err)
|
||||
return
|
||||
}
|
||||
msg, err := json.Marshal(s)
|
||||
if err != nil {
|
||||
tlog.Warn("error making message")
|
||||
r.log.Warn("Error making message", "err", err)
|
||||
return
|
||||
}
|
||||
err = r.glive.Publish(r.opts.OrgID, "grafana/broadcast/crawler", msg)
|
||||
if err != nil {
|
||||
tlog.Warn("error Publish message")
|
||||
r.log.Warn("Error Publish message", "err", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme models.Theme, thumbnailKind models.ThumbnailKind) (crawlStatus, error) {
|
||||
if r.status.State == running {
|
||||
tlog.Info("already running")
|
||||
return r.Status()
|
||||
}
|
||||
|
||||
func (r *simpleCrawler) Run(ctx context.Context, authOpts rendering.AuthOpts, mode CrawlerMode, theme models.Theme, thumbnailKind models.ThumbnailKind) error {
|
||||
r.queueMutex.Lock()
|
||||
defer r.queueMutex.Unlock()
|
||||
if r.IsRunning() {
|
||||
r.queueMutex.Unlock()
|
||||
r.log.Info("Already running")
|
||||
return nil
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
|
||||
ctx := c.Req.Context()
|
||||
items, err := r.thumbnailRepo.findDashboardsWithStaleThumbnails(ctx)
|
||||
items, err := r.thumbnailRepo.findDashboardsWithStaleThumbnails(ctx, theme, thumbnailKind)
|
||||
if err != nil {
|
||||
tlog.Error("error when fetching dashboards with stale thumbnails", "err", err.Error())
|
||||
return crawlStatus{
|
||||
Started: now,
|
||||
Finished: now,
|
||||
Last: now,
|
||||
State: stopped,
|
||||
Complete: 0,
|
||||
}, err
|
||||
r.log.Error("Error when fetching dashboards with stale thumbnails", "err", err.Error())
|
||||
r.queueMutex.Unlock()
|
||||
return err
|
||||
}
|
||||
|
||||
if len(items) == 0 {
|
||||
return crawlStatus{
|
||||
Started: now,
|
||||
Finished: now,
|
||||
Last: now,
|
||||
State: stopped,
|
||||
Complete: 0,
|
||||
}, err
|
||||
r.queueMutex.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
r.mode = mode
|
||||
r.thumbnailKind = thumbnailKind
|
||||
r.opts = rendering.Opts{
|
||||
AuthOpts: rendering.AuthOpts{
|
||||
OrgID: c.OrgId,
|
||||
UserID: c.UserId,
|
||||
OrgRole: c.OrgRole,
|
||||
},
|
||||
AuthOpts: authOpts,
|
||||
TimeoutOpts: rendering.TimeoutOpts{
|
||||
Timeout: 10 * time.Second,
|
||||
RequestTimeoutMultiplier: 3,
|
||||
@ -126,19 +115,14 @@ func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme mode
|
||||
Theme: theme,
|
||||
ConcurrentLimit: 10,
|
||||
}
|
||||
renderingSession, err := r.renderService.CreateRenderingSession(context.Background(), r.opts.AuthOpts, rendering.SessionOpts{
|
||||
renderingSession, err := r.renderService.CreateRenderingSession(ctx, r.opts.AuthOpts, rendering.SessionOpts{
|
||||
Expiry: 5 * time.Minute,
|
||||
RefreshExpiryOnEachRequest: true,
|
||||
})
|
||||
if err != nil {
|
||||
tlog.Error("error when creating rendering session", "err", err.Error())
|
||||
return crawlStatus{
|
||||
Started: now,
|
||||
Finished: now,
|
||||
Last: now,
|
||||
State: stopped,
|
||||
Complete: 0,
|
||||
}, err
|
||||
r.log.Error("Error when creating rendering session", "err", err.Error())
|
||||
r.queueMutex.Unlock()
|
||||
return err
|
||||
}
|
||||
|
||||
r.renderingSession = renderingSession
|
||||
@ -149,14 +133,34 @@ func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme mode
|
||||
Complete: 0,
|
||||
}
|
||||
r.broadcastStatus()
|
||||
r.queueMutex.Unlock()
|
||||
|
||||
tlog.Info("Starting dashboard crawler", "dashboardsToCrawl", len(items))
|
||||
r.log.Info("Starting dashboard crawler", "dashboardsToCrawl", len(items), "mode", string(mode), "theme", string(theme), "kind", string(thumbnailKind))
|
||||
|
||||
group, gCtx := errgroup.WithContext(ctx)
|
||||
// create a pool of workers
|
||||
for i := 0; i < r.threadCount; i++ {
|
||||
go r.walk(ctx)
|
||||
walkerId := i
|
||||
group.Go(func() error {
|
||||
r.walk(gCtx, walkerId)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return r.Status()
|
||||
|
||||
err = group.Wait()
|
||||
if err != nil {
|
||||
r.log.Error("Crawl ended with an error", "err", err)
|
||||
}
|
||||
|
||||
r.crawlFinished()
|
||||
r.broadcastStatus()
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *simpleCrawler) IsRunning() bool {
|
||||
r.statusMutex.Lock()
|
||||
defer r.statusMutex.Unlock()
|
||||
return r.status.State == running
|
||||
}
|
||||
|
||||
func (r *simpleCrawler) Stop() (crawlStatus, error) {
|
||||
@ -200,13 +204,13 @@ func (r *simpleCrawler) newSuccessResult() {
|
||||
r.status.Last = time.Now()
|
||||
}
|
||||
|
||||
func (r *simpleCrawler) walkFinished() {
|
||||
func (r *simpleCrawler) crawlFinished() {
|
||||
r.statusMutex.Lock()
|
||||
defer r.statusMutex.Unlock()
|
||||
|
||||
r.status.State = stopped
|
||||
r.status.Finished = time.Now()
|
||||
tlog.Info("Crawler finished", "startTime", r.status.Started, "endTime", r.status.Finished, "durationInSeconds", int64(time.Since(r.status.Started)/time.Second))
|
||||
r.log.Info("Crawler finished", "startTime", r.status.Started, "endTime", r.status.Finished, "durationInSeconds", int64(time.Since(r.status.Started)/time.Second))
|
||||
}
|
||||
|
||||
func (r *simpleCrawler) shouldWalk() bool {
|
||||
@ -216,7 +220,7 @@ func (r *simpleCrawler) shouldWalk() bool {
|
||||
return r.status.State == running
|
||||
}
|
||||
|
||||
func (r *simpleCrawler) walk(ctx context.Context) {
|
||||
func (r *simpleCrawler) walk(ctx context.Context, id int) {
|
||||
for {
|
||||
if !r.shouldWalk() {
|
||||
break
|
||||
@ -227,10 +231,10 @@ func (r *simpleCrawler) walk(ctx context.Context) {
|
||||
break
|
||||
}
|
||||
|
||||
url := models.GetKioskModeDashboardUrl(item.Uid, item.Slug)
|
||||
tlog.Info("Getting dashboard thumbnail", "dashboardUID", item.Uid, "url", url)
|
||||
url := models.GetKioskModeDashboardUrl(item.Uid, item.Slug, r.opts.Theme)
|
||||
r.log.Info("Getting dashboard thumbnail", "walkerId", id, "dashboardUID", item.Uid, "url", url)
|
||||
|
||||
res, err := r.renderService.Render(context.Background(), rendering.Opts{
|
||||
res, err := r.renderService.Render(ctx, rendering.Opts{
|
||||
Width: 320,
|
||||
Height: 240,
|
||||
Path: strings.TrimPrefix(url, "/"),
|
||||
@ -241,13 +245,13 @@ func (r *simpleCrawler) walk(ctx context.Context) {
|
||||
DeviceScaleFactor: -5, // negative numbers will render larger and then scale down.
|
||||
}, r.renderingSession)
|
||||
if err != nil {
|
||||
tlog.Warn("error getting image", "dashboardUID", item.Uid, "url", url, "err", err)
|
||||
r.log.Warn("Error getting image", "walkerId", id, "dashboardUID", item.Uid, "url", url, "err", err)
|
||||
r.newErrorResult()
|
||||
} else if res.FilePath == "" {
|
||||
tlog.Warn("error getting image... no response", "dashboardUID", item.Uid, "url", url)
|
||||
r.log.Warn("Error getting image... no response", "walkerId", id, "dashboardUID", item.Uid, "url", url)
|
||||
r.newErrorResult()
|
||||
} else if strings.Contains(res.FilePath, "public/img") {
|
||||
tlog.Warn("error getting image... internal result", "dashboardUID", item.Uid, "url", url, "img", res.FilePath)
|
||||
r.log.Warn("Error getting image... internal result", "walkerId", id, "dashboardUID", item.Uid, "url", url, "img", res.FilePath)
|
||||
// rendering service returned a static error image - we should not remove that file
|
||||
r.newErrorResult()
|
||||
} else {
|
||||
@ -255,7 +259,7 @@ func (r *simpleCrawler) walk(ctx context.Context) {
|
||||
defer func() {
|
||||
err := os.Remove(res.FilePath)
|
||||
if err != nil {
|
||||
tlog.Error("failed to remove thumbnail temp file", "dashboardUID", item.Uid, "url", url, "err", err)
|
||||
r.log.Error("Failed to remove thumbnail temp file", "walkerId", id, "dashboardUID", item.Uid, "url", url, "err", err)
|
||||
}
|
||||
}()
|
||||
|
||||
@ -267,10 +271,10 @@ func (r *simpleCrawler) walk(ctx context.Context) {
|
||||
}, item.Version)
|
||||
|
||||
if err != nil {
|
||||
tlog.Warn("error saving image image", "dashboardUID", item.Uid, "url", url, "err", err)
|
||||
r.log.Warn("Error saving image image", "walkerId", id, "dashboardUID", item.Uid, "url", url, "err", err)
|
||||
r.newErrorResult()
|
||||
} else {
|
||||
tlog.Info("saved thumbnail", "dashboardUID", item.Uid, "url", url, "thumbnailId", thumbnailId)
|
||||
r.log.Info("Saved thumbnail", "walkerId", id, "dashboardUID", item.Uid, "url", url, "thumbnailId", thumbnailId)
|
||||
r.newSuccessResult()
|
||||
}
|
||||
}()
|
||||
@ -278,6 +282,5 @@ func (r *simpleCrawler) walk(ctx context.Context) {
|
||||
r.broadcastStatus()
|
||||
}
|
||||
|
||||
r.walkFinished()
|
||||
r.broadcastStatus()
|
||||
r.log.Info("Walker finished", "walkerId", id)
|
||||
}
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
package thumbs
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/grafana/grafana/pkg/api/response"
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
)
|
||||
@ -41,3 +43,7 @@ func (ds *dummyService) CrawlerStatus(c *models.ReqContext) response.Response {
|
||||
result["error"] = "Not enabled"
|
||||
return response.JSON(200, result)
|
||||
}
|
||||
|
||||
func (ds *dummyService) Run(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
"github.com/grafana/grafana/pkg/services/rendering"
|
||||
)
|
||||
|
||||
type CrawlerMode string
|
||||
@ -55,13 +56,15 @@ type crawlStatus struct {
|
||||
type dashRenderer interface {
|
||||
|
||||
// Run Assumes you have already authenticated as admin.
|
||||
Start(c *models.ReqContext, mode CrawlerMode, theme models.Theme, kind models.ThumbnailKind) (crawlStatus, error)
|
||||
Run(ctx context.Context, authOpts rendering.AuthOpts, mode CrawlerMode, theme models.Theme, kind models.ThumbnailKind) error
|
||||
|
||||
// Assumes you have already authenticated as admin.
|
||||
Stop() (crawlStatus, error)
|
||||
|
||||
// Assumes you have already authenticated as admin.
|
||||
Status() (crawlStatus, error)
|
||||
|
||||
IsRunning() bool
|
||||
}
|
||||
|
||||
type thumbnailRepo interface {
|
||||
@ -69,5 +72,5 @@ type thumbnailRepo interface {
|
||||
saveFromFile(ctx context.Context, filePath string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error)
|
||||
saveFromBytes(ctx context.Context, bytes []byte, mimeType string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error)
|
||||
getThumbnail(ctx context.Context, meta models.DashboardThumbnailMeta) (*models.DashboardThumbnail, error)
|
||||
findDashboardsWithStaleThumbnails(ctx context.Context) ([]*models.DashboardWithStaleThumbnail, error)
|
||||
findDashboardsWithStaleThumbnails(ctx context.Context, theme models.Theme, thumbnailKind models.ThumbnailKind) ([]*models.DashboardWithStaleThumbnail, error)
|
||||
}
|
||||
|
||||
@ -7,6 +7,7 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore"
|
||||
)
|
||||
@ -14,12 +15,14 @@ import (
|
||||
func newThumbnailRepo(store *sqlstore.SQLStore) thumbnailRepo {
|
||||
repo := &sqlThumbnailRepository{
|
||||
store: store,
|
||||
log: log.New("thumbnails_repo"),
|
||||
}
|
||||
return repo
|
||||
}
|
||||
|
||||
type sqlThumbnailRepository struct {
|
||||
store *sqlstore.SQLStore
|
||||
log log.Logger
|
||||
}
|
||||
|
||||
func (r *sqlThumbnailRepository) saveFromFile(ctx context.Context, filePath string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error) {
|
||||
@ -28,14 +31,14 @@ func (r *sqlThumbnailRepository) saveFromFile(ctx context.Context, filePath stri
|
||||
// 2. the rendering service, when image-renderer returns a screenshot
|
||||
|
||||
if !filepath.IsAbs(filePath) {
|
||||
tlog.Error("Received relative path", "dashboardUID", meta.DashboardUID, "err", filePath)
|
||||
r.log.Error("Received relative path", "dashboardUID", meta.DashboardUID, "err", filePath)
|
||||
return 0, errors.New("relative paths are not supported")
|
||||
}
|
||||
|
||||
content, err := os.ReadFile(filepath.Clean(filePath))
|
||||
|
||||
if err != nil {
|
||||
tlog.Error("error reading file", "dashboardUID", meta.DashboardUID, "err", err)
|
||||
r.log.Error("error reading file", "dashboardUID", meta.DashboardUID, "err", err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
@ -60,7 +63,7 @@ func (r *sqlThumbnailRepository) saveFromBytes(ctx context.Context, content []by
|
||||
|
||||
_, err := r.store.SaveThumbnail(ctx, cmd)
|
||||
if err != nil {
|
||||
tlog.Error("error saving to the db", "dashboardUID", meta.DashboardUID, "err", err)
|
||||
r.log.Error("error saving to the db", "dashboardUID", meta.DashboardUID, "err", err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
@ -81,8 +84,10 @@ func (r *sqlThumbnailRepository) getThumbnail(ctx context.Context, meta models.D
|
||||
return r.store.GetThumbnail(ctx, query)
|
||||
}
|
||||
|
||||
func (r *sqlThumbnailRepository) findDashboardsWithStaleThumbnails(ctx context.Context) ([]*models.DashboardWithStaleThumbnail, error) {
|
||||
func (r *sqlThumbnailRepository) findDashboardsWithStaleThumbnails(ctx context.Context, theme models.Theme, kind models.ThumbnailKind) ([]*models.DashboardWithStaleThumbnail, error) {
|
||||
return r.store.FindDashboardsWithStaleThumbnails(ctx, &models.FindDashboardsWithStaleThumbnailsCommand{
|
||||
IncludeManuallyUploadedThumbnails: false,
|
||||
Theme: theme,
|
||||
Kind: kind,
|
||||
})
|
||||
}
|
||||
|
||||
@ -1,14 +1,17 @@
|
||||
package thumbs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"time"
|
||||
|
||||
"github.com/grafana/grafana/pkg/api/response"
|
||||
"github.com/grafana/grafana/pkg/bus"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/infra/serverlock"
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/services/guardian"
|
||||
@ -20,11 +23,8 @@ import (
|
||||
"github.com/segmentio/encoding/json"
|
||||
)
|
||||
|
||||
var (
|
||||
tlog log.Logger = log.New("thumbnails")
|
||||
)
|
||||
|
||||
type Service interface {
|
||||
Run(ctx context.Context) error
|
||||
Enabled() bool
|
||||
GetImage(c *models.ReqContext)
|
||||
|
||||
@ -38,25 +38,60 @@ type Service interface {
|
||||
CrawlerStatus(c *models.ReqContext) response.Response
|
||||
}
|
||||
|
||||
func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, renderService rendering.Service, gl *live.GrafanaLive, store *sqlstore.SQLStore) Service {
|
||||
type thumbService struct {
|
||||
scheduleOptions crawlerScheduleOptions
|
||||
renderer dashRenderer
|
||||
thumbnailRepo thumbnailRepo
|
||||
lockService *serverlock.ServerLockService
|
||||
features featuremgmt.FeatureToggles
|
||||
crawlLockServiceActionName string
|
||||
log log.Logger
|
||||
}
|
||||
|
||||
type crawlerScheduleOptions struct {
|
||||
crawlInterval time.Duration
|
||||
tickerInterval time.Duration
|
||||
maxCrawlDuration time.Duration
|
||||
crawlerMode CrawlerMode
|
||||
thumbnailKind models.ThumbnailKind
|
||||
auth rendering.AuthOpts
|
||||
themes []models.Theme
|
||||
}
|
||||
|
||||
func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockService *serverlock.ServerLockService, renderService rendering.Service, gl *live.GrafanaLive, store *sqlstore.SQLStore) Service {
|
||||
if !features.IsEnabled(featuremgmt.FlagDashboardPreviews) {
|
||||
return &dummyService{}
|
||||
}
|
||||
|
||||
thumbnailRepo := newThumbnailRepo(store)
|
||||
|
||||
authOpts := rendering.AuthOpts{
|
||||
OrgID: 0,
|
||||
UserID: 0,
|
||||
OrgRole: models.ROLE_ADMIN,
|
||||
}
|
||||
return &thumbService{
|
||||
renderer: newSimpleCrawler(renderService, gl, thumbnailRepo),
|
||||
thumbnailRepo: thumbnailRepo,
|
||||
renderer: newSimpleCrawler(renderService, gl, thumbnailRepo),
|
||||
thumbnailRepo: thumbnailRepo,
|
||||
features: features,
|
||||
lockService: lockService,
|
||||
crawlLockServiceActionName: "dashboard-crawler",
|
||||
log: log.New("thumbnails_service"),
|
||||
|
||||
scheduleOptions: crawlerScheduleOptions{
|
||||
tickerInterval: time.Hour,
|
||||
crawlInterval: time.Hour * 12,
|
||||
maxCrawlDuration: time.Hour,
|
||||
crawlerMode: CrawlerModeThumbs,
|
||||
thumbnailKind: models.ThumbnailKindDefault,
|
||||
themes: []models.Theme{models.ThemeDark, models.ThemeLight},
|
||||
auth: authOpts,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type thumbService struct {
|
||||
renderer dashRenderer
|
||||
thumbnailRepo thumbnailRepo
|
||||
}
|
||||
|
||||
func (hs *thumbService) Enabled() bool {
|
||||
return true
|
||||
return hs.features.IsEnabled(featuremgmt.FlagDashboardPreviews)
|
||||
}
|
||||
|
||||
func (hs *thumbService) parseImageReq(c *models.ReqContext, checkSave bool) *previewRequest {
|
||||
@ -109,7 +144,7 @@ func (hs *thumbService) UpdateThumbnailState(c *models.ReqContext) {
|
||||
|
||||
err := web.Bind(c.Req, body)
|
||||
if err != nil {
|
||||
tlog.Error("Error parsing update thumbnail state request", "dashboardUid", req.UID, "err", err.Error())
|
||||
hs.log.Error("Error parsing update thumbnail state request", "dashboardUid", req.UID, "err", err.Error())
|
||||
c.JSON(500, map[string]string{"dashboardUID": req.UID, "error": "unknown"})
|
||||
return
|
||||
}
|
||||
@ -122,12 +157,12 @@ func (hs *thumbService) UpdateThumbnailState(c *models.ReqContext) {
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
tlog.Error("Error when trying to update thumbnail state", "dashboardUid", req.UID, "err", err.Error(), "newState", body.State)
|
||||
hs.log.Error("Error when trying to update thumbnail state", "dashboardUid", req.UID, "err", err.Error(), "newState", body.State)
|
||||
c.JSON(500, map[string]string{"dashboardUID": req.UID, "error": "unknown"})
|
||||
return
|
||||
}
|
||||
|
||||
tlog.Info("Updated dashboard thumbnail state", "dashboardUid", req.UID, "theme", req.Theme, "newState", body.State)
|
||||
hs.log.Info("Updated dashboard thumbnail state", "dashboardUid", req.UID, "theme", req.Theme, "newState", body.State)
|
||||
c.JSON(200, map[string]string{"success": "true"})
|
||||
}
|
||||
|
||||
@ -150,14 +185,14 @@ func (hs *thumbService) GetImage(c *models.ReqContext) {
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
tlog.Error("Error when retrieving thumbnail", "dashboardUid", req.UID, "err", err.Error())
|
||||
hs.log.Error("Error when retrieving thumbnail", "dashboardUid", req.UID, "err", err.Error())
|
||||
c.JSON(500, map[string]string{"dashboardUID": req.UID, "error": "unknown"})
|
||||
return
|
||||
}
|
||||
|
||||
c.Resp.Header().Set("Content-Type", res.MimeType)
|
||||
if _, err := c.Resp.Write(res.Image); err != nil {
|
||||
tlog.Error("Error writing to response", "dashboardUid", req.UID, "err", err)
|
||||
hs.log.Error("Error writing to response", "dashboardUid", req.UID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -190,9 +225,9 @@ func (hs *thumbService) SetImage(c *models.ReqContext) {
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
}()
|
||||
tlog.Info("Uploaded File: %+v\n", handler.Filename)
|
||||
tlog.Info("File Size: %+v\n", handler.Size)
|
||||
tlog.Info("MIME Header: %+v\n", handler.Header)
|
||||
hs.log.Info("Uploaded File: %+v\n", handler.Filename)
|
||||
hs.log.Info("File Size: %+v\n", handler.Size)
|
||||
hs.log.Info("MIME Header: %+v\n", handler.Header)
|
||||
|
||||
fileBytes, err := ioutil.ReadAll(file)
|
||||
if err != nil {
|
||||
@ -230,11 +265,19 @@ func (hs *thumbService) StartCrawler(c *models.ReqContext) response.Response {
|
||||
if cmd.Mode == "" {
|
||||
cmd.Mode = CrawlerModeThumbs
|
||||
}
|
||||
msg, err := hs.renderer.Start(c, cmd.Mode, cmd.Theme, models.ThumbnailKindDefault)
|
||||
|
||||
go hs.runOnDemandCrawl(context.Background(), cmd.Theme, cmd.Mode, models.ThumbnailKindDefault, rendering.AuthOpts{
|
||||
OrgID: c.OrgId,
|
||||
UserID: c.UserId,
|
||||
OrgRole: c.OrgRole,
|
||||
})
|
||||
|
||||
status, err := hs.renderer.Status()
|
||||
if err != nil {
|
||||
return response.Error(500, "error starting", err)
|
||||
}
|
||||
return response.JSON(200, msg)
|
||||
|
||||
return response.JSON(200, status)
|
||||
}
|
||||
|
||||
func (hs *thumbService) StopCrawler(c *models.ReqContext) response.Response {
|
||||
@ -284,3 +327,57 @@ func (hs *thumbService) getDashboardId(c *models.ReqContext, uid string) (int64,
|
||||
|
||||
return query.Result.Id, nil
|
||||
}
|
||||
|
||||
func (hs *thumbService) runOnDemandCrawl(parentCtx context.Context, theme models.Theme, mode CrawlerMode, kind models.ThumbnailKind, authOpts rendering.AuthOpts) {
|
||||
crawlerCtx, cancel := context.WithTimeout(parentCtx, hs.scheduleOptions.maxCrawlDuration)
|
||||
defer cancel()
|
||||
|
||||
// wait for at least a minute after the last completed run
|
||||
interval := time.Minute
|
||||
err := hs.lockService.LockAndExecute(crawlerCtx, hs.crawlLockServiceActionName, interval, func(ctx context.Context) {
|
||||
if err := hs.renderer.Run(crawlerCtx, authOpts, mode, theme, kind); err != nil {
|
||||
hs.log.Error("On demand crawl error", "mode", mode, "theme", theme, "kind", kind, "userId", authOpts.UserID, "orgId", authOpts.OrgID, "orgRole", authOpts.OrgRole)
|
||||
}
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
hs.log.Error("On demand crawl lock error", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (hs *thumbService) runScheduledCrawl(parentCtx context.Context) {
|
||||
crawlerCtx, cancel := context.WithTimeout(parentCtx, hs.scheduleOptions.maxCrawlDuration)
|
||||
defer cancel()
|
||||
|
||||
err := hs.lockService.LockAndExecute(crawlerCtx, hs.crawlLockServiceActionName, hs.scheduleOptions.crawlInterval, func(ctx context.Context) {
|
||||
for _, theme := range hs.scheduleOptions.themes {
|
||||
if err := hs.renderer.Run(crawlerCtx, hs.scheduleOptions.auth, hs.scheduleOptions.crawlerMode, theme, hs.scheduleOptions.thumbnailKind); err != nil {
|
||||
hs.log.Error("Scheduled crawl error", "theme", theme, "kind", hs.scheduleOptions.thumbnailKind, "err", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
hs.log.Error("Scheduled crawl lock error", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (hs *thumbService) Run(ctx context.Context) error {
|
||||
if !hs.features.IsEnabled(featuremgmt.FlagDashboardPreviewsScheduler) {
|
||||
return nil
|
||||
}
|
||||
|
||||
gc := time.NewTicker(hs.scheduleOptions.tickerInterval)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-gc.C:
|
||||
go hs.runScheduledCrawl(ctx)
|
||||
case <-ctx.Done():
|
||||
hs.log.Debug("Grafana is shutting down - stopping dashboard crawler")
|
||||
gc.Stop()
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user