diff --git a/conf/defaults.ini b/conf/defaults.ini index 4bb5376c233..024a7066a1c 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -1176,3 +1176,22 @@ default_baselayer_config = # Enable or disable loading other base map layers enable_custom_baselayers = true + +#################################### Dashboard previews ##################################### + +[dashboard_previews.crawler] +# Number of dashboards rendered in parallel. Default is 6. +thread_count = + +# Timeout passed down to the Image Renderer plugin. It is used in two separate places within a single rendering request: +# First during the initial navigation to the dashboard and then when waiting for all the panels to load. Default is 20s. +# This setting should be expressed as a duration. Examples: 10s (seconds), 1m (minutes). +rendering_timeout = + +# Maximum duration of a single crawl. Default is 1h. +# This setting should be expressed as a duration. Examples: 10s (seconds), 1m (minutes). +max_crawl_duration = + +# Minimum interval between two subsequent scheduler runs. Default is 12h. +# This setting should be expressed as a duration. Examples: 10s (seconds), 1m (minutes). +scheduler_interval = diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index e5b4cb3e532..080394dd1a9 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -22,7 +22,6 @@ export interface FeatureToggles { serviceAccounts?: boolean; database_metrics?: boolean; dashboardPreviews?: boolean; - dashboardPreviewsScheduler?: boolean; dashboardPreviewsAdmin?: boolean; ['live-config']?: boolean; ['live-pipeline']?: boolean; diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index 6c18fb2a947..a96c4249061 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -39,14 +39,10 @@ var ( State: FeatureStateAlpha, }, { - Name: "dashboardPreviewsScheduler", - Description: "Schedule automatic updates to dashboard previews", - State: FeatureStateAlpha, - }, - { - Name: "dashboardPreviewsAdmin", - Description: "Manage the dashboard previews crawler process from the UI", - State: FeatureStateAlpha, + Name: "dashboardPreviewsAdmin", + Description: "Manage the dashboard previews crawler process from the UI", + State: FeatureStateAlpha, + RequiresDevMode: true, }, { Name: "live-config", diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index 0d762f26555..31978faca12 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -31,10 +31,6 @@ const ( // Create and show thumbnails for dashboard search results FlagDashboardPreviews = "dashboardPreviews" - // FlagDashboardPreviewsScheduler - // Schedule automatic updates to dashboard previews - FlagDashboardPreviewsScheduler = "dashboardPreviewsScheduler" - // FlagDashboardPreviewsAdmin // Manage the dashboard previews crawler process from the UI FlagDashboardPreviewsAdmin = "dashboardPreviewsAdmin" diff --git a/pkg/services/thumbs/crawler.go b/pkg/services/thumbs/crawler.go index c641455728c..7e65a3590cf 100644 --- a/pkg/services/thumbs/crawler.go +++ b/pkg/services/thumbs/crawler.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/grafana/grafana/pkg/setting" "golang.org/x/sync/errgroup" "github.com/grafana/grafana/pkg/infra/log" @@ -19,8 +20,10 @@ import ( ) type simpleCrawler struct { - renderService rendering.Service - threadCount int + renderService rendering.Service + threadCount int + concurrentLimit int + renderingTimeout time.Duration glive *live.GrafanaLive thumbnailRepo thumbnailRepo @@ -36,13 +39,17 @@ type simpleCrawler struct { renderingSessionByOrgId map[int64]rendering.Session } -func newSimpleCrawler(renderService rendering.Service, gl *live.GrafanaLive, repo thumbnailRepo) dashRenderer { +func newSimpleCrawler(renderService rendering.Service, gl *live.GrafanaLive, repo thumbnailRepo, cfg *setting.Cfg, settings setting.DashboardPreviewsSettings) dashRenderer { + threadCount := int(settings.CrawlThreadCount) c := &simpleCrawler{ - renderService: renderService, - threadCount: 6, - glive: gl, - thumbnailRepo: repo, - log: log.New("thumbnails_crawler"), + // temporarily increases the concurrentLimit from the 'cfg.RendererConcurrentRequestLimit' to 'cfg.RendererConcurrentRequestLimit + crawlerThreadCount' + concurrentLimit: cfg.RendererConcurrentRequestLimit + threadCount, + renderingTimeout: settings.RenderingTimeout, + renderService: renderService, + threadCount: threadCount, + glive: gl, + thumbnailRepo: repo, + log: log.New("thumbnails_crawler"), status: crawlStatus{ State: initializing, Complete: 0, @@ -154,11 +161,11 @@ func (r *simpleCrawler) Run(ctx context.Context, auth CrawlerAuth, mode CrawlerM r.auth = auth r.opts = rendering.Opts{ TimeoutOpts: rendering.TimeoutOpts{ - Timeout: 20 * time.Second, + Timeout: r.renderingTimeout, RequestTimeoutMultiplier: 3, }, Theme: theme, - ConcurrentLimit: 10, + ConcurrentLimit: r.concurrentLimit, } r.renderingSessionByOrgId = make(map[int64]rendering.Session) diff --git a/pkg/services/thumbs/service.go b/pkg/services/thumbs/service.go index 45cba2828aa..72e0ccb450f 100644 --- a/pkg/services/thumbs/service.go +++ b/pkg/services/thumbs/service.go @@ -51,6 +51,8 @@ type thumbService struct { crawlLockServiceActionName string log log.Logger usageStatsService usagestats.Service + canRunCrawler bool + settings setting.DashboardPreviewsSettings } type crawlerScheduleOptions struct { @@ -67,30 +69,34 @@ func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockS if !features.IsEnabled(featuremgmt.FlagDashboardPreviews) { return &dummyService{} } - logger := log.New("thumbnails_service") + logger := log.New("previews_service") + logger.Info("initialized thumb", "settings", cfg.DashboardPreviews) thumbnailRepo := newThumbnailRepo(store) + canRunCrawler := true crawlerAuth, err := authSetupService.Setup(context.Background()) if err != nil { - logger.Error("Failed to setup auth for the dashboard previews crawler", "err", err) - return &dummyService{} + logger.Error("failed to setup auth for the dashboard previews crawler", "err", err) + canRunCrawler = false } t := &thumbService{ usageStatsService: usageStatsService, renderingService: renderService, - renderer: newSimpleCrawler(renderService, gl, thumbnailRepo), + renderer: newSimpleCrawler(renderService, gl, thumbnailRepo, cfg, cfg.DashboardPreviews), thumbnailRepo: thumbnailRepo, store: store, features: features, lockService: lockService, crawlLockServiceActionName: "dashboard-crawler", log: logger, + canRunCrawler: canRunCrawler, + settings: cfg.DashboardPreviews, scheduleOptions: crawlerScheduleOptions{ - tickerInterval: time.Hour, - crawlInterval: time.Hour * 12, - maxCrawlDuration: time.Hour, + tickerInterval: 5 * time.Minute, + crawlInterval: cfg.DashboardPreviews.SchedulerInterval, + maxCrawlDuration: cfg.DashboardPreviews.MaxCrawlDuration, crawlerMode: CrawlerModeThumbs, thumbnailKind: models.ThumbnailKindDefault, themes: []models.Theme{models.ThemeDark, models.ThemeLight}, @@ -401,6 +407,10 @@ func (hs *thumbService) getDashboardId(c *models.ReqContext, uid string) (int64, } func (hs *thumbService) runOnDemandCrawl(parentCtx context.Context, theme models.Theme, mode CrawlerMode, kind models.ThumbnailKind, authOpts rendering.AuthOpts) { + if !hs.canRunCrawler { + return + } + crawlerCtx, cancel := context.WithTimeout(parentCtx, hs.scheduleOptions.maxCrawlDuration) defer cancel() @@ -435,7 +445,7 @@ func (hs *thumbService) runScheduledCrawl(parentCtx context.Context) { } func (hs *thumbService) Run(ctx context.Context) error { - if !hs.features.IsEnabled(featuremgmt.FlagDashboardPreviewsScheduler) { + if !hs.canRunCrawler { return nil } diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 58d695f41fc..43974b76986 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -439,6 +439,8 @@ type Cfg struct { // Query history QueryHistoryEnabled bool + + DashboardPreviews DashboardPreviewsSettings } type CommandLineArgs struct { @@ -1001,6 +1003,8 @@ func (cfg *Cfg) Load(args CommandLineArgs) error { cfg.readDataSourcesSettings() + cfg.DashboardPreviews = readDashboardPreviewsSettings(iniFile) + if VerifyEmailEnabled && !cfg.Smtp.Enabled { cfg.Logger.Warn("require_email_validation is enabled but smtp is disabled") } diff --git a/pkg/setting/setting_dashboard_previews.go b/pkg/setting/setting_dashboard_previews.go new file mode 100644 index 00000000000..f73e046e99f --- /dev/null +++ b/pkg/setting/setting_dashboard_previews.go @@ -0,0 +1,31 @@ +package setting + +import ( + "time" + + "gopkg.in/ini.v1" +) + +type DashboardPreviewsSettings struct { + SchedulerInterval time.Duration + MaxCrawlDuration time.Duration + RenderingTimeout time.Duration + CrawlThreadCount uint32 +} + +func readDashboardPreviewsSettings(iniFile *ini.File) DashboardPreviewsSettings { + maxThreadCount := uint32(20) + + s := DashboardPreviewsSettings{} + + previewsCrawlerSection := iniFile.Section("dashboard_previews.crawler") + s.CrawlThreadCount = uint32(previewsCrawlerSection.Key("thread_count").MustUint(6)) + if s.CrawlThreadCount > maxThreadCount { + s.CrawlThreadCount = maxThreadCount + } + + s.SchedulerInterval = previewsCrawlerSection.Key("scheduler_interval").MustDuration(12 * time.Hour) + s.MaxCrawlDuration = previewsCrawlerSection.Key("max_crawl_duration").MustDuration(1 * time.Hour) + s.RenderingTimeout = previewsCrawlerSection.Key("rendering_timeout").MustDuration(20 * time.Second) + return s +} diff --git a/public/app/features/search/hooks/useShowDashboardPreviews.ts b/public/app/features/search/hooks/useShowDashboardPreviews.ts index ebddd8b607b..80e50f37e7b 100644 --- a/public/app/features/search/hooks/useShowDashboardPreviews.ts +++ b/public/app/features/search/hooks/useShowDashboardPreviews.ts @@ -6,7 +6,7 @@ import { PREVIEWS_LOCAL_STORAGE_KEY } from '../constants'; export const useShowDashboardPreviews = () => { const previewFeatureEnabled = Boolean(config.featureToggles.dashboardPreviews); - const [showPreviews, setShowPreviews] = useLocalStorage(PREVIEWS_LOCAL_STORAGE_KEY, previewFeatureEnabled); + const [showPreviews, setShowPreviews] = useLocalStorage(PREVIEWS_LOCAL_STORAGE_KEY, false); return { showPreviews: Boolean(showPreviews && previewFeatureEnabled), previewFeatureEnabled, setShowPreviews }; };