Previews: refactor (#47728)

* #44449: return standard thumb service even if auth setup fails

* #44449: remove dashboardPreviewsScheduler feature flag

* #44449: externalize dashboardPreviews config

* #44449: disable previews by default

* #44449: rename logger

* #44449: dashboardPreviewsAdmin feature requires dev mode

* #44449: retrigger CII

pkg/services/thumbs/crawler.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/grafana/grafana/pkg/setting"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -19,8 +20,10 @@ import (
 )
 
 type simpleCrawler struct {
-	renderService rendering.Service
-	threadCount   int
+	renderService    rendering.Service
+	threadCount      int
+	concurrentLimit  int
+	renderingTimeout time.Duration
 
 	glive                   *live.GrafanaLive
 	thumbnailRepo           thumbnailRepo
@@ -36,13 +39,17 @@ type simpleCrawler struct {
 	renderingSessionByOrgId map[int64]rendering.Session
 }
 
-func newSimpleCrawler(renderService rendering.Service, gl *live.GrafanaLive, repo thumbnailRepo) dashRenderer {
+func newSimpleCrawler(renderService rendering.Service, gl *live.GrafanaLive, repo thumbnailRepo, cfg *setting.Cfg, settings setting.DashboardPreviewsSettings) dashRenderer {
+	threadCount := int(settings.CrawlThreadCount)
 	c := &simpleCrawler{
-		renderService: renderService,
-		threadCount:   6,
-		glive:         gl,
-		thumbnailRepo: repo,
-		log:           log.New("thumbnails_crawler"),
+		// temporarily increases the concurrentLimit from the 'cfg.RendererConcurrentRequestLimit' to 'cfg.RendererConcurrentRequestLimit + crawlerThreadCount'
+		concurrentLimit:  cfg.RendererConcurrentRequestLimit + threadCount,
+		renderingTimeout: settings.RenderingTimeout,
+		renderService:    renderService,
+		threadCount:      threadCount,
+		glive:            gl,
+		thumbnailRepo:    repo,
+		log:              log.New("thumbnails_crawler"),
 		status: crawlStatus{
 			State:    initializing,
 			Complete: 0,
@@ -154,11 +161,11 @@ func (r *simpleCrawler) Run(ctx context.Context, auth CrawlerAuth, mode CrawlerM
 	r.auth = auth
 	r.opts = rendering.Opts{
 		TimeoutOpts: rendering.TimeoutOpts{
-			Timeout:                  20 * time.Second,
+			Timeout:                  r.renderingTimeout,
 			RequestTimeoutMultiplier: 3,
 		},
 		Theme:           theme,
-		ConcurrentLimit: 10,
+		ConcurrentLimit: r.concurrentLimit,
 	}
 
 	r.renderingSessionByOrgId = make(map[int64]rendering.Session)

pkg/services/thumbs/service.go

@@ -51,6 +51,8 @@ type thumbService struct {
 	crawlLockServiceActionName string
 	log                        log.Logger
 	usageStatsService          usagestats.Service
+	canRunCrawler              bool
+	settings                   setting.DashboardPreviewsSettings
 }
 
 type crawlerScheduleOptions struct {
@@ -67,30 +69,34 @@ func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockS
 	if !features.IsEnabled(featuremgmt.FlagDashboardPreviews) {
 		return &dummyService{}
 	}
-	logger := log.New("thumbnails_service")
+	logger := log.New("previews_service")
 
+	logger.Info("initialized thumb", "settings", cfg.DashboardPreviews)
 	thumbnailRepo := newThumbnailRepo(store)
 
+	canRunCrawler := true
 	crawlerAuth, err := authSetupService.Setup(context.Background())
 	if err != nil {
-		logger.Error("Failed to setup auth for the dashboard previews crawler", "err", err)
-		return &dummyService{}
+		logger.Error("failed to setup auth for the dashboard previews crawler", "err", err)
+		canRunCrawler = false
 	}
 	t := &thumbService{
 		usageStatsService:          usageStatsService,
 		renderingService:           renderService,
-		renderer:                   newSimpleCrawler(renderService, gl, thumbnailRepo),
+		renderer:                   newSimpleCrawler(renderService, gl, thumbnailRepo, cfg, cfg.DashboardPreviews),
 		thumbnailRepo:              thumbnailRepo,
 		store:                      store,
 		features:                   features,
 		lockService:                lockService,
 		crawlLockServiceActionName: "dashboard-crawler",
 		log:                        logger,
+		canRunCrawler:              canRunCrawler,
+		settings:                   cfg.DashboardPreviews,
 
 		scheduleOptions: crawlerScheduleOptions{
-			tickerInterval:   time.Hour,
-			crawlInterval:    time.Hour * 12,
-			maxCrawlDuration: time.Hour,
+			tickerInterval:   5 * time.Minute,
+			crawlInterval:    cfg.DashboardPreviews.SchedulerInterval,
+			maxCrawlDuration: cfg.DashboardPreviews.MaxCrawlDuration,
 			crawlerMode:      CrawlerModeThumbs,
 			thumbnailKind:    models.ThumbnailKindDefault,
 			themes:           []models.Theme{models.ThemeDark, models.ThemeLight},
@@ -401,6 +407,10 @@ func (hs *thumbService) getDashboardId(c *models.ReqContext, uid string) (int64,
 }
 
 func (hs *thumbService) runOnDemandCrawl(parentCtx context.Context, theme models.Theme, mode CrawlerMode, kind models.ThumbnailKind, authOpts rendering.AuthOpts) {
+	if !hs.canRunCrawler {
+		return
+	}
+
 	crawlerCtx, cancel := context.WithTimeout(parentCtx, hs.scheduleOptions.maxCrawlDuration)
 	defer cancel()
 
@@ -435,7 +445,7 @@ func (hs *thumbService) runScheduledCrawl(parentCtx context.Context) {
 }
 
 func (hs *thumbService) Run(ctx context.Context) error {
-	if !hs.features.IsEnabled(featuremgmt.FlagDashboardPreviewsScheduler) {
+	if !hs.canRunCrawler {
 		return nil
 	}