Previews: crawler as a background service (#44891)

* add SQL migrations * dashboard previews from sql: poc * added todos * refactor: use the same enums where possible * use useEffect, always return json * added todo * refactor + delete files after use * refactor + fix manual thumbnail upload * refactor: move all interactions with sqlStore to thumbnail repo * refactor: remove file operations in thumb crawler/service * refactor: fix dashboard_thumbs sql store * refactor: extracted thumbnail fetching/updating to a hook * refactor: store thumbnails in redux store * refactor: store thumbnails in redux store * refactor: private'd repo methods * removed redux storage, saving images as blobs * allow for configurable rendering timeouts * added 1) query for dashboards with stale thumbnails, 2) command for marking thumbnails as stale * use sql-based queue in crawler * ui for marking thumbnails as stale * replaced `stale` boolean prop with `state` enum * introduce rendering session * compilation errors * fix crawler stop button * rename thumbnail state frozen to locked * #44449: fix merge conflicts * #44449: remove thumb methods from `Store` interface * #44449: clean filepath, defer file closing * #44449: fix rendering.Theme cyclic import * #44449: linting * #44449: linting * #44449: mutex'd crawlerStatus access * #44449: added integration tests for `sqlstore.dashboard_thumbs` * #44449: added comments to explain the `ThumbnailState` enum * #44449: use os.ReadFile rather then os.Open * #44449: always enable dashboardPreviews feature during integration tests * #44449: remove sleep time, adjust number of threads * #44449: review fix: add `orgId` to `DashboardThumbnailMeta` * #44449: review fix: automatic parsing of thumbnailState * #44449: lint fixes * #44449: crawler as a background service v0.1 * #44449: use ServerLockService * #44449: use ServerLockService * #44449: review fix: prefer `WithDbSession` over `WithTransactionalDbSession` * #44449: review fix: add a comment explaining source of the filepath * #44449: review fix: added filepath validation * #44449: fix FindDashboardsWithStaleThumbnails to include `theme` and `kind` in search params * #44449: fix FindDashboardsWithStaleThumbnails to include `theme` and `kind` in search params * #44449: create function for crawler on demand * #44449: improve crawler logging * #44449: fix wire * #44449: uncomment dummy thumb service, fix ticker interval * #44449: prevent race condition * #44449: improve logging * #44449: fix theme * #44449: review fixes https://github.com/grafana/grafana/pull/45063/files @fzambia * #44449: add missing unlock * #44449: merge * #44449: review fix - logger @fzambia https://github.com/grafana/grafana/pull/45063/files * #44449: formatting * #44449: merge conflict fix * #44449: merge conflict fix * #44449: merge conflict fix * #44449: naming fix * #44449: update authOpts * #44449: change authOpts.role back to admin * #44449: fix `walk` signature, move ctx to a first argument * #44449: add `dashboardPreviewsScheduler` feature flag Co-authored-by: Ryan McKinley <ryantxu@gmail.com> Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
2026-08-18 17:15:08 -05:00 · 2022-02-10 22:45:00 +04:00
parent 586b89f776
commit 0276b029fc
13 changed files with 259 additions and 105 deletions
@@ -8,6 +8,9 @@ import (
 	"sync"
 	"time"

+	"golang.org/x/sync/errgroup"
+
+	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/live"
 	"github.com/grafana/grafana/pkg/services/rendering"
@@ -26,6 +29,7 @@ type simpleCrawler struct {
 	statusMutex      sync.RWMutex
 	queue            []*models.DashboardWithStaleThumbnail
 	queueMutex       sync.Mutex
+	log              log.Logger
 	renderingSession rendering.Session
 }

@@ -35,6 +39,7 @@ func newSimpleCrawler(renderService rendering.Service, gl *live.GrafanaLive, rep
 		threadCount:   6,
 		glive:         gl,
 		thumbnailRepo: repo,
+		log:           log.New("thumbnails_crawler"),
 		status: crawlStatus{
 			State:    initializing,
 			Complete: 0,
@@ -62,63 +67,47 @@ func (r *simpleCrawler) next() *models.DashboardWithStaleThumbnail {
 func (r *simpleCrawler) broadcastStatus() {
 	s, err := r.Status()
 	if err != nil {
-		tlog.Warn("error reading status")
+		r.log.Warn("Error reading status", "err", err)
 		return
 	}
 	msg, err := json.Marshal(s)
 	if err != nil {
-		tlog.Warn("error making message")
+		r.log.Warn("Error making message", "err", err)
 		return
 	}
 	err = r.glive.Publish(r.opts.OrgID, "grafana/broadcast/crawler", msg)
 	if err != nil {
-		tlog.Warn("error Publish message")
+		r.log.Warn("Error Publish message", "err", err)
 		return
 	}
 }

-func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme models.Theme, thumbnailKind models.ThumbnailKind) (crawlStatus, error) {
-	if r.status.State == running {
-		tlog.Info("already running")
-		return r.Status()
-	}
-
+func (r *simpleCrawler) Run(ctx context.Context, authOpts rendering.AuthOpts, mode CrawlerMode, theme models.Theme, thumbnailKind models.ThumbnailKind) error {
 	r.queueMutex.Lock()
-	defer r.queueMutex.Unlock()
+	if r.IsRunning() {
+		r.queueMutex.Unlock()
+		r.log.Info("Already running")
+		return nil
+	}

 	now := time.Now()

-	ctx := c.Req.Context()
-	items, err := r.thumbnailRepo.findDashboardsWithStaleThumbnails(ctx)
+	items, err := r.thumbnailRepo.findDashboardsWithStaleThumbnails(ctx, theme, thumbnailKind)
 	if err != nil {
-		tlog.Error("error when fetching dashboards with stale thumbnails", "err", err.Error())
-		return crawlStatus{
-			Started:  now,
-			Finished: now,
-			Last:     now,
-			State:    stopped,
-			Complete: 0,
-		}, err
+		r.log.Error("Error when fetching dashboards with stale thumbnails", "err", err.Error())
+		r.queueMutex.Unlock()
+		return err
 	}

 	if len(items) == 0 {
-		return crawlStatus{
-			Started:  now,
-			Finished: now,
-			Last:     now,
-			State:    stopped,
-			Complete: 0,
-		}, err
+		r.queueMutex.Unlock()
+		return nil
 	}

 	r.mode = mode
 	r.thumbnailKind = thumbnailKind
 	r.opts = rendering.Opts{
-		AuthOpts: rendering.AuthOpts{
-			OrgID:   c.OrgId,
-			UserID:  c.UserId,
-			OrgRole: c.OrgRole,
-		},
+		AuthOpts: authOpts,
 		TimeoutOpts: rendering.TimeoutOpts{
 			Timeout:                  10 * time.Second,
 			RequestTimeoutMultiplier: 3,
@@ -126,19 +115,14 @@ func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme mode
 		Theme:           theme,
 		ConcurrentLimit: 10,
 	}
-	renderingSession, err := r.renderService.CreateRenderingSession(context.Background(), r.opts.AuthOpts, rendering.SessionOpts{
+	renderingSession, err := r.renderService.CreateRenderingSession(ctx, r.opts.AuthOpts, rendering.SessionOpts{
 		Expiry:                     5 * time.Minute,
 		RefreshExpiryOnEachRequest: true,
 	})
 	if err != nil {
-		tlog.Error("error when creating rendering session", "err", err.Error())
-		return crawlStatus{
-			Started:  now,
-			Finished: now,
-			Last:     now,
-			State:    stopped,
-			Complete: 0,
-		}, err
+		r.log.Error("Error when creating rendering session", "err", err.Error())
+		r.queueMutex.Unlock()
+		return err
 	}

 	r.renderingSession = renderingSession
@@ -149,14 +133,34 @@ func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme mode
 		Complete: 0,
 	}
 	r.broadcastStatus()
+	r.queueMutex.Unlock()

-	tlog.Info("Starting dashboard crawler", "dashboardsToCrawl", len(items))
+	r.log.Info("Starting dashboard crawler", "dashboardsToCrawl", len(items), "mode", string(mode), "theme", string(theme), "kind", string(thumbnailKind))

+	group, gCtx := errgroup.WithContext(ctx)
 	// create a pool of workers
 	for i := 0; i < r.threadCount; i++ {
-		go r.walk(ctx)
+		walkerId := i
+		group.Go(func() error {
+			r.walk(gCtx, walkerId)
+			return nil
+		})
 	}
-	return r.Status()
+
+	err = group.Wait()
+	if err != nil {
+		r.log.Error("Crawl ended with an error", "err", err)
+	}
+
+	r.crawlFinished()
+	r.broadcastStatus()
+	return err
+}
+
+func (r *simpleCrawler) IsRunning() bool {
+	r.statusMutex.Lock()
+	defer r.statusMutex.Unlock()
+	return r.status.State == running
 }

 func (r *simpleCrawler) Stop() (crawlStatus, error) {
@@ -200,13 +204,13 @@ func (r *simpleCrawler) newSuccessResult() {
 	r.status.Last = time.Now()
 }

-func (r *simpleCrawler) walkFinished() {
+func (r *simpleCrawler) crawlFinished() {
 	r.statusMutex.Lock()
 	defer r.statusMutex.Unlock()

 	r.status.State = stopped
 	r.status.Finished = time.Now()
-	tlog.Info("Crawler finished", "startTime", r.status.Started, "endTime", r.status.Finished, "durationInSeconds", int64(time.Since(r.status.Started)/time.Second))
+	r.log.Info("Crawler finished", "startTime", r.status.Started, "endTime", r.status.Finished, "durationInSeconds", int64(time.Since(r.status.Started)/time.Second))
 }

 func (r *simpleCrawler) shouldWalk() bool {
@@ -216,7 +220,7 @@ func (r *simpleCrawler) shouldWalk() bool {
 	return r.status.State == running
 }

-func (r *simpleCrawler) walk(ctx context.Context) {
+func (r *simpleCrawler) walk(ctx context.Context, id int) {
 	for {
 		if !r.shouldWalk() {
 			break
@@ -227,10 +231,10 @@ func (r *simpleCrawler) walk(ctx context.Context) {
 			break
 		}

-		url := models.GetKioskModeDashboardUrl(item.Uid, item.Slug)
-		tlog.Info("Getting dashboard thumbnail", "dashboardUID", item.Uid, "url", url)
+		url := models.GetKioskModeDashboardUrl(item.Uid, item.Slug, r.opts.Theme)
+		r.log.Info("Getting dashboard thumbnail", "walkerId", id, "dashboardUID", item.Uid, "url", url)

-		res, err := r.renderService.Render(context.Background(), rendering.Opts{
+		res, err := r.renderService.Render(ctx, rendering.Opts{
 			Width:             320,
 			Height:            240,
 			Path:              strings.TrimPrefix(url, "/"),
@@ -241,13 +245,13 @@ func (r *simpleCrawler) walk(ctx context.Context) {
 			DeviceScaleFactor: -5, // negative numbers will render larger and then scale down.
 		}, r.renderingSession)
 		if err != nil {
-			tlog.Warn("error getting image", "dashboardUID", item.Uid, "url", url, "err", err)
+			r.log.Warn("Error getting image", "walkerId", id, "dashboardUID", item.Uid, "url", url, "err", err)
 			r.newErrorResult()
 		} else if res.FilePath == "" {
-			tlog.Warn("error getting image... no response", "dashboardUID", item.Uid, "url", url)
+			r.log.Warn("Error getting image... no response", "walkerId", id, "dashboardUID", item.Uid, "url", url)
 			r.newErrorResult()
 		} else if strings.Contains(res.FilePath, "public/img") {
-			tlog.Warn("error getting image... internal result", "dashboardUID", item.Uid, "url", url, "img", res.FilePath)
+			r.log.Warn("Error getting image... internal result", "walkerId", id, "dashboardUID", item.Uid, "url", url, "img", res.FilePath)
 			// rendering service returned a static error image - we should not remove that file
 			r.newErrorResult()
 		} else {
@@ -255,7 +259,7 @@ func (r *simpleCrawler) walk(ctx context.Context) {
 				defer func() {
 					err := os.Remove(res.FilePath)
 					if err != nil {
-						tlog.Error("failed to remove thumbnail temp file", "dashboardUID", item.Uid, "url", url, "err", err)
+						r.log.Error("Failed to remove thumbnail temp file", "walkerId", id, "dashboardUID", item.Uid, "url", url, "err", err)
 					}
 				}()

@@ -267,10 +271,10 @@ func (r *simpleCrawler) walk(ctx context.Context) {
 				}, item.Version)

 				if err != nil {
-					tlog.Warn("error saving image image", "dashboardUID", item.Uid, "url", url, "err", err)
+					r.log.Warn("Error saving image image", "walkerId", id, "dashboardUID", item.Uid, "url", url, "err", err)
 					r.newErrorResult()
 				} else {
-					tlog.Info("saved thumbnail", "dashboardUID", item.Uid, "url", url, "thumbnailId", thumbnailId)
+					r.log.Info("Saved thumbnail", "walkerId", id, "dashboardUID", item.Uid, "url", url, "thumbnailId", thumbnailId)
 					r.newSuccessResult()
 				}
 			}()
@@ -278,6 +282,5 @@ func (r *simpleCrawler) walk(ctx context.Context) {
 		r.broadcastStatus()
 	}

-	r.walkFinished()
-	r.broadcastStatus()
+	r.log.Info("Walker finished", "walkerId", id)
 }
@@ -1,6 +1,8 @@
 package thumbs

 import (
+	"context"
+
 	"github.com/grafana/grafana/pkg/api/response"
 	"github.com/grafana/grafana/pkg/models"
 )
@@ -41,3 +43,7 @@ func (ds *dummyService) CrawlerStatus(c *models.ReqContext) response.Response {
 	result["error"] = "Not enabled"
 	return response.JSON(200, result)
 }
+
+func (ds *dummyService) Run(ctx context.Context) error {
+	return nil
+}
@@ -5,6 +5,7 @@ import (
 	"time"

 	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/rendering"
 )

 type CrawlerMode string
@@ -55,13 +56,15 @@ type crawlStatus struct {
 type dashRenderer interface {

 	// Run Assumes you have already authenticated as admin.
-	Start(c *models.ReqContext, mode CrawlerMode, theme models.Theme, kind models.ThumbnailKind) (crawlStatus, error)
+	Run(ctx context.Context, authOpts rendering.AuthOpts, mode CrawlerMode, theme models.Theme, kind models.ThumbnailKind) error

 	// Assumes you have already authenticated as admin.
 	Stop() (crawlStatus, error)

 	// Assumes you have already authenticated as admin.
 	Status() (crawlStatus, error)
+
+	IsRunning() bool
 }

 type thumbnailRepo interface {
@@ -69,5 +72,5 @@ type thumbnailRepo interface {
 	saveFromFile(ctx context.Context, filePath string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error)
 	saveFromBytes(ctx context.Context, bytes []byte, mimeType string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error)
 	getThumbnail(ctx context.Context, meta models.DashboardThumbnailMeta) (*models.DashboardThumbnail, error)
-	findDashboardsWithStaleThumbnails(ctx context.Context) ([]*models.DashboardWithStaleThumbnail, error)
+	findDashboardsWithStaleThumbnails(ctx context.Context, theme models.Theme, thumbnailKind models.ThumbnailKind) ([]*models.DashboardWithStaleThumbnail, error)
 }
@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"strings"

+	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 )
@@ -14,12 +15,14 @@ import (
 func newThumbnailRepo(store *sqlstore.SQLStore) thumbnailRepo {
 	repo := &sqlThumbnailRepository{
 		store: store,
+		log:   log.New("thumbnails_repo"),
 	}
 	return repo
 }

 type sqlThumbnailRepository struct {
 	store *sqlstore.SQLStore
+	log   log.Logger
 }

 func (r *sqlThumbnailRepository) saveFromFile(ctx context.Context, filePath string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error) {
@@ -28,14 +31,14 @@ func (r *sqlThumbnailRepository) saveFromFile(ctx context.Context, filePath stri
 	//   2. the rendering service, when image-renderer returns a screenshot

 	if !filepath.IsAbs(filePath) {
-		tlog.Error("Received relative path", "dashboardUID", meta.DashboardUID, "err", filePath)
+		r.log.Error("Received relative path", "dashboardUID", meta.DashboardUID, "err", filePath)
 		return 0, errors.New("relative paths are not supported")
 	}

 	content, err := os.ReadFile(filepath.Clean(filePath))

 	if err != nil {
-		tlog.Error("error reading file", "dashboardUID", meta.DashboardUID, "err", err)
+		r.log.Error("error reading file", "dashboardUID", meta.DashboardUID, "err", err)
 		return 0, err
 	}

@@ -60,7 +63,7 @@ func (r *sqlThumbnailRepository) saveFromBytes(ctx context.Context, content []by

 	_, err := r.store.SaveThumbnail(ctx, cmd)
 	if err != nil {
-		tlog.Error("error saving to the db", "dashboardUID", meta.DashboardUID, "err", err)
+		r.log.Error("error saving to the db", "dashboardUID", meta.DashboardUID, "err", err)
 		return 0, err
 	}

@@ -81,8 +84,10 @@ func (r *sqlThumbnailRepository) getThumbnail(ctx context.Context, meta models.D
 	return r.store.GetThumbnail(ctx, query)
 }

-func (r *sqlThumbnailRepository) findDashboardsWithStaleThumbnails(ctx context.Context) ([]*models.DashboardWithStaleThumbnail, error) {
+func (r *sqlThumbnailRepository) findDashboardsWithStaleThumbnails(ctx context.Context, theme models.Theme, kind models.ThumbnailKind) ([]*models.DashboardWithStaleThumbnail, error) {
 	return r.store.FindDashboardsWithStaleThumbnails(ctx, &models.FindDashboardsWithStaleThumbnailsCommand{
 		IncludeManuallyUploadedThumbnails: false,
+		Theme:                             theme,
+		Kind:                              kind,
 	})
 }
@@ -1,14 +1,17 @@
 package thumbs

 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"time"

 	"github.com/grafana/grafana/pkg/api/response"
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/serverlock"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/guardian"
@@ -20,11 +23,8 @@ import (
 	"github.com/segmentio/encoding/json"
 )

-var (
-	tlog log.Logger = log.New("thumbnails")
-)
-
 type Service interface {
+	Run(ctx context.Context) error
 	Enabled() bool
 	GetImage(c *models.ReqContext)

@@ -38,25 +38,60 @@ type Service interface {
 	CrawlerStatus(c *models.ReqContext) response.Response
 }

-func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, renderService rendering.Service, gl *live.GrafanaLive, store *sqlstore.SQLStore) Service {
+type thumbService struct {
+	scheduleOptions            crawlerScheduleOptions
+	renderer                   dashRenderer
+	thumbnailRepo              thumbnailRepo
+	lockService                *serverlock.ServerLockService
+	features                   featuremgmt.FeatureToggles
+	crawlLockServiceActionName string
+	log                        log.Logger
+}
+
+type crawlerScheduleOptions struct {
+	crawlInterval    time.Duration
+	tickerInterval   time.Duration
+	maxCrawlDuration time.Duration
+	crawlerMode      CrawlerMode
+	thumbnailKind    models.ThumbnailKind
+	auth             rendering.AuthOpts
+	themes           []models.Theme
+}
+
+func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockService *serverlock.ServerLockService, renderService rendering.Service, gl *live.GrafanaLive, store *sqlstore.SQLStore) Service {
 	if !features.IsEnabled(featuremgmt.FlagDashboardPreviews) {
 		return &dummyService{}
 	}

 	thumbnailRepo := newThumbnailRepo(store)
+
+	authOpts := rendering.AuthOpts{
+		OrgID:   0,
+		UserID:  0,
+		OrgRole: models.ROLE_ADMIN,
+	}
 	return &thumbService{
-		renderer:      newSimpleCrawler(renderService, gl, thumbnailRepo),
-		thumbnailRepo: thumbnailRepo,
+		renderer:                   newSimpleCrawler(renderService, gl, thumbnailRepo),
+		thumbnailRepo:              thumbnailRepo,
+		features:                   features,
+		lockService:                lockService,
+		crawlLockServiceActionName: "dashboard-crawler",
+		log:                        log.New("thumbnails_service"),
+
+		scheduleOptions: crawlerScheduleOptions{
+			tickerInterval:   time.Hour,
+			crawlInterval:    time.Hour * 12,
+			maxCrawlDuration: time.Hour,
+			crawlerMode:      CrawlerModeThumbs,
+			thumbnailKind:    models.ThumbnailKindDefault,
+			themes:           []models.Theme{models.ThemeDark, models.ThemeLight},
+			auth:             authOpts,
+		},
 	}
 }

-type thumbService struct {
-	renderer      dashRenderer
-	thumbnailRepo thumbnailRepo
-}
-
 func (hs *thumbService) Enabled() bool {
-	return true
+	return hs.features.IsEnabled(featuremgmt.FlagDashboardPreviews)
 }

 func (hs *thumbService) parseImageReq(c *models.ReqContext, checkSave bool) *previewRequest {
@@ -109,7 +144,7 @@ func (hs *thumbService) UpdateThumbnailState(c *models.ReqContext) {

 	err := web.Bind(c.Req, body)
 	if err != nil {
-		tlog.Error("Error parsing update thumbnail state request", "dashboardUid", req.UID, "err", err.Error())
+		hs.log.Error("Error parsing update thumbnail state request", "dashboardUid", req.UID, "err", err.Error())
 		c.JSON(500, map[string]string{"dashboardUID": req.UID, "error": "unknown"})
 		return
 	}
@@ -122,12 +157,12 @@ func (hs *thumbService) UpdateThumbnailState(c *models.ReqContext) {
 	})

 	if err != nil {
-		tlog.Error("Error when trying to update thumbnail state", "dashboardUid", req.UID, "err", err.Error(), "newState", body.State)
+		hs.log.Error("Error when trying to update thumbnail state", "dashboardUid", req.UID, "err", err.Error(), "newState", body.State)
 		c.JSON(500, map[string]string{"dashboardUID": req.UID, "error": "unknown"})
 		return
 	}

-	tlog.Info("Updated dashboard thumbnail state", "dashboardUid", req.UID, "theme", req.Theme, "newState", body.State)
+	hs.log.Info("Updated dashboard thumbnail state", "dashboardUid", req.UID, "theme", req.Theme, "newState", body.State)
 	c.JSON(200, map[string]string{"success": "true"})
 }

@@ -150,14 +185,14 @@ func (hs *thumbService) GetImage(c *models.ReqContext) {
 	}

 	if err != nil {
-		tlog.Error("Error when retrieving thumbnail", "dashboardUid", req.UID, "err", err.Error())
+		hs.log.Error("Error when retrieving thumbnail", "dashboardUid", req.UID, "err", err.Error())
 		c.JSON(500, map[string]string{"dashboardUID": req.UID, "error": "unknown"})
 		return
 	}

 	c.Resp.Header().Set("Content-Type", res.MimeType)
 	if _, err := c.Resp.Write(res.Image); err != nil {
-		tlog.Error("Error writing to response", "dashboardUid", req.UID, "err", err)
+		hs.log.Error("Error writing to response", "dashboardUid", req.UID, "err", err)
 	}
 }

@@ -190,9 +225,9 @@ func (hs *thumbService) SetImage(c *models.ReqContext) {
 	defer func() {
 		_ = file.Close()
 	}()
-	tlog.Info("Uploaded File: %+v\n", handler.Filename)
-	tlog.Info("File Size: %+v\n", handler.Size)
-	tlog.Info("MIME Header: %+v\n", handler.Header)
+	hs.log.Info("Uploaded File: %+v\n", handler.Filename)
+	hs.log.Info("File Size: %+v\n", handler.Size)
+	hs.log.Info("MIME Header: %+v\n", handler.Header)

 	fileBytes, err := ioutil.ReadAll(file)
 	if err != nil {
@@ -230,11 +265,19 @@ func (hs *thumbService) StartCrawler(c *models.ReqContext) response.Response {
 	if cmd.Mode == "" {
 		cmd.Mode = CrawlerModeThumbs
 	}
-	msg, err := hs.renderer.Start(c, cmd.Mode, cmd.Theme, models.ThumbnailKindDefault)
+
+	go hs.runOnDemandCrawl(context.Background(), cmd.Theme, cmd.Mode, models.ThumbnailKindDefault, rendering.AuthOpts{
+		OrgID:   c.OrgId,
+		UserID:  c.UserId,
+		OrgRole: c.OrgRole,
+	})
+
+	status, err := hs.renderer.Status()
 	if err != nil {
 		return response.Error(500, "error starting", err)
 	}
-	return response.JSON(200, msg)
+
+	return response.JSON(200, status)
 }

 func (hs *thumbService) StopCrawler(c *models.ReqContext) response.Response {
@@ -284,3 +327,57 @@ func (hs *thumbService) getDashboardId(c *models.ReqContext, uid string) (int64,

 	return query.Result.Id, nil
 }
+
+func (hs *thumbService) runOnDemandCrawl(parentCtx context.Context, theme models.Theme, mode CrawlerMode, kind models.ThumbnailKind, authOpts rendering.AuthOpts) {
+	crawlerCtx, cancel := context.WithTimeout(parentCtx, hs.scheduleOptions.maxCrawlDuration)
+	defer cancel()
+
+	// wait for at least a minute after the last completed run
+	interval := time.Minute
+	err := hs.lockService.LockAndExecute(crawlerCtx, hs.crawlLockServiceActionName, interval, func(ctx context.Context) {
+		if err := hs.renderer.Run(crawlerCtx, authOpts, mode, theme, kind); err != nil {
+			hs.log.Error("On demand crawl error", "mode", mode, "theme", theme, "kind", kind, "userId", authOpts.UserID, "orgId", authOpts.OrgID, "orgRole", authOpts.OrgRole)
+		}
+	})
+
+	if err != nil {
+		hs.log.Error("On demand crawl lock error", "err", err)
+	}
+}
+
+func (hs *thumbService) runScheduledCrawl(parentCtx context.Context) {
+	crawlerCtx, cancel := context.WithTimeout(parentCtx, hs.scheduleOptions.maxCrawlDuration)
+	defer cancel()
+
+	err := hs.lockService.LockAndExecute(crawlerCtx, hs.crawlLockServiceActionName, hs.scheduleOptions.crawlInterval, func(ctx context.Context) {
+		for _, theme := range hs.scheduleOptions.themes {
+			if err := hs.renderer.Run(crawlerCtx, hs.scheduleOptions.auth, hs.scheduleOptions.crawlerMode, theme, hs.scheduleOptions.thumbnailKind); err != nil {
+				hs.log.Error("Scheduled crawl error", "theme", theme, "kind", hs.scheduleOptions.thumbnailKind, "err", err)
+			}
+		}
+	})
+
+	if err != nil {
+		hs.log.Error("Scheduled crawl lock error", "err", err)
+	}
+}
+
+func (hs *thumbService) Run(ctx context.Context) error {
+	if !hs.features.IsEnabled(featuremgmt.FlagDashboardPreviewsScheduler) {
+		return nil
+	}
+
+	gc := time.NewTicker(hs.scheduleOptions.tickerInterval)
+
+	for {
+		select {
+		case <-gc.C:
+			go hs.runScheduledCrawl(ctx)
+		case <-ctx.Done():
+			hs.log.Debug("Grafana is shutting down - stopping dashboard crawler")
+			gc.Stop()
+
+			return nil
+		}
+	}
+}