Dash previews: populate crawler queue from SQL query (#44083)

* add SQL migrations

* dashboard previews from sql: poc

* added todos

* refactor: use the same enums where possible

* use useEffect, always return json

* added todo

* refactor + delete files after use

* refactor + fix manual thumbnail upload

* refactor: move all interactions with sqlStore to thumbnail repo

* refactor: remove file operations in thumb crawler/service

* refactor: fix dashboard_thumbs sql store

* refactor: extracted thumbnail fetching/updating to a hook

* refactor: store thumbnails in redux store

* refactor: store thumbnails in redux store

* refactor: private'd repo methods

* removed redux storage, saving images as blobs

* allow for configurable rendering timeouts

* added 1) query for dashboards with stale thumbnails, 2) command for marking thumbnails as stale

* use sql-based queue in crawler

* ui for marking thumbnails as stale

* replaced `stale` boolean prop with `state` enum

* introduce rendering session

* compilation errors

* fix crawler stop button

* rename thumbnail state frozen to locked

* #44449: fix merge conflicts

* #44449: remove thumb methods from `Store` interface

* #44449: clean filepath, defer file closing

* #44449: fix rendering.Theme cyclic import

* #44449: linting

* #44449: linting

* #44449: mutex'd crawlerStatus access

* #44449: added integration tests for `sqlstore.dashboard_thumbs`

* #44449: added comments to explain the `ThumbnailState` enum

* #44449: use os.ReadFile rather then os.Open

* #44449: always enable dashboardPreviews feature during integration tests

* #44449: remove sleep time, adjust number of threads

* #44449: review fix: add `orgId` to `DashboardThumbnailMeta`

* #44449: review fix: automatic parsing of thumbnailState

* #44449: lint fixes

* #44449: review fix: prefer `WithDbSession` over `WithTransactionalDbSession`

* #44449: review fix: add a comment explaining source of the filepath

* #44449: review fix: added filepath validation

* #44449: review fixes https://github.com/grafana/grafana/pull/45063/files @fzambia

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
Co-authored-by: Alexander Emelin <frvzmb@gmail.com>

pkg/services/thumbs/crawler.go

@@ -3,66 +3,60 @@ package thumbs
 import (
 	"context"
 	"encoding/json"
-	"errors"
-	"fmt"
-	"math/rand"
 	"os"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/live"
 	"github.com/grafana/grafana/pkg/services/rendering"
-	"github.com/grafana/grafana/pkg/services/search"
 )
 
-type dashItem struct {
-	uid string
-	url string
-}
-
 type simpleCrawler struct {
-	screenshotsFolder string
-	renderService     rendering.Service
-	threadCount       int
+	renderService rendering.Service
+	threadCount   int
 
-	glive  *live.GrafanaLive
-	mode   CrawlerMode
-	opts   rendering.Opts
-	status crawlStatus
-	queue  []dashItem
-	mu     sync.Mutex
+	glive            *live.GrafanaLive
+	thumbnailRepo    thumbnailRepo
+	mode             CrawlerMode
+	thumbnailKind    models.ThumbnailKind
+	opts             rendering.Opts
+	status           crawlStatus
+	statusMutex      sync.RWMutex
+	queue            []*models.DashboardWithStaleThumbnail
+	queueMutex       sync.Mutex
+	renderingSession rendering.Session
 }
 
-func newSimpleCrawler(folder string, renderService rendering.Service, gl *live.GrafanaLive) dashRenderer {
+func newSimpleCrawler(renderService rendering.Service, gl *live.GrafanaLive, repo thumbnailRepo) dashRenderer {
 	c := &simpleCrawler{
-		screenshotsFolder: folder,
-		renderService:     renderService,
-		threadCount:       5,
-		glive:             gl,
+		renderService: renderService,
+		threadCount:   6,
+		glive:         gl,
+		thumbnailRepo: repo,
 		status: crawlStatus{
-			State:    "init",
+			State:    initializing,
 			Complete: 0,
 			Queue:    0,
 		},
-		queue: make([]dashItem, 0),
+		queue: nil,
 	}
 	c.broadcastStatus()
 	return c
 }
 
-func (r *simpleCrawler) next() *dashItem {
-	if len(r.queue) < 1 {
+func (r *simpleCrawler) next() *models.DashboardWithStaleThumbnail {
+	r.queueMutex.Lock()
+	defer r.queueMutex.Unlock()
+
+	if r.queue == nil || len(r.queue) < 1 {
 		return nil
 	}
-	r.mu.Lock()
-	defer r.mu.Unlock()
 
 	v := r.queue[0]
 	r.queue = r.queue[1:]
-	return &v
+	return v
 }
 
 func (r *simpleCrawler) broadcastStatus() {
@@ -83,61 +77,42 @@ func (r *simpleCrawler) broadcastStatus() {
 	}
 }
 
-func (r *simpleCrawler) GetPreview(req *previewRequest) *previewResponse {
-	p := getFilePath(r.screenshotsFolder, req)
-	if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) {
-		return r.queueRender(p, req)
-	}
-
-	return &previewResponse{
-		Path: p,
-		Code: 200,
-	}
-}
-
-func (r *simpleCrawler) queueRender(p string, req *previewRequest) *previewResponse {
-	go func() {
-		fmt.Printf("todo? queue")
-	}()
-
-	return &previewResponse{
-		Code: 202,
-		Path: p,
-	}
-}
-
-func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme rendering.Theme) (crawlStatus, error) {
-	if r.status.State == "running" {
+func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme models.Theme, thumbnailKind models.ThumbnailKind) (crawlStatus, error) {
+	if r.status.State == running {
 		tlog.Info("already running")
 		return r.Status()
 	}
 
-	r.mu.Lock()
-	defer r.mu.Unlock()
+	r.queueMutex.Lock()
+	defer r.queueMutex.Unlock()
 
-	searchQuery := search.Query{
-		SignedInUser: c.SignedInUser,
-		OrgId:        c.OrgId,
-	}
+	now := time.Now()
 
-	err := bus.Dispatch(context.Background(), &searchQuery)
+	ctx := c.Req.Context()
+	items, err := r.thumbnailRepo.findDashboardsWithStaleThumbnails(ctx)
 	if err != nil {
-		return crawlStatus{}, err
+		tlog.Error("error when fetching dashboards with stale thumbnails", "err", err.Error())
+		return crawlStatus{
+			Started:  now,
+			Finished: now,
+			Last:     now,
+			State:    stopped,
+			Complete: 0,
+		}, err
 	}
 
-	queue := make([]dashItem, 0, len(searchQuery.Result))
-	for _, v := range searchQuery.Result {
-		if v.Type == search.DashHitDB {
-			queue = append(queue, dashItem{
-				uid: v.UID,
-				url: v.URL,
-			})
-		}
+	if len(items) == 0 {
+		return crawlStatus{
+			Started:  now,
+			Finished: now,
+			Last:     now,
+			State:    stopped,
+			Complete: 0,
+		}, err
 	}
-	rand.Seed(time.Now().UnixNano())
-	rand.Shuffle(len(queue), func(i, j int) { queue[i], queue[j] = queue[j], queue[i] })
 
 	r.mode = mode
+	r.thumbnailKind = thumbnailKind
 	r.opts = rendering.Opts{
 		AuthOpts: rendering.AuthOpts{
 			OrgID:   c.OrgId,
@@ -151,35 +126,53 @@ func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme rend
 		Theme:           theme,
 		ConcurrentLimit: 10,
 	}
-	r.queue = queue
+	renderingSession, err := r.renderService.CreateRenderingSession(context.Background(), r.opts.AuthOpts, rendering.SessionOpts{
+		Expiry:                     5 * time.Minute,
+		RefreshExpiryOnEachRequest: true,
+	})
+	if err != nil {
+		tlog.Error("error when creating rendering session", "err", err.Error())
+		return crawlStatus{
+			Started:  now,
+			Finished: now,
+			Last:     now,
+			State:    stopped,
+			Complete: 0,
+		}, err
+	}
+
+	r.renderingSession = renderingSession
+	r.queue = items
 	r.status = crawlStatus{
-		Started:  time.Now(),
-		State:    "running",
+		Started:  now,
+		State:    running,
 		Complete: 0,
 	}
 	r.broadcastStatus()
 
+	tlog.Info("Starting dashboard crawler", "dashboardsToCrawl", len(items))
+
 	// create a pool of workers
 	for i := 0; i < r.threadCount; i++ {
-		go r.walk()
-
-		// wait 1/2 second before starting a new thread
-		time.Sleep(500 * time.Millisecond)
+		go r.walk(ctx)
 	}
-
-	r.broadcastStatus()
 	return r.Status()
 }
 
 func (r *simpleCrawler) Stop() (crawlStatus, error) {
-	// cheap hack!
-	if r.status.State == "running" {
-		r.status.State = "stopping"
+	r.statusMutex.Lock()
+	if r.status.State == running {
+		r.status.State = stopping
 	}
+	r.statusMutex.Unlock()
+
 	return r.Status()
 }
 
 func (r *simpleCrawler) Status() (crawlStatus, error) {
+	r.statusMutex.RLock()
+	defer r.statusMutex.RUnlock()
+
 	status := crawlStatus{
 		State:    r.status.State,
 		Started:  r.status.Started,
@@ -191,9 +184,41 @@ func (r *simpleCrawler) Status() (crawlStatus, error) {
 	return status, nil
 }
 
-func (r *simpleCrawler) walk() {
+func (r *simpleCrawler) newErrorResult() {
+	r.statusMutex.Lock()
+	defer r.statusMutex.Unlock()
+
+	r.status.Errors++
+	r.status.Last = time.Now()
+}
+
+func (r *simpleCrawler) newSuccessResult() {
+	r.statusMutex.Lock()
+	defer r.statusMutex.Unlock()
+
+	r.status.Complete++
+	r.status.Last = time.Now()
+}
+
+func (r *simpleCrawler) walkFinished() {
+	r.statusMutex.Lock()
+	defer r.statusMutex.Unlock()
+
+	r.status.State = stopped
+	r.status.Finished = time.Now()
+	tlog.Info("Crawler finished", "startTime", r.status.Started, "endTime", r.status.Finished, "durationInSeconds", int64(time.Since(r.status.Started)/time.Second))
+}
+
+func (r *simpleCrawler) shouldWalk() bool {
+	r.statusMutex.RLock()
+	defer r.statusMutex.RUnlock()
+
+	return r.status.State == running
+}
+
+func (r *simpleCrawler) walk(ctx context.Context) {
 	for {
-		if r.status.State == "stopping" {
+		if !r.shouldWalk() {
 			break
 		}
 
@@ -202,51 +227,57 @@ func (r *simpleCrawler) walk() {
 			break
 		}
 
-		tlog.Info("GET THUMBNAIL", "url", item.url)
+		url := models.GetKioskModeDashboardUrl(item.Uid, item.Slug)
+		tlog.Info("Getting dashboard thumbnail", "dashboardUID", item.Uid, "url", url)
 
-		// Hack (for now) pick a URL that will render
-		panelURL := strings.TrimPrefix(item.url, "/") + "?kiosk"
 		res, err := r.renderService.Render(context.Background(), rendering.Opts{
 			Width:             320,
 			Height:            240,
-			Path:              panelURL,
+			Path:              strings.TrimPrefix(url, "/"),
 			AuthOpts:          r.opts.AuthOpts,
 			TimeoutOpts:       r.opts.TimeoutOpts,
+			ConcurrentLimit:   r.opts.ConcurrentLimit,
 			Theme:             r.opts.Theme,
-			DeviceScaleFactor: -5, // negative numbers will render larger then scale down
-		}, nil)
+			DeviceScaleFactor: -5, // negative numbers will render larger and then scale down.
+		}, r.renderingSession)
 		if err != nil {
-			tlog.Warn("error getting image", "err", err)
-			r.status.Errors++
+			tlog.Warn("error getting image", "dashboardUID", item.Uid, "url", url, "err", err)
+			r.newErrorResult()
 		} else if res.FilePath == "" {
-			tlog.Warn("error getting image... no response")
-			r.status.Errors++
+			tlog.Warn("error getting image... no response", "dashboardUID", item.Uid, "url", url)
+			r.newErrorResult()
 		} else if strings.Contains(res.FilePath, "public/img") {
-			tlog.Warn("error getting image... internal result", "img", res.FilePath)
-			r.status.Errors++
+			tlog.Warn("error getting image... internal result", "dashboardUID", item.Uid, "url", url, "img", res.FilePath)
+			// rendering service returned a static error image - we should not remove that file
+			r.newErrorResult()
 		} else {
-			p := getFilePath(r.screenshotsFolder, &previewRequest{
-				UID:   item.uid,
-				OrgID: r.opts.OrgID,
-				Theme: r.opts.Theme,
-				Size:  PreviewSizeThumb,
-			})
-			err = os.Rename(res.FilePath, p)
-			if err != nil {
-				r.status.Errors++
-				tlog.Warn("error moving image", "err", err)
-			} else {
-				r.status.Complete++
-				tlog.Info("saved thumbnail", "img", item.url)
-			}
-		}
+			func() {
+				defer func() {
+					err := os.Remove(res.FilePath)
+					if err != nil {
+						tlog.Error("failed to remove thumbnail temp file", "dashboardUID", item.Uid, "url", url, "err", err)
+					}
+				}()
 
-		time.Sleep(5 * time.Second)
-		r.status.Last = time.Now()
+				thumbnailId, err := r.thumbnailRepo.saveFromFile(ctx, res.FilePath, models.DashboardThumbnailMeta{
+					DashboardUID: item.Uid,
+					OrgId:        item.OrgId,
+					Theme:        r.opts.Theme,
+					Kind:         r.thumbnailKind,
+				}, item.Version)
+
+				if err != nil {
+					tlog.Warn("error saving image image", "dashboardUID", item.Uid, "url", url, "err", err)
+					r.newErrorResult()
+				} else {
+					tlog.Info("saved thumbnail", "dashboardUID", item.Uid, "url", url, "thumbnailId", thumbnailId)
+					r.newSuccessResult()
+				}
+			}()
+		}
 		r.broadcastStatus()
 	}
 
-	r.status.State = "stopped"
-	r.status.Finished = time.Now()
+	r.walkFinished()
 	r.broadcastStatus()
 }

pkg/services/thumbs/dummy.go

@@ -12,6 +12,10 @@ func (ds *dummyService) GetImage(c *models.ReqContext) {
 	c.JSON(400, map[string]string{"error": "invalid size"})
 }
 
+func (ds *dummyService) UpdateThumbnailState(c *models.ReqContext) {
+	c.JSON(400, map[string]string{"error": "invalid size"})
+}
+
 func (ds *dummyService) SetImage(c *models.ReqContext) {
 	c.JSON(400, map[string]string{"error": "invalid size"})
 }

pkg/services/thumbs/models.go

@@ -1,107 +1,73 @@
 package thumbs
 
 import (
+	"context"
 	"time"
 
 	"github.com/grafana/grafana/pkg/models"
-	"github.com/grafana/grafana/pkg/services/rendering"
 )
 
-type PreviewSize string
 type CrawlerMode string
 
 const (
-	// PreviewSizeThumb is a small 320x240 preview
-	PreviewSizeThumb PreviewSize = "thumb"
 
-	// PreviewSizeLarge is a large image 2000x1500
-	PreviewSizeLarge PreviewSize = "large"
-
-	// PreviewSizeLarge is a large image 512x????
-	PreviewSizeTall PreviewSize = "tall"
-
-	// CrawlerModeThumbs will create small thumbnails for everything
+	// CrawlerModeThumbs will create small thumbnails for everything.
 	CrawlerModeThumbs CrawlerMode = "thumbs"
 
-	// CrawlerModeAnalytics will get full page results for everythign
+	// CrawlerModeAnalytics will get full page results for everything.
 	CrawlerModeAnalytics CrawlerMode = "analytics"
 
-	// CrawlerModeMigrate will migrate all dashboards with old schema
+	// CrawlerModeMigrate will migrate all dashboards with old schema.
 	CrawlerModeMigrate CrawlerMode = "migrate"
 )
 
-// IsKnownSize checks if the value is a standard size
-func (p PreviewSize) IsKnownSize() bool {
-	switch p {
-	case
-		PreviewSizeThumb,
-		PreviewSizeLarge,
-		PreviewSizeTall:
-		return true
-	}
-	return false
-}
+type crawlerState string
 
-func getPreviewSize(str string) (PreviewSize, bool) {
-	switch str {
-	case string(PreviewSizeThumb):
-		return PreviewSizeThumb, true
-	case string(PreviewSizeLarge):
-		return PreviewSizeLarge, true
-	case string(PreviewSizeTall):
-		return PreviewSizeTall, true
-	}
-	return PreviewSizeThumb, false
-}
-
-func getTheme(str string) (rendering.Theme, bool) {
-	switch str {
-	case "light":
-		return rendering.ThemeLight, true
-	case "dark":
-		return rendering.ThemeDark, true
-	}
-	return rendering.ThemeDark, false
-}
+const (
+	initializing crawlerState = "initializing"
+	running      crawlerState = "running"
+	stopping     crawlerState = "stopping"
+	stopped      crawlerState = "stopped"
+)
 
 type previewRequest struct {
-	OrgID int64           `json:"orgId"`
-	UID   string          `json:"uid"`
-	Size  PreviewSize     `json:"size"`
-	Theme rendering.Theme `json:"theme"`
-}
-
-type previewResponse struct {
-	Code int    `json:"code"` // 200 | 202
-	Path string `json:"path"` // local file path to serve
-	URL  string `json:"url"`  // redirect to this URL
+	OrgID int64                `json:"orgId"`
+	UID   string               `json:"uid"`
+	Kind  models.ThumbnailKind `json:"kind"`
+	Theme models.Theme         `json:"theme"`
 }
 
 type crawlCmd struct {
-	Mode  CrawlerMode     `json:"mode"`  // thumbs | analytics | migrate
-	Theme rendering.Theme `json:"theme"` // light | dark
+	Mode  CrawlerMode  `json:"mode"`  // thumbs | analytics | migrate
+	Theme models.Theme `json:"theme"` // light | dark
 }
 
 type crawlStatus struct {
-	State    string    `json:"state"`
-	Started  time.Time `json:"started,omitempty"`
-	Finished time.Time `json:"finished,omitempty"`
-	Complete int       `json:"complete"`
-	Errors   int       `json:"errors"`
-	Queue    int       `json:"queue"`
-	Last     time.Time `json:"last,omitempty"`
+	State    crawlerState `json:"state"`
+	Started  time.Time    `json:"started,omitempty"`
+	Finished time.Time    `json:"finished,omitempty"`
+	Complete int          `json:"complete"`
+	Errors   int          `json:"errors"`
+	Queue    int          `json:"queue"`
+	Last     time.Time    `json:"last,omitempty"`
 }
 
 type dashRenderer interface {
-	// Assumes you have already authenticated as admin
-	GetPreview(req *previewRequest) *previewResponse
 
-	// Assumes you have already authenticated as admin
-	Start(c *models.ReqContext, mode CrawlerMode, theme rendering.Theme) (crawlStatus, error)
+	// Run Assumes you have already authenticated as admin.
+	Start(c *models.ReqContext, mode CrawlerMode, theme models.Theme, kind models.ThumbnailKind) (crawlStatus, error)
 
-	// Assumes you have already authenticated as admin
+	// Assumes you have already authenticated as admin.
 	Stop() (crawlStatus, error)
 
-	// Assumes you have already authenticated as admin
+	// Assumes you have already authenticated as admin.
 	Status() (crawlStatus, error)
 }
+
+type thumbnailRepo interface {
+	updateThumbnailState(ctx context.Context, state models.ThumbnailState, meta models.DashboardThumbnailMeta) error
+	saveFromFile(ctx context.Context, filePath string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error)
+	saveFromBytes(ctx context.Context, bytes []byte, mimeType string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error)
+	getThumbnail(ctx context.Context, meta models.DashboardThumbnailMeta) (*models.DashboardThumbnail, error)
+	findDashboardsWithStaleThumbnails(ctx context.Context) ([]*models.DashboardWithStaleThumbnail, error)
+}

pkg/services/thumbs/repo.go

@@ -0,0 +1,88 @@
+package thumbs
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+)
+
+func newThumbnailRepo(store *sqlstore.SQLStore) thumbnailRepo {
+	repo := &sqlThumbnailRepository{
+		store: store,
+	}
+	return repo
+}
+
+type sqlThumbnailRepository struct {
+	store *sqlstore.SQLStore
+}
+
+func (r *sqlThumbnailRepository) saveFromFile(ctx context.Context, filePath string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error) {
+	// the filePath variable is never set by the user. it refers to a temporary file created either in
+	//   1. thumbs/service.go, when user uploads a thumbnail
+	//   2. the rendering service, when image-renderer returns a screenshot
+
+	if !filepath.IsAbs(filePath) {
+		tlog.Error("Received relative path", "dashboardUID", meta.DashboardUID, "err", filePath)
+		return 0, errors.New("relative paths are not supported")
+	}
+
+	content, err := os.ReadFile(filepath.Clean(filePath))
+
+	if err != nil {
+		tlog.Error("error reading file", "dashboardUID", meta.DashboardUID, "err", err)
+		return 0, err
+	}
+
+	return r.saveFromBytes(ctx, content, getMimeType(filePath), meta, dashboardVersion)
+}
+
+func getMimeType(filePath string) string {
+	if strings.HasSuffix(filePath, ".webp") {
+		return "image/webp"
+	}
+
+	return "image/png"
+}
+
+func (r *sqlThumbnailRepository) saveFromBytes(ctx context.Context, content []byte, mimeType string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error) {
+	cmd := &models.SaveDashboardThumbnailCommand{
+		DashboardThumbnailMeta: meta,
+		Image:                  content,
+		MimeType:               mimeType,
+		DashboardVersion:       dashboardVersion,
+	}
+
+	_, err := r.store.SaveThumbnail(ctx, cmd)
+	if err != nil {
+		tlog.Error("error saving to the db", "dashboardUID", meta.DashboardUID, "err", err)
+		return 0, err
+	}
+
+	return cmd.Result.Id, nil
+}
+
+func (r *sqlThumbnailRepository) updateThumbnailState(ctx context.Context, state models.ThumbnailState, meta models.DashboardThumbnailMeta) error {
+	return r.store.UpdateThumbnailState(ctx, &models.UpdateThumbnailStateCommand{
+		State:                  state,
+		DashboardThumbnailMeta: meta,
+	})
+}
+
+func (r *sqlThumbnailRepository) getThumbnail(ctx context.Context, meta models.DashboardThumbnailMeta) (*models.DashboardThumbnail, error) {
+	query := &models.GetDashboardThumbnailCommand{
+		DashboardThumbnailMeta: meta,
+	}
+	return r.store.GetThumbnail(ctx, query)
+}
+
+func (r *sqlThumbnailRepository) findDashboardsWithStaleThumbnails(ctx context.Context) ([]*models.DashboardWithStaleThumbnail, error) {
+	return r.store.FindDashboardsWithStaleThumbnails(ctx, &models.FindDashboardsWithStaleThumbnailsCommand{
+		IncludeManuallyUploadedThumbnails: false,
+	})
+}

pkg/services/thumbs/service.go

@@ -1,23 +1,20 @@
 package thumbs
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/grafana/grafana/pkg/infra/log"
 
 	"github.com/grafana/grafana/pkg/api/response"
 	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/guardian"
 	"github.com/grafana/grafana/pkg/services/live"
 	"github.com/grafana/grafana/pkg/services/rendering"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/web"
 	"github.com/segmentio/encoding/json"
@@ -31,8 +28,9 @@ type Service interface {
 	Enabled() bool
 	GetImage(c *models.ReqContext)
 
-	// Form post (from dashboard page)
-	SetImage(c *models.ReqContext)
+	// from dashboard page
+	SetImage(c *models.ReqContext) // form post
+	UpdateThumbnailState(c *models.ReqContext)
 
 	// Must be admin
 	StartCrawler(c *models.ReqContext) response.Response
@@ -40,28 +38,21 @@ type Service interface {
 	CrawlerStatus(c *models.ReqContext) response.Response
 }
 
-func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, renderService rendering.Service, gl *live.GrafanaLive) Service {
+func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, renderService rendering.Service, gl *live.GrafanaLive, store *sqlstore.SQLStore) Service {
 	if !features.IsEnabled(featuremgmt.FlagDashboardPreviews) {
 		return &dummyService{}
 	}
 
-	root := filepath.Join(cfg.DataPath, "crawler", "preview")
-	tempdir := filepath.Join(cfg.DataPath, "temp")
-	_ = os.MkdirAll(root, 0700)
-	_ = os.MkdirAll(tempdir, 0700)
-
-	renderer := newSimpleCrawler(root, renderService, gl)
+	thumbnailRepo := newThumbnailRepo(store)
 	return &thumbService{
-		renderer: renderer,
-		root:     root,
-		tempdir:  tempdir,
+		renderer:      newSimpleCrawler(renderService, gl, thumbnailRepo),
+		thumbnailRepo: thumbnailRepo,
 	}
 }
 
 type thumbService struct {
-	renderer dashRenderer
-	root     string
-	tempdir  string
+	renderer      dashRenderer
+	thumbnailRepo thumbnailRepo
 }
 
 func (hs *thumbService) Enabled() bool {
@@ -71,14 +62,14 @@ func (hs *thumbService) Enabled() bool {
 func (hs *thumbService) parseImageReq(c *models.ReqContext, checkSave bool) *previewRequest {
 	params := web.Params(c.Req)
 
-	size, ok := getPreviewSize(params[":size"])
-	if !ok {
+	kind, err := models.ParseThumbnailKind(params[":kind"])
+	if err != nil {
 		c.JSON(400, map[string]string{"error": "invalid size"})
 		return nil
 	}
 
-	theme, ok := getTheme(params[":theme"])
-	if !ok {
+	theme, err := models.ParseTheme(params[":theme"])
+	if err != nil {
 		c.JSON(400, map[string]string{"error": "invalid theme"})
 		return nil
 	}
@@ -87,7 +78,7 @@ func (hs *thumbService) parseImageReq(c *models.ReqContext, checkSave bool) *pre
 		OrgID: c.OrgId,
 		UID:   params[":uid"],
 		Theme: theme,
-		Size:  size,
+		Kind:  kind,
 	}
 
 	if len(req.UID) < 1 {
@@ -104,36 +95,70 @@ func (hs *thumbService) parseImageReq(c *models.ReqContext, checkSave bool) *pre
 	return req
 }
 
+type updateThumbnailStateRequest struct {
+	State models.ThumbnailState `json:"state" binding:"Required"`
+}
+
+func (hs *thumbService) UpdateThumbnailState(c *models.ReqContext) {
+	req := hs.parseImageReq(c, false)
+	if req == nil {
+		return // already returned value
+	}
+
+	var body = &updateThumbnailStateRequest{}
+
+	err := web.Bind(c.Req, body)
+	if err != nil {
+		tlog.Error("Error parsing update thumbnail state request", "dashboardUid", req.UID, "err", err.Error())
+		c.JSON(500, map[string]string{"dashboardUID": req.UID, "error": "unknown"})
+		return
+	}
+
+	err = hs.thumbnailRepo.updateThumbnailState(c.Req.Context(), body.State, models.DashboardThumbnailMeta{
+		DashboardUID: req.UID,
+		OrgId:        req.OrgID,
+		Theme:        req.Theme,
+		Kind:         models.ThumbnailKindDefault,
+	})
+
+	if err != nil {
+		tlog.Error("Error when trying to update thumbnail state", "dashboardUid", req.UID, "err", err.Error(), "newState", body.State)
+		c.JSON(500, map[string]string{"dashboardUID": req.UID, "error": "unknown"})
+		return
+	}
+
+	tlog.Info("Updated dashboard thumbnail state", "dashboardUid", req.UID, "theme", req.Theme, "newState", body.State)
+	c.JSON(200, map[string]string{"success": "true"})
+}
+
 func (hs *thumbService) GetImage(c *models.ReqContext) {
 	req := hs.parseImageReq(c, false)
 	if req == nil {
 		return // already returned value
 	}
 
-	rsp := hs.renderer.GetPreview(req)
-	if rsp.Code == 200 {
-		if rsp.Path != "" {
-			if strings.HasSuffix(rsp.Path, ".webp") {
-				c.Resp.Header().Set("Content-Type", "image/webp")
-			} else if strings.HasSuffix(rsp.Path, ".png") {
-				c.Resp.Header().Set("Content-Type", "image/png")
-			}
-			c.Resp.Header().Set("Content-Type", "image/png")
-			http.ServeFile(c.Resp, c.Req, rsp.Path)
-			return
-		}
-		if rsp.URL != "" {
-			// todo redirect
-			fmt.Printf("TODO redirect: %s\n", rsp.URL)
-		}
-	}
+	res, err := hs.thumbnailRepo.getThumbnail(c.Req.Context(), models.DashboardThumbnailMeta{
+		DashboardUID: req.UID,
+		OrgId:        req.OrgID,
+		Theme:        req.Theme,
+		Kind:         models.ThumbnailKindDefault,
+	})
 
-	if rsp.Code == 202 {
-		c.JSON(202, map[string]string{"path": rsp.Path, "todo": "queue processing"})
+	if errors.Is(err, models.ErrDashboardThumbnailNotFound) {
+		c.Resp.WriteHeader(404)
 		return
 	}
 
-	c.JSON(500, map[string]string{"path": rsp.Path, "error": "unknown!"})
+	if err != nil {
+		tlog.Error("Error when retrieving thumbnail", "dashboardUid", req.UID, "err", err.Error())
+		c.JSON(500, map[string]string{"dashboardUID": req.UID, "error": "unknown"})
+		return
+	}
+
+	c.Resp.Header().Set("Content-Type", res.MimeType)
+	if _, err := c.Resp.Write(res.Image); err != nil {
+		tlog.Error("Error writing to response", "dashboardUid", req.UID, "err", err)
+	}
 }
 
 // Hack for now -- lets you upload images explicitly
@@ -169,38 +194,23 @@ func (hs *thumbService) SetImage(c *models.ReqContext) {
 	tlog.Info("File Size: %+v\n", handler.Size)
 	tlog.Info("MIME Header: %+v\n", handler.Header)
 
-	// Create a temporary file within our temp-images directory that follows
-	// a particular naming pattern
-	tempFile, err := ioutil.TempFile(hs.tempdir, "upload-*")
-	if err != nil {
-		c.JSON(400, map[string]string{"error": "error creating temp file"})
-		fmt.Println("error", err)
-		tlog.Info("ERROR", "err", handler.Header)
-		return
-	}
-	defer func() {
-		_ = tempFile.Close()
-	}()
-
-	// read all of the contents of our uploaded file into a
-	// byte array
 	fileBytes, err := ioutil.ReadAll(file)
 	if err != nil {
 		fmt.Println(err)
-	}
-	// write this byte array to our temporary file
-	_, err = tempFile.Write(fileBytes)
-	if err != nil {
-		c.JSON(400, map[string]string{"error": "error writing file"})
-		fmt.Println("error", err)
-
+		c.JSON(400, map[string]string{"error": "error reading file"})
 		return
 	}
 
-	p := getFilePath(hs.root, req)
-	err = os.Rename(tempFile.Name(), p)
+	_, err = hs.thumbnailRepo.saveFromBytes(c.Req.Context(), fileBytes, getMimeType(handler.Filename), models.DashboardThumbnailMeta{
+		DashboardUID: req.UID,
+		OrgId:        req.OrgID,
+		Theme:        req.Theme,
+		Kind:         req.Kind,
+	}, models.DashboardVersionForManualThumbnailUpload)
+
 	if err != nil {
-		c.JSON(400, map[string]string{"error": "unable to rename file"})
+		c.JSON(400, map[string]string{"error": "error saving thumbnail file"})
+		fmt.Println("error", err)
 		return
 	}
 
@@ -220,7 +230,7 @@ func (hs *thumbService) StartCrawler(c *models.ReqContext) response.Response {
 	if cmd.Mode == "" {
 		cmd.Mode = CrawlerModeThumbs
 	}
-	msg, err := hs.renderer.Start(c, cmd.Mode, cmd.Theme)
+	msg, err := hs.renderer.Start(c, cmd.Mode, cmd.Theme, models.ThumbnailKindDefault)
 	if err != nil {
 		return response.Error(500, "error starting", err)
 	}
@@ -245,15 +255,12 @@ func (hs *thumbService) CrawlerStatus(c *models.ReqContext) response.Response {
 
 // Ideally this service would not require first looking up the full dashboard just to bet the id!
 func (hs *thumbService) getStatus(c *models.ReqContext, uid string, checkSave bool) int {
-	query := models.GetDashboardQuery{Uid: uid, OrgId: c.OrgId}
-
-	if err := bus.Dispatch(c.Req.Context(), &query); err != nil {
-		return 404 // not found
+	dashboardID, err := hs.getDashboardId(c, uid)
+	if err != nil {
+		return 404
 	}
 
-	dash := query.Result
-
-	guardian := guardian.New(c.Req.Context(), dash.Id, c.OrgId, c.SignedInUser)
+	guardian := guardian.New(c.Req.Context(), dashboardID, c.OrgId, c.SignedInUser)
 	if checkSave {
 		if canSave, err := guardian.CanSave(); err != nil || !canSave {
 			return 403 // forbidden
@@ -267,3 +274,13 @@ func (hs *thumbService) getStatus(c *models.ReqContext, uid string, checkSave bo
 
 	return 200 // found and OK
 }
+
+func (hs *thumbService) getDashboardId(c *models.ReqContext, uid string) (int64, error) {
+	query := models.GetDashboardQuery{Uid: uid, OrgId: c.OrgId}
+
+	if err := bus.Dispatch(c.Req.Context(), &query); err != nil {
+		return 0, err
+	}
+
+	return query.Result.Id, nil
+}

pkg/services/thumbs/utils.go

@@ -1,14 +0,0 @@
-package thumbs
-
-import (
-	"fmt"
-	"path/filepath"
-)
-
-func getFilePath(root string, req *previewRequest) string {
-	ext := "webp"
-	if req.Size != PreviewSizeThumb {
-		ext = "png"
-	}
-	return filepath.Join(root, fmt.Sprintf("%s-%s-%s.%s", req.UID, req.Size, req.Theme, ext))
-}