DashboardPreviews: add dashboard previews behind feature flag (#43226)

Co-authored-by: Ashley Harrison <ashley.harrison@grafana.com>
Co-authored-by: Artur Wierzbicki <artur@arturwierzbicki.com>

pkg/services/sqlstore/migrations/migrations.go

@@ -56,9 +56,12 @@ func (*OSSMigrations) AddMigration(mg *Migrator) {
 	ualert.AddTablesMigrations(mg)
 	ualert.AddDashAlertMigration(mg)
 	addLibraryElementsMigrations(mg)
-	if mg.Cfg != nil || mg.Cfg.IsLiveConfigEnabled() {
-		addLiveChannelMigrations(mg)
+	if mg.Cfg != nil {
+		if mg.Cfg.IsLiveConfigEnabled() {
+			addLiveChannelMigrations(mg)
+		}
 	}
+
 	ualert.RerunDashAlertMigration(mg)
 	addSecretsMigration(mg)
 	addKVStoreMigrations(mg)

pkg/services/thumbs/crawler_http.go

@@ -0,0 +1,72 @@
+package thumbs
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+)
+
+type renderHttp struct {
+	crawlerURL string
+	config     crawConfig
+}
+
+func newRenderHttp(crawlerURL string, cfg crawConfig) dashRenderer {
+	return &renderHttp{
+		crawlerURL: crawlerURL,
+		config:     cfg,
+	}
+}
+
+func (r *renderHttp) GetPreview(req *previewRequest) *previewResponse {
+	p := getFilePath(r.config.ScreenshotsFolder, req)
+	if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) {
+		return r.queueRender(p, req)
+	}
+
+	return &previewResponse{
+		Path: p,
+		Code: 200,
+	}
+}
+
+func (r *renderHttp) CrawlerCmd(cfg *crawlCmd) (json.RawMessage, error) {
+	cmd := r.config
+	cmd.crawlCmd = *cfg
+
+	jsonData, err := json.Marshal(cmd)
+	if err != nil {
+		return nil, err
+	}
+	request, err := http.NewRequest("POST", r.crawlerURL, bytes.NewBuffer(jsonData))
+	if err != nil {
+		return nil, err
+	}
+	request.Header.Set("Content-Type", "application/json; charset=UTF-8")
+
+	client := &http.Client{}
+	response, error := client.Do(request)
+	if error != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = response.Body.Close()
+	}()
+
+	return ioutil.ReadAll(response.Body)
+}
+
+func (r *renderHttp) queueRender(p string, req *previewRequest) *previewResponse {
+	go func() {
+		fmt.Printf("todo? queue")
+	}()
+
+	return &previewResponse{
+		Code: 202,
+		Path: p,
+	}
+}

pkg/services/thumbs/dummy.go

@@ -0,0 +1,32 @@
+package thumbs
+
+import (
+	"github.com/grafana/grafana/pkg/api/response"
+	"github.com/grafana/grafana/pkg/models"
+)
+
+// When the feature flag is not enabled we just implement a dummy service
+type dummyService struct{}
+
+func (ds *dummyService) GetImage(c *models.ReqContext) {
+	c.JSON(400, map[string]string{"error": "invalid size"})
+}
+
+func (ds *dummyService) SetImage(c *models.ReqContext) {
+	c.JSON(400, map[string]string{"error": "invalid size"})
+}
+
+func (ds *dummyService) Enabled() bool {
+	return false
+}
+
+func (ds *dummyService) StartCrawler(c *models.ReqContext) response.Response {
+	result := make(map[string]string)
+	result["error"] = "Not enabled"
+	return response.JSON(200, result)
+}
+func (ds *dummyService) StopCrawler(c *models.ReqContext) response.Response {
+	result := make(map[string]string)
+	result["error"] = "Not enabled"
+	return response.JSON(200, result)
+}

pkg/services/thumbs/models.go

@@ -0,0 +1,103 @@
+package thumbs
+
+import "encoding/json"
+
+type PreviewSize string
+
+const (
+	// PreviewSizeThumb is a small 320x240 preview
+	PreviewSizeThumb PreviewSize = "thumb"
+
+	// PreviewSizeLarge is a large image 2000x1500
+	PreviewSizeLarge PreviewSize = "large"
+
+	// PreviewSizeLarge is a large image 512x????
+	PreviewSizeTall PreviewSize = "tall"
+)
+
+// IsKnownSize checks if the value is a standard size
+func (p PreviewSize) IsKnownSize() bool {
+	switch p {
+	case
+		PreviewSizeThumb,
+		PreviewSizeLarge,
+		PreviewSizeTall:
+		return true
+	}
+	return false
+}
+
+func getPreviewSize(str string) (PreviewSize, bool) {
+	switch str {
+	case string(PreviewSizeThumb):
+		return PreviewSizeThumb, true
+	case string(PreviewSizeLarge):
+		return PreviewSizeLarge, true
+	case string(PreviewSizeTall):
+		return PreviewSizeTall, true
+	}
+	return PreviewSizeThumb, false
+}
+
+func getTheme(str string) (string, bool) {
+	switch str {
+	case "light":
+		return str, true
+	case "dark":
+		return str, true
+	}
+	return "dark", false
+}
+
+type previewRequest struct {
+	Kind  string      `json:"kind"`
+	OrgID int64       `json:"orgId"`
+	UID   string      `json:"uid"`
+	Size  PreviewSize `json:"size"`
+	Theme string      `json:"theme"`
+}
+
+type previewResponse struct {
+	Code int    `json:"code"` // 200 | 202
+	Path string `json:"path"` // local file path to serve
+	URL  string `json:"url"`  // redirect to this URL
+}
+
+// export enum CrawlerMode {
+// 	Thumbs = 'thumbs',
+// 	Analytics = 'analytics', // Enterprise only
+// 	Migrate = 'migrate',
+//   }
+
+//   export enum CrawlerAction {
+// 	Run = 'run',
+// 	Stop = 'stop',
+// 	Queue = 'queue', // TODO (later!) move some to the front
+//   }
+
+type crawlCmd struct {
+	Mode        string `json:"mode"`        // thumbs | analytics | migrate
+	Action      string `json:"action"`      // run | stop | queue
+	Theme       string `json:"theme"`       // light | dark
+	User        string `json:"user"`        // :(
+	Password    string `json:"password"`    // :(
+	Concurrency int    `json:"concurrency"` // number of pages to run in parallel
+
+	Path string `json:"path"` // eventually for queue
+}
+
+type crawConfig struct {
+	crawlCmd
+
+	// Sent to the crawler with each command
+	URL               string `json:"url"`
+	ScreenshotsFolder string `json:"screenshotsFolder"`
+}
+
+type dashRenderer interface {
+	// Assumes you have already authenticated as admin
+	GetPreview(req *previewRequest) *previewResponse
+
+	// Assumes you have already authenticated as admin
+	CrawlerCmd(cfg *crawlCmd) (json.RawMessage, error)
+}

pkg/services/thumbs/service.go

@@ -0,0 +1,268 @@
+package thumbs
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+
+	"github.com/grafana/grafana/pkg/api/response"
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/guardian"
+	"github.com/grafana/grafana/pkg/services/rendering"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/grafana/grafana/pkg/web"
+	"github.com/segmentio/encoding/json"
+)
+
+var (
+	tlog log.Logger = log.New("thumbnails")
+)
+
+type Service interface {
+	Enabled() bool
+	GetImage(c *models.ReqContext)
+	SetImage(c *models.ReqContext)
+
+	// Must be admin
+	StartCrawler(c *models.ReqContext) response.Response
+	StopCrawler(c *models.ReqContext) response.Response
+}
+
+func ProvideService(cfg *setting.Cfg, renderService rendering.Service) Service {
+	if !cfg.IsDashboardPreviesEnabled() {
+		return &dummyService{}
+	}
+
+	root := filepath.Join(cfg.DataPath, "crawler", "preview")
+	url := strings.TrimSuffix(cfg.RendererUrl, "/render") + "/scan"
+
+	renderer := newRenderHttp(url, crawConfig{
+		URL:               strings.TrimSuffix(cfg.RendererCallbackUrl, "/"),
+		ScreenshotsFolder: root,
+	})
+
+	tempdir := filepath.Join(cfg.DataPath, "temp")
+	_ = os.MkdirAll(tempdir, 0700)
+
+	return &thumbService{
+		renderer: renderer,
+		root:     root,
+		tempdir:  tempdir,
+	}
+}
+
+type thumbService struct {
+	renderer dashRenderer
+	root     string
+	tempdir  string
+}
+
+func (hs *thumbService) Enabled() bool {
+	return true
+}
+
+func (hs *thumbService) parseImageReq(c *models.ReqContext, checkSave bool) *previewRequest {
+	params := web.Params(c.Req)
+
+	size, ok := getPreviewSize(params[":size"])
+	if !ok {
+		c.JSON(400, map[string]string{"error": "invalid size"})
+		return nil
+	}
+
+	theme, ok := getTheme(params[":theme"])
+	if !ok {
+		c.JSON(400, map[string]string{"error": "invalid theme"})
+		return nil
+	}
+
+	req := &previewRequest{
+		Kind:  "dash",
+		OrgID: c.OrgId,
+		UID:   params[":uid"],
+		Theme: theme,
+		Size:  size,
+	}
+
+	if len(req.UID) < 1 {
+		c.JSON(400, map[string]string{"error": "missing UID"})
+		return nil
+	}
+
+	// Check permissions and status
+	status := hs.getStatus(c, req.UID, checkSave)
+	if status != 200 {
+		c.JSON(status, map[string]string{"error": fmt.Sprintf("code: %d", status)})
+		return nil
+	}
+	return req
+}
+
+func (hs *thumbService) GetImage(c *models.ReqContext) {
+	req := hs.parseImageReq(c, false)
+	if req == nil {
+		return // already returned value
+	}
+
+	rsp := hs.renderer.GetPreview(req)
+	if rsp.Code == 200 {
+		if rsp.Path != "" {
+			if strings.HasSuffix(rsp.Path, ".webp") {
+				c.Resp.Header().Set("Content-Type", "image/webp")
+			} else if strings.HasSuffix(rsp.Path, ".png") {
+				c.Resp.Header().Set("Content-Type", "image/png")
+			}
+			c.Resp.Header().Set("Content-Type", "image/png")
+			http.ServeFile(c.Resp, c.Req, rsp.Path)
+			return
+		}
+		if rsp.URL != "" {
+			// todo redirect
+			fmt.Printf("TODO redirect: %s\n", rsp.URL)
+		}
+	}
+
+	if rsp.Code == 202 {
+		c.JSON(202, map[string]string{"path": rsp.Path, "todo": "queue processing"})
+		return
+	}
+
+	c.JSON(500, map[string]string{"path": rsp.Path, "error": "unknown!"})
+}
+
+func (hs *thumbService) SetImage(c *models.ReqContext) {
+	req := hs.parseImageReq(c, false)
+	if req == nil {
+		return // already returned value
+	}
+
+	r := c.Req
+
+	// Parse our multipart form, 10 << 20 specifies a maximum
+	// upload of 10 MB files.
+	err := r.ParseMultipartForm(10 << 20)
+	if err != nil {
+		c.JSON(400, map[string]string{"error": "invalid upload size"})
+		return
+	}
+
+	// FormFile returns the first file for the given key `myFile`
+	// it also returns the FileHeader so we can get the Filename,
+	// the Header and the size of the file
+	file, handler, err := r.FormFile("file")
+	if err != nil {
+		c.JSON(400, map[string]string{"error": "missing multi-part form field named 'file'"})
+		fmt.Println("error", err)
+		return
+	}
+	defer func() {
+		_ = file.Close()
+	}()
+	tlog.Info("Uploaded File: %+v\n", handler.Filename)
+	tlog.Info("File Size: %+v\n", handler.Size)
+	tlog.Info("MIME Header: %+v\n", handler.Header)
+
+	// Create a temporary file within our temp-images directory that follows
+	// a particular naming pattern
+	tempFile, err := ioutil.TempFile(hs.tempdir, "upload-*")
+	if err != nil {
+		c.JSON(400, map[string]string{"error": "error creating temp file"})
+		fmt.Println("error", err)
+		tlog.Info("ERROR", "err", handler.Header)
+		return
+	}
+	defer func() {
+		_ = tempFile.Close()
+	}()
+
+	// read all of the contents of our uploaded file into a
+	// byte array
+	fileBytes, err := ioutil.ReadAll(file)
+	if err != nil {
+		fmt.Println(err)
+	}
+	// write this byte array to our temporary file
+	_, err = tempFile.Write(fileBytes)
+	if err != nil {
+		c.JSON(400, map[string]string{"error": "error writing file"})
+		fmt.Println("error", err)
+
+		return
+	}
+
+	p := getFilePath(hs.root, req)
+	err = os.Rename(tempFile.Name(), p)
+	if err != nil {
+		c.JSON(400, map[string]string{"error": "unable to rename file"})
+		return
+	}
+
+	c.JSON(200, map[string]int{"OK": len(fileBytes)})
+}
+
+func (hs *thumbService) StartCrawler(c *models.ReqContext) response.Response {
+	body, err := io.ReadAll(c.Req.Body)
+	if err != nil {
+		return response.Error(500, "error reading bytes", err)
+	}
+	cmd := &crawlCmd{}
+	err = json.Unmarshal(body, cmd)
+	if err != nil {
+		return response.Error(500, "error parsing bytes", err)
+	}
+	cmd.Action = "start"
+
+	msg, err := hs.renderer.CrawlerCmd(cmd)
+	if err != nil {
+		return response.Error(500, "error starting", err)
+	}
+
+	header := make(http.Header)
+	header.Set("Content-Type", "application/json")
+	return response.CreateNormalResponse(header, msg, 200)
+}
+
+func (hs *thumbService) StopCrawler(c *models.ReqContext) response.Response {
+	_, err := hs.renderer.CrawlerCmd(&crawlCmd{
+		Action: "stop",
+	})
+	if err != nil {
+		return response.Error(500, "error stopping crawler", err)
+	}
+
+	result := make(map[string]string)
+	result["message"] = "Stopping..."
+	return response.JSON(200, result)
+}
+
+// Ideally this service would not require first looking up the full dashboard just to bet the id!
+func (hs *thumbService) getStatus(c *models.ReqContext, uid string, checkSave bool) int {
+	query := models.GetDashboardQuery{Uid: uid, OrgId: c.OrgId}
+
+	if err := bus.DispatchCtx(c.Req.Context(), &query); err != nil {
+		return 404 // not found
+	}
+
+	dash := query.Result
+
+	guardian := guardian.New(c.Req.Context(), dash.Id, c.OrgId, c.SignedInUser)
+	if checkSave {
+		if canSave, err := guardian.CanSave(); err != nil || !canSave {
+			return 403 // forbidden
+		}
+		return 200
+	}
+
+	if canView, err := guardian.CanView(); err != nil || !canView {
+		return 403 // forbidden
+	}
+
+	return 200 // found and OK
+}

pkg/services/thumbs/utils.go

@@ -0,0 +1,14 @@
+package thumbs
+
+import (
+	"fmt"
+	"path/filepath"
+)
+
+func getFilePath(root string, req *previewRequest) string {
+	ext := "webp"
+	if req.Size != PreviewSizeThumb {
+		ext = "png"
+	}
+	return filepath.Join(root, fmt.Sprintf("%s-%s-%s.%s", req.UID, req.Size, req.Theme, ext))
+}