IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Dash previews: populate crawler queue from SQL query (#44083)

Browse Source 
* add SQL migrations

* dashboard previews from sql: poc

* added todos

* refactor: use the same enums where possible

* use useEffect, always return json

* added todo

* refactor + delete files after use

* refactor + fix manual thumbnail upload

* refactor: move all interactions with sqlStore to thumbnail repo

* refactor: remove file operations in thumb crawler/service

* refactor: fix dashboard_thumbs sql store

* refactor: extracted thumbnail fetching/updating to a hook

* refactor: store thumbnails in redux store

* refactor: store thumbnails in redux store

* refactor: private'd repo methods

* removed redux storage, saving images as blobs

* allow for configurable rendering timeouts

* added 1) query for dashboards with stale thumbnails, 2) command for marking thumbnails as stale

* use sql-based queue in crawler

* ui for marking thumbnails as stale

* replaced `stale` boolean prop with `state` enum

* introduce rendering session

* compilation errors

* fix crawler stop button

* rename thumbnail state frozen to locked

* #44449: fix merge conflicts

* #44449: remove thumb methods from `Store` interface

* #44449: clean filepath, defer file closing

* #44449: fix rendering.Theme cyclic import

* #44449: linting

* #44449: linting

* #44449: mutex'd crawlerStatus access

* #44449: added integration tests for `sqlstore.dashboard_thumbs`

* #44449: added comments to explain the `ThumbnailState` enum

* #44449: use os.ReadFile rather then os.Open

* #44449: always enable dashboardPreviews feature during integration tests

* #44449: remove sleep time, adjust number of threads

* #44449: review fix: add `orgId` to `DashboardThumbnailMeta`

* #44449: review fix: automatic parsing of thumbnailState

* #44449: lint fixes

* #44449: review fix: prefer `WithDbSession` over `WithTransactionalDbSession`

* #44449: review fix: add a comment explaining source of the filepath

* #44449: review fix: added filepath validation

* #44449: review fixes https://github.com/grafana/grafana/pull/45063/files @fzambia

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
This commit is contained in:
Artur Wierzbicki 2022-02-09 13:23:32 +04:00 committed by GitHub
parent 4e3a72fc2a
commit a025109647
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 1003 additions and 306 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pkg/api/api.go
View File

@ -335,8 +335,9 @@ func (hs *HTTPServer) registerRoutes() {
dashboardRoute.Delete("/uid/:uid", routing.Wrap(hs.DeleteDashboardByUID))
if hs.ThumbService != nil {
dashboardRoute.Get("/uid/:uid/img/:size/:theme", hs.ThumbService.GetImage)
dashboardRoute.Post("/uid/:uid/img/:size/:theme", hs.ThumbService.SetImage)
dashboardRoute.Get("/uid/:uid/img/:kind/:theme", hs.ThumbService.GetImage)
dashboardRoute.Post("/uid/:uid/img/:kind/:theme", hs.ThumbService.SetImage)
dashboardRoute.Put("/uid/:uid/img/:kind/:theme", hs.ThumbService.UpdateThumbnailState)
}
dashboardRoute.Post("/calculate-diff", routing.Wrap(CalculateDashboardDiff))

2
pkg/api/render.go
View File

@ -69,7 +69,7 @@ func (hs *HTTPServer) RenderToPng(c *models.ReqContext) {
ConcurrentLimit: hs.Cfg.RendererConcurrentRequestLimit,
DeviceScaleFactor: scale,
Headers: headers,
Theme: rendering.ThemeDark,
Theme: models.ThemeDark,
}, nil)
if err != nil {
if errors.Is(err, rendering.ErrTimeout) {

137
pkg/models/dashboard_thumbs.go Normal file
View File

@ -0,0 +1,137 @@
package models
import (
"encoding/json"
"errors"
"fmt"
"time"
)
type ThumbnailKind string
type ThumbnailState string
type CrawlerMode string
const (
// ThumbnailKindDefault is a small 320x240 preview
ThumbnailKindDefault ThumbnailKind = "thumb"
// unsupported for now
// - ThumbnailKindLarge ThumbnailKind = "large"
// - ThumbnailKindTall ThumbnailKind = "tall"
)
const (
// ThumbnailStateDefault is the initial state for all thumbnails. Thumbnails in the "default" state will be considered stale,
// and thus refreshed by the crawler, if the dashboard version from the time of taking the thumbnail is different from the current dashboard version
ThumbnailStateDefault ThumbnailState = "default"
// ThumbnailStateStale is a manually assigned state. Thumbnails in the "stale" state will be refreshed on the next crawler run
ThumbnailStateStale ThumbnailState = "stale"
// ThumbnailStateLocked is a manually assigned state. Thumbnails in the "locked" state will not be refreshed by the crawler as long as they remain in the "locked" state.
ThumbnailStateLocked ThumbnailState = "locked"
)
func (s ThumbnailState) IsValid() bool {
return s == ThumbnailStateDefault || s == ThumbnailStateStale || s == ThumbnailStateLocked
}
func (s *ThumbnailState) UnmarshalJSON(data []byte) error {
var str string
err := json.Unmarshal(data, &str)
if err != nil {
return err
}
*s = ThumbnailState(str)
if !s.IsValid() {
if (*s) != "" {
return fmt.Errorf("JSON validation error: invalid thumbnail state value: %s", *s)
}
*s = ThumbnailStateDefault
}
return nil
}
// IsKnownThumbnailKind checks if the value is supported
func (p ThumbnailKind) IsKnownThumbnailKind() bool {
switch p {
case
ThumbnailKindDefault:
return true
}
return false
}
func ParseThumbnailKind(str string) (ThumbnailKind, error) {
switch str {
case string(ThumbnailKindDefault):
return ThumbnailKindDefault, nil
}
return ThumbnailKindDefault, errors.New("unknown thumbnail kind " + str)
}
// A DashboardThumbnail includes all metadata for a dashboard thumbnail
type DashboardThumbnail struct {
Id int64 `json:"id"`
DashboardId int64 `json:"dashboardId"`
DashboardVersion int `json:"dashboardVersion"`
State ThumbnailState `json:"state"`
PanelId int64 `json:"panelId,omitempty"`
Kind ThumbnailKind `json:"kind"`
Theme Theme `json:"theme"`
Image []byte `json:"image"`
MimeType string `json:"mimeType"`
Updated time.Time `json:"updated"`
}
//
// Commands
//
// DashboardThumbnailMeta uniquely identifies a thumbnail; a natural key
type DashboardThumbnailMeta struct {
DashboardUID string
OrgId int64
PanelID int64
Kind ThumbnailKind
Theme Theme
}
type GetDashboardThumbnailCommand struct {
DashboardThumbnailMeta
Result *DashboardThumbnail
}
const DashboardVersionForManualThumbnailUpload = -1
type DashboardWithStaleThumbnail struct {
Id int64
OrgId int64
Uid string
Version int
Slug string
}
type FindDashboardsWithStaleThumbnailsCommand struct {
IncludeManuallyUploadedThumbnails bool
Result []*DashboardWithStaleThumbnail
}
type SaveDashboardThumbnailCommand struct {
DashboardThumbnailMeta
DashboardVersion int
Image []byte
MimeType string
Result *DashboardThumbnail
}
type UpdateThumbnailStateCommand struct {
State ThumbnailState
DashboardThumbnailMeta
}

10
pkg/models/dashboards.go
View File

@ -108,6 +108,11 @@ var (
StatusCode: 404,
Status: "not-found",
}
ErrDashboardThumbnailNotFound = DashboardErr{
Reason: "Dashboard thumbnail not found",
StatusCode: 404,
Status: "not-found",
}
)
// DashboardErr represents a dashboard error.
@ -317,6 +322,11 @@ func GetDashboardUrl(uid string, slug string) string {
return fmt.Sprintf("%s/d/%s/%s", setting.AppSubUrl, uid, slug)
}
// GetKioskModeDashboardUrl returns the HTML url for a dashboard in kiosk mode.
func GetKioskModeDashboardUrl(uid string, slug string) string {
return fmt.Sprintf("%s?kiosk", GetDashboardUrl(uid, slug))
}
// GetFullDashboardUrl returns the full URL for a dashboard.
func GetFullDashboardUrl(uid string, slug string) string {
return fmt.Sprintf("%sd/%s/%s", setting.AppUrl, uid, slug)

20
pkg/models/theme.go Normal file
View File

@ -0,0 +1,20 @@
package models
import "errors"
type Theme string
const (
ThemeLight Theme = "light"
ThemeDark Theme = "dark"
)
func ParseTheme(str string) (Theme, error) {
switch str {
case string(ThemeLight):
return ThemeLight, nil
case string(ThemeDark):
return ThemeDark, nil
}
return ThemeDark, errors.New("unknown theme " + str)
}

2
pkg/services/alerting/notifier.go
View File

@ -214,7 +214,7 @@ func (n *notificationService) renderAndUploadImage(evalCtx *EvalContext, timeout
Width: 1000,
Height: 500,
ConcurrentLimit: setting.AlertingRenderLimit,
Theme: rendering.ThemeDark,
Theme: models.ThemeDark,
}
ref, err := evalCtx.GetDashboardUID()

2
pkg/services/alerting/notifier_test.go
View File

@ -373,7 +373,7 @@ func (s *testRenderService) RenderCSV(ctx context.Context, opts rendering.CSVOpt
return nil, nil
}
func (s *testRenderService) RenderErrorImage(theme rendering.Theme, err error) (*rendering.RenderResult, error) {
func (s *testRenderService) RenderErrorImage(theme models.Theme, err error) (*rendering.RenderResult, error) {
if s.renderErrorImageProvider != nil {
return s.renderErrorImageProvider(err)
}

11
pkg/services/rendering/interface.go
View File

@ -19,13 +19,6 @@ const (
RenderPNG RenderType = "png"
)
type Theme string
const (
ThemeLight Theme = "light"
ThemeDark Theme = "dark"
)
type TimeoutOpts struct {
Timeout time.Duration // Timeout param passed to image-renderer service
RequestTimeoutMultiplier time.Duration // RequestTimeoutMultiplier used for plugin/HTTP request context timeout
@ -56,7 +49,7 @@ type Opts struct {
ConcurrentLimit int
DeviceScaleFactor float64
Headers map[string][]string
Theme Theme
Theme models.Theme
}
type CSVOpts struct {
@ -106,7 +99,7 @@ type Service interface {
Version() string
Render(ctx context.Context, opts Opts, session Session) (*RenderResult, error)
RenderCSV(ctx context.Context, opts CSVOpts, session Session) (*RenderCSVResult, error)
RenderErrorImage(theme Theme, error error) (*RenderResult, error)
RenderErrorImage(theme models.Theme, error error) (*RenderResult, error)
GetRenderUser(ctx context.Context, key string) (*RenderUser, bool)
HasCapability(capability CapabilityName) (CapabilitySupportRequestResult, error)
CreateRenderingSession(ctx context.Context, authOpts AuthOpts, sessionOpts SessionOpts) (Session, error)

7
pkg/services/rendering/rendering.go
View File

@ -17,6 +17,7 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/metrics"
"github.com/grafana/grafana/pkg/infra/remotecache"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
@ -176,9 +177,9 @@ func (rs *RenderingService) Version() string {
return rs.version
}
func (rs *RenderingService) RenderErrorImage(theme Theme, err error) (*RenderResult, error) {
func (rs *RenderingService) RenderErrorImage(theme models.Theme, err error) (*RenderResult, error) {
if theme == "" {
theme = ThemeDark
theme = models.ThemeDark
}
imgUrl := "public/img/rendering_%s_%s.png"
if errors.Is(err, ErrTimeout) {
@ -224,7 +225,7 @@ func (rs *RenderingService) render(ctx context.Context, opts Opts, renderKeyProv
if int(atomic.LoadInt32(&rs.inProgressCount)) > opts.ConcurrentLimit {
rs.log.Warn("Could not render image, hit the currency limit", "concurrencyLimit", opts.ConcurrentLimit, "path", opts.Path)
theme := ThemeDark
theme := models.ThemeDark
if opts.Theme != "" {
theme = opts.Theme
}

11
pkg/services/rendering/rendering_test.go
View File

@ -10,6 +10,7 @@ import (
"time"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/setting"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -83,13 +84,13 @@ func TestRenderErrorImage(t *testing.T) {
})
t.Run("Timeout error returns timeout error image", func(t *testing.T) {
result, err := rs.RenderErrorImage(ThemeLight, ErrTimeout)
result, err := rs.RenderErrorImage(models.ThemeLight, ErrTimeout)
require.NoError(t, err)
assert.Equal(t, result.FilePath, path+"/public/img/rendering_timeout_light.png")
})
t.Run("Generic error returns error image", func(t *testing.T) {
result, err := rs.RenderErrorImage(ThemeLight, errors.New("an error"))
result, err := rs.RenderErrorImage(models.ThemeLight, errors.New("an error"))
require.NoError(t, err)
assert.Equal(t, result.FilePath, path+"/public/img/rendering_error_light.png")
})
@ -115,17 +116,17 @@ func TestRenderLimitImage(t *testing.T) {
tests := []struct {
name string
theme Theme
theme models.Theme
expected string
}{
{
name: "Light theme returns light image",
theme: ThemeLight,
theme: models.ThemeLight,
expected: path + "/public/img/rendering_limit_light.png",
},
{
name: "Dark theme returns dark image",
theme: ThemeDark,
theme: models.ThemeDark,
expected: path + "/public/img/rendering_limit_dark.png",
},
{

167
pkg/services/sqlstore/dashboard_thumbs.go Normal file
View File

@ -0,0 +1,167 @@
package sqlstore
import (
"context"
"errors"
"time"
"github.com/grafana/grafana/pkg/models"
)
func (ss *SQLStore) GetThumbnail(ctx context.Context, query *models.GetDashboardThumbnailCommand) (*models.DashboardThumbnail, error) {
err := ss.WithDbSession(ctx, func(sess *DBSession) error {
result, err := findThumbnailByMeta(sess, query.DashboardThumbnailMeta)
if err != nil {
return err
}
query.Result = result
return nil
})
return query.Result, err
}
func (ss *SQLStore) SaveThumbnail(ctx context.Context, cmd *models.SaveDashboardThumbnailCommand) (*models.DashboardThumbnail, error) {
err := ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error {
existing, err := findThumbnailByMeta(sess, cmd.DashboardThumbnailMeta)
if err != nil && !errors.Is(err, models.ErrDashboardThumbnailNotFound) {
return err
}
if existing != nil {
existing.Image = cmd.Image
existing.MimeType = cmd.MimeType
existing.Updated = time.Now()
existing.DashboardVersion = cmd.DashboardVersion
existing.State = models.ThumbnailStateDefault
_, err = sess.ID(existing.Id).Update(existing)
cmd.Result = existing
return err
}
thumb := &models.DashboardThumbnail{}
dash, err := findDashboardIdByThumbMeta(sess, cmd.DashboardThumbnailMeta)
if err != nil {
return err
}
thumb.Updated = time.Now()
thumb.Theme = cmd.Theme
thumb.Kind = cmd.Kind
thumb.Image = cmd.Image
thumb.MimeType = cmd.MimeType
thumb.DashboardId = dash.Id
thumb.DashboardVersion = cmd.DashboardVersion
thumb.State = models.ThumbnailStateDefault
thumb.PanelId = cmd.PanelID
_, err = sess.Insert(thumb)
cmd.Result = thumb
return err
})
return cmd.Result, err
}
func (ss *SQLStore) UpdateThumbnailState(ctx context.Context, cmd *models.UpdateThumbnailStateCommand) error {
err := ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error {
existing, err := findThumbnailByMeta(sess, cmd.DashboardThumbnailMeta)
if err != nil {
return err
}
existing.State = cmd.State
_, err = sess.ID(existing.Id).Update(existing)
return err
})
return err
}
func (ss *SQLStore) FindDashboardsWithStaleThumbnails(ctx context.Context, cmd *models.FindDashboardsWithStaleThumbnailsCommand) ([]*models.DashboardWithStaleThumbnail, error) {
err := ss.WithDbSession(ctx, func(sess *DBSession) error {
sess.Table("dashboard")
sess.Join("LEFT", "dashboard_thumbnail", "dashboard.id = dashboard_thumbnail.dashboard_id")
sess.Where("dashboard.is_folder = ?", dialect.BooleanStr(false))
sess.Where("(dashboard.version != dashboard_thumbnail.dashboard_version "+
"OR dashboard_thumbnail.state = ? "+
"OR dashboard_thumbnail.id IS NULL)", models.ThumbnailStateStale)
if !cmd.IncludeManuallyUploadedThumbnails {
sess.Where("(dashboard_thumbnail.id is not null AND dashboard_thumbnail.dashboard_version != ?) "+
"OR dashboard_thumbnail.id is null "+
"OR dashboard_thumbnail.state = ?", models.DashboardVersionForManualThumbnailUpload, models.ThumbnailStateStale)
}
sess.Where("(dashboard_thumbnail.id IS NULL OR dashboard_thumbnail.state != ?)", models.ThumbnailStateLocked)
sess.Cols("dashboard.id",
"dashboard.uid",
"dashboard.org_id",
"dashboard.version",
"dashboard.slug")
var dashboards = make([]*models.DashboardWithStaleThumbnail, 0)
err := sess.Find(&dashboards)
if err != nil {
return err
}
cmd.Result = dashboards
return err
})
return cmd.Result, err
}
func findThumbnailByMeta(sess *DBSession, meta models.DashboardThumbnailMeta) (*models.DashboardThumbnail, error) {
result := &models.DashboardThumbnail{}
sess.Table("dashboard_thumbnail")
sess.Join("INNER", "dashboard", "dashboard.id = dashboard_thumbnail.dashboard_id")
sess.Where("dashboard.uid = ? AND dashboard.org_id = ? AND panel_id = ? AND kind = ? AND theme = ?", meta.DashboardUID, meta.OrgId, meta.PanelID, meta.Kind, meta.Theme)
sess.Cols("dashboard_thumbnail.id",
"dashboard_thumbnail.dashboard_id",
"dashboard_thumbnail.panel_id",
"dashboard_thumbnail.image",
"dashboard_thumbnail.dashboard_version",
"dashboard_thumbnail.state",
"dashboard_thumbnail.kind",
"dashboard_thumbnail.mime_type",
"dashboard_thumbnail.theme",
"dashboard_thumbnail.updated")
exists, err := sess.Get(result)
if !exists {
return nil, models.ErrDashboardThumbnailNotFound
}
if err != nil {
return nil, err
}
return result, nil
}
type dash struct {
Id int64
}
func findDashboardIdByThumbMeta(sess *DBSession, meta models.DashboardThumbnailMeta) (*dash, error) {
result := &dash{}
sess.Table("dashboard").Where("dashboard.uid = ? AND dashboard.org_id = ?", meta.DashboardUID, meta.OrgId).Cols("id")
exists, err := sess.Get(result)
if err != nil {
return nil, err
}
if !exists {
return nil, models.ErrDashboardNotFound
}
return result, err
}

216
pkg/services/sqlstore/dashboard_thumbs_test.go Normal file
View File

@ -0,0 +1,216 @@
//go:build integration
// +build integration
package sqlstore
import (
"context"
"testing"
"github.com/grafana/grafana/pkg/models"
"github.com/stretchr/testify/require"
)
func TestSqlStorage(t *testing.T) {
var sqlStore *SQLStore
var savedFolder *models.Dashboard
setup := func() {
sqlStore = InitTestDB(t)
savedFolder = insertTestDashboard(t, sqlStore, "1 test dash folder", 1, 0, true, "prod", "webapp")
}
t.Run("Should insert dashboard in default state", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
thumb := getThumbnail(t, sqlStore, dash.Uid, dash.OrgId)
require.Positive(t, thumb.Id)
require.Equal(t, models.ThumbnailStateDefault, thumb.State)
require.Equal(t, dash.Version, thumb.DashboardVersion)
})
t.Run("Should be able to update the thumbnail", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
thumb := getThumbnail(t, sqlStore, dash.Uid, dash.OrgId)
insertedThumbnailId := thumb.Id
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version+1)
updatedThumb := getThumbnail(t, sqlStore, dash.Uid, dash.OrgId)
require.Equal(t, insertedThumbnailId, updatedThumb.Id)
require.Equal(t, dash.Version+1, updatedThumb.DashboardVersion)
})
t.Run("Should return empty array if all dashboards have thumbnails", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 0)
})
t.Run("Should return dashboards with thumbnails marked as stale", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateStale)
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 1)
require.Equal(t, dash.Id, res[0].Id)
})
t.Run("Should not return dashboards with updated thumbnails that had been marked as stale", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateStale)
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 0)
})
t.Run("Should find dashboards without thumbnails", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 1)
require.Equal(t, dash.Id, res[0].Id)
})
t.Run("Should find dashboards with outdated thumbnails", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
updateTestDashboard(t, sqlStore, dash, map[string]interface{}{
"tags": "different-tag",
})
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 1)
require.Equal(t, dash.Id, res[0].Id)
})
t.Run("Should not return dashboards with locked thumbnails even if they are outdated", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version)
updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateLocked)
updateTestDashboard(t, sqlStore, dash, map[string]interface{}{
"tags": "different-tag",
})
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 0)
})
t.Run("Should not return dashboards with manually uploaded thumbnails by default", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, models.DashboardVersionForManualThumbnailUpload)
updateTestDashboard(t, sqlStore, dash, map[string]interface{}{
"tags": "different-tag",
})
cmd := models.FindDashboardsWithStaleThumbnailsCommand{}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 0)
})
t.Run("Should return dashboards with manually uploaded thumbnails if requested", func(t *testing.T) {
setup()
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp")
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, models.DashboardVersionForManualThumbnailUpload)
updateTestDashboard(t, sqlStore, dash, map[string]interface{}{
"tags": "different-tag",
})
cmd := models.FindDashboardsWithStaleThumbnailsCommand{
IncludeManuallyUploadedThumbnails: true,
}
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd)
require.NoError(t, err)
require.Len(t, res, 1)
require.Equal(t, dash.Id, res[0].Id)
})
}
func getThumbnail(t *testing.T, sqlStore *SQLStore, dashboardUID string, orgId int64) *models.DashboardThumbnail {
t.Helper()
cmd := models.GetDashboardThumbnailCommand{
DashboardThumbnailMeta: models.DashboardThumbnailMeta{
DashboardUID: dashboardUID,
OrgId: orgId,
PanelID: 0,
Kind: models.ThumbnailKindDefault,
Theme: models.ThemeDark,
},
}
thumb, err := sqlStore.GetThumbnail(context.Background(), &cmd)
require.NoError(t, err)
return thumb
}
func upsertTestDashboardThumbnail(t *testing.T, sqlStore *SQLStore, dashboardUID string, orgId int64, dashboardVersion int) *models.DashboardThumbnail {
t.Helper()
cmd := models.SaveDashboardThumbnailCommand{
DashboardThumbnailMeta: models.DashboardThumbnailMeta{
DashboardUID: dashboardUID,
OrgId: orgId,
PanelID: 0,
Kind: models.ThumbnailKindDefault,
Theme: models.ThemeDark,
},
DashboardVersion: dashboardVersion,
Image: make([]byte, 0),
MimeType: "image/png",
}
dash, err := sqlStore.SaveThumbnail(context.Background(), &cmd)
require.NoError(t, err)
require.NotNil(t, dash)
return dash
}
func updateThumbnailState(t *testing.T, sqlStore *SQLStore, dashboardUID string, orgId int64, state models.ThumbnailState) {
t.Helper()
cmd := models.UpdateThumbnailStateCommand{
DashboardThumbnailMeta: models.DashboardThumbnailMeta{
DashboardUID: dashboardUID,
OrgId: orgId,
PanelID: 0,
Kind: models.ThumbnailKindDefault,
Theme: models.ThemeDark,
},
State: state,
}
err := sqlStore.UpdateThumbnailState(context.Background(), &cmd)
require.NoError(t, err)
}

27
pkg/services/sqlstore/migrations/dashboard_thumbs_mig.go Normal file
View File

@ -0,0 +1,27 @@
package migrations
import "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
func addDashboardThumbsMigrations(mg *migrator.Migrator) {
dashThumbs := migrator.Table{
Name: "dashboard_thumbnail",
Columns: []*migrator.Column{
{Name: "id", Type: migrator.DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
{Name: "dashboard_id", Type: migrator.DB_BigInt, Nullable: false}, // can join with dashboard table
{Name: "dashboard_version", Type: migrator.DB_Int, Nullable: false}, // screenshoted version of the dashboard
{Name: "state", Type: migrator.DB_NVarchar, Length: 10, Nullable: false}, // stale | locked
{Name: "panel_id", Type: migrator.DB_SmallInt, Nullable: false, Default: "0"}, // for panel thumbnails
{Name: "image", Type: migrator.DB_MediumBlob, Nullable: false}, // image stored as blob. MediumBlob has a max limit of 16mb in MySQL
{Name: "mime_type", Type: migrator.DB_NVarchar, Length: 255, Nullable: false}, // e.g. image/png, image/webp
{Name: "kind", Type: migrator.DB_NVarchar, Length: 8, Nullable: false}, // thumb | tall
{Name: "theme", Type: migrator.DB_NVarchar, Length: 8, Nullable: false}, // light|dark
{Name: "updated", Type: migrator.DB_DateTime, Nullable: false},
},
Indices: []*migrator.Index{
{Cols: []string{"dashboard_id", "panel_id", "kind", "theme"}, Type: migrator.UniqueIndex},
},
}
mg.AddMigration("create dashboard_thumbnail table", migrator.NewAddTableMigration(dashThumbs))
mg.AddMigration("add unique indexes for dashboard_thumbnail", migrator.NewAddIndexMigration(dashThumbs, dashThumbs.Indices[0]))
}

3
pkg/services/sqlstore/migrations/migrations.go
View File

@ -61,6 +61,9 @@ func (*OSSMigrations) AddMigration(mg *Migrator) {
if mg.Cfg.IsFeatureToggleEnabled(featuremgmt.FlagLiveConfig) {
addLiveChannelMigrations(mg)
}
if mg.Cfg.IsFeatureToggleEnabled(featuremgmt.FlagDashboardPreviews) {
addDashboardThumbsMigrations(mg)
}
}
ualert.RerunDashAlertMigration(mg)

14
pkg/services/sqlstore/sqlstore.go
View File

@ -463,6 +463,10 @@ type InitTestDBOpt struct {
EnsureDefaultOrgAndUser bool
}
var featuresEnabledDuringTests = []string{
featuremgmt.FlagDashboardPreviews,
}
// InitTestDBWithMigration initializes the test DB given custom migrations.
func InitTestDBWithMigration(t ITestDB, migration registry.DatabaseMigrator, opts ...InitTestDBOpt) *SQLStore {
t.Helper()
@ -500,7 +504,15 @@ func initTestDB(migration registry.DatabaseMigrator, opts ...InitTestDBOpt) (*SQ
// set test db config
cfg := setting.NewCfg()
cfg.IsFeatureToggleEnabled = func(key string) bool { return false }
cfg.IsFeatureToggleEnabled = func(requestedFeature string) bool {
for _, enabledFeature := range featuresEnabledDuringTests {
if enabledFeature == requestedFeature {
return true
}
}
return false
}
sec, err := cfg.Raw.NewSection("database")
if err != nil {
return nil, err

269
pkg/services/thumbs/crawler.go
View File

@ -3,66 +3,60 @@ package thumbs
import (
"context"
"encoding/json"
"errors"
"fmt"
"math/rand"
"os"
"strings"
"sync"
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/live"
"github.com/grafana/grafana/pkg/services/rendering"
"github.com/grafana/grafana/pkg/services/search"
)
type dashItem struct {
uid string
url string
}
type simpleCrawler struct {
screenshotsFolder string
renderService rendering.Service
threadCount int
renderService rendering.Service
threadCount int
glive *live.GrafanaLive
mode CrawlerMode
opts rendering.Opts
status crawlStatus
queue []dashItem
mu sync.Mutex
glive *live.GrafanaLive
thumbnailRepo thumbnailRepo
mode CrawlerMode
thumbnailKind models.ThumbnailKind
opts rendering.Opts
status crawlStatus
statusMutex sync.RWMutex
queue []*models.DashboardWithStaleThumbnail
queueMutex sync.Mutex
renderingSession rendering.Session
}
func newSimpleCrawler(folder string, renderService rendering.Service, gl *live.GrafanaLive) dashRenderer {
func newSimpleCrawler(renderService rendering.Service, gl *live.GrafanaLive, repo thumbnailRepo) dashRenderer {
c := &simpleCrawler{
screenshotsFolder: folder,
renderService: renderService,
threadCount: 5,
glive: gl,
renderService: renderService,
threadCount: 6,
glive: gl,
thumbnailRepo: repo,
status: crawlStatus{
State: "init",
State: initializing,
Complete: 0,
Queue: 0,
},
queue: make([]dashItem, 0),
queue: nil,
}
c.broadcastStatus()
return c
}
func (r *simpleCrawler) next() *dashItem {
if len(r.queue) < 1 {
func (r *simpleCrawler) next() *models.DashboardWithStaleThumbnail {
r.queueMutex.Lock()
defer r.queueMutex.Unlock()
if r.queue == nil || len(r.queue) < 1 {
return nil
}
r.mu.Lock()
defer r.mu.Unlock()
v := r.queue[0]
r.queue = r.queue[1:]
return &v
return v
}
func (r *simpleCrawler) broadcastStatus() {
@ -83,61 +77,42 @@ func (r *simpleCrawler) broadcastStatus() {
}
}
func (r *simpleCrawler) GetPreview(req *previewRequest) *previewResponse {
p := getFilePath(r.screenshotsFolder, req)
if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) {
return r.queueRender(p, req)
}
return &previewResponse{
Path: p,
Code: 200,
}
}
func (r *simpleCrawler) queueRender(p string, req *previewRequest) *previewResponse {
go func() {
fmt.Printf("todo? queue")
}()
return &previewResponse{
Code: 202,
Path: p,
}
}
func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme rendering.Theme) (crawlStatus, error) {
if r.status.State == "running" {
func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme models.Theme, thumbnailKind models.ThumbnailKind) (crawlStatus, error) {
if r.status.State == running {
tlog.Info("already running")
return r.Status()
}
r.mu.Lock()
defer r.mu.Unlock()
r.queueMutex.Lock()
defer r.queueMutex.Unlock()
searchQuery := search.Query{
SignedInUser: c.SignedInUser,
OrgId: c.OrgId,
}
now := time.Now()
err := bus.Dispatch(context.Background(), &searchQuery)
ctx := c.Req.Context()
items, err := r.thumbnailRepo.findDashboardsWithStaleThumbnails(ctx)
if err != nil {
return crawlStatus{}, err
tlog.Error("error when fetching dashboards with stale thumbnails", "err", err.Error())
return crawlStatus{
Started: now,
Finished: now,
Last: now,
State: stopped,
Complete: 0,
}, err
}
queue := make([]dashItem, 0, len(searchQuery.Result))
for _, v := range searchQuery.Result {
if v.Type == search.DashHitDB {
queue = append(queue, dashItem{
uid: v.UID,
url: v.URL,
})
}
if len(items) == 0 {
return crawlStatus{
Started: now,
Finished: now,
Last: now,
State: stopped,
Complete: 0,
}, err
}
rand.Seed(time.Now().UnixNano())
rand.Shuffle(len(queue), func(i, j int) { queue[i], queue[j] = queue[j], queue[i] })
r.mode = mode
r.thumbnailKind = thumbnailKind
r.opts = rendering.Opts{
AuthOpts: rendering.AuthOpts{
OrgID: c.OrgId,
@ -151,35 +126,53 @@ func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme rend
Theme: theme,
ConcurrentLimit: 10,
}
r.queue = queue
renderingSession, err := r.renderService.CreateRenderingSession(context.Background(), r.opts.AuthOpts, rendering.SessionOpts{
Expiry: 5 * time.Minute,
RefreshExpiryOnEachRequest: true,
})
if err != nil {
tlog.Error("error when creating rendering session", "err", err.Error())
return crawlStatus{
Started: now,
Finished: now,
Last: now,
State: stopped,
Complete: 0,
}, err
}
r.renderingSession = renderingSession
r.queue = items
r.status = crawlStatus{
Started: time.Now(),
State: "running",
Started: now,
State: running,
Complete: 0,
}
r.broadcastStatus()
tlog.Info("Starting dashboard crawler", "dashboardsToCrawl", len(items))
// create a pool of workers
for i := 0; i < r.threadCount; i++ {
go r.walk()
// wait 1/2 second before starting a new thread
time.Sleep(500 * time.Millisecond)
go r.walk(ctx)
}
r.broadcastStatus()
return r.Status()
}
func (r *simpleCrawler) Stop() (crawlStatus, error) {
// cheap hack!
if r.status.State == "running" {
r.status.State = "stopping"
r.statusMutex.Lock()
if r.status.State == running {
r.status.State = stopping
}
r.statusMutex.Unlock()
return r.Status()
}
func (r *simpleCrawler) Status() (crawlStatus, error) {
r.statusMutex.RLock()
defer r.statusMutex.RUnlock()
status := crawlStatus{
State: r.status.State,
Started: r.status.Started,
@ -191,9 +184,41 @@ func (r *simpleCrawler) Status() (crawlStatus, error) {
return status, nil
}
func (r *simpleCrawler) walk() {
func (r *simpleCrawler) newErrorResult() {
r.statusMutex.Lock()
defer r.statusMutex.Unlock()
r.status.Errors++
r.status.Last = time.Now()
}
func (r *simpleCrawler) newSuccessResult() {
r.statusMutex.Lock()
defer r.statusMutex.Unlock()
r.status.Complete++
r.status.Last = time.Now()
}
func (r *simpleCrawler) walkFinished() {
r.statusMutex.Lock()
defer r.statusMutex.Unlock()
r.status.State = stopped
r.status.Finished = time.Now()
tlog.Info("Crawler finished", "startTime", r.status.Started, "endTime", r.status.Finished, "durationInSeconds", int64(time.Since(r.status.Started)/time.Second))
}
func (r *simpleCrawler) shouldWalk() bool {
r.statusMutex.RLock()
defer r.statusMutex.RUnlock()
return r.status.State == running
}
func (r *simpleCrawler) walk(ctx context.Context) {
for {
if r.status.State == "stopping" {
if !r.shouldWalk() {
break
}
@ -202,51 +227,57 @@ func (r *simpleCrawler) walk() {
break
}
tlog.Info("GET THUMBNAIL", "url", item.url)
url := models.GetKioskModeDashboardUrl(item.Uid, item.Slug)
tlog.Info("Getting dashboard thumbnail", "dashboardUID", item.Uid, "url", url)
// Hack (for now) pick a URL that will render
panelURL := strings.TrimPrefix(item.url, "/") + "?kiosk"
res, err := r.renderService.Render(context.Background(), rendering.Opts{
Width: 320,
Height: 240,
Path: panelURL,
Path: strings.TrimPrefix(url, "/"),
AuthOpts: r.opts.AuthOpts,
TimeoutOpts: r.opts.TimeoutOpts,
ConcurrentLimit: r.opts.ConcurrentLimit,
Theme: r.opts.Theme,
DeviceScaleFactor: -5, // negative numbers will render larger then scale down
}, nil)
DeviceScaleFactor: -5, // negative numbers will render larger and then scale down.
}, r.renderingSession)
if err != nil {
tlog.Warn("error getting image", "err", err)
r.status.Errors++
tlog.Warn("error getting image", "dashboardUID", item.Uid, "url", url, "err", err)
r.newErrorResult()
} else if res.FilePath == "" {
tlog.Warn("error getting image... no response")
r.status.Errors++
tlog.Warn("error getting image... no response", "dashboardUID", item.Uid, "url", url)
r.newErrorResult()
} else if strings.Contains(res.FilePath, "public/img") {
tlog.Warn("error getting image... internal result", "img", res.FilePath)
r.status.Errors++
tlog.Warn("error getting image... internal result", "dashboardUID", item.Uid, "url", url, "img", res.FilePath)
// rendering service returned a static error image - we should not remove that file
r.newErrorResult()
} else {
p := getFilePath(r.screenshotsFolder, &previewRequest{
UID: item.uid,
OrgID: r.opts.OrgID,
Theme: r.opts.Theme,
Size: PreviewSizeThumb,
})
err = os.Rename(res.FilePath, p)
if err != nil {
r.status.Errors++
tlog.Warn("error moving image", "err", err)
} else {
r.status.Complete++
tlog.Info("saved thumbnail", "img", item.url)
}
}
func() {
defer func() {
err := os.Remove(res.FilePath)
if err != nil {
tlog.Error("failed to remove thumbnail temp file", "dashboardUID", item.Uid, "url", url, "err", err)
}
}()
time.Sleep(5 * time.Second)
r.status.Last = time.Now()
thumbnailId, err := r.thumbnailRepo.saveFromFile(ctx, res.FilePath, models.DashboardThumbnailMeta{
DashboardUID: item.Uid,
OrgId: item.OrgId,
Theme: r.opts.Theme,
Kind: r.thumbnailKind,
}, item.Version)
if err != nil {
tlog.Warn("error saving image image", "dashboardUID", item.Uid, "url", url, "err", err)
r.newErrorResult()
} else {
tlog.Info("saved thumbnail", "dashboardUID", item.Uid, "url", url, "thumbnailId", thumbnailId)
r.newSuccessResult()
}
}()
}
r.broadcastStatus()
}
r.status.State = "stopped"
r.status.Finished = time.Now()
r.walkFinished()
r.broadcastStatus()
}

4
pkg/services/thumbs/dummy.go
View File

@ -12,6 +12,10 @@ func (ds *dummyService) GetImage(c *models.ReqContext) {
c.JSON(400, map[string]string{"error": "invalid size"})
}
func (ds *dummyService) UpdateThumbnailState(c *models.ReqContext) {
c.JSON(400, map[string]string{"error": "invalid size"})
}
func (ds *dummyService) SetImage(c *models.ReqContext) {
c.JSON(400, map[string]string{"error": "invalid size"})
}

106
pkg/services/thumbs/models.go
View File

@ -1,107 +1,73 @@
package thumbs
import (
"context"
"time"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/rendering"
)
type PreviewSize string
type CrawlerMode string
const (
// PreviewSizeThumb is a small 320x240 preview
PreviewSizeThumb PreviewSize = "thumb"
// PreviewSizeLarge is a large image 2000x1500
PreviewSizeLarge PreviewSize = "large"
// PreviewSizeLarge is a large image 512x????
PreviewSizeTall PreviewSize = "tall"
// CrawlerModeThumbs will create small thumbnails for everything
// CrawlerModeThumbs will create small thumbnails for everything.
CrawlerModeThumbs CrawlerMode = "thumbs"
// CrawlerModeAnalytics will get full page results for everythign
// CrawlerModeAnalytics will get full page results for everything.
CrawlerModeAnalytics CrawlerMode = "analytics"
// CrawlerModeMigrate will migrate all dashboards with old schema
// CrawlerModeMigrate will migrate all dashboards with old schema.
CrawlerModeMigrate CrawlerMode = "migrate"
)
// IsKnownSize checks if the value is a standard size
func (p PreviewSize) IsKnownSize() bool {
switch p {
case
PreviewSizeThumb,
PreviewSizeLarge,
PreviewSizeTall:
return true
}
return false
}
type crawlerState string
func getPreviewSize(str string) (PreviewSize, bool) {
switch str {
case string(PreviewSizeThumb):
return PreviewSizeThumb, true
case string(PreviewSizeLarge):
return PreviewSizeLarge, true
case string(PreviewSizeTall):
return PreviewSizeTall, true
}
return PreviewSizeThumb, false
}
func getTheme(str string) (rendering.Theme, bool) {
switch str {
case "light":
return rendering.ThemeLight, true
case "dark":
return rendering.ThemeDark, true
}
return rendering.ThemeDark, false
}
const (
initializing crawlerState = "initializing"
running crawlerState = "running"
stopping crawlerState = "stopping"
stopped crawlerState = "stopped"
)
type previewRequest struct {
OrgID int64 `json:"orgId"`
UID string `json:"uid"`
Size PreviewSize `json:"size"`
Theme rendering.Theme `json:"theme"`
}
type previewResponse struct {
Code int `json:"code"` // 200 | 202
Path string `json:"path"` // local file path to serve
URL string `json:"url"` // redirect to this URL
OrgID int64 `json:"orgId"`
UID string `json:"uid"`
Kind models.ThumbnailKind `json:"kind"`
Theme models.Theme `json:"theme"`
}
type crawlCmd struct {
Mode CrawlerMode `json:"mode"` // thumbs | analytics | migrate
Theme rendering.Theme `json:"theme"` // light | dark
Mode CrawlerMode `json:"mode"` // thumbs | analytics | migrate
Theme models.Theme `json:"theme"` // light | dark
}
type crawlStatus struct {
State string `json:"state"`
Started time.Time `json:"started,omitempty"`
Finished time.Time `json:"finished,omitempty"`
Complete int `json:"complete"`
Errors int `json:"errors"`
Queue int `json:"queue"`
Last time.Time `json:"last,omitempty"`
State crawlerState `json:"state"`
Started time.Time `json:"started,omitempty"`
Finished time.Time `json:"finished,omitempty"`
Complete int `json:"complete"`
Errors int `json:"errors"`
Queue int `json:"queue"`
Last time.Time `json:"last,omitempty"`
}
type dashRenderer interface {
// Assumes you have already authenticated as admin
GetPreview(req *previewRequest) *previewResponse
// Assumes you have already authenticated as admin
Start(c *models.ReqContext, mode CrawlerMode, theme rendering.Theme) (crawlStatus, error)
// Run Assumes you have already authenticated as admin.
Start(c *models.ReqContext, mode CrawlerMode, theme models.Theme, kind models.ThumbnailKind) (crawlStatus, error)
// Assumes you have already authenticated as admin
// Assumes you have already authenticated as admin.
Stop() (crawlStatus, error)
// Assumes you have already authenticated as admin
// Assumes you have already authenticated as admin.
Status() (crawlStatus, error)
}
type thumbnailRepo interface {
updateThumbnailState(ctx context.Context, state models.ThumbnailState, meta models.DashboardThumbnailMeta) error
saveFromFile(ctx context.Context, filePath string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error)
saveFromBytes(ctx context.Context, bytes []byte, mimeType string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error)
getThumbnail(ctx context.Context, meta models.DashboardThumbnailMeta) (*models.DashboardThumbnail, error)
findDashboardsWithStaleThumbnails(ctx context.Context) ([]*models.DashboardWithStaleThumbnail, error)
}

88
pkg/services/thumbs/repo.go Normal file
View File

@ -0,0 +1,88 @@
package thumbs
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/sqlstore"
)
func newThumbnailRepo(store *sqlstore.SQLStore) thumbnailRepo {
repo := &sqlThumbnailRepository{
store: store,
}
return repo
}
type sqlThumbnailRepository struct {
store *sqlstore.SQLStore
}
func (r *sqlThumbnailRepository) saveFromFile(ctx context.Context, filePath string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error) {
// the filePath variable is never set by the user. it refers to a temporary file created either in
// 1. thumbs/service.go, when user uploads a thumbnail
// 2. the rendering service, when image-renderer returns a screenshot
if !filepath.IsAbs(filePath) {
tlog.Error("Received relative path", "dashboardUID", meta.DashboardUID, "err", filePath)
return 0, errors.New("relative paths are not supported")
}
content, err := os.ReadFile(filepath.Clean(filePath))
if err != nil {
tlog.Error("error reading file", "dashboardUID", meta.DashboardUID, "err", err)
return 0, err
}
return r.saveFromBytes(ctx, content, getMimeType(filePath), meta, dashboardVersion)
}
func getMimeType(filePath string) string {
if strings.HasSuffix(filePath, ".webp") {
return "image/webp"
}
return "image/png"
}
func (r *sqlThumbnailRepository) saveFromBytes(ctx context.Context, content []byte, mimeType string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error) {
cmd := &models.SaveDashboardThumbnailCommand{
DashboardThumbnailMeta: meta,
Image: content,
MimeType: mimeType,
DashboardVersion: dashboardVersion,
}
_, err := r.store.SaveThumbnail(ctx, cmd)
if err != nil {
tlog.Error("error saving to the db", "dashboardUID", meta.DashboardUID, "err", err)
return 0, err
}
return cmd.Result.Id, nil
}
func (r *sqlThumbnailRepository) updateThumbnailState(ctx context.Context, state models.ThumbnailState, meta models.DashboardThumbnailMeta) error {
return r.store.UpdateThumbnailState(ctx, &models.UpdateThumbnailStateCommand{
State: state,
DashboardThumbnailMeta: meta,
})
}
func (r *sqlThumbnailRepository) getThumbnail(ctx context.Context, meta models.DashboardThumbnailMeta) (*models.DashboardThumbnail, error) {
query := &models.GetDashboardThumbnailCommand{
DashboardThumbnailMeta: meta,
}
return r.store.GetThumbnail(ctx, query)
}
func (r *sqlThumbnailRepository) findDashboardsWithStaleThumbnails(ctx context.Context) ([]*models.DashboardWithStaleThumbnail, error) {
return r.store.FindDashboardsWithStaleThumbnails(ctx, &models.FindDashboardsWithStaleThumbnailsCommand{
IncludeManuallyUploadedThumbnails: false,
})
}

175
pkg/services/thumbs/service.go
View File

@ -1,23 +1,20 @@
package thumbs
import (
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strings"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/guardian"
"github.com/grafana/grafana/pkg/services/live"
"github.com/grafana/grafana/pkg/services/rendering"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/web"
"github.com/segmentio/encoding/json"
@ -31,8 +28,9 @@ type Service interface {
Enabled() bool
GetImage(c *models.ReqContext)
// Form post (from dashboard page)
SetImage(c *models.ReqContext)
// from dashboard page
SetImage(c *models.ReqContext) // form post
UpdateThumbnailState(c *models.ReqContext)
// Must be admin
StartCrawler(c *models.ReqContext) response.Response
@ -40,28 +38,21 @@ type Service interface {
CrawlerStatus(c *models.ReqContext) response.Response
}
func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, renderService rendering.Service, gl *live.GrafanaLive) Service {
func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, renderService rendering.Service, gl *live.GrafanaLive, store *sqlstore.SQLStore) Service {
if !features.IsEnabled(featuremgmt.FlagDashboardPreviews) {
return &dummyService{}
}
root := filepath.Join(cfg.DataPath, "crawler", "preview")
tempdir := filepath.Join(cfg.DataPath, "temp")
_ = os.MkdirAll(root, 0700)
_ = os.MkdirAll(tempdir, 0700)
renderer := newSimpleCrawler(root, renderService, gl)
thumbnailRepo := newThumbnailRepo(store)
return &thumbService{
renderer: renderer,
root: root,
tempdir: tempdir,
renderer: newSimpleCrawler(renderService, gl, thumbnailRepo),
thumbnailRepo: thumbnailRepo,
}
}
type thumbService struct {
renderer dashRenderer
root string
tempdir string
renderer dashRenderer
thumbnailRepo thumbnailRepo
}
func (hs *thumbService) Enabled() bool {
@ -71,14 +62,14 @@ func (hs *thumbService) Enabled() bool {
func (hs *thumbService) parseImageReq(c *models.ReqContext, checkSave bool) *previewRequest {
params := web.Params(c.Req)
size, ok := getPreviewSize(params[":size"])
if !ok {
kind, err := models.ParseThumbnailKind(params[":kind"])
if err != nil {
c.JSON(400, map[string]string{"error": "invalid size"})
return nil
}
theme, ok := getTheme(params[":theme"])
if !ok {
theme, err := models.ParseTheme(params[":theme"])
if err != nil {
c.JSON(400, map[string]string{"error": "invalid theme"})
return nil
}
@ -87,7 +78,7 @@ func (hs *thumbService) parseImageReq(c *models.ReqContext, checkSave bool) *pre
OrgID: c.OrgId,
UID: params[":uid"],
Theme: theme,
Size: size,
Kind: kind,
}
if len(req.UID) < 1 {
@ -104,36 +95,70 @@ func (hs *thumbService) parseImageReq(c *models.ReqContext, checkSave bool) *pre
return req
}
type updateThumbnailStateRequest struct {
State models.ThumbnailState `json:"state" binding:"Required"`
}
func (hs *thumbService) UpdateThumbnailState(c *models.ReqContext) {
req := hs.parseImageReq(c, false)
if req == nil {
return // already returned value
}
var body = &updateThumbnailStateRequest{}
err := web.Bind(c.Req, body)
if err != nil {
tlog.Error("Error parsing update thumbnail state request", "dashboardUid", req.UID, "err", err.Error())
c.JSON(500, map[string]string{"dashboardUID": req.UID, "error": "unknown"})
return
}
err = hs.thumbnailRepo.updateThumbnailState(c.Req.Context(), body.State, models.DashboardThumbnailMeta{
DashboardUID: req.UID,
OrgId: req.OrgID,
Theme: req.Theme,
Kind: models.ThumbnailKindDefault,
})
if err != nil {
tlog.Error("Error when trying to update thumbnail state", "dashboardUid", req.UID, "err", err.Error(), "newState", body.State)
c.JSON(500, map[string]string{"dashboardUID": req.UID, "error": "unknown"})
return
}
tlog.Info("Updated dashboard thumbnail state", "dashboardUid", req.UID, "theme", req.Theme, "newState", body.State)
c.JSON(200, map[string]string{"success": "true"})
}
func (hs *thumbService) GetImage(c *models.ReqContext) {
req := hs.parseImageReq(c, false)
if req == nil {
return // already returned value
}
rsp := hs.renderer.GetPreview(req)
if rsp.Code == 200 {
if rsp.Path != "" {
if strings.HasSuffix(rsp.Path, ".webp") {
c.Resp.Header().Set("Content-Type", "image/webp")
} else if strings.HasSuffix(rsp.Path, ".png") {
c.Resp.Header().Set("Content-Type", "image/png")
}
c.Resp.Header().Set("Content-Type", "image/png")
http.ServeFile(c.Resp, c.Req, rsp.Path)
return
}
if rsp.URL != "" {
// todo redirect
fmt.Printf("TODO redirect: %s\n", rsp.URL)
}
}
res, err := hs.thumbnailRepo.getThumbnail(c.Req.Context(), models.DashboardThumbnailMeta{
DashboardUID: req.UID,
OrgId: req.OrgID,
Theme: req.Theme,
Kind: models.ThumbnailKindDefault,
})
if rsp.Code == 202 {
c.JSON(202, map[string]string{"path": rsp.Path, "todo": "queue processing"})
if errors.Is(err, models.ErrDashboardThumbnailNotFound) {
c.Resp.WriteHeader(404)
return
}
c.JSON(500, map[string]string{"path": rsp.Path, "error": "unknown!"})
if err != nil {
tlog.Error("Error when retrieving thumbnail", "dashboardUid", req.UID, "err", err.Error())
c.JSON(500, map[string]string{"dashboardUID": req.UID, "error": "unknown"})
return
}
c.Resp.Header().Set("Content-Type", res.MimeType)
if _, err := c.Resp.Write(res.Image); err != nil {
tlog.Error("Error writing to response", "dashboardUid", req.UID, "err", err)
}
}
// Hack for now -- lets you upload images explicitly
@ -169,38 +194,23 @@ func (hs *thumbService) SetImage(c *models.ReqContext) {
tlog.Info("File Size: %+v\n", handler.Size)
tlog.Info("MIME Header: %+v\n", handler.Header)
// Create a temporary file within our temp-images directory that follows
// a particular naming pattern
tempFile, err := ioutil.TempFile(hs.tempdir, "upload-*")
if err != nil {
c.JSON(400, map[string]string{"error": "error creating temp file"})
fmt.Println("error", err)
tlog.Info("ERROR", "err", handler.Header)
return
}
defer func() {
_ = tempFile.Close()
}()
// read all of the contents of our uploaded file into a
// byte array
fileBytes, err := ioutil.ReadAll(file)
if err != nil {
fmt.Println(err)
}
// write this byte array to our temporary file
_, err = tempFile.Write(fileBytes)
if err != nil {
c.JSON(400, map[string]string{"error": "error writing file"})
fmt.Println("error", err)
c.JSON(400, map[string]string{"error": "error reading file"})
return
}
p := getFilePath(hs.root, req)
err = os.Rename(tempFile.Name(), p)
_, err = hs.thumbnailRepo.saveFromBytes(c.Req.Context(), fileBytes, getMimeType(handler.Filename), models.DashboardThumbnailMeta{
DashboardUID: req.UID,
OrgId: req.OrgID,
Theme: req.Theme,
Kind: req.Kind,
}, models.DashboardVersionForManualThumbnailUpload)
if err != nil {
c.JSON(400, map[string]string{"error": "unable to rename file"})
c.JSON(400, map[string]string{"error": "error saving thumbnail file"})
fmt.Println("error", err)
return
}
@ -220,7 +230,7 @@ func (hs *thumbService) StartCrawler(c *models.ReqContext) response.Response {
if cmd.Mode == "" {
cmd.Mode = CrawlerModeThumbs
}
msg, err := hs.renderer.Start(c, cmd.Mode, cmd.Theme)
msg, err := hs.renderer.Start(c, cmd.Mode, cmd.Theme, models.ThumbnailKindDefault)
if err != nil {
return response.Error(500, "error starting", err)
}
@ -245,15 +255,12 @@ func (hs *thumbService) CrawlerStatus(c *models.ReqContext) response.Response {
// Ideally this service would not require first looking up the full dashboard just to bet the id!
func (hs *thumbService) getStatus(c *models.ReqContext, uid string, checkSave bool) int {
query := models.GetDashboardQuery{Uid: uid, OrgId: c.OrgId}
if err := bus.Dispatch(c.Req.Context(), &query); err != nil {
return 404 // not found
dashboardID, err := hs.getDashboardId(c, uid)
if err != nil {
return 404
}
dash := query.Result
guardian := guardian.New(c.Req.Context(), dash.Id, c.OrgId, c.SignedInUser)
guardian := guardian.New(c.Req.Context(), dashboardID, c.OrgId, c.SignedInUser)
if checkSave {
if canSave, err := guardian.CanSave(); err != nil || !canSave {
return 403 // forbidden
@ -267,3 +274,13 @@ func (hs *thumbService) getStatus(c *models.ReqContext, uid string, checkSave bo
return 200 // found and OK
}
func (hs *thumbService) getDashboardId(c *models.ReqContext, uid string) (int64, error) {
query := models.GetDashboardQuery{Uid: uid, OrgId: c.OrgId}
if err := bus.Dispatch(c.Req.Context(), &query); err != nil {
return 0, err
}
return query.Result.Id, nil
}

14
pkg/services/thumbs/utils.go
View File

@ -1,14 +0,0 @@
package thumbs
import (
"fmt"
"path/filepath"
)
func getFilePath(root string, req *previewRequest) string {
ext := "webp"
if req.Size != PreviewSizeThumb {
ext = "png"
}
return filepath.Join(root, fmt.Sprintf("%s-%s-%s.%s", req.UID, req.Size, req.Theme, ext))
}

19
public/app/features/dashboard/components/DashboardSettings/PreviewSettings.tsx
View File

@ -1,6 +1,7 @@
import React, { PureComponent } from 'react';
import { CollapsableSection, FileUpload } from '@grafana/ui';
import { Button, CollapsableSection, FileUpload } from '@grafana/ui';
import { getThumbnailURL } from 'app/features/search/components/SearchCard';
import { getBackendSrv } from 'app/core/services/backend_srv';
interface Props {
uid: string;
@ -36,6 +37,10 @@ export class PreviewSettings extends PureComponent<Props, State> {
});
};
markAsStale = (isLight: boolean) => async () => {
return getBackendSrv().put(getThumbnailURL(this.props.uid, isLight), { state: 'stale' });
};
render() {
const { uid } = this.props;
const imgstyle = { maxWidth: 300, maxHeight: 300 };
@ -50,6 +55,18 @@ export class PreviewSettings extends PureComponent<Props, State> {
</tr>
</thead>
<tbody>
<tr>
<td>
<Button type="button" variant="primary" onClick={this.markAsStale(false)} fill="outline">
Mark as stale
</Button>
</td>
<td>
<Button type="button" variant="primary" onClick={this.markAsStale(true)} fill="outline">
Mark as stale
</Button>
</td>
</tr>
<tr>
<td>
<img src={getThumbnailURL(uid, false)} style={imgstyle} />