grafana/pkg/services/sqlstore/dashboard_thumbs.go

182 lines
5.2 KiB
Go
Raw Normal View History

Dash previews: populate crawler queue from SQL query (#44083) * add SQL migrations * dashboard previews from sql: poc * added todos * refactor: use the same enums where possible * use useEffect, always return json * added todo * refactor + delete files after use * refactor + fix manual thumbnail upload * refactor: move all interactions with sqlStore to thumbnail repo * refactor: remove file operations in thumb crawler/service * refactor: fix dashboard_thumbs sql store * refactor: extracted thumbnail fetching/updating to a hook * refactor: store thumbnails in redux store * refactor: store thumbnails in redux store * refactor: private'd repo methods * removed redux storage, saving images as blobs * allow for configurable rendering timeouts * added 1) query for dashboards with stale thumbnails, 2) command for marking thumbnails as stale * use sql-based queue in crawler * ui for marking thumbnails as stale * replaced `stale` boolean prop with `state` enum * introduce rendering session * compilation errors * fix crawler stop button * rename thumbnail state frozen to locked * #44449: fix merge conflicts * #44449: remove thumb methods from `Store` interface * #44449: clean filepath, defer file closing * #44449: fix rendering.Theme cyclic import * #44449: linting * #44449: linting * #44449: mutex'd crawlerStatus access * #44449: added integration tests for `sqlstore.dashboard_thumbs` * #44449: added comments to explain the `ThumbnailState` enum * #44449: use os.ReadFile rather then os.Open * #44449: always enable dashboardPreviews feature during integration tests * #44449: remove sleep time, adjust number of threads * #44449: review fix: add `orgId` to `DashboardThumbnailMeta` * #44449: review fix: automatic parsing of thumbnailState * #44449: lint fixes * #44449: review fix: prefer `WithDbSession` over `WithTransactionalDbSession` * #44449: review fix: add a comment explaining source of the filepath * #44449: review fix: added filepath validation * #44449: review fixes https://github.com/grafana/grafana/pull/45063/files @fzambia Co-authored-by: Ryan McKinley <ryantxu@gmail.com> Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
2022-02-09 03:23:32 -06:00
package sqlstore
import (
"context"
"errors"
"time"
"github.com/grafana/grafana/pkg/models"
)
func (ss *SQLStore) GetThumbnail(ctx context.Context, query *models.GetDashboardThumbnailCommand) (*models.DashboardThumbnail, error) {
err := ss.WithDbSession(ctx, func(sess *DBSession) error {
result, err := findThumbnailByMeta(sess, query.DashboardThumbnailMeta)
if err != nil {
return err
}
query.Result = result
return nil
})
return query.Result, err
}
func (ss *SQLStore) SaveThumbnail(ctx context.Context, cmd *models.SaveDashboardThumbnailCommand) (*models.DashboardThumbnail, error) {
err := ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error {
existing, err := findThumbnailByMeta(sess, cmd.DashboardThumbnailMeta)
if err != nil && !errors.Is(err, models.ErrDashboardThumbnailNotFound) {
return err
}
if existing != nil {
existing.Image = cmd.Image
existing.MimeType = cmd.MimeType
existing.Updated = time.Now()
existing.DashboardVersion = cmd.DashboardVersion
existing.State = models.ThumbnailStateDefault
_, err = sess.ID(existing.Id).Update(existing)
cmd.Result = existing
return err
}
thumb := &models.DashboardThumbnail{}
dash, err := findDashboardIdByThumbMeta(sess, cmd.DashboardThumbnailMeta)
if err != nil {
return err
}
thumb.Updated = time.Now()
thumb.Theme = cmd.Theme
thumb.Kind = cmd.Kind
thumb.Image = cmd.Image
thumb.MimeType = cmd.MimeType
thumb.DashboardId = dash.Id
thumb.DashboardVersion = cmd.DashboardVersion
thumb.State = models.ThumbnailStateDefault
thumb.PanelId = cmd.PanelID
_, err = sess.Insert(thumb)
cmd.Result = thumb
return err
})
return cmd.Result, err
}
func (ss *SQLStore) UpdateThumbnailState(ctx context.Context, cmd *models.UpdateThumbnailStateCommand) error {
err := ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error {
existing, err := findThumbnailByMeta(sess, cmd.DashboardThumbnailMeta)
if err != nil {
return err
}
existing.State = cmd.State
_, err = sess.ID(existing.Id).Update(existing)
return err
})
return err
}
Previews: capability check (#44601) * add SQL migrations * dashboard previews from sql: poc * added todos * refactor: use the same enums where possible * use useEffect, always return json * added todo * refactor + delete files after use * refactor + fix manual thumbnail upload * refactor: move all interactions with sqlStore to thumbnail repo * refactor: remove file operations in thumb crawler/service * refactor: fix dashboard_thumbs sql store * refactor: extracted thumbnail fetching/updating to a hook * refactor: store thumbnails in redux store * refactor: store thumbnails in redux store * refactor: private'd repo methods * removed redux storage, saving images as blobs * allow for configurable rendering timeouts * added 1) query for dashboards with stale thumbnails, 2) command for marking thumbnails as stale * use sql-based queue in crawler * ui for marking thumbnails as stale * replaced `stale` boolean prop with `state` enum * introduce rendering session * compilation errors * fix crawler stop button * rename thumbnail state frozen to locked * #44449: fix merge conflicts * #44449: remove thumb methods from `Store` interface * #44449: clean filepath, defer file closing * #44449: fix rendering.Theme cyclic import * #44449: linting * #44449: linting * #44449: mutex'd crawlerStatus access * #44449: added integration tests for `sqlstore.dashboard_thumbs` * #44449: added comments to explain the `ThumbnailState` enum * #44449: use os.ReadFile rather then os.Open * #44449: always enable dashboardPreviews feature during integration tests * #44449: add /previews/system-requirements API * #44449: remove sleep time, adjust number of threads * #44449: review fix: add `orgId` to `DashboardThumbnailMeta` * #44449: review fix: automatic parsing of thumbnailState * #44449: update returned json * #44449: UI changes - dashboard previews sytem req check * #44449: lint fixes * #44449: fix tests * #44449: typo * #44449: fix getSystemRequirements API: return 200 even if we plugin version is invalid * #44449: fix getSystemRequirements API: don't return SemverConstraint on error * #44449: fix getSystemRequirements API * #44449: fix previews sytem requirements text * #44449: add `doThumbnailsExist` to repo * #44449: remove redux api * #44449: add missing model * #44449: implement frontedsettings-driven capability check * #44449: simplify * #44449: revert test changes * #44449: add dummy setup settings * #44449: implicit typing over `FC<Props>` * #44449: refactor conditionals * #44449: replace `getText` with a react component * #44449: fix component interface * #44449: add onRemove to `PreviewsSystemRequirements` alert * #44449: add bottom/top margin to previewSystemRequirements modal * #44449: merge conflict fix * #44449: remove console.log Co-authored-by: Ryan McKinley <ryantxu@gmail.com> Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
2022-02-16 11:49:50 -06:00
func (ss *SQLStore) FindThumbnailCount(ctx context.Context, cmd *models.FindDashboardThumbnailCountCommand) (int64, error) {
err := ss.WithDbSession(ctx, func(sess *DBSession) error {
count, err := sess.Count(&models.DashboardThumbnail{})
if err != nil {
return err
}
cmd.Result = count
return nil
})
return cmd.Result, err
}
Dash previews: populate crawler queue from SQL query (#44083) * add SQL migrations * dashboard previews from sql: poc * added todos * refactor: use the same enums where possible * use useEffect, always return json * added todo * refactor + delete files after use * refactor + fix manual thumbnail upload * refactor: move all interactions with sqlStore to thumbnail repo * refactor: remove file operations in thumb crawler/service * refactor: fix dashboard_thumbs sql store * refactor: extracted thumbnail fetching/updating to a hook * refactor: store thumbnails in redux store * refactor: store thumbnails in redux store * refactor: private'd repo methods * removed redux storage, saving images as blobs * allow for configurable rendering timeouts * added 1) query for dashboards with stale thumbnails, 2) command for marking thumbnails as stale * use sql-based queue in crawler * ui for marking thumbnails as stale * replaced `stale` boolean prop with `state` enum * introduce rendering session * compilation errors * fix crawler stop button * rename thumbnail state frozen to locked * #44449: fix merge conflicts * #44449: remove thumb methods from `Store` interface * #44449: clean filepath, defer file closing * #44449: fix rendering.Theme cyclic import * #44449: linting * #44449: linting * #44449: mutex'd crawlerStatus access * #44449: added integration tests for `sqlstore.dashboard_thumbs` * #44449: added comments to explain the `ThumbnailState` enum * #44449: use os.ReadFile rather then os.Open * #44449: always enable dashboardPreviews feature during integration tests * #44449: remove sleep time, adjust number of threads * #44449: review fix: add `orgId` to `DashboardThumbnailMeta` * #44449: review fix: automatic parsing of thumbnailState * #44449: lint fixes * #44449: review fix: prefer `WithDbSession` over `WithTransactionalDbSession` * #44449: review fix: add a comment explaining source of the filepath * #44449: review fix: added filepath validation * #44449: review fixes https://github.com/grafana/grafana/pull/45063/files @fzambia Co-authored-by: Ryan McKinley <ryantxu@gmail.com> Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
2022-02-09 03:23:32 -06:00
func (ss *SQLStore) FindDashboardsWithStaleThumbnails(ctx context.Context, cmd *models.FindDashboardsWithStaleThumbnailsCommand) ([]*models.DashboardWithStaleThumbnail, error) {
err := ss.WithDbSession(ctx, func(sess *DBSession) error {
sess.Table("dashboard")
Previews: crawler as a background service (#44891) * add SQL migrations * dashboard previews from sql: poc * added todos * refactor: use the same enums where possible * use useEffect, always return json * added todo * refactor + delete files after use * refactor + fix manual thumbnail upload * refactor: move all interactions with sqlStore to thumbnail repo * refactor: remove file operations in thumb crawler/service * refactor: fix dashboard_thumbs sql store * refactor: extracted thumbnail fetching/updating to a hook * refactor: store thumbnails in redux store * refactor: store thumbnails in redux store * refactor: private'd repo methods * removed redux storage, saving images as blobs * allow for configurable rendering timeouts * added 1) query for dashboards with stale thumbnails, 2) command for marking thumbnails as stale * use sql-based queue in crawler * ui for marking thumbnails as stale * replaced `stale` boolean prop with `state` enum * introduce rendering session * compilation errors * fix crawler stop button * rename thumbnail state frozen to locked * #44449: fix merge conflicts * #44449: remove thumb methods from `Store` interface * #44449: clean filepath, defer file closing * #44449: fix rendering.Theme cyclic import * #44449: linting * #44449: linting * #44449: mutex'd crawlerStatus access * #44449: added integration tests for `sqlstore.dashboard_thumbs` * #44449: added comments to explain the `ThumbnailState` enum * #44449: use os.ReadFile rather then os.Open * #44449: always enable dashboardPreviews feature during integration tests * #44449: remove sleep time, adjust number of threads * #44449: review fix: add `orgId` to `DashboardThumbnailMeta` * #44449: review fix: automatic parsing of thumbnailState * #44449: lint fixes * #44449: crawler as a background service v0.1 * #44449: use ServerLockService * #44449: use ServerLockService * #44449: review fix: prefer `WithDbSession` over `WithTransactionalDbSession` * #44449: review fix: add a comment explaining source of the filepath * #44449: review fix: added filepath validation * #44449: fix FindDashboardsWithStaleThumbnails to include `theme` and `kind` in search params * #44449: fix FindDashboardsWithStaleThumbnails to include `theme` and `kind` in search params * #44449: create function for crawler on demand * #44449: improve crawler logging * #44449: fix wire * #44449: uncomment dummy thumb service, fix ticker interval * #44449: prevent race condition * #44449: improve logging * #44449: fix theme * #44449: review fixes https://github.com/grafana/grafana/pull/45063/files @fzambia * #44449: add missing unlock * #44449: merge * #44449: review fix - logger @fzambia https://github.com/grafana/grafana/pull/45063/files * #44449: formatting * #44449: merge conflict fix * #44449: merge conflict fix * #44449: merge conflict fix * #44449: naming fix * #44449: update authOpts * #44449: change authOpts.role back to admin * #44449: fix `walk` signature, move ctx to a first argument * #44449: add `dashboardPreviewsScheduler` feature flag Co-authored-by: Ryan McKinley <ryantxu@gmail.com> Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
2022-02-10 12:45:00 -06:00
sess.Join("LEFT", "dashboard_thumbnail", "dashboard.id = dashboard_thumbnail.dashboard_id AND dashboard_thumbnail.theme = ? AND dashboard_thumbnail.kind = ?", cmd.Theme, cmd.Kind)
Dash previews: populate crawler queue from SQL query (#44083) * add SQL migrations * dashboard previews from sql: poc * added todos * refactor: use the same enums where possible * use useEffect, always return json * added todo * refactor + delete files after use * refactor + fix manual thumbnail upload * refactor: move all interactions with sqlStore to thumbnail repo * refactor: remove file operations in thumb crawler/service * refactor: fix dashboard_thumbs sql store * refactor: extracted thumbnail fetching/updating to a hook * refactor: store thumbnails in redux store * refactor: store thumbnails in redux store * refactor: private'd repo methods * removed redux storage, saving images as blobs * allow for configurable rendering timeouts * added 1) query for dashboards with stale thumbnails, 2) command for marking thumbnails as stale * use sql-based queue in crawler * ui for marking thumbnails as stale * replaced `stale` boolean prop with `state` enum * introduce rendering session * compilation errors * fix crawler stop button * rename thumbnail state frozen to locked * #44449: fix merge conflicts * #44449: remove thumb methods from `Store` interface * #44449: clean filepath, defer file closing * #44449: fix rendering.Theme cyclic import * #44449: linting * #44449: linting * #44449: mutex'd crawlerStatus access * #44449: added integration tests for `sqlstore.dashboard_thumbs` * #44449: added comments to explain the `ThumbnailState` enum * #44449: use os.ReadFile rather then os.Open * #44449: always enable dashboardPreviews feature during integration tests * #44449: remove sleep time, adjust number of threads * #44449: review fix: add `orgId` to `DashboardThumbnailMeta` * #44449: review fix: automatic parsing of thumbnailState * #44449: lint fixes * #44449: review fix: prefer `WithDbSession` over `WithTransactionalDbSession` * #44449: review fix: add a comment explaining source of the filepath * #44449: review fix: added filepath validation * #44449: review fixes https://github.com/grafana/grafana/pull/45063/files @fzambia Co-authored-by: Ryan McKinley <ryantxu@gmail.com> Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
2022-02-09 03:23:32 -06:00
sess.Where("dashboard.is_folder = ?", dialect.BooleanStr(false))
sess.Where("(dashboard.version != dashboard_thumbnail.dashboard_version "+
"OR dashboard_thumbnail.state = ? "+
"OR dashboard_thumbnail.id IS NULL)", models.ThumbnailStateStale)
if !cmd.IncludeManuallyUploadedThumbnails {
sess.Where("(dashboard_thumbnail.id is not null AND dashboard_thumbnail.dashboard_version != ?) "+
"OR dashboard_thumbnail.id is null "+
"OR dashboard_thumbnail.state = ?", models.DashboardVersionForManualThumbnailUpload, models.ThumbnailStateStale)
}
sess.Where("(dashboard_thumbnail.id IS NULL OR dashboard_thumbnail.state != ?)", models.ThumbnailStateLocked)
sess.Cols("dashboard.id",
"dashboard.uid",
"dashboard.org_id",
"dashboard.version",
"dashboard.slug")
var dashboards = make([]*models.DashboardWithStaleThumbnail, 0)
err := sess.Find(&dashboards)
if err != nil {
return err
}
cmd.Result = dashboards
return err
})
return cmd.Result, err
}
func findThumbnailByMeta(sess *DBSession, meta models.DashboardThumbnailMeta) (*models.DashboardThumbnail, error) {
result := &models.DashboardThumbnail{}
sess.Table("dashboard_thumbnail")
sess.Join("INNER", "dashboard", "dashboard.id = dashboard_thumbnail.dashboard_id")
sess.Where("dashboard.uid = ? AND dashboard.org_id = ? AND panel_id = ? AND kind = ? AND theme = ?", meta.DashboardUID, meta.OrgId, meta.PanelID, meta.Kind, meta.Theme)
sess.Cols("dashboard_thumbnail.id",
"dashboard_thumbnail.dashboard_id",
"dashboard_thumbnail.panel_id",
"dashboard_thumbnail.image",
"dashboard_thumbnail.dashboard_version",
"dashboard_thumbnail.state",
"dashboard_thumbnail.kind",
"dashboard_thumbnail.mime_type",
"dashboard_thumbnail.theme",
"dashboard_thumbnail.updated")
exists, err := sess.Get(result)
if !exists {
return nil, models.ErrDashboardThumbnailNotFound
}
if err != nil {
return nil, err
}
return result, nil
}
type dash struct {
Id int64
}
func findDashboardIdByThumbMeta(sess *DBSession, meta models.DashboardThumbnailMeta) (*dash, error) {
result := &dash{}
sess.Table("dashboard").Where("dashboard.uid = ? AND dashboard.org_id = ?", meta.DashboardUID, meta.OrgId).Cols("id")
exists, err := sess.Get(result)
if err != nil {
return nil, err
}
if !exists {
return nil, models.ErrDashboardNotFound
}
return result, err
}