IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Crawler: use existing render service to generate dashboard thumbnails (#43515)

Browse Source 
Co-authored-by: Artur Wierzbicki <artur@arturwierzbicki.com>
This commit is contained in:
Ryan McKinley 2022-01-10 08:21:35 -08:00 committed by GitHub
parent cc9e70be5c
commit b404aae9c3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 475 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contribute/architecture/backend/communication.md
View File

@ -109,7 +109,7 @@ ctx := req.Request.Context()
query := &models.FindDashboardQuery{
ID: "foo",
}
if err := bus.DispatchCtx(ctx, query); err != nil {
if err := bus.Dispatch(ctx, query); err != nil {
return err
}
// The query now contains a result.

6
pkg/api/api.go
View File

@ -461,6 +461,12 @@ func (hs *HTTPServer) registerRoutes() {
adminRoute.Get("/stats", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionServerStatsRead)), routing.Wrap(AdminGetStats))
adminRoute.Post("/pause-all-alerts", reqGrafanaAdmin, routing.Wrap(PauseAllAlerts))
if hs.ThumbService != nil {
adminRoute.Post("/crawler/start", reqGrafanaAdmin, routing.Wrap(hs.ThumbService.StartCrawler))
adminRoute.Post("/crawler/stop", reqGrafanaAdmin, routing.Wrap(hs.ThumbService.StopCrawler))
adminRoute.Get("/crawler/status", reqGrafanaAdmin, routing.Wrap(hs.ThumbService.CrawlerStatus))
}
adminRoute.Post("/provisioning/dashboards/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersDashboards)), routing.Wrap(hs.AdminProvisioningReloadDashboards))
adminRoute.Post("/provisioning/plugins/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersPlugins)), routing.Wrap(hs.AdminProvisioningReloadPlugins))
adminRoute.Post("/provisioning/datasources/reload", authorize(reqGrafanaAdmin, ac.EvalPermission(ActionProvisioningReload, ScopeProvisionersDatasources)), routing.Wrap(hs.AdminProvisioningReloadDatasources))

4
pkg/services/rendering/rendering.go
View File

@ -207,6 +207,8 @@ func (rs *RenderingService) Render(ctx context.Context, opts Opts) (*RenderResul
func (rs *RenderingService) render(ctx context.Context, opts Opts) (*RenderResult, error) {
if int(atomic.LoadInt32(&rs.inProgressCount)) > opts.ConcurrentLimit {
rs.log.Warn("Could not render image, hit the currency limit", "concurrencyLimit", opts.ConcurrentLimit, "path", opts.Path)
theme := ThemeDark
if opts.Theme != "" {
theme = opts.Theme
@ -225,7 +227,7 @@ func (rs *RenderingService) render(ctx context.Context, opts Opts) (*RenderResul
}
rs.log.Info("Rendering", "path", opts.Path)
if math.IsInf(opts.DeviceScaleFactor, 0) || math.IsNaN(opts.DeviceScaleFactor) || opts.DeviceScaleFactor <= 0 {
if math.IsInf(opts.DeviceScaleFactor, 0) || math.IsNaN(opts.DeviceScaleFactor) || opts.DeviceScaleFactor == 0 {
opts.DeviceScaleFactor = 1
}
renderKey, err := rs.generateAndStoreRenderKey(ctx, opts.OrgID, opts.UserID, opts.OrgRole)

1
pkg/services/rendering/rendering_test.go
View File

@ -110,6 +110,7 @@ func TestRenderLimitImage(t *testing.T) {
HomePath: path,
},
inProgressCount: 2,
log: log.New("test"),
}
tests := []struct {

249
pkg/services/thumbs/crawler.go Normal file
View File

@ -0,0 +1,249 @@
package thumbs
import (
"context"
"encoding/json"
"errors"
"fmt"
"math/rand"
"os"
"strings"
"sync"
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/live"
"github.com/grafana/grafana/pkg/services/rendering"
"github.com/grafana/grafana/pkg/services/search"
)
type dashItem struct {
uid string
url string
}
type simpleCrawler struct {
screenshotsFolder string
renderService rendering.Service
threadCount int
glive *live.GrafanaLive
mode CrawlerMode
opts rendering.Opts
status crawlStatus
queue []dashItem
mu sync.Mutex
}
func newSimpleCrawler(folder string, renderService rendering.Service, gl *live.GrafanaLive) dashRenderer {
c := &simpleCrawler{
screenshotsFolder: folder,
renderService: renderService,
threadCount: 5,
glive: gl,
status: crawlStatus{
State: "init",
Complete: 0,
Queue: 0,
},
queue: make([]dashItem, 0),
}
c.broadcastStatus()
return c
}
func (r *simpleCrawler) next() *dashItem {
if len(r.queue) < 1 {
return nil
}
r.mu.Lock()
defer r.mu.Unlock()
v := r.queue[0]
r.queue = r.queue[1:]
return &v
}
func (r *simpleCrawler) broadcastStatus() {
s, err := r.Status()
if err != nil {
tlog.Warn("error reading status")
return
}
msg, err := json.Marshal(s)
if err != nil {
tlog.Warn("error making message")
return
}
err = r.glive.Publish(r.opts.OrgID, "grafana/broadcast/crawler", msg)
if err != nil {
tlog.Warn("error Publish message")
return
}
}
func (r *simpleCrawler) GetPreview(req *previewRequest) *previewResponse {
p := getFilePath(r.screenshotsFolder, req)
if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) {
return r.queueRender(p, req)
}
return &previewResponse{
Path: p,
Code: 200,
}
}
func (r *simpleCrawler) queueRender(p string, req *previewRequest) *previewResponse {
go func() {
fmt.Printf("todo? queue")
}()
return &previewResponse{
Code: 202,
Path: p,
}
}
func (r *simpleCrawler) Start(c *models.ReqContext, mode CrawlerMode, theme rendering.Theme) (crawlStatus, error) {
if r.status.State == "running" {
tlog.Info("already running")
return r.Status()
}
r.mu.Lock()
defer r.mu.Unlock()
searchQuery := search.Query{
SignedInUser: c.SignedInUser,
OrgId: c.OrgId,
}
err := bus.Dispatch(context.Background(), &searchQuery)
if err != nil {
return crawlStatus{}, err
}
queue := make([]dashItem, 0, len(searchQuery.Result))
for _, v := range searchQuery.Result {
if v.Type == search.DashHitDB {
queue = append(queue, dashItem{
uid: v.UID,
url: v.URL,
})
}
}
rand.Seed(time.Now().UnixNano())
rand.Shuffle(len(queue), func(i, j int) { queue[i], queue[j] = queue[j], queue[i] })
r.mode = mode
r.opts = rendering.Opts{
OrgID: c.OrgId,
UserID: c.UserId,
OrgRole: c.OrgRole,
Theme: theme,
ConcurrentLimit: 10,
}
r.queue = queue
r.status = crawlStatus{
Started: time.Now(),
State: "running",
Complete: 0,
}
r.broadcastStatus()
// create a pool of workers
for i := 0; i < r.threadCount; i++ {
go r.walk()
// wait 1/2 second before starting a new thread
time.Sleep(500 * time.Millisecond)
}
r.broadcastStatus()
return r.Status()
}
func (r *simpleCrawler) Stop() (crawlStatus, error) {
// cheap hack!
if r.status.State == "running" {
r.status.State = "stopping"
}
return r.Status()
}
func (r *simpleCrawler) Status() (crawlStatus, error) {
status := crawlStatus{
State: r.status.State,
Started: r.status.Started,
Complete: r.status.Complete,
Errors: r.status.Errors,
Queue: len(r.queue),
Last: r.status.Last,
}
return status, nil
}
func (r *simpleCrawler) walk() {
for {
if r.status.State == "stopping" {
break
}
item := r.next()
if item == nil {
break
}
tlog.Info("GET THUMBNAIL", "url", item.url)
// Hack (for now) pick a URL that will render
panelURL := strings.TrimPrefix(item.url, "/") + "?kiosk"
res, err := r.renderService.Render(context.Background(), rendering.Opts{
Width: 320,
Height: 240,
Path: panelURL,
OrgID: r.opts.OrgID,
UserID: r.opts.UserID,
ConcurrentLimit: r.opts.ConcurrentLimit,
OrgRole: r.opts.OrgRole,
Theme: r.opts.Theme,
Timeout: 10 * time.Second,
DeviceScaleFactor: -5, // negative numbers will render larger then scale down
})
if err != nil {
tlog.Warn("error getting image", "err", err)
r.status.Errors++
} else if res.FilePath == "" {
tlog.Warn("error getting image... no response")
r.status.Errors++
} else if strings.Contains(res.FilePath, "public/img") {
tlog.Warn("error getting image... internal result", "img", res.FilePath)
r.status.Errors++
} else {
p := getFilePath(r.screenshotsFolder, &previewRequest{
UID: item.uid,
OrgID: r.opts.OrgID,
Theme: r.opts.Theme,
Size: PreviewSizeThumb,
})
err = os.Rename(res.FilePath, p)
if err != nil {
r.status.Errors++
tlog.Warn("error moving image", "err", err)
} else {
r.status.Complete++
tlog.Info("saved thumbnail", "img", item.url)
}
}
time.Sleep(5 * time.Second)
r.status.Last = time.Now()
r.broadcastStatus()
}
r.status.State = "stopped"
r.status.Finished = time.Now()
r.broadcastStatus()
}

72
pkg/services/thumbs/crawler_http.go
View File

@ -1,72 +0,0 @@
package thumbs
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"os"
)
type renderHttp struct {
crawlerURL string
config crawConfig
}
func newRenderHttp(crawlerURL string, cfg crawConfig) dashRenderer {
return &renderHttp{
crawlerURL: crawlerURL,
config: cfg,
}
}
func (r *renderHttp) GetPreview(req *previewRequest) *previewResponse {
p := getFilePath(r.config.ScreenshotsFolder, req)
if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) {
return r.queueRender(p, req)
}
return &previewResponse{
Path: p,
Code: 200,
}
}
func (r *renderHttp) CrawlerCmd(cfg *crawlCmd) (json.RawMessage, error) {
cmd := r.config
cmd.crawlCmd = *cfg
jsonData, err := json.Marshal(cmd)
if err != nil {
return nil, err
}
request, err := http.NewRequest("POST", r.crawlerURL, bytes.NewBuffer(jsonData))
if err != nil {
return nil, err
}
request.Header.Set("Content-Type", "application/json; charset=UTF-8")
client := &http.Client{}
response, error := client.Do(request)
if error != nil {
return nil, err
}
defer func() {
_ = response.Body.Close()
}()
return ioutil.ReadAll(response.Body)
}
func (r *renderHttp) queueRender(p string, req *previewRequest) *previewResponse {
go func() {
fmt.Printf("todo? queue")
}()
return &previewResponse{
Code: 202,
Path: p,
}
}

7
pkg/services/thumbs/dummy.go
View File

@ -25,8 +25,15 @@ func (ds *dummyService) StartCrawler(c *models.ReqContext) response.Response {
result["error"] = "Not enabled"
return response.JSON(200, result)
}
func (ds *dummyService) StopCrawler(c *models.ReqContext) response.Response {
result := make(map[string]string)
result["error"] = "Not enabled"
return response.JSON(200, result)
}
func (ds *dummyService) CrawlerStatus(c *models.ReqContext) response.Response {
result := make(map[string]string)
result["error"] = "Not enabled"
return response.JSON(200, result)
}

78
pkg/services/thumbs/models.go
View File

@ -1,8 +1,14 @@
package thumbs
import "encoding/json"
import (
"time"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/rendering"
)
type PreviewSize string
type CrawlerMode string
const (
// PreviewSizeThumb is a small 320x240 preview
@ -13,6 +19,15 @@ const (
// PreviewSizeLarge is a large image 512x????
PreviewSizeTall PreviewSize = "tall"
// CrawlerModeThumbs will create small thumbnails for everything
CrawlerModeThumbs CrawlerMode = "thumbs"
// CrawlerModeAnalytics will get full page results for everythign
CrawlerModeAnalytics CrawlerMode = "analytics"
// CrawlerModeMigrate will migrate all dashboards with old schema
CrawlerModeMigrate CrawlerMode = "migrate"
)
// IsKnownSize checks if the value is a standard size
@ -39,22 +54,21 @@ func getPreviewSize(str string) (PreviewSize, bool) {
return PreviewSizeThumb, false
}
func getTheme(str string) (string, bool) {
func getTheme(str string) (rendering.Theme, bool) {
switch str {
case "light":
return str, true
return rendering.ThemeLight, true
case "dark":
return str, true
return rendering.ThemeDark, true
}
return "dark", false
return rendering.ThemeDark, false
}
type previewRequest struct {
Kind string `json:"kind"`
OrgID int64 `json:"orgId"`
UID string `json:"uid"`
Size PreviewSize `json:"size"`
Theme string `json:"theme"`
OrgID int64 `json:"orgId"`
UID string `json:"uid"`
Size PreviewSize `json:"size"`
Theme rendering.Theme `json:"theme"`
}
type previewResponse struct {
@ -63,35 +77,19 @@ type previewResponse struct {
URL string `json:"url"` // redirect to this URL
}
// export enum CrawlerMode {
// Thumbs = 'thumbs',
// Analytics = 'analytics', // Enterprise only
// Migrate = 'migrate',
// }
// export enum CrawlerAction {
// Run = 'run',
// Stop = 'stop',
// Queue = 'queue', // TODO (later!) move some to the front
// }
type crawlCmd struct {
Mode string `json:"mode"` // thumbs | analytics | migrate
Action string `json:"action"` // run | stop | queue
Theme string `json:"theme"` // light | dark
User string `json:"user"` // :(
Password string `json:"password"` // :(
Concurrency int `json:"concurrency"` // number of pages to run in parallel
Path string `json:"path"` // eventually for queue
Mode CrawlerMode `json:"mode"` // thumbs | analytics | migrate
Theme rendering.Theme `json:"theme"` // light | dark
}
type crawConfig struct {
crawlCmd
// Sent to the crawler with each command
URL string `json:"url"`
ScreenshotsFolder string `json:"screenshotsFolder"`
type crawlStatus struct {
State string `json:"state"`
Started time.Time `json:"started,omitempty"`
Finished time.Time `json:"finished,omitempty"`
Complete int `json:"complete"`
Errors int `json:"errors"`
Queue int `json:"queue"`
Last time.Time `json:"last,omitempty"`
}
type dashRenderer interface {
@ -99,5 +97,11 @@ type dashRenderer interface {
GetPreview(req *previewRequest) *previewResponse
// Assumes you have already authenticated as admin
CrawlerCmd(cfg *crawlCmd) (json.RawMessage, error)
Start(c *models.ReqContext, mode CrawlerMode, theme rendering.Theme) (crawlStatus, error)
// Assumes you have already authenticated as admin
Stop() (crawlStatus, error)
// Assumes you have already authenticated as admin
Status() (crawlStatus, error)
}

46
pkg/services/thumbs/service.go
View File

@ -15,6 +15,7 @@ import (
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/guardian"
"github.com/grafana/grafana/pkg/services/live"
"github.com/grafana/grafana/pkg/services/rendering"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/web"
@ -28,29 +29,27 @@ var (
type Service interface {
Enabled() bool
GetImage(c *models.ReqContext)
// Form post (from dashboard page)
SetImage(c *models.ReqContext)
// Must be admin
StartCrawler(c *models.ReqContext) response.Response
StopCrawler(c *models.ReqContext) response.Response
CrawlerStatus(c *models.ReqContext) response.Response
}
func ProvideService(cfg *setting.Cfg, renderService rendering.Service) Service {
func ProvideService(cfg *setting.Cfg, renderService rendering.Service, gl *live.GrafanaLive) Service {
if !cfg.IsDashboardPreviesEnabled() {
return &dummyService{}
}
root := filepath.Join(cfg.DataPath, "crawler", "preview")
url := strings.TrimSuffix(cfg.RendererUrl, "/render") + "/scan"
renderer := newRenderHttp(url, crawConfig{
URL: strings.TrimSuffix(cfg.RendererCallbackUrl, "/"),
ScreenshotsFolder: root,
})
tempdir := filepath.Join(cfg.DataPath, "temp")
_ = os.MkdirAll(root, 0700)
_ = os.MkdirAll(tempdir, 0700)
renderer := newSimpleCrawler(root, renderService, gl)
return &thumbService{
renderer: renderer,
root: root,
@ -84,7 +83,6 @@ func (hs *thumbService) parseImageReq(c *models.ReqContext, checkSave bool) *pre
}
req := &previewRequest{
Kind: "dash",
OrgID: c.OrgId,
UID: params[":uid"],
Theme: theme,
@ -137,6 +135,7 @@ func (hs *thumbService) GetImage(c *models.ReqContext) {
c.JSON(500, map[string]string{"path": rsp.Path, "error": "unknown!"})
}
// Hack for now -- lets you upload images explicitly
func (hs *thumbService) SetImage(c *models.ReqContext) {
req := hs.parseImageReq(c, false)
if req == nil {
@ -217,29 +216,30 @@ func (hs *thumbService) StartCrawler(c *models.ReqContext) response.Response {
if err != nil {
return response.Error(500, "error parsing bytes", err)
}
cmd.Action = "start"
msg, err := hs.renderer.CrawlerCmd(cmd)
if cmd.Mode == "" {
cmd.Mode = CrawlerModeThumbs
}
msg, err := hs.renderer.Start(c, cmd.Mode, cmd.Theme)
if err != nil {
return response.Error(500, "error starting", err)
}
header := make(http.Header)
header.Set("Content-Type", "application/json")
return response.CreateNormalResponse(header, msg, 200)
return response.JSON(200, msg)
}
func (hs *thumbService) StopCrawler(c *models.ReqContext) response.Response {
_, err := hs.renderer.CrawlerCmd(&crawlCmd{
Action: "stop",
})
msg, err := hs.renderer.Stop()
if err != nil {
return response.Error(500, "error stopping crawler", err)
return response.Error(500, "error starting", err)
}
return response.JSON(200, msg)
}
result := make(map[string]string)
result["message"] = "Stopping..."
return response.JSON(200, result)
func (hs *thumbService) CrawlerStatus(c *models.ReqContext) response.Response {
msg, err := hs.renderer.Status()
if err != nil {
return response.Error(500, "error starting", err)
}
return response.JSON(200, msg)
}
// Ideally this service would not require first looking up the full dashboard just to bet the id!

61
public/app/features/admin/CrawlerStartButton.tsx Normal file
View File

@ -0,0 +1,61 @@
import React, { useState } from 'react';
import { css } from '@emotion/css';
import { Button, CodeEditor, Modal, useTheme2 } from '@grafana/ui';
import { GrafanaTheme2 } from '@grafana/data';
import { getBackendSrv, config } from '@grafana/runtime';
export const CrawlerStartButton = () => {
const styles = getStyles(useTheme2());
const [open, setOpen] = useState(false);
const [body, setBody] = useState({
mode: 'thumbs',
theme: config.theme2.isLight ? 'light' : 'dark',
});
const onDismiss = () => setOpen(false);
const doStart = () => {
getBackendSrv()
.post('/api/admin/crawler/start', body)
.then((v) => {
console.log('GOT', v);
onDismiss();
});
};
return (
<>
<Modal title={'Start crawler'} isOpen={open} onDismiss={onDismiss}>
<div className={styles.wrap}>
<CodeEditor
height={200}
value={JSON.stringify(body, null, 2) ?? ''}
showLineNumbers={false}
readOnly={false}
language="json"
showMiniMap={false}
onBlur={(text: string) => {
setBody(JSON.parse(text)); // force JSON?
}}
/>
</div>
<Modal.ButtonRow>
<Button onClick={doStart}>Start</Button>
<Button variant="secondary" onClick={onDismiss}>
Cancel
</Button>
</Modal.ButtonRow>
</Modal>
<Button onClick={() => setOpen(true)} variant="primary">
Start
</Button>
</>
);
};
const getStyles = (theme: GrafanaTheme2) => {
return {
wrap: css`
border: 2px solid #111;
`,
};
};

79
public/app/features/admin/CrawlerStatus.tsx Normal file
View File

@ -0,0 +1,79 @@
import React, { useEffect, useState } from 'react';
import { css } from '@emotion/css';
import { Button, useTheme2 } from '@grafana/ui';
import { GrafanaTheme2, isLiveChannelMessageEvent, isLiveChannelStatusEvent, LiveChannelScope } from '@grafana/data';
import { getBackendSrv, getGrafanaLiveSrv } from '@grafana/runtime';
import { CrawlerStartButton } from './CrawlerStartButton';
interface CrawlerStatusMessage {
state: string;
started: string;
finished: string;
complete: number;
queue: number;
last: string;
}
export const CrawlerStatus = () => {
const styles = getStyles(useTheme2());
const [status, setStatus] = useState<CrawlerStatusMessage>();
useEffect(() => {
const subscription = getGrafanaLiveSrv()
.getStream<CrawlerStatusMessage>({
scope: LiveChannelScope.Grafana,
namespace: 'broadcast',
path: 'crawler',
})
.subscribe({
next: (evt) => {
if (isLiveChannelMessageEvent(evt)) {
setStatus(evt.message);
} else if (isLiveChannelStatusEvent(evt)) {
setStatus(evt.message);
}
},
});
return () => {
subscription.unsubscribe();
};
}, []);
if (!status) {
return (
<div className={styles.wrap}>
No status (never run)
<br />
<CrawlerStartButton />
</div>
);
}
return (
<div className={styles.wrap}>
<pre>{JSON.stringify(status, null, 2)}</pre>
{status.state !== 'running' && <CrawlerStartButton />}
{status.state !== 'stopped' && (
<Button
variant="secondary"
onClick={() => {
getBackendSrv().post('/api/admin/crawler/stop');
}}
>
Stop
</Button>
)}
</div>
);
};
const getStyles = (theme: GrafanaTheme2) => {
return {
wrap: css`
border: 4px solid red;
`,
running: css`
border: 4px solid green;
`,
};
};

4
public/app/features/admin/ServerStats.tsx
View File

@ -6,6 +6,8 @@ import { AccessControlAction } from 'app/types';
import { getServerStats, ServerStat } from './state/apis';
import { contextSrv } from '../../core/services/context_srv';
import { Loader } from '../plugins/admin/components/Loader';
import { config } from '@grafana/runtime';
import { CrawlerStatus } from './CrawlerStatus';
export const ServerStats = () => {
const [stats, setStats] = useState<ServerStat | null>(null);
@ -84,6 +86,8 @@ export const ServerStats = () => {
) : (
<p className={styles.notFound}>No stats found.</p>
)}
{config.featureToggles.dashboardPreviews && <CrawlerStatus />}
</>
);
};