IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Previews: create crawler auth setup service (#47349)

Browse Source 
* #46968: add `RetrieveServiceAccountIdByName` to serviceaccounts service

* #46968: improve error logging in rendering service

* #46968: add oss crawler account setup

* #46968: fix tests

* #46968: switch back to ROLE_ADMIN

* #46968: rename to crawlerAuth

* comment crawler_auth.go
This commit is contained in:
Artur Wierzbicki 2022-04-12 19:34:04 +02:00 committed by GitHub
parent 5cb5141c72
commit a4381ebc91
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 153 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
pkg/server/wireexts_oss.go
View File

@ -31,6 +31,7 @@ import (
"github.com/grafana/grafana/pkg/services/searchusers"
"github.com/grafana/grafana/pkg/services/searchusers/filters"
"github.com/grafana/grafana/pkg/services/sqlstore/migrations"
"github.com/grafana/grafana/pkg/services/thumbs"
"github.com/grafana/grafana/pkg/services/validations"
"github.com/grafana/grafana/pkg/setting"
)
@ -46,6 +47,8 @@ var wireExtsBasicSet = wire.NewSet(
ossaccesscontrol.ProvideService,
wire.Bind(new(accesscontrol.RoleRegistry), new(*ossaccesscontrol.OSSAccessControlService)),
wire.Bind(new(accesscontrol.AccessControl), new(*ossaccesscontrol.OSSAccessControlService)),
thumbs.ProvideCrawlerAuthSetupService,
wire.Bind(new(thumbs.CrawlerAuthSetupService), new(*thumbs.OSSCrawlerAuthSetupService)),
validations.ProvideValidator,
wire.Bind(new(models.PluginRequestValidator), new(*validations.OSSPluginRequestValidator)),
provisioning.ProvideService,

14
pkg/services/rendering/http_mode.go
View File

@ -46,7 +46,8 @@ func (rs *RenderingService) renderViaHTTP(ctx context.Context, renderKey string,
}
queryParams := rendererURL.Query()
queryParams.Add("url", rs.getURL(opts.Path))
url := rs.getURL(opts.Path)
queryParams.Add("url", url)
queryParams.Add("renderKey", renderKey)
queryParams.Add("width", strconv.Itoa(opts.Width))
queryParams.Add("height", strconv.Itoa(opts.Height))
@ -74,7 +75,7 @@ func (rs *RenderingService) renderViaHTTP(ctx context.Context, renderKey string,
}
}()
err = rs.readFileResponse(reqContext, resp, filePath)
err = rs.readFileResponse(reqContext, resp, filePath, url)
if err != nil {
return nil, err
}
@ -94,7 +95,8 @@ func (rs *RenderingService) renderCSVViaHTTP(ctx context.Context, renderKey stri
}
queryParams := rendererURL.Query()
queryParams.Add("url", rs.getURL(opts.Path))
url := rs.getURL(opts.Path)
queryParams.Add("url", url)
queryParams.Add("renderKey", renderKey)
queryParams.Add("domain", rs.domain)
queryParams.Add("timezone", isoTimeOffsetToPosixTz(opts.Timezone))
@ -125,7 +127,7 @@ func (rs *RenderingService) renderCSVViaHTTP(ctx context.Context, renderKey stri
}
downloadFileName := params["filename"]
err = rs.readFileResponse(reqContext, resp, filePath)
err = rs.readFileResponse(reqContext, resp, filePath, url)
if err != nil {
return nil, err
}
@ -156,7 +158,7 @@ func (rs *RenderingService) doRequest(ctx context.Context, url *url.URL, headers
return resp, nil
}
func (rs *RenderingService) readFileResponse(ctx context.Context, resp *http.Response, filePath string) error {
func (rs *RenderingService) readFileResponse(ctx context.Context, resp *http.Response, filePath string, url string) error {
// check for timeout first
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
rs.log.Info("Rendering timed out")
@ -165,7 +167,7 @@ func (rs *RenderingService) readFileResponse(ctx context.Context, resp *http.Res
// if we didn't get a 200 response, something went wrong.
if resp.StatusCode != http.StatusOK {
rs.log.Error("Remote rendering request failed", "error", resp.Status)
rs.log.Error("Remote rendering request failed", "error", resp.Status, "url", url)
return fmt.Errorf("remote rendering request failed, status code: %d, status: %s", resp.StatusCode,
resp.Status)
}

41
pkg/services/serviceaccounts/database/database.go
View File

@ -231,6 +231,47 @@ func (s *ServiceAccountsStoreImpl) RetrieveServiceAccount(ctx context.Context, o
return serviceAccount, nil
}
func (s *ServiceAccountsStoreImpl) RetrieveServiceAccountIdByName(ctx context.Context, orgID int64, name string) (int64, error) {
serviceAccount := &struct {
Id int64
}{}
err := s.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error {
sess := dbSession.Table("user")
whereConditions := []string{
fmt.Sprintf("%s.name = ?",
s.sqlStore.Dialect.Quote("user")),
fmt.Sprintf("%s.org_id = ?",
s.sqlStore.Dialect.Quote("user")),
fmt.Sprintf("%s.is_service_account = %s",
s.sqlStore.Dialect.Quote("user"),
s.sqlStore.Dialect.BooleanStr(true)),
}
whereParams := []interface{}{name, orgID}
sess.Where(strings.Join(whereConditions, " AND "), whereParams...)
sess.Cols(
"user.id",
)
if ok, err := sess.Get(serviceAccount); err != nil {
return err
} else if !ok {
return serviceaccounts.ErrServiceAccountNotFound
}
return nil
})
if err != nil {
return 0, err
}
return serviceAccount.Id, nil
}
func (s *ServiceAccountsStoreImpl) UpdateServiceAccount(ctx context.Context,
orgID, serviceAccountID int64,
saForm *serviceaccounts.UpdateServiceAccountForm) (*serviceaccounts.ServiceAccountProfileDTO, error) {

17
pkg/services/serviceaccounts/database/database_test.go
View File

@ -15,17 +15,24 @@ import (
func TestStore_CreateServiceAccount(t *testing.T) {
_, store := setupTestDatabase(t)
t.Run("create service account", func(t *testing.T) {
saDTO, err := store.CreateServiceAccount(context.Background(), 1, "new Service Account")
serviceAccountName := "new Service Account"
serviceAccountOrgId := int64(1)
saDTO, err := store.CreateServiceAccount(context.Background(), serviceAccountOrgId, serviceAccountName)
require.NoError(t, err)
assert.Equal(t, "sa-new-service-account", saDTO.Login)
assert.Equal(t, "new Service Account", saDTO.Name)
assert.Equal(t, serviceAccountName, saDTO.Name)
assert.Equal(t, 0, int(saDTO.Tokens))
retrieved, err := store.RetrieveServiceAccount(context.Background(), 1, saDTO.Id)
retrieved, err := store.RetrieveServiceAccount(context.Background(), serviceAccountOrgId, saDTO.Id)
require.NoError(t, err)
assert.Equal(t, "sa-new-service-account", retrieved.Login)
assert.Equal(t, "new Service Account", retrieved.Name)
assert.Equal(t, 1, int(retrieved.OrgId))
assert.Equal(t, serviceAccountName, retrieved.Name)
assert.Equal(t, serviceAccountOrgId, retrieved.OrgId)
retrievedId, err := store.RetrieveServiceAccountIdByName(context.Background(), serviceAccountOrgId, serviceAccountName)
require.NoError(t, err)
assert.Equal(t, saDTO.Id, retrievedId)
})
}

8
pkg/services/serviceaccounts/manager/service.go
View File

@ -68,3 +68,11 @@ func (sa *ServiceAccountsService) DeleteServiceAccount(ctx context.Context, orgI
}
return sa.store.DeleteServiceAccount(ctx, orgID, serviceAccountID)
}
func (sa *ServiceAccountsService) RetrieveServiceAccountIdByName(ctx context.Context, orgID int64, name string) (int64, error) {
if !sa.features.IsEnabled(featuremgmt.FlagServiceAccounts) {
sa.log.Debug(ServiceAccountFeatureToggleNotFound)
return 0, nil
}
return sa.store.RetrieveServiceAccountIdByName(ctx, orgID, name)
}

2
pkg/services/serviceaccounts/serviceaccounts.go
View File

@ -10,6 +10,7 @@ import (
type Service interface {
CreateServiceAccount(ctx context.Context, orgID int64, name string) (*ServiceAccountDTO, error)
DeleteServiceAccount(ctx context.Context, orgID, serviceAccountID int64) error
RetrieveServiceAccountIdByName(ctx context.Context, orgID int64, name string) (int64, error)
}
type Store interface {
@ -19,6 +20,7 @@ type Store interface {
UpdateServiceAccount(ctx context.Context, orgID, serviceAccountID int64,
saForm *UpdateServiceAccountForm) (*ServiceAccountProfileDTO, error)
RetrieveServiceAccount(ctx context.Context, orgID, serviceAccountID int64) (*ServiceAccountProfileDTO, error)
RetrieveServiceAccountIdByName(ctx context.Context, orgID int64, name string) (int64, error)
DeleteServiceAccount(ctx context.Context, orgID, serviceAccountID int64) error
UpgradeServiceAccounts(ctx context.Context) error
ConvertToServiceAccounts(ctx context.Context, keys []int64) error

31
pkg/services/serviceaccounts/tests/common.go
View File

@ -38,6 +38,10 @@ func SetupUserServiceAccount(t *testing.T, sqlStore *sqlstore.SQLStore, testUser
// create mock for serviceaccountservice
type ServiceAccountMock struct{}
func (s *ServiceAccountMock) RetrieveServiceAccountIdByName(ctx context.Context, orgID int64, name string) (int64, error) {
return 0, nil
}
func (s *ServiceAccountMock) CreateServiceAccount(ctx context.Context, orgID int64, name string) (*serviceaccounts.ServiceAccountDTO, error) {
return nil, nil
}
@ -65,24 +69,31 @@ func SetupMockAccesscontrol(t *testing.T,
// this is a way to see
// that the Mock implements the store interface
var _ serviceaccounts.Store = new(ServiceAccountsStoreMock)
var _ serviceaccounts.Service = new(ServiceAccountMock)
type Calls struct {
CreateServiceAccount []interface{}
RetrieveServiceAccount []interface{}
DeleteServiceAccount []interface{}
UpgradeServiceAccounts []interface{}
ConvertServiceAccounts []interface{}
ListTokens []interface{}
DeleteServiceAccountToken []interface{}
UpdateServiceAccount []interface{}
AddServiceAccountToken []interface{}
SearchOrgServiceAccounts []interface{}
CreateServiceAccount []interface{}
RetrieveServiceAccount []interface{}
DeleteServiceAccount []interface{}
UpgradeServiceAccounts []interface{}
ConvertServiceAccounts []interface{}
ListTokens []interface{}
DeleteServiceAccountToken []interface{}
UpdateServiceAccount []interface{}
AddServiceAccountToken []interface{}
SearchOrgServiceAccounts []interface{}
RetrieveServiceAccountIdByName []interface{}
}
type ServiceAccountsStoreMock struct {
Calls Calls
}
func (s *ServiceAccountsStoreMock) RetrieveServiceAccountIdByName(ctx context.Context, orgID int64, name string) (int64, error) {
s.Calls.RetrieveServiceAccountIdByName = append(s.Calls.RetrieveServiceAccountIdByName, []interface{}{ctx, orgID, name})
return 0, nil
}
func (s *ServiceAccountsStoreMock) CreateServiceAccount(ctx context.Context, orgID int64, name string) (*serviceaccounts.ServiceAccountDTO, error) {
// now we can test that the mock has these calls when we call the function
s.Calls.CreateServiceAccount = append(s.Calls.CreateServiceAccount, []interface{}{ctx, orgID, name})

9
pkg/services/thumbs/crawler.go
View File

@ -26,6 +26,7 @@ type simpleCrawler struct {
thumbnailRepo thumbnailRepo
mode CrawlerMode
thumbnailKind models.ThumbnailKind
auth CrawlerAuth
opts rendering.Opts
status crawlStatus
statusMutex sync.RWMutex
@ -67,8 +68,8 @@ func (r *simpleCrawler) next(ctx context.Context) (*models.DashboardWithStaleThu
authOpts := rendering.AuthOpts{
OrgID: v.OrgId,
UserID: r.opts.AuthOpts.UserID,
OrgRole: r.opts.AuthOpts.OrgRole,
UserID: r.auth.GetUserId(v.OrgId),
OrgRole: r.auth.GetOrgRole(),
}
if renderingSession, ok := r.renderingSessionByOrgId[v.OrgId]; ok {
@ -112,7 +113,7 @@ func (d byOrgId) Len() int { return len(d) }
func (d byOrgId) Less(i, j int) bool { return d[i].OrgId > d[j].OrgId }
func (d byOrgId) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (r *simpleCrawler) Run(ctx context.Context, authOpts rendering.AuthOpts, mode CrawlerMode, theme models.Theme, thumbnailKind models.ThumbnailKind) error {
func (r *simpleCrawler) Run(ctx context.Context, auth CrawlerAuth, mode CrawlerMode, theme models.Theme, thumbnailKind models.ThumbnailKind) error {
res, err := r.renderService.HasCapability(rendering.ScalingDownImages)
if err != nil {
return err
@ -150,8 +151,8 @@ func (r *simpleCrawler) Run(ctx context.Context, authOpts rendering.AuthOpts, mo
r.mode = mode
r.thumbnailKind = thumbnailKind
r.auth = auth
r.opts = rendering.Opts{
AuthOpts: authOpts,
TimeoutOpts: rendering.TimeoutOpts{
Timeout: 20 * time.Second,
RequestTimeoutMultiplier: 3,

41
pkg/services/thumbs/crawler_auth.go Normal file
View File

@ -0,0 +1,41 @@
package thumbs
import (
"context"
"github.com/grafana/grafana/pkg/models"
)
type CrawlerAuthSetupService interface {
Setup(ctx context.Context) (CrawlerAuth, error)
}
func ProvideCrawlerAuthSetupService() *OSSCrawlerAuthSetupService {
return &OSSCrawlerAuthSetupService{}
}
type OSSCrawlerAuthSetupService struct{}
type CrawlerAuth interface {
GetUserId(orgId int64) int64
GetOrgRole() models.RoleType
}
type staticCrawlerAuth struct {
userId int64
orgRole models.RoleType
}
func (o *staticCrawlerAuth) GetOrgRole() models.RoleType {
return o.orgRole
}
func (o *staticCrawlerAuth) GetUserId(orgId int64) int64 {
return o.userId
}
func (o *OSSCrawlerAuthSetupService) Setup(ctx context.Context) (CrawlerAuth, error) {
// userId:0 and ROLE_ADMIN grants the crawler process permissions to view all dashboards in all folders & orgs
// the process doesn't and shouldn't actually need to edit/modify any resources from the UI
return &staticCrawlerAuth{userId: 0, orgRole: models.ROLE_ADMIN}, nil
}

3
pkg/services/thumbs/models.go
View File

@ -5,7 +5,6 @@ import (
"time"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/rendering"
)
type CrawlerMode string
@ -66,7 +65,7 @@ type dashboardPreviewsSetupConfig struct {
type dashRenderer interface {
// Run Assumes you have already authenticated as admin.
Run(ctx context.Context, authOpts rendering.AuthOpts, mode CrawlerMode, theme models.Theme, kind models.ThumbnailKind) error
Run(ctx context.Context, auth CrawlerAuth, mode CrawlerMode, theme models.Theme, kind models.ThumbnailKind) error
// Assumes you have already authenticated as admin.
Stop() (crawlStatus, error)

20
pkg/services/thumbs/service.go
View File

@ -56,22 +56,24 @@ type crawlerScheduleOptions struct {
maxCrawlDuration time.Duration
crawlerMode CrawlerMode
thumbnailKind models.ThumbnailKind
auth rendering.AuthOpts
themes []models.Theme
auth CrawlerAuth
}
func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockService *serverlock.ServerLockService, renderService rendering.Service, gl *live.GrafanaLive, store *sqlstore.SQLStore) Service {
func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockService *serverlock.ServerLockService, renderService rendering.Service, gl *live.GrafanaLive, store *sqlstore.SQLStore, authSetupService CrawlerAuthSetupService) Service {
if !features.IsEnabled(featuremgmt.FlagDashboardPreviews) {
return &dummyService{}
}
logger := log.New("thumbnails_service")
thumbnailRepo := newThumbnailRepo(store)
authOpts := rendering.AuthOpts{
OrgID: 0,
UserID: 0,
OrgRole: models.ROLE_ADMIN,
crawlerAuth, err := authSetupService.Setup(context.Background())
if err != nil {
logger.Error("Failed to setup auth for the dashboard previews crawler", "err", err)
return &dummyService{}
}
return &thumbService{
renderingService: renderService,
renderer: newSimpleCrawler(renderService, gl, thumbnailRepo),
@ -80,7 +82,7 @@ func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockS
features: features,
lockService: lockService,
crawlLockServiceActionName: "dashboard-crawler",
log: log.New("thumbnails_service"),
log: logger,
scheduleOptions: crawlerScheduleOptions{
tickerInterval: time.Hour,
@ -89,7 +91,7 @@ func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockS
crawlerMode: CrawlerModeThumbs,
thumbnailKind: models.ThumbnailKindDefault,
themes: []models.Theme{models.ThemeDark, models.ThemeLight},
auth: authOpts,
auth: crawlerAuth,
},
}
}
@ -377,7 +379,7 @@ func (hs *thumbService) runOnDemandCrawl(parentCtx context.Context, theme models
// wait for at least a minute after the last completed run
interval := time.Minute
err := hs.lockService.LockAndExecute(crawlerCtx, hs.crawlLockServiceActionName, interval, func(ctx context.Context) {
if err := hs.renderer.Run(crawlerCtx, authOpts, mode, theme, kind); err != nil {
if err := hs.renderer.Run(crawlerCtx, hs.scheduleOptions.auth, mode, theme, kind); err != nil {
hs.log.Error("On demand crawl error", "mode", mode, "theme", theme, "kind", kind, "userId", authOpts.UserID, "orgId", authOpts.OrgID, "orgRole", authOpts.OrgRole)
}
})