grafana/pkg/services/store/entity_events.go

172 lines
4.7 KiB
Go
Raw Normal View History

Search: in-memory index (#47709) * #45498: add entity events table * #45498: add entity events service * #45498: hook up entity events service to http server * #45498: use `dashboards.id` rather than `uid` and `org_id` in grn * Update pkg/services/entityevents/service.go Co-authored-by: Ryan McKinley <ryantxu@gmail.com> * #45498: move entityeventsservice to services/store * #45498: add null check * #45498: rename * #45498: fix comment * #45498: switch grn back to uid * Search: listen for updates (#47719) * #45498: wire entity event service with searchv2 * load last event id before building index for org 1 * fix service init in integration tests * depend on required subset of event store methods * Update pkg/services/sqlstore/migrations/entity_events_mig.go Co-authored-by: Alexander Emelin <frvzmb@gmail.com> * #45498: pointer receiver * #45498: mockery! * #45498: add entity events service to background services * dashboard query pagination, allow queries while re-indexing * log level cleanups, use rlock, add comments * fix lint, check feature toggle in search v2 service * use unix time for event created column * add missing changes for created column * fix integration tests init * log re-index execution times on info level * #45498: fix entityEventsService tests * #45498: save events on dashboard delete * use camel case for log labels * formatting * #45498: rename grn to entityid * #45498: add `IsDisabled` to entityEventsService * #45498: remove feature flag from migration * better context usage, fix capacity, comments/cleanups * replace print with logger * Revert "#45498: remove feature flag from migration" This reverts commit ed23968898e27d65cfc5187acbcb1e8976c848a5. * revert:revert:revert conditional feature flag Co-authored-by: Ryan McKinley <ryantxu@gmail.com> Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
2022-04-27 03:29:39 -05:00
package store
import (
"context"
"fmt"
"time"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/registry"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
)
type EntityEventType string
const (
EntityEventTypeDelete EntityEventType = "delete"
EntityEventTypeCreate EntityEventType = "create"
EntityEventTypeUpdate EntityEventType = "update"
)
type EntityType string
const (
EntityTypeDashboard EntityType = "dashboard"
EntityTypeFolder EntityType = "folder"
EntityTypeImage EntityType = "image"
EntityTypeJSON EntityType = "json"
Search: in-memory index (#47709) * #45498: add entity events table * #45498: add entity events service * #45498: hook up entity events service to http server * #45498: use `dashboards.id` rather than `uid` and `org_id` in grn * Update pkg/services/entityevents/service.go Co-authored-by: Ryan McKinley <ryantxu@gmail.com> * #45498: move entityeventsservice to services/store * #45498: add null check * #45498: rename * #45498: fix comment * #45498: switch grn back to uid * Search: listen for updates (#47719) * #45498: wire entity event service with searchv2 * load last event id before building index for org 1 * fix service init in integration tests * depend on required subset of event store methods * Update pkg/services/sqlstore/migrations/entity_events_mig.go Co-authored-by: Alexander Emelin <frvzmb@gmail.com> * #45498: pointer receiver * #45498: mockery! * #45498: add entity events service to background services * dashboard query pagination, allow queries while re-indexing * log level cleanups, use rlock, add comments * fix lint, check feature toggle in search v2 service * use unix time for event created column * add missing changes for created column * fix integration tests init * log re-index execution times on info level * #45498: fix entityEventsService tests * #45498: save events on dashboard delete * use camel case for log labels * formatting * #45498: rename grn to entityid * #45498: add `IsDisabled` to entityEventsService * #45498: remove feature flag from migration * better context usage, fix capacity, comments/cleanups * replace print with logger * Revert "#45498: remove feature flag from migration" This reverts commit ed23968898e27d65cfc5187acbcb1e8976c848a5. * revert:revert:revert conditional feature flag Co-authored-by: Ryan McKinley <ryantxu@gmail.com> Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
2022-04-27 03:29:39 -05:00
)
// CreateDatabaseEntityId creates entityId for entities stored in the existing SQL tables
func CreateDatabaseEntityId(internalId interface{}, orgId int64, entityType EntityType) string {
var internalIdAsString string
switch id := internalId.(type) {
case string:
internalIdAsString = id
default:
internalIdAsString = fmt.Sprintf("%#v", internalId)
}
return fmt.Sprintf("database/%d/%s/%s", orgId, entityType, internalIdAsString)
}
type EntityEvent struct {
Id int64
EventType EntityEventType
EntityId string
Created int64
}
type SaveEventCmd struct {
EntityId string
EventType EntityEventType
}
type EventHandler func(ctx context.Context, e *EntityEvent) error
Search: in-memory index (#47709) * #45498: add entity events table * #45498: add entity events service * #45498: hook up entity events service to http server * #45498: use `dashboards.id` rather than `uid` and `org_id` in grn * Update pkg/services/entityevents/service.go Co-authored-by: Ryan McKinley <ryantxu@gmail.com> * #45498: move entityeventsservice to services/store * #45498: add null check * #45498: rename * #45498: fix comment * #45498: switch grn back to uid * Search: listen for updates (#47719) * #45498: wire entity event service with searchv2 * load last event id before building index for org 1 * fix service init in integration tests * depend on required subset of event store methods * Update pkg/services/sqlstore/migrations/entity_events_mig.go Co-authored-by: Alexander Emelin <frvzmb@gmail.com> * #45498: pointer receiver * #45498: mockery! * #45498: add entity events service to background services * dashboard query pagination, allow queries while re-indexing * log level cleanups, use rlock, add comments * fix lint, check feature toggle in search v2 service * use unix time for event created column * add missing changes for created column * fix integration tests init * log re-index execution times on info level * #45498: fix entityEventsService tests * #45498: save events on dashboard delete * use camel case for log labels * formatting * #45498: rename grn to entityid * #45498: add `IsDisabled` to entityEventsService * #45498: remove feature flag from migration * better context usage, fix capacity, comments/cleanups * replace print with logger * Revert "#45498: remove feature flag from migration" This reverts commit ed23968898e27d65cfc5187acbcb1e8976c848a5. * revert:revert:revert conditional feature flag Co-authored-by: Ryan McKinley <ryantxu@gmail.com> Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
2022-04-27 03:29:39 -05:00
// EntityEventsService is a temporary solution to support change notifications in an HA setup
// With this service each system can query for any events that have happened since a fixed time
//
Search: in-memory index (#47709) * #45498: add entity events table * #45498: add entity events service * #45498: hook up entity events service to http server * #45498: use `dashboards.id` rather than `uid` and `org_id` in grn * Update pkg/services/entityevents/service.go Co-authored-by: Ryan McKinley <ryantxu@gmail.com> * #45498: move entityeventsservice to services/store * #45498: add null check * #45498: rename * #45498: fix comment * #45498: switch grn back to uid * Search: listen for updates (#47719) * #45498: wire entity event service with searchv2 * load last event id before building index for org 1 * fix service init in integration tests * depend on required subset of event store methods * Update pkg/services/sqlstore/migrations/entity_events_mig.go Co-authored-by: Alexander Emelin <frvzmb@gmail.com> * #45498: pointer receiver * #45498: mockery! * #45498: add entity events service to background services * dashboard query pagination, allow queries while re-indexing * log level cleanups, use rlock, add comments * fix lint, check feature toggle in search v2 service * use unix time for event created column * add missing changes for created column * fix integration tests init * log re-index execution times on info level * #45498: fix entityEventsService tests * #45498: save events on dashboard delete * use camel case for log labels * formatting * #45498: rename grn to entityid * #45498: add `IsDisabled` to entityEventsService * #45498: remove feature flag from migration * better context usage, fix capacity, comments/cleanups * replace print with logger * Revert "#45498: remove feature flag from migration" This reverts commit ed23968898e27d65cfc5187acbcb1e8976c848a5. * revert:revert:revert conditional feature flag Co-authored-by: Ryan McKinley <ryantxu@gmail.com> Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
2022-04-27 03:29:39 -05:00
//go:generate mockery --name EntityEventsService --structname MockEntityEventsService --inpackage --filename entity_events_mock.go
type EntityEventsService interface {
registry.BackgroundService
registry.CanBeDisabled
GetLastEvent(ctx context.Context) (*EntityEvent, error)
GetAllEventsAfter(ctx context.Context, id int64) ([]*EntityEvent, error)
deleteEventsOlderThan(ctx context.Context, duration time.Duration) error
}
func ProvideEntityEventsService(cfg *setting.Cfg, sqlStore *sqlstore.SQLStore, features featuremgmt.FeatureToggles) EntityEventsService {
if !features.IsEnabled(featuremgmt.FlagPanelTitleSearch) {
return &dummyEntityEventsService{}
}
return &entityEventService{
sql: sqlStore,
features: features,
log: log.New("entity-events"),
eventHandlers: make([]EventHandler, 0),
Search: in-memory index (#47709) * #45498: add entity events table * #45498: add entity events service * #45498: hook up entity events service to http server * #45498: use `dashboards.id` rather than `uid` and `org_id` in grn * Update pkg/services/entityevents/service.go Co-authored-by: Ryan McKinley <ryantxu@gmail.com> * #45498: move entityeventsservice to services/store * #45498: add null check * #45498: rename * #45498: fix comment * #45498: switch grn back to uid * Search: listen for updates (#47719) * #45498: wire entity event service with searchv2 * load last event id before building index for org 1 * fix service init in integration tests * depend on required subset of event store methods * Update pkg/services/sqlstore/migrations/entity_events_mig.go Co-authored-by: Alexander Emelin <frvzmb@gmail.com> * #45498: pointer receiver * #45498: mockery! * #45498: add entity events service to background services * dashboard query pagination, allow queries while re-indexing * log level cleanups, use rlock, add comments * fix lint, check feature toggle in search v2 service * use unix time for event created column * add missing changes for created column * fix integration tests init * log re-index execution times on info level * #45498: fix entityEventsService tests * #45498: save events on dashboard delete * use camel case for log labels * formatting * #45498: rename grn to entityid * #45498: add `IsDisabled` to entityEventsService * #45498: remove feature flag from migration * better context usage, fix capacity, comments/cleanups * replace print with logger * Revert "#45498: remove feature flag from migration" This reverts commit ed23968898e27d65cfc5187acbcb1e8976c848a5. * revert:revert:revert conditional feature flag Co-authored-by: Ryan McKinley <ryantxu@gmail.com> Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
2022-04-27 03:29:39 -05:00
}
}
type entityEventService struct {
sql *sqlstore.SQLStore
log log.Logger
features featuremgmt.FeatureToggles
eventHandlers []EventHandler
Search: in-memory index (#47709) * #45498: add entity events table * #45498: add entity events service * #45498: hook up entity events service to http server * #45498: use `dashboards.id` rather than `uid` and `org_id` in grn * Update pkg/services/entityevents/service.go Co-authored-by: Ryan McKinley <ryantxu@gmail.com> * #45498: move entityeventsservice to services/store * #45498: add null check * #45498: rename * #45498: fix comment * #45498: switch grn back to uid * Search: listen for updates (#47719) * #45498: wire entity event service with searchv2 * load last event id before building index for org 1 * fix service init in integration tests * depend on required subset of event store methods * Update pkg/services/sqlstore/migrations/entity_events_mig.go Co-authored-by: Alexander Emelin <frvzmb@gmail.com> * #45498: pointer receiver * #45498: mockery! * #45498: add entity events service to background services * dashboard query pagination, allow queries while re-indexing * log level cleanups, use rlock, add comments * fix lint, check feature toggle in search v2 service * use unix time for event created column * add missing changes for created column * fix integration tests init * log re-index execution times on info level * #45498: fix entityEventsService tests * #45498: save events on dashboard delete * use camel case for log labels * formatting * #45498: rename grn to entityid * #45498: add `IsDisabled` to entityEventsService * #45498: remove feature flag from migration * better context usage, fix capacity, comments/cleanups * replace print with logger * Revert "#45498: remove feature flag from migration" This reverts commit ed23968898e27d65cfc5187acbcb1e8976c848a5. * revert:revert:revert conditional feature flag Co-authored-by: Ryan McKinley <ryantxu@gmail.com> Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
2022-04-27 03:29:39 -05:00
}
func (e *entityEventService) GetLastEvent(ctx context.Context) (*EntityEvent, error) {
var entityEvent *EntityEvent
err := e.sql.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
bean := &EntityEvent{}
found, err := sess.OrderBy("id desc").Get(bean)
if found {
entityEvent = bean
}
return err
})
return entityEvent, err
}
func (e *entityEventService) GetAllEventsAfter(ctx context.Context, id int64) ([]*EntityEvent, error) {
var evs = make([]*EntityEvent, 0)
err := e.sql.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
return sess.OrderBy("id asc").Where("id > ?", id).Find(&evs)
})
return evs, err
}
func (e *entityEventService) deleteEventsOlderThan(ctx context.Context, duration time.Duration) error {
return e.sql.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
maxCreated := time.Now().Add(-duration)
deletedCount, err := sess.Where("created < ?", maxCreated.Unix()).Delete(&EntityEvent{})
e.log.Info("deleting old events", "count", deletedCount, "maxCreated", maxCreated)
return err
})
}
func (e *entityEventService) IsDisabled() bool {
return false
}
func (e *entityEventService) Run(ctx context.Context) error {
clean := time.NewTicker(1 * time.Hour)
for {
select {
case <-clean.C:
go func() {
err := e.deleteEventsOlderThan(context.Background(), 24*time.Hour)
if err != nil {
e.log.Info("failed to delete old entity events", "error", err)
}
}()
case <-ctx.Done():
e.log.Debug("Grafana is shutting down - stopping entity events service")
clean.Stop()
return nil
}
}
}
type dummyEntityEventsService struct {
}
func (d dummyEntityEventsService) Run(ctx context.Context) error {
return nil
}
func (d dummyEntityEventsService) IsDisabled() bool {
return false
}
func (d dummyEntityEventsService) GetLastEvent(ctx context.Context) (*EntityEvent, error) {
return nil, nil
}
func (d dummyEntityEventsService) GetAllEventsAfter(ctx context.Context, id int64) ([]*EntityEvent, error) {
return make([]*EntityEvent, 0), nil
}
func (d dummyEntityEventsService) deleteEventsOlderThan(ctx context.Context, duration time.Duration) error {
return nil
}
var _ EntityEventsService = &dummyEntityEventsService{}