From de3737b5de033889c7fc673b890f3e58c5bd0288 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Wed, 19 Oct 2022 10:33:26 -0400 Subject: [PATCH] Store: Add resolver service (#57112) --- pkg/server/wire.go | 2 + pkg/services/store/auth.go | 13 ++ pkg/services/store/resolver/ds_cache.go | 136 +++++++++++++++++++ pkg/services/store/resolver/service.go | 125 ++++++++++++++++++ pkg/services/store/resolver/service_test.go | 137 ++++++++++++++++++++ 5 files changed, 413 insertions(+) create mode 100644 pkg/services/store/resolver/ds_cache.go create mode 100644 pkg/services/store/resolver/service.go create mode 100644 pkg/services/store/resolver/service_test.go diff --git a/pkg/server/wire.go b/pkg/server/wire.go index b801d76d167..4672542132e 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -128,6 +128,7 @@ import ( "github.com/grafana/grafana/pkg/services/store/kind" "github.com/grafana/grafana/pkg/services/store/object" objectdummyserver "github.com/grafana/grafana/pkg/services/store/object/dummy" + "github.com/grafana/grafana/pkg/services/store/resolver" "github.com/grafana/grafana/pkg/services/store/sanitizer" "github.com/grafana/grafana/pkg/services/tag" "github.com/grafana/grafana/pkg/services/tag/tagimpl" @@ -357,6 +358,7 @@ var wireBasicSet = wire.NewSet( interceptors.ProvideAuthenticator, kind.ProvideService, // The registry known kinds objectdummyserver.ProvideDummyObjectServer, + resolver.ProvideObjectReferenceResolver, object.ProvideHTTPObjectStore, teamimpl.ProvideService, tempuserimpl.ProvideService, diff --git a/pkg/services/store/auth.go b/pkg/services/store/auth.go index cd4ebe8b1c1..d7258280f89 100644 --- a/pkg/services/store/auth.go +++ b/pkg/services/store/auth.go @@ -10,6 +10,12 @@ import ( "github.com/grafana/grafana/pkg/services/user" ) +type testUserKey struct{} + +func ContextWithUser(ctx context.Context, data *user.SignedInUser) context.Context { + return context.WithValue(ctx, testUserKey{}, data) +} + // UserFromContext ** Experimental ** // TODO: move to global infra package / new auth service func UserFromContext(ctx context.Context) *user.SignedInUser { @@ -18,6 +24,13 @@ func UserFromContext(ctx context.Context) *user.SignedInUser { return grpcCtx.SignedInUser } + // Explicitly set in context + u, ok := ctx.Value(testUserKey{}).(*user.SignedInUser) + if ok && u != nil { + return u + } + + // From the HTTP request c, ok := ctxkey.Get(ctx).(*models.ReqContext) if !ok || c == nil || c.SignedInUser == nil { return nil diff --git a/pkg/services/store/resolver/ds_cache.go b/pkg/services/store/resolver/ds_cache.go new file mode 100644 index 00000000000..632fe39da56 --- /dev/null +++ b/pkg/services/store/resolver/ds_cache.go @@ -0,0 +1,136 @@ +package resolver + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/grafana/grafana/pkg/plugins/manager/registry" + "github.com/grafana/grafana/pkg/services/datasources" + "github.com/grafana/grafana/pkg/services/store" + "github.com/grafana/grafana/pkg/tsdb/grafanads" +) + +type dsVal struct { + InternalID int64 + IsDefault bool + Name string + Type string + UID string + PluginExists bool // type exists +} + +type dsCache struct { + ds datasources.DataSourceService + pluginRegistry registry.Service + cache map[int64]map[string]*dsVal + timestamp time.Time // across all orgIDs + mu sync.Mutex +} + +func (c *dsCache) refreshCache(ctx context.Context) error { + old := c.timestamp + + c.mu.Lock() + defer c.mu.Unlock() + + if c.timestamp != old { + return nil // already updated while we waited! + } + + cache := make(map[int64]map[string]*dsVal, 0) + defaultDS := make(map[int64]*dsVal, 0) + + q := &datasources.GetAllDataSourcesQuery{} + err := c.ds.GetAllDataSources(ctx, q) + if err != nil { + return err + } + + for _, ds := range q.Result { + val := &dsVal{ + InternalID: ds.Id, + Name: ds.Name, + UID: ds.Uid, + Type: ds.Type, + IsDefault: ds.IsDefault, + } + _, ok := c.pluginRegistry.Plugin(ctx, val.Type) + val.PluginExists = ok + + orgCache, ok := cache[ds.OrgId] + if !ok { + orgCache = make(map[string]*dsVal, 0) + cache[ds.OrgId] = orgCache + } + + orgCache[val.UID] = val + + // Empty string or + if val.IsDefault { + defaultDS[ds.OrgId] = val + } + } + + for orgID, orgDSCache := range cache { + // modifies the cache we are iterating over? + for _, ds := range orgDSCache { + // Lookup by internal ID + id := fmt.Sprintf("%d", ds.InternalID) + _, ok := orgDSCache[id] + if !ok { + orgDSCache[id] = ds + } + + // Lookup by name + _, ok = orgDSCache[ds.Name] + if !ok { + orgDSCache[ds.Name] = ds + } + } + + // Register the internal builtin grafana datasource + gds := &dsVal{ + Name: grafanads.DatasourceName, + UID: grafanads.DatasourceUID, + Type: grafanads.DatasourceUID, + PluginExists: true, + } + orgDSCache[gds.UID] = gds + ds, ok := defaultDS[orgID] + if !ok { + ds = gds // use the internal grafana datasource + } + orgDSCache[""] = ds + if orgDSCache["default"] == nil { + orgDSCache["default"] = ds + } + } + + c.cache = cache + c.timestamp = getNow() + return nil +} + +func (c *dsCache) getDS(ctx context.Context, uid string) (*dsVal, error) { + // refresh cache every 1 min + if c.cache == nil || c.timestamp.Before(getNow().Add(time.Minute*-1)) { + err := c.refreshCache(ctx) + if err != nil { + return nil, err + } + } + + orgID := store.UserFromContext(ctx).OrgID + + v, ok := c.cache[orgID] + if !ok { + return nil, nil // org not found + } + ds, ok := v[uid] + if !ok { + return nil, nil // data source not found + } + return ds, nil +} diff --git a/pkg/services/store/resolver/service.go b/pkg/services/store/resolver/service.go new file mode 100644 index 00000000000..956b911e873 --- /dev/null +++ b/pkg/services/store/resolver/service.go @@ -0,0 +1,125 @@ +package resolver + +import ( + "context" + "fmt" + "time" + + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/plugins/manager/registry" + "github.com/grafana/grafana/pkg/services/datasources" +) + +const ( + WarningNotImplemented = "not implemented" + WarningDatasourcePluginNotFound = "datasource plugin not found" + WarningTypeNotSpecified = "type not specified" + WarningPluginNotFound = "plugin not found" +) + +// for testing +var getNow = func() time.Time { return time.Now() } + +type ResolutionInfo struct { + OK bool `json:"ok"` + Key string `json:"key,omitempty"` // GRN? UID? + Warning string `json:"kind,omitempty"` // old syntax? (name>uid) references a renamed object? + Timestamp time.Time `json:"timestamp,omitempty"` +} + +type ObjectReferenceResolver interface { + Resolve(ctx context.Context, ref *models.ObjectExternalReference) (ResolutionInfo, error) +} + +func ProvideObjectReferenceResolver(ds datasources.DataSourceService, pluginRegistry registry.Service) ObjectReferenceResolver { + return &standardReferenceResolver{ + pluginRegistry: pluginRegistry, + ds: dsCache{ + ds: ds, + pluginRegistry: pluginRegistry, + }, + } +} + +type standardReferenceResolver struct { + pluginRegistry registry.Service + ds dsCache +} + +func (r *standardReferenceResolver) Resolve(ctx context.Context, ref *models.ObjectExternalReference) (ResolutionInfo, error) { + if ref == nil { + return ResolutionInfo{OK: false, Timestamp: getNow()}, fmt.Errorf("ref is nil") + } + + switch ref.Kind { + case models.StandardKindDataSource: + return r.resolveDatasource(ctx, ref) + + case models.ExternalEntityReferencePlugin: + return r.resolvePlugin(ctx, ref) + + // case models.ExternalEntityReferenceRuntime: + // return ResolutionInfo{ + // OK: false, + // Timestamp: getNow(), + // Warning: WarningNotImplemented, + // }, nil + } + + return ResolutionInfo{ + OK: false, + Timestamp: getNow(), + Warning: WarningNotImplemented, + }, nil +} + +func (r *standardReferenceResolver) resolveDatasource(ctx context.Context, ref *models.ObjectExternalReference) (ResolutionInfo, error) { + ds, err := r.ds.getDS(ctx, ref.UID) + if err != nil || ds == nil || ds.UID == "" { + return ResolutionInfo{ + OK: false, + Timestamp: r.ds.timestamp, + }, err + } + + res := ResolutionInfo{ + OK: true, + Timestamp: r.ds.timestamp, + Key: ds.UID, // TODO! + } + if !ds.PluginExists { + res.OK = false + res.Warning = WarningDatasourcePluginNotFound + } else if ref.Type == "" { + ref.Type = ds.Type // awkward! but makes the reporting accurate for dashboards before schemaVersion 36 + res.Warning = WarningTypeNotSpecified + } else if ref.Type != ds.Type { + res.Warning = fmt.Sprintf("type mismatch (expect:%s, found:%s)", ref.Type, ds.Type) + } + return res, nil +} + +func (r *standardReferenceResolver) resolvePlugin(ctx context.Context, ref *models.ObjectExternalReference) (ResolutionInfo, error) { + p, ok := r.pluginRegistry.Plugin(ctx, ref.UID) + if !ok || p == nil { + return ResolutionInfo{ + OK: false, + Timestamp: getNow(), + Warning: WarningPluginNotFound, + }, nil + } + + if p.Type != plugins.Type(ref.Type) { + return ResolutionInfo{ + OK: false, + Timestamp: getNow(), + Warning: fmt.Sprintf("expected type: %s, found%s", ref.Type, p.Type), + }, nil + } + + return ResolutionInfo{ + OK: true, + Timestamp: getNow(), + }, nil +} diff --git a/pkg/services/store/resolver/service_test.go b/pkg/services/store/resolver/service_test.go new file mode 100644 index 00000000000..0f324989d80 --- /dev/null +++ b/pkg/services/store/resolver/service_test.go @@ -0,0 +1,137 @@ +package resolver + +import ( + "context" + "testing" + + "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/plugins" + "github.com/grafana/grafana/pkg/plugins/manager/registry" + "github.com/grafana/grafana/pkg/services/datasources" + fakeDatasources "github.com/grafana/grafana/pkg/services/datasources/fakes" + "github.com/grafana/grafana/pkg/services/store" + "github.com/grafana/grafana/pkg/services/user" + "github.com/stretchr/testify/require" +) + +func TestResolver(t *testing.T) { + ctxOrg1 := store.ContextWithUser(context.Background(), &user.SignedInUser{OrgID: 1}) + + ds := &fakeDatasources.FakeDataSourceService{ + DataSources: []*datasources.DataSource{ + { + Id: 123, + OrgId: 1, + Type: "influx", + Uid: "influx-uid", + IsDefault: true, + }, + { + Id: 234, + OrgId: 1, + Type: "influx", + Uid: "influx-uid2", + Name: "Influx2", + }, + }, + } + + p1 := &plugins.Plugin{} + p2 := &plugins.Plugin{} + p3 := &plugins.Plugin{} + + p1.ID = "influx" + p2.ID = "heatmap" + p3.ID = "xyz" + + pluginRegistry := registry.ProvideService() + _ = pluginRegistry.Add(ctxOrg1, p1) + _ = pluginRegistry.Add(ctxOrg1, p2) + _ = pluginRegistry.Add(ctxOrg1, p3) + + provider := ProvideObjectReferenceResolver(ds, pluginRegistry) + + scenarios := []struct { + name string + given *models.ObjectExternalReference + expect ResolutionInfo + err string + ctx context.Context + }{ + { + name: "Missing datasource without type", + given: &models.ObjectExternalReference{ + Kind: models.StandardKindDataSource, + UID: "xyz", + }, + expect: ResolutionInfo{OK: false}, + ctx: ctxOrg1, + }, + { + name: "OK datasource", + given: &models.ObjectExternalReference{ + Kind: models.StandardKindDataSource, + Type: "influx", + UID: "influx-uid", + }, + expect: ResolutionInfo{OK: true, Key: "influx-uid"}, + ctx: ctxOrg1, + }, + { + name: "Get the default datasource", + given: &models.ObjectExternalReference{ + Kind: models.StandardKindDataSource, + }, + expect: ResolutionInfo{ + OK: true, + Key: "influx-uid", + Warning: "type not specified", + }, + ctx: ctxOrg1, + }, + { + name: "Get the default datasource (with type)", + given: &models.ObjectExternalReference{ + Kind: models.StandardKindDataSource, + Type: "influx", + }, + expect: ResolutionInfo{ + OK: true, + Key: "influx-uid", + }, + ctx: ctxOrg1, + }, + { + name: "Lookup by name", + given: &models.ObjectExternalReference{ + Kind: models.StandardKindDataSource, + UID: "Influx2", + }, + expect: ResolutionInfo{ + OK: true, + Key: "influx-uid2", + Warning: "type not specified", + }, + ctx: ctxOrg1, + }, + { + name: "invalid input", + given: nil, + expect: ResolutionInfo{OK: false}, + err: "ref is nil", + ctx: ctxOrg1, + }, + } + + for _, scenario := range scenarios { + res, err := provider.Resolve(scenario.ctx, scenario.given) + + require.Equal(t, scenario.expect.OK, res.OK, scenario.name) + require.Equal(t, scenario.expect.Key, res.Key, scenario.name) + require.Equal(t, scenario.expect.Warning, res.Warning, scenario.name) + + if scenario.err != "" { + require.Equal(t, scenario.err, err.Error(), scenario.name) + } + } +}