diff --git a/pkg/registry/apis/folders/legacy_storage.go b/pkg/registry/apis/folders/legacy_storage.go index ac258217ccc..82a4f097a32 100644 --- a/pkg/registry/apis/folders/legacy_storage.go +++ b/pkg/registry/apis/folders/legacy_storage.go @@ -67,15 +67,16 @@ func (s *legacyStorage) List(ctx context.Context, options *internalversion.ListO } parentUID := "" - fieldRequirements, fieldSelector, err := entity.ReadFieldRequirements(options.FieldSelector) + // translate grafana.app/* label selectors into field requirements + requirements, newSelector, err := entity.ReadLabelSelectors(options.LabelSelector) if err != nil { return nil, err } - if fieldRequirements.Folder != nil { - parentUID = *fieldRequirements.Folder + if requirements.Folder != nil { + parentUID = *requirements.Folder } - // Update the field selector to remove the unneeded selectors - options.FieldSelector = fieldSelector + // Update the selector to remove the unneeded requirements + options.LabelSelector = newSelector paging, err := readContinueToken(options) if err != nil { diff --git a/pkg/server/test_env.go b/pkg/server/test_env.go index 2d54760fd5a..6420fd63bb0 100644 --- a/pkg/server/test_env.go +++ b/pkg/server/test_env.go @@ -3,6 +3,7 @@ package server import ( "github.com/grafana/grafana/pkg/infra/httpclient" "github.com/grafana/grafana/pkg/plugins/manager/registry" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/grpcserver" "github.com/grafana/grafana/pkg/services/notifications" "github.com/grafana/grafana/pkg/services/oauthtoken/oauthtokentest" @@ -18,6 +19,7 @@ func ProvideTestEnv( pluginRegistry registry.Service, httpClientProvider httpclient.Provider, oAuthTokenService *oauthtokentest.Service, + featureMgmt featuremgmt.FeatureToggles, ) (*TestEnv, error) { return &TestEnv{ Server: server, @@ -27,6 +29,7 @@ func ProvideTestEnv( PluginRegistry: pluginRegistry, HTTPClientProvider: httpClientProvider, OAuthTokenService: oAuthTokenService, + FeatureToggles: featureMgmt, }, nil } @@ -39,4 +42,5 @@ type TestEnv struct { HTTPClientProvider httpclient.Provider OAuthTokenService *oauthtokentest.Service RequestMiddleware web.Middleware + FeatureToggles featuremgmt.FeatureToggles } diff --git a/pkg/services/apiserver/service.go b/pkg/services/apiserver/service.go index c547fb21e10..7e85e8704b1 100644 --- a/pkg/services/apiserver/service.go +++ b/pkg/services/apiserver/service.go @@ -277,12 +277,6 @@ func (s *service) start(ctx context.Context) error { return err } - // support folder selection - err = entitystorage.RegisterFieldSelectorSupport(Scheme) - if err != nil { - return err - } - // Create the server server, err := serverConfig.Complete().New("grafana-apiserver", genericapiserver.NewEmptyDelegate()) if err != nil { diff --git a/pkg/services/apiserver/storage/entity/fieldRequirements.go b/pkg/services/apiserver/storage/entity/fieldRequirements.go deleted file mode 100644 index 72aab2595d8..00000000000 --- a/pkg/services/apiserver/storage/entity/fieldRequirements.go +++ /dev/null @@ -1,76 +0,0 @@ -package entity - -import ( - "fmt" - "strings" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/selection" -) - -const folderAnnoKey = "grafana.app/folder" - -type FieldRequirements struct { - // Equals folder - Folder *string -} - -func ReadFieldRequirements(selector fields.Selector) (FieldRequirements, fields.Selector, error) { - requirements := FieldRequirements{} - - if selector == nil { - return requirements, selector, nil - } - - for _, r := range selector.Requirements() { - switch r.Field { - case folderAnnoKey: - if (r.Operator != selection.Equals) && (r.Operator != selection.DoubleEquals) { - return requirements, selector, apierrors.NewBadRequest("only equality is supported in the selectors") - } - folder := r.Value - requirements.Folder = &folder - } - } - - // use Transform function to remove grafana.app/folder field selector - selector, err := selector.Transform(func(field, value string) (string, string, error) { - switch field { - case folderAnnoKey: - return "", "", nil - } - return field, value, nil - }) - - return requirements, selector, err -} - -func RegisterFieldSelectorSupport(scheme *runtime.Scheme) error { - grafanaFieldSupport := runtime.FieldLabelConversionFunc( - func(field, value string) (string, string, error) { - if strings.HasPrefix(field, "grafana.app/") { - return field, value, nil - } - return "", "", getBadSelectorError(field) - }, - ) - - // Register all the internal types - for gvk := range scheme.AllKnownTypes() { - if strings.HasSuffix(gvk.Group, ".grafana.app") { - err := scheme.AddFieldLabelConversionFunc(gvk, grafanaFieldSupport) - if err != nil { - return err - } - } - } - return nil -} - -func getBadSelectorError(f string) error { - return apierrors.NewBadRequest( - fmt.Sprintf("%q is not a known field selector: only %q works", f, folderAnnoKey), - ) -} diff --git a/pkg/services/apiserver/storage/entity/selector.go b/pkg/services/apiserver/storage/entity/selector.go new file mode 100644 index 00000000000..463ebe60782 --- /dev/null +++ b/pkg/services/apiserver/storage/entity/selector.go @@ -0,0 +1,49 @@ +package entity + +import ( + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" +) + +const folderAnnoKey = "grafana.app/folder" +const sortByKey = "grafana.app/sortBy" + +type Requirements struct { + // Equals folder + Folder *string + // SortBy is a list of fields to sort by + SortBy []string +} + +func ReadLabelSelectors(selector labels.Selector) (Requirements, labels.Selector, error) { + requirements := Requirements{} + newSelector := labels.NewSelector() + + if selector == nil { + return requirements, newSelector, nil + } + + labelSelectors, _ := selector.Requirements() + + for _, r := range labelSelectors { + switch r.Key() { + case folderAnnoKey: + if (r.Operator() != selection.Equals) && (r.Operator() != selection.DoubleEquals) { + return requirements, newSelector, apierrors.NewBadRequest(folderAnnoKey + " label selector only supports equality") + } + folder := r.Values().List()[0] + requirements.Folder = &folder + case sortByKey: + if r.Operator() != selection.In { + return requirements, newSelector, apierrors.NewBadRequest(sortByKey + " label selector only supports in") + } + requirements.SortBy = r.Values().List() + // add all unregonized label selectors to the new selector list, these will be processed by the entity store + default: + newSelector = newSelector.Add(r) + } + } + + return requirements, newSelector, nil +} diff --git a/pkg/services/apiserver/storage/entity/storage.go b/pkg/services/apiserver/storage/entity/storage.go index b11a029d6a2..5ddfda401a4 100644 --- a/pkg/services/apiserver/storage/entity/storage.go +++ b/pkg/services/apiserver/storage/entity/storage.go @@ -229,29 +229,32 @@ func (s *Storage) GetList(ctx context.Context, key string, opts storage.ListOpti // TODO push label/field matching down to storage } + // translate grafana.app/* label selectors into field requirements + requirements, newSelector, err := ReadLabelSelectors(opts.Predicate.Label) + if err != nil { + return err + } + if requirements.Folder != nil { + req.Folder = *requirements.Folder + } + if len(requirements.SortBy) > 0 { + req.Sort = requirements.SortBy + } + // Update the selector to remove the unneeded requirements + opts.Predicate.Label = newSelector + // translate "equals" label selectors to storage label conditions - requirements, selectable := opts.Predicate.Label.Requirements() + labelRequirements, selectable := opts.Predicate.Label.Requirements() if !selectable { return apierrors.NewBadRequest("label selector is not selectable") } - for _, r := range requirements { + for _, r := range labelRequirements { if r.Operator() == selection.Equals { req.Labels[r.Key()] = r.Values().List()[0] } } - // translate grafana.app/folder field selector to the folder condition - fieldRequirements, fieldSelector, err := ReadFieldRequirements(opts.Predicate.Field) - if err != nil { - return err - } - if fieldRequirements.Folder != nil { - req.Folder = *fieldRequirements.Folder - } - // Update the field selector to remove the unneeded selectors - opts.Predicate.Field = fieldSelector - rsp, err := s.store.List(ctx, req) if err != nil { return apierrors.NewInternalError(err) diff --git a/pkg/services/sqlstore/migrator/migrator.go b/pkg/services/sqlstore/migrator/migrator.go index 30e2fc74142..c58da4729b2 100644 --- a/pkg/services/sqlstore/migrator/migrator.go +++ b/pkg/services/sqlstore/migrator/migrator.go @@ -1,13 +1,14 @@ package migrator import ( + "errors" "fmt" "time" _ "github.com/go-sql-driver/mysql" "github.com/golang-migrate/migrate/v4/database" _ "github.com/lib/pq" - _ "github.com/mattn/go-sqlite3" + "github.com/mattn/go-sqlite3" "go.uber.org/atomic" "xorm.io/xorm" @@ -208,6 +209,13 @@ func (mg *Migrator) run() (err error) { err := mg.InTransaction(func(sess *xorm.Session) error { err := mg.exec(m, sess) + // if we get an sqlite busy/locked error, sleep 100ms and try again + if errors.Is(err, sqlite3.ErrLocked) || errors.Is(err, sqlite3.ErrBusy) { + mg.Logger.Debug("Database locked, sleeping then retrying", "error", err, "sql", sql) + time.Sleep(100 * time.Millisecond) + err = mg.exec(m, sess) + } + if err != nil { mg.Logger.Error("Exec failed", "error", err, "sql", sql) record.Error = err.Error() diff --git a/pkg/services/store/entity/db/migrations/entity_store_mig.go b/pkg/services/store/entity/db/migrations/entity_store_mig.go index feb317d8614..10c766ee093 100644 --- a/pkg/services/store/entity/db/migrations/entity_store_mig.go +++ b/pkg/services/store/entity/db/migrations/entity_store_mig.go @@ -7,7 +7,7 @@ import ( ) func initEntityTables(mg *migrator.Migrator) string { - marker := "Initialize entity tables (v12)" // changing this key wipe+rewrite everything + marker := "Initialize entity tables (v13)" // changing this key wipe+rewrite everything mg.AddMigration(marker, &migrator.RawSQLMigration{}) tables := []migrator.Table{} @@ -120,7 +120,11 @@ func initEntityTables(mg *migrator.Migrator) string { }, Indices: []*migrator.Index{ {Cols: []string{"guid", "resource_version"}, Type: migrator.UniqueIndex}, - {Cols: []string{"namespace", "group", "resource", "name", "resource_version"}, Type: migrator.UniqueIndex}, + { + Cols: []string{"namespace", "group", "resource", "name", "resource_version"}, + Type: migrator.UniqueIndex, + Name: "UQE_entity_history_namespace_group_name_version", + }, }, }) diff --git a/pkg/services/store/entity/sqlstash/querybuilder.go b/pkg/services/store/entity/sqlstash/querybuilder.go index fe5e2d5b237..588b8961753 100644 --- a/pkg/services/store/entity/sqlstash/querybuilder.go +++ b/pkg/services/store/entity/sqlstash/querybuilder.go @@ -6,15 +6,33 @@ import ( "github.com/grafana/grafana/pkg/services/sqlstore/migrator" ) +type Direction int + +const ( + Ascending Direction = iota + Descending +) + +func (d Direction) String() string { + if d == Descending { + return "DESC" + } + return "ASC" +} + type selectQuery struct { dialect migrator.Dialect fields []string // SELECT xyz from string // FROM object + offset int64 limit int64 oneExtra bool where []string args []any + + orderBy []string + direction []Direction } func (q *selectQuery) addWhere(f string, val ...any) { @@ -53,6 +71,11 @@ func (q *selectQuery) addWhereIn(f string, vals []string) { } } +func (q *selectQuery) addOrderBy(field string, direction Direction) { + q.orderBy = append(q.orderBy, field) + q.direction = append(q.direction, direction) +} + func (q *selectQuery) toQuery() (string, []any) { args := q.args sb := strings.Builder{} @@ -77,17 +100,27 @@ func (q *selectQuery) toQuery() (string, []any) { } } - if q.limit > 0 || q.oneExtra { - limit := q.limit - if limit < 1 { - limit = 20 - q.limit = limit + if len(q.orderBy) > 0 && len(q.direction) == len(q.orderBy) { + sb.WriteString(" ORDER BY ") + for i, f := range q.orderBy { + if i > 0 { + sb.WriteString(",") + } + sb.WriteString(q.dialect.Quote(f)) + sb.WriteString(" ") + sb.WriteString(q.direction[i].String()) } - if q.oneExtra { - limit = limit + 1 - } - sb.WriteString(" LIMIT ?") - args = append(args, limit) } + + limit := q.limit + if limit < 1 { + limit = 20 + q.limit = limit + } + if q.oneExtra { + limit = limit + 1 + } + sb.WriteString(q.dialect.LimitOffset(limit, q.offset)) + return sb.String(), args } diff --git a/pkg/services/store/entity/sqlstash/sql_storage_server.go b/pkg/services/store/entity/sqlstash/sql_storage_server.go index 0286ea340d8..655eed92838 100644 --- a/pkg/services/store/entity/sqlstash/sql_storage_server.go +++ b/pkg/services/store/entity/sqlstash/sql_storage_server.go @@ -3,10 +3,12 @@ package sqlstash import ( "context" "database/sql" + "encoding/base64" "encoding/json" "errors" "fmt" "math/rand" + "slices" "strings" "time" @@ -89,6 +91,7 @@ func (s *sqlEntityServer) getReadFields(r *entity.ReadEntityRequest) []string { "origin", "origin_key", "origin_ts", "meta", "title", "slug", "description", "labels", "fields", + "message", } if r.WithBody { @@ -134,6 +137,7 @@ func (s *sqlEntityServer) rowToEntity(ctx context.Context, rows *sql.Rows, r *en &raw.Origin.Source, &raw.Origin.Key, &raw.Origin.Time, &raw.Meta, &raw.Title, &raw.Slug, &raw.Description, &labels, &fields, + &raw.Message, } if r.WithBody { args = append(args, &raw.Body) @@ -147,10 +151,6 @@ func (s *sqlEntityServer) rowToEntity(ctx context.Context, rows *sql.Rows, r *en return nil, err } - if raw.Origin.Source == "" { - raw.Origin = nil - } - // unmarshal json labels if labels != "" { if err := json.Unmarshal([]byte(labels), &raw.Labels); err != nil { @@ -158,6 +158,17 @@ func (s *sqlEntityServer) rowToEntity(ctx context.Context, rows *sql.Rows, r *en } } + // set empty body, meta or status to nil + if raw.Body != nil && len(raw.Body) == 0 { + raw.Body = nil + } + if raw.Meta != nil && len(raw.Meta) == 0 { + raw.Meta = nil + } + if raw.Status != nil && len(raw.Status) == 0 { + raw.Status = nil + } + return raw, nil } @@ -278,6 +289,10 @@ func (s *sqlEntityServer) Create(ctx context.Context, r *entity.CreateEntityRequ } createdAt := r.Entity.CreatedAt + if createdAt < 1000 { + createdAt = time.Now().UnixMilli() + } + createdBy := r.Entity.CreatedBy if createdBy == "" { modifier, err := appcontext.User(ctx) @@ -289,6 +304,7 @@ func (s *sqlEntityServer) Create(ctx context.Context, r *entity.CreateEntityRequ } createdBy = store.GetUserIDString(modifier) } + updatedAt := r.Entity.UpdatedAt updatedBy := r.Entity.UpdatedBy @@ -315,6 +331,10 @@ func (s *sqlEntityServer) Create(ctx context.Context, r *entity.CreateEntityRequ // generate guid for new entity current.Guid = uuid.New().String() + // set created at/by + current.CreatedAt = createdAt + current.CreatedBy = createdBy + // parse provided key key, err := entity.ParseKey(r.Entity.Key) if err != nil { @@ -350,6 +370,7 @@ func (s *sqlEntityServer) Create(ctx context.Context, r *entity.CreateEntityRequ etag := createContentsHash(current.Body, current.Meta, current.Status) current.ETag = etag + current.UpdatedAt = updatedAt current.UpdatedBy = updatedBy @@ -365,18 +386,21 @@ func (s *sqlEntityServer) Create(ctx context.Context, r *entity.CreateEntityRequ s.log.Error("error marshalling labels", "msg", err.Error()) return err } + current.Labels = r.Entity.Labels fields, err := json.Marshal(r.Entity.Fields) if err != nil { s.log.Error("error marshalling fields", "msg", err.Error()) return err } + current.Fields = r.Entity.Fields errors, err := json.Marshal(r.Entity.Errors) if err != nil { s.log.Error("error marshalling errors", "msg", err.Error()) return err } + current.Errors = r.Entity.Errors if current.Origin == nil { current.Origin = &entity.EntityOriginInfo{} @@ -409,13 +433,13 @@ func (s *sqlEntityServer) Create(ctx context.Context, r *entity.CreateEntityRequ "group": current.Group, "resource": current.Resource, "name": current.Name, - "created_at": createdAt, - "created_by": createdBy, + "created_at": current.CreatedAt, + "created_by": current.CreatedBy, "group_version": current.GroupVersion, "folder": current.Folder, "slug": current.Slug, - "updated_at": updatedAt, - "updated_by": updatedBy, + "updated_at": current.UpdatedAt, + "updated_by": current.UpdatedBy, "body": current.Body, "meta": current.Meta, "status": current.Status, @@ -459,7 +483,7 @@ func (s *sqlEntityServer) Create(ctx context.Context, r *entity.CreateEntityRequ rsp.Entity = current - return nil // s.writeSearchInfo(ctx, tx, current) + return s.setLabels(ctx, tx, current.Guid, current.Labels) }) if err != nil { s.log.Error("error creating entity", "msg", err.Error()) @@ -475,8 +499,11 @@ func (s *sqlEntityServer) Update(ctx context.Context, r *entity.UpdateEntityRequ return nil, err } - timestamp := time.Now().UnixMilli() updatedAt := r.Entity.UpdatedAt + if updatedAt < 1000 { + updatedAt = time.Now().UnixMilli() + } + updatedBy := r.Entity.UpdatedBy if updatedBy == "" { modifier, err := appcontext.User(ctx) @@ -488,9 +515,6 @@ func (s *sqlEntityServer) Update(ctx context.Context, r *entity.UpdateEntityRequ } updatedBy = store.GetUserIDString(modifier) } - if updatedAt < 1000 { - updatedAt = timestamp - } rsp := &entity.UpdateEntityResponse{ Entity: &entity.Entity{}, @@ -519,10 +543,7 @@ func (s *sqlEntityServer) Update(ctx context.Context, r *entity.UpdateEntityRequ rsp.Entity.Guid = current.Guid - // Clear the labels+refs - if _, err := tx.Exec(ctx, "DELETE FROM entity_labels WHERE guid=?", rsp.Entity.Guid); err != nil { - return err - } + // Clear the refs if _, err := tx.Exec(ctx, "DELETE FROM entity_ref WHERE guid=?", rsp.Entity.Guid); err != nil { return err } @@ -553,6 +574,7 @@ func (s *sqlEntityServer) Update(ctx context.Context, r *entity.UpdateEntityRequ etag := createContentsHash(current.Body, current.Meta, current.Status) current.ETag = etag + current.UpdatedAt = updatedAt current.UpdatedBy = updatedBy @@ -568,18 +590,21 @@ func (s *sqlEntityServer) Update(ctx context.Context, r *entity.UpdateEntityRequ s.log.Error("error marshalling labels", "msg", err.Error()) return err } + current.Labels = r.Entity.Labels fields, err := json.Marshal(r.Entity.Fields) if err != nil { s.log.Error("error marshalling fields", "msg", err.Error()) return err } + current.Fields = r.Entity.Fields errors, err := json.Marshal(r.Entity.Errors) if err != nil { s.log.Error("error marshalling errors", "msg", err.Error()) return err } + current.Errors = r.Entity.Errors if current.Origin == nil { current.Origin = &entity.EntityOriginInfo{} @@ -619,8 +644,8 @@ func (s *sqlEntityServer) Update(ctx context.Context, r *entity.UpdateEntityRequ "group_version": current.GroupVersion, "folder": current.Folder, "slug": current.Slug, - "updated_at": updatedAt, - "updated_by": updatedBy, + "updated_at": current.UpdatedAt, + "updated_by": current.UpdatedBy, "body": current.Body, "meta": current.Meta, "status": current.Status, @@ -684,7 +709,7 @@ func (s *sqlEntityServer) Update(ctx context.Context, r *entity.UpdateEntityRequ rsp.Entity = current - return nil // s.writeSearchInfo(ctx, tx, current) + return s.setLabels(ctx, tx, current.Guid, current.Labels) }) if err != nil { s.log.Error("error updating entity", "msg", err.Error()) @@ -694,23 +719,22 @@ func (s *sqlEntityServer) Update(ctx context.Context, r *entity.UpdateEntityRequ return rsp, err } -/* -func (s *sqlEntityServer) writeSearchInfo( - ctx context.Context, - tx *session.SessionTx, - current *entity.Entity, -) error { - // parent_key := current.getParentKey() +func (s *sqlEntityServer) setLabels(ctx context.Context, tx *session.SessionTx, guid string, labels map[string]string) error { + s.log.Debug("setLabels", "guid", guid, "labels", labels) - // Add the labels rows - for k, v := range current.Labels { + // Clear the old labels + if _, err := tx.Exec(ctx, "DELETE FROM entity_labels WHERE guid=?", guid); err != nil { + return err + } + + // Add the new labels + for k, v := range labels { query, args, err := s.dialect.InsertQuery( "entity_labels", map[string]any{ - "key": current.Key, + "guid": guid, "label": k, "value": v, - // "parent_key": parent_key, }, ) if err != nil { @@ -725,7 +749,6 @@ func (s *sqlEntityServer) writeSearchInfo( return nil } -*/ func (s *sqlEntityServer) Delete(ctx context.Context, r *entity.DeleteEntityRequest) (*entity.DeleteEntityResponse, error) { if err := s.Init(); err != nil { @@ -816,7 +839,7 @@ func (s *sqlEntityServer) History(ctx context.Context, r *entity.EntityHistoryRe rr := &entity.ReadEntityRequest{ Key: r.Key, WithBody: true, - WithStatus: false, + WithStatus: true, } query, err := s.getReadSelect(rr) @@ -879,6 +902,75 @@ func (s *sqlEntityServer) History(ctx context.Context, r *entity.EntityHistoryRe return rsp, err } +type ContinueToken struct { + Sort []string `json:"s"` + StartOffset int64 `json:"o"` +} + +func (c *ContinueToken) String() string { + b, _ := json.Marshal(c) + return base64.StdEncoding.EncodeToString(b) +} + +func GetContinueToken(r *entity.EntityListRequest) (*ContinueToken, error) { + if r.NextPageToken == "" { + return nil, nil + } + + continueVal, err := base64.StdEncoding.DecodeString(r.NextPageToken) + if err != nil { + return nil, fmt.Errorf("error decoding continue token") + } + + t := &ContinueToken{} + err = json.Unmarshal(continueVal, t) + if err != nil { + return nil, err + } + + if !slices.Equal(t.Sort, r.Sort) { + return nil, fmt.Errorf("sort order changed") + } + + return t, nil +} + +var sortByFields = []string{ + "guid", + "key", + "namespace", "group", "group_version", "resource", "name", "folder", + "resource_version", "size", "etag", + "created_at", "created_by", + "updated_at", "updated_by", + "origin", "origin_key", "origin_ts", + "title", "slug", "description", +} + +type SortBy struct { + Field string + Direction Direction +} + +func ParseSortBy(sort string) (*SortBy, error) { + sortBy := &SortBy{ + Field: "guid", + Direction: Ascending, + } + + if strings.HasSuffix(sort, "_desc") { + sortBy.Field = sort[:len(sort)-5] + sortBy.Direction = Descending + } else { + sortBy.Field = sort + } + + if !slices.Contains(sortByFields, sortBy.Field) { + return nil, fmt.Errorf("invalid sort field '%s', valid fields: %v", sortBy.Field, sortByFields) + } + + return sortBy, nil +} + func (s *sqlEntityServer) List(ctx context.Context, r *entity.EntityListRequest) (*entity.EntityListResponse, error) { if err := s.Init(); err != nil { return nil, err @@ -892,10 +984,6 @@ func (s *sqlEntityServer) List(ctx context.Context, r *entity.EntityListRequest) return nil, fmt.Errorf("missing user in context") } - if r.NextPageToken != "" || len(r.Sort) > 0 { - return nil, fmt.Errorf("not yet supported") - } - rr := &entity.ReadEntityRequest{ WithBody: r.WithBody, WithStatus: r.WithStatus, @@ -909,6 +997,7 @@ func (s *sqlEntityServer) List(ctx context.Context, r *entity.EntityListRequest) from: "entity", // the table args: []any{}, limit: r.Limit, + offset: 0, oneExtra: true, // request one more than the limit (and show next token if it exists) } @@ -951,8 +1040,13 @@ func (s *sqlEntityServer) List(ctx context.Context, r *entity.EntityListRequest) entityQuery.addWhere("folder", r.Folder) } - if r.NextPageToken != "" { - entityQuery.addWhere("guid>?", r.NextPageToken) + // if we have a page token, use that to specify the first record + continueToken, err := GetContinueToken(r) + if err != nil { + return nil, err + } + if continueToken != nil { + entityQuery.offset = continueToken.StartOffset } if len(r.Labels) > 0 { @@ -971,6 +1065,14 @@ func (s *sqlEntityServer) List(ctx context.Context, r *entity.EntityListRequest) entityQuery.addWhereInSubquery("guid", query, args) } + for _, sort := range r.Sort { + sortBy, err := ParseSortBy(sort) + if err != nil { + return nil, err + } + entityQuery.addOrderBy(sortBy.Field, sortBy.Direction) + } + entityQuery.addOrderBy("guid", Ascending) query, args := entityQuery.toQuery() @@ -990,8 +1092,11 @@ func (s *sqlEntityServer) List(ctx context.Context, r *entity.EntityListRequest) // found more than requested if int64(len(rsp.Results)) >= entityQuery.limit { - // TODO? this only works if we sort by guid - rsp.NextPageToken = result.Guid + continueToken := &ContinueToken{ + Sort: r.Sort, + StartOffset: entityQuery.offset + entityQuery.limit, + } + rsp.NextPageToken = continueToken.String() break } diff --git a/pkg/services/store/entity/sqlstash/sql_storage_server_test.go b/pkg/services/store/entity/sqlstash/sql_storage_server_test.go index 430731131b3..7851939deb8 100644 --- a/pkg/services/store/entity/sqlstash/sql_storage_server_test.go +++ b/pkg/services/store/entity/sqlstash/sql_storage_server_test.go @@ -31,6 +31,7 @@ func TestCreate(t *testing.T) { Name: "set-minimum-uid", Key: "/playlist.grafana.app/playlists/default/set-minimum-uid", CreatedBy: "set-minimum-creator", + Origin: &entity.EntityOriginInfo{}, }, false, true, @@ -103,7 +104,7 @@ func TestCreate(t *testing.T) { require.Equal(t, tc.ent.Status, read.Status) require.Equal(t, tc.ent.Title, read.Title) require.Equal(t, tc.ent.Size, read.Size) - require.Equal(t, tc.ent.CreatedAt, read.CreatedAt) + require.Greater(t, read.CreatedAt, int64(0)) require.Equal(t, tc.ent.CreatedBy, read.CreatedBy) require.Equal(t, tc.ent.UpdatedAt, read.UpdatedAt) require.Equal(t, tc.ent.UpdatedBy, read.UpdatedBy) diff --git a/pkg/services/store/entity/tests/common.go b/pkg/services/store/entity/tests/common.go index b22980edd58..7bd8b4fc9c2 100644 --- a/pkg/services/store/entity/tests/common.go +++ b/pkg/services/store/entity/tests/common.go @@ -5,8 +5,6 @@ import ( "testing" "github.com/stretchr/testify/require" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" "github.com/grafana/grafana/pkg/components/satokengen" "github.com/grafana/grafana/pkg/infra/appcontext" @@ -16,6 +14,8 @@ import ( saAPI "github.com/grafana/grafana/pkg/services/serviceaccounts/api" saTests "github.com/grafana/grafana/pkg/services/serviceaccounts/tests" "github.com/grafana/grafana/pkg/services/store/entity" + "github.com/grafana/grafana/pkg/services/store/entity/db/dbimpl" + "github.com/grafana/grafana/pkg/services/store/entity/sqlstash" "github.com/grafana/grafana/pkg/services/user" "github.com/grafana/grafana/pkg/tests/testinfra" ) @@ -53,7 +53,7 @@ func createServiceAccountAdminToken(t *testing.T, env *server.TestEnv) (string, type testContext struct { authToken string - client entity.EntityStoreClient + client entity.EntityStoreServer user *user.SignedInUser ctx context.Context } @@ -74,17 +74,18 @@ func createTestContext(t *testing.T) testContext { authToken, serviceAccountUser := createServiceAccountAdminToken(t, env) - conn, err := grpc.Dial( - env.GRPCServer.GetAddress(), - grpc.WithTransportCredentials(insecure.NewCredentials()), - ) + eDB, err := dbimpl.ProvideEntityDB(env.SQLStore, env.SQLStore.Cfg, env.FeatureToggles) require.NoError(t, err) - client := entity.NewEntityStoreClient(conn) + err = eDB.Init() + require.NoError(t, err) + + store, err := sqlstash.ProvideSQLEntityServer(eDB) + require.NoError(t, err) return testContext{ authToken: authToken, - client: client, + client: store, user: serviceAccountUser, ctx: appcontext.WithUser(context.Background(), serviceAccountUser), } diff --git a/pkg/services/store/entity/tests/server_integration_test.go b/pkg/services/store/entity/tests/server_integration_test.go index ad23dfe5d0a..2ac11cb6821 100644 --- a/pkg/services/store/entity/tests/server_integration_test.go +++ b/pkg/services/store/entity/tests/server_integration_test.go @@ -9,8 +9,8 @@ import ( "time" "github.com/stretchr/testify/require" - "google.golang.org/grpc/metadata" + "github.com/grafana/grafana/pkg/infra/appcontext" "github.com/grafana/grafana/pkg/services/store" "github.com/grafana/grafana/pkg/services/store/entity" ) @@ -64,11 +64,11 @@ func requireEntityMatch(t *testing.T, obj *entity.Entity, m rawEntityMatcher) { } if m.createdBy != "" && m.createdBy != obj.CreatedBy { - mismatches += fmt.Sprintf("createdBy: expected:%s, found:%s\n", m.createdBy, obj.CreatedBy) + mismatches += fmt.Sprintf("createdBy: expected: '%s', found: '%s'\n", m.createdBy, obj.CreatedBy) } if m.updatedBy != "" && m.updatedBy != obj.UpdatedBy { - mismatches += fmt.Sprintf("updatedBy: expected:%s, found:%s\n", m.updatedBy, obj.UpdatedBy) + mismatches += fmt.Sprintf("updatedBy: expected: '%s', found: '%s'\n", m.updatedBy, obj.UpdatedBy) } if len(m.body) > 0 { @@ -99,7 +99,7 @@ func requireVersionMatch(t *testing.T, obj *entity.Entity, m objectVersionMatche } if m.updatedBy != "" && m.updatedBy != obj.UpdatedBy { - mismatches += fmt.Sprintf("updatedBy: expected:%s, found:%s\n", m.updatedBy, obj.UpdatedBy) + mismatches += fmt.Sprintf("updatedBy: expected: '%s', found: '%s'\n", m.updatedBy, obj.UpdatedBy) } if m.version != 0 && m.version != obj.ResourceVersion { @@ -111,7 +111,7 @@ func requireVersionMatch(t *testing.T, obj *entity.Entity, m objectVersionMatche func TestIntegrationEntityServer(t *testing.T) { if true { - // FIXME + // TODO: enable this test once we fix test "database locked" issues t.Skip() } @@ -120,7 +120,7 @@ func TestIntegrationEntityServer(t *testing.T) { } testCtx := createTestContext(t) - ctx := metadata.AppendToOutgoingContext(testCtx.ctx, "authorization", fmt.Sprintf("Bearer %s", testCtx.authToken)) + ctx := appcontext.WithUser(testCtx.ctx, testCtx.user) fakeUser := store.GetUserIDString(testCtx.user) firstVersion := int64(0) @@ -130,6 +130,7 @@ func TestIntegrationEntityServer(t *testing.T) { namespace := "default" name := "my-test-entity" testKey := "/" + group + "/" + resource + "/" + namespace + "/" + name + testKey2 := "/" + group + "/" + resource2 + "/" + namespace + "/" + name body := []byte("{\"name\":\"John\"}") t.Run("should not retrieve non-existent objects", func(t *testing.T) { @@ -158,11 +159,18 @@ func TestIntegrationEntityServer(t *testing.T) { createResp, err := testCtx.client.Create(ctx, createReq) require.NoError(t, err) + // clean up in case test fails + t.Cleanup(func() { + _, _ = testCtx.client.Delete(ctx, &entity.DeleteEntityRequest{ + Key: testKey, + }) + }) + versionMatcher := objectVersionMatcher{ - updatedRange: []time.Time{before, time.Now()}, - updatedBy: fakeUser, - version: firstVersion, - comment: &createReq.Entity.Message, + // updatedRange: []time.Time{before, time.Now()}, + // updatedBy: fakeUser, + version: firstVersion, + comment: &createReq.Entity.Message, } requireVersionMatch(t, createResp.Entity, versionMatcher) @@ -182,11 +190,11 @@ func TestIntegrationEntityServer(t *testing.T) { objectMatcher := rawEntityMatcher{ key: testKey, createdRange: []time.Time{before, time.Now()}, - updatedRange: []time.Time{before, time.Now()}, - createdBy: fakeUser, - updatedBy: fakeUser, - body: body, - version: firstVersion, + // updatedRange: []time.Time{before, time.Now()}, + createdBy: fakeUser, + // updatedBy: fakeUser, + body: body, + version: firstVersion, } requireEntityMatch(t, readResp, objectMatcher) @@ -222,6 +230,14 @@ func TestIntegrationEntityServer(t *testing.T) { } createResp, err := testCtx.client.Create(ctx, createReq) require.NoError(t, err) + + // clean up in case test fails + t.Cleanup(func() { + _, _ = testCtx.client.Delete(ctx, &entity.DeleteEntityRequest{ + Key: testKey, + }) + }) + require.Equal(t, entity.CreateEntityResponse_CREATED, createResp.Status) body2 := []byte("{\"name\":\"John2\"}") @@ -238,12 +254,14 @@ func TestIntegrationEntityServer(t *testing.T) { require.NotEqual(t, createResp.Entity.ResourceVersion, updateResp.Entity.ResourceVersion) // Duplicate write (no change) - writeDupRsp, err := testCtx.client.Update(ctx, updateReq) - require.NoError(t, err) - require.Nil(t, writeDupRsp.Error) - require.Equal(t, entity.UpdateEntityResponse_UNCHANGED, writeDupRsp.Status) - require.Equal(t, updateResp.Entity.ResourceVersion, writeDupRsp.Entity.ResourceVersion) - require.Equal(t, updateResp.Entity.ETag, writeDupRsp.Entity.ETag) + /* + writeDupRsp, err := testCtx.client.Update(ctx, updateReq) + require.NoError(t, err) + require.Nil(t, writeDupRsp.Error) + require.Equal(t, entity.UpdateEntityResponse_UNCHANGED, writeDupRsp.Status) + require.Equal(t, updateResp.Entity.ResourceVersion, writeDupRsp.Entity.ResourceVersion) + require.Equal(t, updateResp.Entity.ETag, writeDupRsp.Entity.ETag) + */ body3 := []byte("{\"name\":\"John3\"}") writeReq3 := &entity.UpdateEntityRequest{ @@ -255,6 +273,7 @@ func TestIntegrationEntityServer(t *testing.T) { } writeResp3, err := testCtx.client.Update(ctx, writeReq3) require.NoError(t, err) + require.Equal(t, entity.UpdateEntityResponse_UPDATED, writeResp3.Status) require.NotEqual(t, writeResp3.Entity.ResourceVersion, updateResp.Entity.ResourceVersion) latestMatcher := rawEntityMatcher{ @@ -285,9 +304,7 @@ func TestIntegrationEntityServer(t *testing.T) { requireEntityMatch(t, readRespFirstVer, rawEntityMatcher{ key: testKey, createdRange: []time.Time{before, time.Now()}, - updatedRange: []time.Time{before, time.Now()}, createdBy: fakeUser, - updatedBy: fakeUser, body: body, version: 0, }) @@ -329,7 +346,7 @@ func TestIntegrationEntityServer(t *testing.T) { w3, err := testCtx.client.Create(ctx, &entity.CreateEntityRequest{ Entity: &entity.Entity{ - Key: testKey + "3", + Key: testKey2 + "3", Body: body, }, }) @@ -337,7 +354,7 @@ func TestIntegrationEntityServer(t *testing.T) { w4, err := testCtx.client.Create(ctx, &entity.CreateEntityRequest{ Entity: &entity.Entity{ - Key: testKey + "4", + Key: testKey2 + "4", Body: body, }, }) @@ -358,18 +375,94 @@ func TestIntegrationEntityServer(t *testing.T) { kinds = append(kinds, res.Resource) version = append(version, res.ResourceVersion) } - require.Equal(t, []string{"my-test-entity", "name2", "name3", "name4"}, names) - require.Equal(t, []string{"jsonobj", "jsonobj", "playlist", "playlist"}, kinds) - require.Equal(t, []int64{ + + // default sort is by guid, so we ignore order + require.ElementsMatch(t, []string{"my-test-entity1", "my-test-entity2", "my-test-entity3", "my-test-entity4"}, names) + require.ElementsMatch(t, []string{"jsonobjs", "jsonobjs", "playlists", "playlists"}, kinds) + require.ElementsMatch(t, []int64{ w1.Entity.ResourceVersion, w2.Entity.ResourceVersion, w3.Entity.ResourceVersion, w4.Entity.ResourceVersion, }, version) + // sorted by name + resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{ + Resource: []string{resource, resource2}, + WithBody: false, + Sort: []string{"name"}, + }) + require.NoError(t, err) + + require.NotNil(t, resp) + require.Equal(t, 4, len(resp.Results)) + + require.Equal(t, "my-test-entity1", resp.Results[0].Name) + require.Equal(t, "my-test-entity2", resp.Results[1].Name) + require.Equal(t, "my-test-entity3", resp.Results[2].Name) + require.Equal(t, "my-test-entity4", resp.Results[3].Name) + + require.Equal(t, "jsonobjs", resp.Results[0].Resource) + require.Equal(t, "jsonobjs", resp.Results[1].Resource) + require.Equal(t, "playlists", resp.Results[2].Resource) + require.Equal(t, "playlists", resp.Results[3].Resource) + + // sorted by name desc + resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{ + Resource: []string{resource, resource2}, + WithBody: false, + Sort: []string{"name_desc"}, + }) + require.NoError(t, err) + + require.NotNil(t, resp) + require.Equal(t, 4, len(resp.Results)) + + require.Equal(t, "my-test-entity1", resp.Results[3].Name) + require.Equal(t, "my-test-entity2", resp.Results[2].Name) + require.Equal(t, "my-test-entity3", resp.Results[1].Name) + require.Equal(t, "my-test-entity4", resp.Results[0].Name) + + require.Equal(t, "jsonobjs", resp.Results[3].Resource) + require.Equal(t, "jsonobjs", resp.Results[2].Resource) + require.Equal(t, "playlists", resp.Results[1].Resource) + require.Equal(t, "playlists", resp.Results[0].Resource) + + // with limit + resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{ + Resource: []string{resource, resource2}, + WithBody: false, + Limit: 2, + Sort: []string{"name"}, + }) + require.NoError(t, err) + + require.NotNil(t, resp) + require.Equal(t, 2, len(resp.Results)) + + require.Equal(t, "my-test-entity1", resp.Results[0].Name) + require.Equal(t, "my-test-entity2", resp.Results[1].Name) + + // with limit & continue + resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{ + Resource: []string{resource, resource2}, + WithBody: false, + Limit: 2, + NextPageToken: resp.NextPageToken, + Sort: []string{"name"}, + }) + require.NoError(t, err) + + require.NotNil(t, resp) + require.Equal(t, 2, len(resp.Results)) + + require.Equal(t, "my-test-entity3", resp.Results[0].Name) + require.Equal(t, "my-test-entity4", resp.Results[1].Name) + // Again with only one kind respKind1, err := testCtx.client.List(ctx, &entity.EntityListRequest{ Resource: []string{resource}, + Sort: []string{"name"}, }) require.NoError(t, err) names = make([]string, 0, len(respKind1.Results)) @@ -380,8 +473,8 @@ func TestIntegrationEntityServer(t *testing.T) { kinds = append(kinds, res.Resource) version = append(version, res.ResourceVersion) } - require.Equal(t, []string{"my-test-entity", "name2"}, names) - require.Equal(t, []string{"jsonobj", "jsonobj"}, kinds) + require.Equal(t, []string{"my-test-entity1", "my-test-entity2"}, names) + require.Equal(t, []string{"jsonobjs", "jsonobjs"}, kinds) require.Equal(t, []int64{ w1.Entity.ResourceVersion, w2.Entity.ResourceVersion, @@ -389,25 +482,32 @@ func TestIntegrationEntityServer(t *testing.T) { }) t.Run("should be able to filter objects based on their labels", func(t *testing.T) { - kind := entity.StandardKindDashboard _, err := testCtx.client.Create(ctx, &entity.CreateEntityRequest{ Entity: &entity.Entity{ - Key: "/grafana/dashboards/blue-green", + Key: "/dashboards.grafana.app/dashboards/default/blue-green", Body: []byte(dashboardWithTagsBlueGreen), + Labels: map[string]string{ + "blue": "", + "green": "", + }, }, }) require.NoError(t, err) _, err = testCtx.client.Create(ctx, &entity.CreateEntityRequest{ Entity: &entity.Entity{ - Key: "/grafana/dashboards/red-green", + Key: "/dashboards.grafana.app/dashboards/default/red-green", Body: []byte(dashboardWithTagsRedGreen), + Labels: map[string]string{ + "red": "", + "green": "", + }, }, }) require.NoError(t, err) resp, err := testCtx.client.List(ctx, &entity.EntityListRequest{ - Key: []string{kind}, + Key: []string{"/dashboards.grafana.app/dashboards/default"}, WithBody: false, Labels: map[string]string{ "red": "", @@ -419,7 +519,7 @@ func TestIntegrationEntityServer(t *testing.T) { require.Equal(t, resp.Results[0].Name, "red-green") resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{ - Key: []string{kind}, + Key: []string{"/dashboards.grafana.app/dashboards/default"}, WithBody: false, Labels: map[string]string{ "red": "", @@ -432,7 +532,7 @@ func TestIntegrationEntityServer(t *testing.T) { require.Equal(t, resp.Results[0].Name, "red-green") resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{ - Key: []string{kind}, + Key: []string{"/dashboards.grafana.app/dashboards/default"}, WithBody: false, Labels: map[string]string{ "red": "invalid", @@ -443,7 +543,7 @@ func TestIntegrationEntityServer(t *testing.T) { require.Len(t, resp.Results, 0) resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{ - Key: []string{kind}, + Key: []string{"/dashboards.grafana.app/dashboards/default"}, WithBody: false, Labels: map[string]string{ "green": "", @@ -454,7 +554,7 @@ func TestIntegrationEntityServer(t *testing.T) { require.Len(t, resp.Results, 2) resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{ - Key: []string{kind}, + Key: []string{"/dashboards.grafana.app/dashboards/default"}, WithBody: false, Labels: map[string]string{ "yellow": "",