IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Storage: Add support for sortBy selector (#80680)

Browse Source 
* add support for sortBy field selector

* use label selectors instead of field selectors

* set entity_labels on create & update

* make entity server integration tests work

* test fixes

* be more consistent with handling of empty body, meta or status

* workaround for database is locked errors during migration

* fix double import of sqlite3

* rename functions and tidy up

* refactor update

* disable integration tests until we can fix the database locking issue
This commit is contained in:
Dan Cech 2024-02-07 15:05:10 -05:00 committed by GitHub
parent bb6db46ecc
commit 1f1461734c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 429 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
pkg/registry/apis/folders/legacy_storage.go
View File

@ -67,15 +67,16 @@ func (s *legacyStorage) List(ctx context.Context, options *internalversion.ListO
}
parentUID := ""
fieldRequirements, fieldSelector, err := entity.ReadFieldRequirements(options.FieldSelector)
// translate grafana.app/* label selectors into field requirements
requirements, newSelector, err := entity.ReadLabelSelectors(options.LabelSelector)
if err != nil {
return nil, err
}
if fieldRequirements.Folder != nil {
parentUID = *fieldRequirements.Folder
if requirements.Folder != nil {
parentUID = *requirements.Folder
}
// Update the field selector to remove the unneeded selectors
options.FieldSelector = fieldSelector
// Update the selector to remove the unneeded requirements
options.LabelSelector = newSelector
paging, err := readContinueToken(options)
if err != nil {

4
pkg/server/test_env.go
View File

@ -3,6 +3,7 @@ package server
import (
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/plugins/manager/registry"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/grpcserver"
"github.com/grafana/grafana/pkg/services/notifications"
"github.com/grafana/grafana/pkg/services/oauthtoken/oauthtokentest"
@ -18,6 +19,7 @@ func ProvideTestEnv(
pluginRegistry registry.Service,
httpClientProvider httpclient.Provider,
oAuthTokenService *oauthtokentest.Service,
featureMgmt featuremgmt.FeatureToggles,
) (*TestEnv, error) {
return &TestEnv{
Server: server,
@ -27,6 +29,7 @@ func ProvideTestEnv(
PluginRegistry: pluginRegistry,
HTTPClientProvider: httpClientProvider,
OAuthTokenService: oAuthTokenService,
FeatureToggles: featureMgmt,
}, nil
}
@ -39,4 +42,5 @@ type TestEnv struct {
HTTPClientProvider httpclient.Provider
OAuthTokenService *oauthtokentest.Service
RequestMiddleware web.Middleware
FeatureToggles featuremgmt.FeatureToggles
}

6
pkg/services/apiserver/service.go
View File

@ -277,12 +277,6 @@ func (s *service) start(ctx context.Context) error {
return err
}
// support folder selection
err = entitystorage.RegisterFieldSelectorSupport(Scheme)
if err != nil {
return err
}
// Create the server
server, err := serverConfig.Complete().New("grafana-apiserver", genericapiserver.NewEmptyDelegate())
if err != nil {

76
pkg/services/apiserver/storage/entity/fieldRequirements.go
View File

@ -1,76 +0,0 @@
package entity
import (
"fmt"
"strings"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/selection"
)
const folderAnnoKey = "grafana.app/folder"
type FieldRequirements struct {
// Equals folder
Folder *string
}
func ReadFieldRequirements(selector fields.Selector) (FieldRequirements, fields.Selector, error) {
requirements := FieldRequirements{}
if selector == nil {
return requirements, selector, nil
}
for _, r := range selector.Requirements() {
switch r.Field {
case folderAnnoKey:
if (r.Operator != selection.Equals) && (r.Operator != selection.DoubleEquals) {
return requirements, selector, apierrors.NewBadRequest("only equality is supported in the selectors")
}
folder := r.Value
requirements.Folder = &folder
}
}
// use Transform function to remove grafana.app/folder field selector
selector, err := selector.Transform(func(field, value string) (string, string, error) {
switch field {
case folderAnnoKey:
return "", "", nil
}
return field, value, nil
})
return requirements, selector, err
}
func RegisterFieldSelectorSupport(scheme *runtime.Scheme) error {
grafanaFieldSupport := runtime.FieldLabelConversionFunc(
func(field, value string) (string, string, error) {
if strings.HasPrefix(field, "grafana.app/") {
return field, value, nil
}
return "", "", getBadSelectorError(field)
},
)
// Register all the internal types
for gvk := range scheme.AllKnownTypes() {
if strings.HasSuffix(gvk.Group, ".grafana.app") {
err := scheme.AddFieldLabelConversionFunc(gvk, grafanaFieldSupport)
if err != nil {
return err
}
}
}
return nil
}
func getBadSelectorError(f string) error {
return apierrors.NewBadRequest(
fmt.Sprintf("%q is not a known field selector: only %q works", f, folderAnnoKey),
)
}

49
pkg/services/apiserver/storage/entity/selector.go Normal file
View File

@ -0,0 +1,49 @@
package entity
import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/selection"
)
const folderAnnoKey = "grafana.app/folder"
const sortByKey = "grafana.app/sortBy"
type Requirements struct {
// Equals folder
Folder *string
// SortBy is a list of fields to sort by
SortBy []string
}
func ReadLabelSelectors(selector labels.Selector) (Requirements, labels.Selector, error) {
requirements := Requirements{}
newSelector := labels.NewSelector()
if selector == nil {
return requirements, newSelector, nil
}
labelSelectors, _ := selector.Requirements()
for _, r := range labelSelectors {
switch r.Key() {
case folderAnnoKey:
if (r.Operator() != selection.Equals) && (r.Operator() != selection.DoubleEquals) {
return requirements, newSelector, apierrors.NewBadRequest(folderAnnoKey + " label selector only supports equality")
}
folder := r.Values().List()[0]
requirements.Folder = &folder
case sortByKey:
if r.Operator() != selection.In {
return requirements, newSelector, apierrors.NewBadRequest(sortByKey + " label selector only supports in")
}
requirements.SortBy = r.Values().List()
// add all unregonized label selectors to the new selector list, these will be processed by the entity store
default:
newSelector = newSelector.Add(r)
}
}
return requirements, newSelector, nil
}

29
pkg/services/apiserver/storage/entity/storage.go
View File

@ -229,29 +229,32 @@ func (s *Storage) GetList(ctx context.Context, key string, opts storage.ListOpti
// TODO push label/field matching down to storage
}
// translate grafana.app/* label selectors into field requirements
requirements, newSelector, err := ReadLabelSelectors(opts.Predicate.Label)
if err != nil {
return err
}
if requirements.Folder != nil {
req.Folder = *requirements.Folder
}
if len(requirements.SortBy) > 0 {
req.Sort = requirements.SortBy
}
// Update the selector to remove the unneeded requirements
opts.Predicate.Label = newSelector
// translate "equals" label selectors to storage label conditions
requirements, selectable := opts.Predicate.Label.Requirements()
labelRequirements, selectable := opts.Predicate.Label.Requirements()
if !selectable {
return apierrors.NewBadRequest("label selector is not selectable")
}
for _, r := range requirements {
for _, r := range labelRequirements {
if r.Operator() == selection.Equals {
req.Labels[r.Key()] = r.Values().List()[0]
}
}
// translate grafana.app/folder field selector to the folder condition
fieldRequirements, fieldSelector, err := ReadFieldRequirements(opts.Predicate.Field)
if err != nil {
return err
}
if fieldRequirements.Folder != nil {
req.Folder = *fieldRequirements.Folder
}
// Update the field selector to remove the unneeded selectors
opts.Predicate.Field = fieldSelector
rsp, err := s.store.List(ctx, req)
if err != nil {
return apierrors.NewInternalError(err)

10
pkg/services/sqlstore/migrator/migrator.go
View File

@ -1,13 +1,14 @@
package migrator
import (
"errors"
"fmt"
"time"
_ "github.com/go-sql-driver/mysql"
"github.com/golang-migrate/migrate/v4/database"
_ "github.com/lib/pq"
_ "github.com/mattn/go-sqlite3"
"github.com/mattn/go-sqlite3"
"go.uber.org/atomic"
"xorm.io/xorm"
@ -208,6 +209,13 @@ func (mg *Migrator) run() (err error) {
err := mg.InTransaction(func(sess *xorm.Session) error {
err := mg.exec(m, sess)
// if we get an sqlite busy/locked error, sleep 100ms and try again
if errors.Is(err, sqlite3.ErrLocked) || errors.Is(err, sqlite3.ErrBusy) {
mg.Logger.Debug("Database locked, sleeping then retrying", "error", err, "sql", sql)
time.Sleep(100 * time.Millisecond)
err = mg.exec(m, sess)
}
if err != nil {
mg.Logger.Error("Exec failed", "error", err, "sql", sql)
record.Error = err.Error()

8
pkg/services/store/entity/db/migrations/entity_store_mig.go
View File

@ -7,7 +7,7 @@ import (
)
func initEntityTables(mg *migrator.Migrator) string {
marker := "Initialize entity tables (v12)" // changing this key wipe+rewrite everything
marker := "Initialize entity tables (v13)" // changing this key wipe+rewrite everything
mg.AddMigration(marker, &migrator.RawSQLMigration{})
tables := []migrator.Table{}
@ -120,7 +120,11 @@ func initEntityTables(mg *migrator.Migrator) string {
},
Indices: []*migrator.Index{
{Cols: []string{"guid", "resource_version"}, Type: migrator.UniqueIndex},
{Cols: []string{"namespace", "group", "resource", "name", "resource_version"}, Type: migrator.UniqueIndex},
{
Cols: []string{"namespace", "group", "resource", "name", "resource_version"},
Type: migrator.UniqueIndex,
Name: "UQE_entity_history_namespace_group_name_version",
},
},
})

53
pkg/services/store/entity/sqlstash/querybuilder.go
View File

@ -6,15 +6,33 @@ import (
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
)
type Direction int
const (
Ascending Direction = iota
Descending
)
func (d Direction) String() string {
if d == Descending {
return "DESC"
}
return "ASC"
}
type selectQuery struct {
dialect migrator.Dialect
fields []string // SELECT xyz
from string // FROM object
offset int64
limit int64
oneExtra bool
where []string
args []any
orderBy []string
direction []Direction
}
func (q *selectQuery) addWhere(f string, val ...any) {
@ -53,6 +71,11 @@ func (q *selectQuery) addWhereIn(f string, vals []string) {
}
}
func (q *selectQuery) addOrderBy(field string, direction Direction) {
q.orderBy = append(q.orderBy, field)
q.direction = append(q.direction, direction)
}
func (q *selectQuery) toQuery() (string, []any) {
args := q.args
sb := strings.Builder{}
@ -77,17 +100,27 @@ func (q *selectQuery) toQuery() (string, []any) {
}
}
if q.limit > 0 || q.oneExtra {
limit := q.limit
if limit < 1 {
limit = 20
q.limit = limit
if len(q.orderBy) > 0 && len(q.direction) == len(q.orderBy) {
sb.WriteString(" ORDER BY ")
for i, f := range q.orderBy {
if i > 0 {
sb.WriteString(",")
}
sb.WriteString(q.dialect.Quote(f))
sb.WriteString(" ")
sb.WriteString(q.direction[i].String())
}
if q.oneExtra {
limit = limit + 1
}
sb.WriteString(" LIMIT ?")
args = append(args, limit)
}
limit := q.limit
if limit < 1 {
limit = 20
q.limit = limit
}
if q.oneExtra {
limit = limit + 1
}
sb.WriteString(q.dialect.LimitOffset(limit, q.offset))
return sb.String(), args
}

187
pkg/services/store/entity/sqlstash/sql_storage_server.go
View File

@ -3,10 +3,12 @@ package sqlstash
import (
"context"
"database/sql"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"math/rand"
"slices"
"strings"
"time"
@ -89,6 +91,7 @@ func (s *sqlEntityServer) getReadFields(r *entity.ReadEntityRequest) []string {
"origin", "origin_key", "origin_ts",
"meta",
"title", "slug", "description", "labels", "fields",
"message",
}
if r.WithBody {
@ -134,6 +137,7 @@ func (s *sqlEntityServer) rowToEntity(ctx context.Context, rows *sql.Rows, r *en
&raw.Origin.Source, &raw.Origin.Key, &raw.Origin.Time,
&raw.Meta,
&raw.Title, &raw.Slug, &raw.Description, &labels, &fields,
&raw.Message,
}
if r.WithBody {
args = append(args, &raw.Body)
@ -147,10 +151,6 @@ func (s *sqlEntityServer) rowToEntity(ctx context.Context, rows *sql.Rows, r *en
return nil, err
}
if raw.Origin.Source == "" {
raw.Origin = nil
}
// unmarshal json labels
if labels != "" {
if err := json.Unmarshal([]byte(labels), &raw.Labels); err != nil {
@ -158,6 +158,17 @@ func (s *sqlEntityServer) rowToEntity(ctx context.Context, rows *sql.Rows, r *en
}
}
// set empty body, meta or status to nil
if raw.Body != nil && len(raw.Body) == 0 {
raw.Body = nil
}
if raw.Meta != nil && len(raw.Meta) == 0 {
raw.Meta = nil
}
if raw.Status != nil && len(raw.Status) == 0 {
raw.Status = nil
}
return raw, nil
}
@ -278,6 +289,10 @@ func (s *sqlEntityServer) Create(ctx context.Context, r *entity.CreateEntityRequ
}
createdAt := r.Entity.CreatedAt
if createdAt < 1000 {
createdAt = time.Now().UnixMilli()
}
createdBy := r.Entity.CreatedBy
if createdBy == "" {
modifier, err := appcontext.User(ctx)
@ -289,6 +304,7 @@ func (s *sqlEntityServer) Create(ctx context.Context, r *entity.CreateEntityRequ
}
createdBy = store.GetUserIDString(modifier)
}
updatedAt := r.Entity.UpdatedAt
updatedBy := r.Entity.UpdatedBy
@ -315,6 +331,10 @@ func (s *sqlEntityServer) Create(ctx context.Context, r *entity.CreateEntityRequ
// generate guid for new entity
current.Guid = uuid.New().String()
// set created at/by
current.CreatedAt = createdAt
current.CreatedBy = createdBy
// parse provided key
key, err := entity.ParseKey(r.Entity.Key)
if err != nil {
@ -350,6 +370,7 @@ func (s *sqlEntityServer) Create(ctx context.Context, r *entity.CreateEntityRequ
etag := createContentsHash(current.Body, current.Meta, current.Status)
current.ETag = etag
current.UpdatedAt = updatedAt
current.UpdatedBy = updatedBy
@ -365,18 +386,21 @@ func (s *sqlEntityServer) Create(ctx context.Context, r *entity.CreateEntityRequ
s.log.Error("error marshalling labels", "msg", err.Error())
return err
}
current.Labels = r.Entity.Labels
fields, err := json.Marshal(r.Entity.Fields)
if err != nil {
s.log.Error("error marshalling fields", "msg", err.Error())
return err
}
current.Fields = r.Entity.Fields
errors, err := json.Marshal(r.Entity.Errors)
if err != nil {
s.log.Error("error marshalling errors", "msg", err.Error())
return err
}
current.Errors = r.Entity.Errors
if current.Origin == nil {
current.Origin = &entity.EntityOriginInfo{}
@ -409,13 +433,13 @@ func (s *sqlEntityServer) Create(ctx context.Context, r *entity.CreateEntityRequ
"group": current.Group,
"resource": current.Resource,
"name": current.Name,
"created_at": createdAt,
"created_by": createdBy,
"created_at": current.CreatedAt,
"created_by": current.CreatedBy,
"group_version": current.GroupVersion,
"folder": current.Folder,
"slug": current.Slug,
"updated_at": updatedAt,
"updated_by": updatedBy,
"updated_at": current.UpdatedAt,
"updated_by": current.UpdatedBy,
"body": current.Body,
"meta": current.Meta,
"status": current.Status,
@ -459,7 +483,7 @@ func (s *sqlEntityServer) Create(ctx context.Context, r *entity.CreateEntityRequ
rsp.Entity = current
return nil // s.writeSearchInfo(ctx, tx, current)
return s.setLabels(ctx, tx, current.Guid, current.Labels)
})
if err != nil {
s.log.Error("error creating entity", "msg", err.Error())
@ -475,8 +499,11 @@ func (s *sqlEntityServer) Update(ctx context.Context, r *entity.UpdateEntityRequ
return nil, err
}
timestamp := time.Now().UnixMilli()
updatedAt := r.Entity.UpdatedAt
if updatedAt < 1000 {
updatedAt = time.Now().UnixMilli()
}
updatedBy := r.Entity.UpdatedBy
if updatedBy == "" {
modifier, err := appcontext.User(ctx)
@ -488,9 +515,6 @@ func (s *sqlEntityServer) Update(ctx context.Context, r *entity.UpdateEntityRequ
}
updatedBy = store.GetUserIDString(modifier)
}
if updatedAt < 1000 {
updatedAt = timestamp
}
rsp := &entity.UpdateEntityResponse{
Entity: &entity.Entity{},
@ -519,10 +543,7 @@ func (s *sqlEntityServer) Update(ctx context.Context, r *entity.UpdateEntityRequ
rsp.Entity.Guid = current.Guid
// Clear the labels+refs
if _, err := tx.Exec(ctx, "DELETE FROM entity_labels WHERE guid=?", rsp.Entity.Guid); err != nil {
return err
}
// Clear the refs
if _, err := tx.Exec(ctx, "DELETE FROM entity_ref WHERE guid=?", rsp.Entity.Guid); err != nil {
return err
}
@ -553,6 +574,7 @@ func (s *sqlEntityServer) Update(ctx context.Context, r *entity.UpdateEntityRequ
etag := createContentsHash(current.Body, current.Meta, current.Status)
current.ETag = etag
current.UpdatedAt = updatedAt
current.UpdatedBy = updatedBy
@ -568,18 +590,21 @@ func (s *sqlEntityServer) Update(ctx context.Context, r *entity.UpdateEntityRequ
s.log.Error("error marshalling labels", "msg", err.Error())
return err
}
current.Labels = r.Entity.Labels
fields, err := json.Marshal(r.Entity.Fields)
if err != nil {
s.log.Error("error marshalling fields", "msg", err.Error())
return err
}
current.Fields = r.Entity.Fields
errors, err := json.Marshal(r.Entity.Errors)
if err != nil {
s.log.Error("error marshalling errors", "msg", err.Error())
return err
}
current.Errors = r.Entity.Errors
if current.Origin == nil {
current.Origin = &entity.EntityOriginInfo{}
@ -619,8 +644,8 @@ func (s *sqlEntityServer) Update(ctx context.Context, r *entity.UpdateEntityRequ
"group_version": current.GroupVersion,
"folder": current.Folder,
"slug": current.Slug,
"updated_at": updatedAt,
"updated_by": updatedBy,
"updated_at": current.UpdatedAt,
"updated_by": current.UpdatedBy,
"body": current.Body,
"meta": current.Meta,
"status": current.Status,
@ -684,7 +709,7 @@ func (s *sqlEntityServer) Update(ctx context.Context, r *entity.UpdateEntityRequ
rsp.Entity = current
return nil // s.writeSearchInfo(ctx, tx, current)
return s.setLabels(ctx, tx, current.Guid, current.Labels)
})
if err != nil {
s.log.Error("error updating entity", "msg", err.Error())
@ -694,23 +719,22 @@ func (s *sqlEntityServer) Update(ctx context.Context, r *entity.UpdateEntityRequ
return rsp, err
}
/*
func (s *sqlEntityServer) writeSearchInfo(
ctx context.Context,
tx *session.SessionTx,
current *entity.Entity,
) error {
// parent_key := current.getParentKey()
func (s *sqlEntityServer) setLabels(ctx context.Context, tx *session.SessionTx, guid string, labels map[string]string) error {
s.log.Debug("setLabels", "guid", guid, "labels", labels)
// Add the labels rows
for k, v := range current.Labels {
// Clear the old labels
if _, err := tx.Exec(ctx, "DELETE FROM entity_labels WHERE guid=?", guid); err != nil {
return err
}
// Add the new labels
for k, v := range labels {
query, args, err := s.dialect.InsertQuery(
"entity_labels",
map[string]any{
"key": current.Key,
"guid": guid,
"label": k,
"value": v,
// "parent_key": parent_key,
},
)
if err != nil {
@ -725,7 +749,6 @@ func (s *sqlEntityServer) writeSearchInfo(
return nil
}
*/
func (s *sqlEntityServer) Delete(ctx context.Context, r *entity.DeleteEntityRequest) (*entity.DeleteEntityResponse, error) {
if err := s.Init(); err != nil {
@ -816,7 +839,7 @@ func (s *sqlEntityServer) History(ctx context.Context, r *entity.EntityHistoryRe
rr := &entity.ReadEntityRequest{
Key: r.Key,
WithBody: true,
WithStatus: false,
WithStatus: true,
}
query, err := s.getReadSelect(rr)
@ -879,6 +902,75 @@ func (s *sqlEntityServer) History(ctx context.Context, r *entity.EntityHistoryRe
return rsp, err
}
type ContinueToken struct {
Sort []string `json:"s"`
StartOffset int64 `json:"o"`
}
func (c *ContinueToken) String() string {
b, _ := json.Marshal(c)
return base64.StdEncoding.EncodeToString(b)
}
func GetContinueToken(r *entity.EntityListRequest) (*ContinueToken, error) {
if r.NextPageToken == "" {
return nil, nil
}
continueVal, err := base64.StdEncoding.DecodeString(r.NextPageToken)
if err != nil {
return nil, fmt.Errorf("error decoding continue token")
}
t := &ContinueToken{}
err = json.Unmarshal(continueVal, t)
if err != nil {
return nil, err
}
if !slices.Equal(t.Sort, r.Sort) {
return nil, fmt.Errorf("sort order changed")
}
return t, nil
}
var sortByFields = []string{
"guid",
"key",
"namespace", "group", "group_version", "resource", "name", "folder",
"resource_version", "size", "etag",
"created_at", "created_by",
"updated_at", "updated_by",
"origin", "origin_key", "origin_ts",
"title", "slug", "description",
}
type SortBy struct {
Field string
Direction Direction
}
func ParseSortBy(sort string) (*SortBy, error) {
sortBy := &SortBy{
Field: "guid",
Direction: Ascending,
}
if strings.HasSuffix(sort, "_desc") {
sortBy.Field = sort[:len(sort)-5]
sortBy.Direction = Descending
} else {
sortBy.Field = sort
}
if !slices.Contains(sortByFields, sortBy.Field) {
return nil, fmt.Errorf("invalid sort field '%s', valid fields: %v", sortBy.Field, sortByFields)
}
return sortBy, nil
}
func (s *sqlEntityServer) List(ctx context.Context, r *entity.EntityListRequest) (*entity.EntityListResponse, error) {
if err := s.Init(); err != nil {
return nil, err
@ -892,10 +984,6 @@ func (s *sqlEntityServer) List(ctx context.Context, r *entity.EntityListRequest)
return nil, fmt.Errorf("missing user in context")
}
if r.NextPageToken != "" || len(r.Sort) > 0 {
return nil, fmt.Errorf("not yet supported")
}
rr := &entity.ReadEntityRequest{
WithBody: r.WithBody,
WithStatus: r.WithStatus,
@ -909,6 +997,7 @@ func (s *sqlEntityServer) List(ctx context.Context, r *entity.EntityListRequest)
from: "entity", // the table
args: []any{},
limit: r.Limit,
offset: 0,
oneExtra: true, // request one more than the limit (and show next token if it exists)
}
@ -951,8 +1040,13 @@ func (s *sqlEntityServer) List(ctx context.Context, r *entity.EntityListRequest)
entityQuery.addWhere("folder", r.Folder)
}
if r.NextPageToken != "" {
entityQuery.addWhere("guid>?", r.NextPageToken)
// if we have a page token, use that to specify the first record
continueToken, err := GetContinueToken(r)
if err != nil {
return nil, err
}
if continueToken != nil {
entityQuery.offset = continueToken.StartOffset
}
if len(r.Labels) > 0 {
@ -971,6 +1065,14 @@ func (s *sqlEntityServer) List(ctx context.Context, r *entity.EntityListRequest)
entityQuery.addWhereInSubquery("guid", query, args)
}
for _, sort := range r.Sort {
sortBy, err := ParseSortBy(sort)
if err != nil {
return nil, err
}
entityQuery.addOrderBy(sortBy.Field, sortBy.Direction)
}
entityQuery.addOrderBy("guid", Ascending)
query, args := entityQuery.toQuery()
@ -990,8 +1092,11 @@ func (s *sqlEntityServer) List(ctx context.Context, r *entity.EntityListRequest)
// found more than requested
if int64(len(rsp.Results)) >= entityQuery.limit {
// TODO? this only works if we sort by guid
rsp.NextPageToken = result.Guid
continueToken := &ContinueToken{
Sort: r.Sort,
StartOffset: entityQuery.offset + entityQuery.limit,
}
rsp.NextPageToken = continueToken.String()
break
}

3
pkg/services/store/entity/sqlstash/sql_storage_server_test.go
View File

@ -31,6 +31,7 @@ func TestCreate(t *testing.T) {
Name: "set-minimum-uid",
Key: "/playlist.grafana.app/playlists/default/set-minimum-uid",
CreatedBy: "set-minimum-creator",
Origin: &entity.EntityOriginInfo{},
},
false,
true,
@ -103,7 +104,7 @@ func TestCreate(t *testing.T) {
require.Equal(t, tc.ent.Status, read.Status)
require.Equal(t, tc.ent.Title, read.Title)
require.Equal(t, tc.ent.Size, read.Size)
require.Equal(t, tc.ent.CreatedAt, read.CreatedAt)
require.Greater(t, read.CreatedAt, int64(0))
require.Equal(t, tc.ent.CreatedBy, read.CreatedBy)
require.Equal(t, tc.ent.UpdatedAt, read.UpdatedAt)
require.Equal(t, tc.ent.UpdatedBy, read.UpdatedBy)

19
pkg/services/store/entity/tests/common.go
View File

@ -5,8 +5,6 @@ import (
"testing"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"github.com/grafana/grafana/pkg/components/satokengen"
"github.com/grafana/grafana/pkg/infra/appcontext"
@ -16,6 +14,8 @@ import (
saAPI "github.com/grafana/grafana/pkg/services/serviceaccounts/api"
saTests "github.com/grafana/grafana/pkg/services/serviceaccounts/tests"
"github.com/grafana/grafana/pkg/services/store/entity"
"github.com/grafana/grafana/pkg/services/store/entity/db/dbimpl"
"github.com/grafana/grafana/pkg/services/store/entity/sqlstash"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/tests/testinfra"
)
@ -53,7 +53,7 @@ func createServiceAccountAdminToken(t *testing.T, env *server.TestEnv) (string,
type testContext struct {
authToken string
client entity.EntityStoreClient
client entity.EntityStoreServer
user *user.SignedInUser
ctx context.Context
}
@ -74,17 +74,18 @@ func createTestContext(t *testing.T) testContext {
authToken, serviceAccountUser := createServiceAccountAdminToken(t, env)
conn, err := grpc.Dial(
env.GRPCServer.GetAddress(),
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
eDB, err := dbimpl.ProvideEntityDB(env.SQLStore, env.SQLStore.Cfg, env.FeatureToggles)
require.NoError(t, err)
client := entity.NewEntityStoreClient(conn)
err = eDB.Init()
require.NoError(t, err)
store, err := sqlstash.ProvideSQLEntityServer(eDB)
require.NoError(t, err)
return testContext{
authToken: authToken,
client: client,
client: store,
user: serviceAccountUser,
ctx: appcontext.WithUser(context.Background(), serviceAccountUser),
}

176
pkg/services/store/entity/tests/server_integration_test.go
View File

@ -9,8 +9,8 @@ import (
"time"
"github.com/stretchr/testify/require"
"google.golang.org/grpc/metadata"
"github.com/grafana/grafana/pkg/infra/appcontext"
"github.com/grafana/grafana/pkg/services/store"
"github.com/grafana/grafana/pkg/services/store/entity"
)
@ -64,11 +64,11 @@ func requireEntityMatch(t *testing.T, obj *entity.Entity, m rawEntityMatcher) {
}
if m.createdBy != "" && m.createdBy != obj.CreatedBy {
mismatches += fmt.Sprintf("createdBy: expected:%s, found:%s\n", m.createdBy, obj.CreatedBy)
mismatches += fmt.Sprintf("createdBy: expected: '%s', found: '%s'\n", m.createdBy, obj.CreatedBy)
}
if m.updatedBy != "" && m.updatedBy != obj.UpdatedBy {
mismatches += fmt.Sprintf("updatedBy: expected:%s, found:%s\n", m.updatedBy, obj.UpdatedBy)
mismatches += fmt.Sprintf("updatedBy: expected: '%s', found: '%s'\n", m.updatedBy, obj.UpdatedBy)
}
if len(m.body) > 0 {
@ -99,7 +99,7 @@ func requireVersionMatch(t *testing.T, obj *entity.Entity, m objectVersionMatche
}
if m.updatedBy != "" && m.updatedBy != obj.UpdatedBy {
mismatches += fmt.Sprintf("updatedBy: expected:%s, found:%s\n", m.updatedBy, obj.UpdatedBy)
mismatches += fmt.Sprintf("updatedBy: expected: '%s', found: '%s'\n", m.updatedBy, obj.UpdatedBy)
}
if m.version != 0 && m.version != obj.ResourceVersion {
@ -111,7 +111,7 @@ func requireVersionMatch(t *testing.T, obj *entity.Entity, m objectVersionMatche
func TestIntegrationEntityServer(t *testing.T) {
if true {
// FIXME
// TODO: enable this test once we fix test "database locked" issues
t.Skip()
}
@ -120,7 +120,7 @@ func TestIntegrationEntityServer(t *testing.T) {
}
testCtx := createTestContext(t)
ctx := metadata.AppendToOutgoingContext(testCtx.ctx, "authorization", fmt.Sprintf("Bearer %s", testCtx.authToken))
ctx := appcontext.WithUser(testCtx.ctx, testCtx.user)
fakeUser := store.GetUserIDString(testCtx.user)
firstVersion := int64(0)
@ -130,6 +130,7 @@ func TestIntegrationEntityServer(t *testing.T) {
namespace := "default"
name := "my-test-entity"
testKey := "/" + group + "/" + resource + "/" + namespace + "/" + name
testKey2 := "/" + group + "/" + resource2 + "/" + namespace + "/" + name
body := []byte("{\"name\":\"John\"}")
t.Run("should not retrieve non-existent objects", func(t *testing.T) {
@ -158,11 +159,18 @@ func TestIntegrationEntityServer(t *testing.T) {
createResp, err := testCtx.client.Create(ctx, createReq)
require.NoError(t, err)
// clean up in case test fails
t.Cleanup(func() {
_, _ = testCtx.client.Delete(ctx, &entity.DeleteEntityRequest{
Key: testKey,
})
})
versionMatcher := objectVersionMatcher{
updatedRange: []time.Time{before, time.Now()},
updatedBy: fakeUser,
version: firstVersion,
comment: &createReq.Entity.Message,
// updatedRange: []time.Time{before, time.Now()},
// updatedBy: fakeUser,
version: firstVersion,
comment: &createReq.Entity.Message,
}
requireVersionMatch(t, createResp.Entity, versionMatcher)
@ -182,11 +190,11 @@ func TestIntegrationEntityServer(t *testing.T) {
objectMatcher := rawEntityMatcher{
key: testKey,
createdRange: []time.Time{before, time.Now()},
updatedRange: []time.Time{before, time.Now()},
createdBy: fakeUser,
updatedBy: fakeUser,
body: body,
version: firstVersion,
// updatedRange: []time.Time{before, time.Now()},
createdBy: fakeUser,
// updatedBy: fakeUser,
body: body,
version: firstVersion,
}
requireEntityMatch(t, readResp, objectMatcher)
@ -222,6 +230,14 @@ func TestIntegrationEntityServer(t *testing.T) {
}
createResp, err := testCtx.client.Create(ctx, createReq)
require.NoError(t, err)
// clean up in case test fails
t.Cleanup(func() {
_, _ = testCtx.client.Delete(ctx, &entity.DeleteEntityRequest{
Key: testKey,
})
})
require.Equal(t, entity.CreateEntityResponse_CREATED, createResp.Status)
body2 := []byte("{\"name\":\"John2\"}")
@ -238,12 +254,14 @@ func TestIntegrationEntityServer(t *testing.T) {
require.NotEqual(t, createResp.Entity.ResourceVersion, updateResp.Entity.ResourceVersion)
// Duplicate write (no change)
writeDupRsp, err := testCtx.client.Update(ctx, updateReq)
require.NoError(t, err)
require.Nil(t, writeDupRsp.Error)
require.Equal(t, entity.UpdateEntityResponse_UNCHANGED, writeDupRsp.Status)
require.Equal(t, updateResp.Entity.ResourceVersion, writeDupRsp.Entity.ResourceVersion)
require.Equal(t, updateResp.Entity.ETag, writeDupRsp.Entity.ETag)
/*
writeDupRsp, err := testCtx.client.Update(ctx, updateReq)
require.NoError(t, err)
require.Nil(t, writeDupRsp.Error)
require.Equal(t, entity.UpdateEntityResponse_UNCHANGED, writeDupRsp.Status)
require.Equal(t, updateResp.Entity.ResourceVersion, writeDupRsp.Entity.ResourceVersion)
require.Equal(t, updateResp.Entity.ETag, writeDupRsp.Entity.ETag)
*/
body3 := []byte("{\"name\":\"John3\"}")
writeReq3 := &entity.UpdateEntityRequest{
@ -255,6 +273,7 @@ func TestIntegrationEntityServer(t *testing.T) {
}
writeResp3, err := testCtx.client.Update(ctx, writeReq3)
require.NoError(t, err)
require.Equal(t, entity.UpdateEntityResponse_UPDATED, writeResp3.Status)
require.NotEqual(t, writeResp3.Entity.ResourceVersion, updateResp.Entity.ResourceVersion)
latestMatcher := rawEntityMatcher{
@ -285,9 +304,7 @@ func TestIntegrationEntityServer(t *testing.T) {
requireEntityMatch(t, readRespFirstVer, rawEntityMatcher{
key: testKey,
createdRange: []time.Time{before, time.Now()},
updatedRange: []time.Time{before, time.Now()},
createdBy: fakeUser,
updatedBy: fakeUser,
body: body,
version: 0,
})
@ -329,7 +346,7 @@ func TestIntegrationEntityServer(t *testing.T) {
w3, err := testCtx.client.Create(ctx, &entity.CreateEntityRequest{
Entity: &entity.Entity{
Key: testKey + "3",
Key: testKey2 + "3",
Body: body,
},
})
@ -337,7 +354,7 @@ func TestIntegrationEntityServer(t *testing.T) {
w4, err := testCtx.client.Create(ctx, &entity.CreateEntityRequest{
Entity: &entity.Entity{
Key: testKey + "4",
Key: testKey2 + "4",
Body: body,
},
})
@ -358,18 +375,94 @@ func TestIntegrationEntityServer(t *testing.T) {
kinds = append(kinds, res.Resource)
version = append(version, res.ResourceVersion)
}
require.Equal(t, []string{"my-test-entity", "name2", "name3", "name4"}, names)
require.Equal(t, []string{"jsonobj", "jsonobj", "playlist", "playlist"}, kinds)
require.Equal(t, []int64{
// default sort is by guid, so we ignore order
require.ElementsMatch(t, []string{"my-test-entity1", "my-test-entity2", "my-test-entity3", "my-test-entity4"}, names)
require.ElementsMatch(t, []string{"jsonobjs", "jsonobjs", "playlists", "playlists"}, kinds)
require.ElementsMatch(t, []int64{
w1.Entity.ResourceVersion,
w2.Entity.ResourceVersion,
w3.Entity.ResourceVersion,
w4.Entity.ResourceVersion,
}, version)
// sorted by name
resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{
Resource: []string{resource, resource2},
WithBody: false,
Sort: []string{"name"},
})
require.NoError(t, err)
require.NotNil(t, resp)
require.Equal(t, 4, len(resp.Results))
require.Equal(t, "my-test-entity1", resp.Results[0].Name)
require.Equal(t, "my-test-entity2", resp.Results[1].Name)
require.Equal(t, "my-test-entity3", resp.Results[2].Name)
require.Equal(t, "my-test-entity4", resp.Results[3].Name)
require.Equal(t, "jsonobjs", resp.Results[0].Resource)
require.Equal(t, "jsonobjs", resp.Results[1].Resource)
require.Equal(t, "playlists", resp.Results[2].Resource)
require.Equal(t, "playlists", resp.Results[3].Resource)
// sorted by name desc
resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{
Resource: []string{resource, resource2},
WithBody: false,
Sort: []string{"name_desc"},
})
require.NoError(t, err)
require.NotNil(t, resp)
require.Equal(t, 4, len(resp.Results))
require.Equal(t, "my-test-entity1", resp.Results[3].Name)
require.Equal(t, "my-test-entity2", resp.Results[2].Name)
require.Equal(t, "my-test-entity3", resp.Results[1].Name)
require.Equal(t, "my-test-entity4", resp.Results[0].Name)
require.Equal(t, "jsonobjs", resp.Results[3].Resource)
require.Equal(t, "jsonobjs", resp.Results[2].Resource)
require.Equal(t, "playlists", resp.Results[1].Resource)
require.Equal(t, "playlists", resp.Results[0].Resource)
// with limit
resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{
Resource: []string{resource, resource2},
WithBody: false,
Limit: 2,
Sort: []string{"name"},
})
require.NoError(t, err)
require.NotNil(t, resp)
require.Equal(t, 2, len(resp.Results))
require.Equal(t, "my-test-entity1", resp.Results[0].Name)
require.Equal(t, "my-test-entity2", resp.Results[1].Name)
// with limit & continue
resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{
Resource: []string{resource, resource2},
WithBody: false,
Limit: 2,
NextPageToken: resp.NextPageToken,
Sort: []string{"name"},
})
require.NoError(t, err)
require.NotNil(t, resp)
require.Equal(t, 2, len(resp.Results))
require.Equal(t, "my-test-entity3", resp.Results[0].Name)
require.Equal(t, "my-test-entity4", resp.Results[1].Name)
// Again with only one kind
respKind1, err := testCtx.client.List(ctx, &entity.EntityListRequest{
Resource: []string{resource},
Sort: []string{"name"},
})
require.NoError(t, err)
names = make([]string, 0, len(respKind1.Results))
@ -380,8 +473,8 @@ func TestIntegrationEntityServer(t *testing.T) {
kinds = append(kinds, res.Resource)
version = append(version, res.ResourceVersion)
}
require.Equal(t, []string{"my-test-entity", "name2"}, names)
require.Equal(t, []string{"jsonobj", "jsonobj"}, kinds)
require.Equal(t, []string{"my-test-entity1", "my-test-entity2"}, names)
require.Equal(t, []string{"jsonobjs", "jsonobjs"}, kinds)
require.Equal(t, []int64{
w1.Entity.ResourceVersion,
w2.Entity.ResourceVersion,
@ -389,25 +482,32 @@ func TestIntegrationEntityServer(t *testing.T) {
})
t.Run("should be able to filter objects based on their labels", func(t *testing.T) {
kind := entity.StandardKindDashboard
_, err := testCtx.client.Create(ctx, &entity.CreateEntityRequest{
Entity: &entity.Entity{
Key: "/grafana/dashboards/blue-green",
Key: "/dashboards.grafana.app/dashboards/default/blue-green",
Body: []byte(dashboardWithTagsBlueGreen),
Labels: map[string]string{
"blue": "",
"green": "",
},
},
})
require.NoError(t, err)
_, err = testCtx.client.Create(ctx, &entity.CreateEntityRequest{
Entity: &entity.Entity{
Key: "/grafana/dashboards/red-green",
Key: "/dashboards.grafana.app/dashboards/default/red-green",
Body: []byte(dashboardWithTagsRedGreen),
Labels: map[string]string{
"red": "",
"green": "",
},
},
})
require.NoError(t, err)
resp, err := testCtx.client.List(ctx, &entity.EntityListRequest{
Key: []string{kind},
Key: []string{"/dashboards.grafana.app/dashboards/default"},
WithBody: false,
Labels: map[string]string{
"red": "",
@ -419,7 +519,7 @@ func TestIntegrationEntityServer(t *testing.T) {
require.Equal(t, resp.Results[0].Name, "red-green")
resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{
Key: []string{kind},
Key: []string{"/dashboards.grafana.app/dashboards/default"},
WithBody: false,
Labels: map[string]string{
"red": "",
@ -432,7 +532,7 @@ func TestIntegrationEntityServer(t *testing.T) {
require.Equal(t, resp.Results[0].Name, "red-green")
resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{
Key: []string{kind},
Key: []string{"/dashboards.grafana.app/dashboards/default"},
WithBody: false,
Labels: map[string]string{
"red": "invalid",
@ -443,7 +543,7 @@ func TestIntegrationEntityServer(t *testing.T) {
require.Len(t, resp.Results, 0)
resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{
Key: []string{kind},
Key: []string{"/dashboards.grafana.app/dashboards/default"},
WithBody: false,
Labels: map[string]string{
"green": "",
@ -454,7 +554,7 @@ func TestIntegrationEntityServer(t *testing.T) {
require.Len(t, resp.Results, 2)
resp, err = testCtx.client.List(ctx, &entity.EntityListRequest{
Key: []string{kind},
Key: []string{"/dashboards.grafana.app/dashboards/default"},
WithBody: false,
Labels: map[string]string{
"yellow": "",