mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
ResourceServer: Resource store sql backend (#90170)
This commit is contained in:
parent
bb40fb342a
commit
08c611c68b
@ -4,13 +4,10 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/grafana/dskit/services"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"gocloud.dev/blob/fileblob"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@ -51,6 +48,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/storage/unified/apistore"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/entitybridge"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -272,27 +270,11 @@ func (s *service) start(ctx context.Context) error {
|
||||
}
|
||||
|
||||
case grafanaapiserveroptions.StorageTypeUnifiedNext:
|
||||
// CDK (for now)
|
||||
dir := filepath.Join(s.cfg.DataPath, "unistore", "resource")
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return err
|
||||
if !s.features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorage) {
|
||||
return fmt.Errorf("unified storage requires the unifiedStorage feature flag")
|
||||
}
|
||||
|
||||
bucket, err := fileblob.OpenBucket(dir, &fileblob.Options{
|
||||
CreateDir: true,
|
||||
Metadata: fileblob.MetadataDontWrite, // skip
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
backend, err := resource.NewCDKBackend(context.Background(), resource.CDKBackendOptions{
|
||||
Tracer: s.tracing,
|
||||
Bucket: bucket,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
server, err := resource.NewResourceServer(resource.ResourceServerOptions{Backend: backend})
|
||||
server, err := sql.ProvideResourceServer(s.db, s.cfg, s.features, s.tracing)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -11,7 +11,7 @@ import (
|
||||
grafanaregistry "github.com/grafana/grafana/pkg/apiserver/registry/generic"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/db"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/sqlstash/sqltemplate"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
)
|
||||
|
||||
func (s *sqlEntityServer) Create(ctx context.Context, r *entity.CreateEntityRequest) (*entity.CreateEntityResponse, error) {
|
||||
|
@ -10,7 +10,7 @@ import (
|
||||
grafanaregistry "github.com/grafana/grafana/pkg/apiserver/registry/generic"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/db"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/sqlstash/sqltemplate"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
)
|
||||
|
||||
func (s *sqlEntityServer) Delete(ctx context.Context, r *entity.DeleteEntityRequest) (*entity.DeleteEntityResponse, error) {
|
||||
|
@ -8,7 +8,7 @@ import (
|
||||
|
||||
folder "github.com/grafana/grafana/pkg/apis/folder/v0alpha1"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/db"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/sqlstash/sqltemplate"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
)
|
||||
|
||||
type folderInfo struct {
|
||||
|
@ -16,7 +16,7 @@ import (
|
||||
grafanaregistry "github.com/grafana/grafana/pkg/apiserver/registry/generic"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/db"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/sqlstash/sqltemplate"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
)
|
||||
|
||||
// Templates setup.
|
||||
|
@ -16,7 +16,7 @@ import (
|
||||
grafanaregistry "github.com/grafana/grafana/pkg/apiserver/registry/generic"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/db"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/sqlstash/sqltemplate"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
"github.com/grafana/grafana/pkg/util/testutil"
|
||||
)
|
||||
|
||||
|
@ -25,8 +25,8 @@ import (
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/session"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/db"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/sqlstash/sqltemplate"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
)
|
||||
|
||||
const entityTable = "entity"
|
||||
|
@ -10,7 +10,7 @@ import (
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/session"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/sqlstash/sqltemplate"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
"github.com/grafana/grafana/pkg/util/testutil"
|
||||
)
|
||||
|
||||
|
@ -11,7 +11,7 @@ import (
|
||||
grafanaregistry "github.com/grafana/grafana/pkg/apiserver/registry/generic"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/db"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/sqlstash/sqltemplate"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
)
|
||||
|
||||
func (s *sqlEntityServer) Update(ctx context.Context, r *entity.UpdateEntityRequest) (*entity.UpdateEntityResponse, error) {
|
||||
|
@ -10,7 +10,7 @@ import (
|
||||
|
||||
"github.com/grafana/grafana/pkg/apimachinery/identity"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/db"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/sqlstash/sqltemplate"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
)
|
||||
|
||||
func createETag(body []byte, meta []byte, status []byte) string {
|
||||
|
@ -16,8 +16,8 @@ import (
|
||||
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/db"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/db/dbimpl"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/sqlstash/sqltemplate"
|
||||
sqltemplateMocks "github.com/grafana/grafana/pkg/services/store/entity/sqlstash/sqltemplate/mocks"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
sqltemplateMocks "github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate/mocks"
|
||||
"github.com/grafana/grafana/pkg/util/testutil"
|
||||
)
|
||||
|
||||
|
733
pkg/storage/unified/sql/backend.go
Normal file
733
pkg/storage/unified/sql/backend.go
Normal file
@ -0,0 +1,733 @@
|
||||
package sql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"go.opentelemetry.io/otel/trace/noop"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/session"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/db"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
)
|
||||
|
||||
const trace_prefix = "sql.resource."
|
||||
|
||||
type backendOptions struct {
|
||||
DB db.ResourceDBInterface
|
||||
Tracer trace.Tracer
|
||||
}
|
||||
|
||||
func NewBackendStore(opts backendOptions) (*backend, error) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
if opts.Tracer == nil {
|
||||
opts.Tracer = noop.NewTracerProvider().Tracer("sql-backend")
|
||||
}
|
||||
|
||||
return &backend{
|
||||
db: opts.DB,
|
||||
log: log.New("sql-resource-server"),
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
tracer: opts.Tracer,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type backend struct {
|
||||
log log.Logger
|
||||
db db.ResourceDBInterface // needed to keep xorm engine in scope
|
||||
sess *session.SessionDB
|
||||
dialect migrator.Dialect
|
||||
ctx context.Context // TODO: remove
|
||||
cancel context.CancelFunc
|
||||
tracer trace.Tracer
|
||||
|
||||
//stream chan *resource.WatchEvent
|
||||
|
||||
sqlDB db.DB
|
||||
sqlDialect sqltemplate.Dialect
|
||||
}
|
||||
|
||||
func (b *backend) Init() error {
|
||||
if b.sess != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if b.db == nil {
|
||||
return errors.New("missing db")
|
||||
}
|
||||
|
||||
err := b.db.Init()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sqlDB, err := b.db.GetDB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.sqlDB = sqlDB
|
||||
|
||||
driverName := sqlDB.DriverName()
|
||||
driverName = strings.TrimSuffix(driverName, "WithHooks")
|
||||
switch driverName {
|
||||
case db.DriverMySQL:
|
||||
b.sqlDialect = sqltemplate.MySQL
|
||||
case db.DriverPostgres:
|
||||
b.sqlDialect = sqltemplate.PostgreSQL
|
||||
case db.DriverSQLite, db.DriverSQLite3:
|
||||
b.sqlDialect = sqltemplate.SQLite
|
||||
default:
|
||||
return fmt.Errorf("no dialect for driver %q", driverName)
|
||||
}
|
||||
|
||||
sess, err := b.db.GetSession()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
engine, err := b.db.GetEngine()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b.sess = sess
|
||||
b.dialect = migrator.NewDialect(engine.DriverName())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *backend) IsHealthy(ctx context.Context, r *resource.HealthCheckRequest) (*resource.HealthCheckResponse, error) {
|
||||
// ctxLogger := s.log.FromContext(log.WithContextualAttributes(ctx, []any{"method", "isHealthy"}))
|
||||
|
||||
if err := b.sqlDB.PingContext(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// TODO: check the status of the watcher implementation as well
|
||||
return &resource.HealthCheckResponse{Status: resource.HealthCheckResponse_SERVING}, nil
|
||||
}
|
||||
|
||||
func (b *backend) Stop() {
|
||||
b.cancel()
|
||||
}
|
||||
|
||||
func (b *backend) WriteEvent(ctx context.Context, event resource.WriteEvent) (int64, error) {
|
||||
_, span := b.tracer.Start(ctx, trace_prefix+"WriteEvent")
|
||||
defer span.End()
|
||||
// TODO: validate key ?
|
||||
if err := b.Init(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
switch event.Type {
|
||||
case resource.WatchEvent_ADDED:
|
||||
return b.create(ctx, event)
|
||||
case resource.WatchEvent_MODIFIED:
|
||||
return b.update(ctx, event)
|
||||
case resource.WatchEvent_DELETED:
|
||||
return b.delete(ctx, event)
|
||||
default:
|
||||
return 0, fmt.Errorf("unsupported event type")
|
||||
}
|
||||
}
|
||||
|
||||
func (b *backend) create(ctx context.Context, event resource.WriteEvent) (int64, error) {
|
||||
ctx, span := b.tracer.Start(ctx, trace_prefix+"Create")
|
||||
defer span.End()
|
||||
var newVersion int64
|
||||
guid := uuid.New().String()
|
||||
err := b.sqlDB.WithTx(ctx, ReadCommitted, func(ctx context.Context, tx db.Tx) error {
|
||||
// TODO: Set the Labels
|
||||
|
||||
// 1. Insert into resource
|
||||
if _, err := exec(ctx, tx, sqlResourceInsert, sqlResourceRequest{
|
||||
SQLTemplate: sqltemplate.New(b.sqlDialect),
|
||||
WriteEvent: event,
|
||||
GUID: guid,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("insert into resource: %w", err)
|
||||
}
|
||||
|
||||
// 2. Insert into resource history
|
||||
if _, err := exec(ctx, tx, sqlResourceHistoryInsert, sqlResourceRequest{
|
||||
SQLTemplate: sqltemplate.New(b.sqlDialect),
|
||||
WriteEvent: event,
|
||||
GUID: guid,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("insert into resource history: %w", err)
|
||||
}
|
||||
|
||||
// 3. TODO: Rebuild the whole folder tree structure if we're creating a folder
|
||||
|
||||
// 4. Atomically increpement resource version for this kind
|
||||
rv, err := resourceVersionAtomicInc(ctx, tx, b.sqlDialect, event.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newVersion = rv
|
||||
|
||||
// 5. Update the RV in both resource and resource_history
|
||||
if _, err = exec(ctx, tx, sqlResourceHistoryUpdateRV, sqlResourceUpdateRVRequest{
|
||||
SQLTemplate: sqltemplate.New(b.sqlDialect),
|
||||
GUID: guid,
|
||||
ResourceVersion: newVersion,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("update history rv: %w", err)
|
||||
}
|
||||
if _, err = exec(ctx, tx, sqlResourceUpdateRV, sqlResourceUpdateRVRequest{
|
||||
SQLTemplate: sqltemplate.New(b.sqlDialect),
|
||||
GUID: guid,
|
||||
ResourceVersion: newVersion,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("update resource rv: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
return newVersion, err
|
||||
}
|
||||
|
||||
func (b *backend) update(ctx context.Context, event resource.WriteEvent) (int64, error) {
|
||||
ctx, span := b.tracer.Start(ctx, trace_prefix+"Update")
|
||||
defer span.End()
|
||||
var newVersion int64
|
||||
guid := uuid.New().String()
|
||||
err := b.sqlDB.WithTx(ctx, ReadCommitted, func(ctx context.Context, tx db.Tx) error {
|
||||
// TODO: Set the Labels
|
||||
|
||||
// 1. Update into resource
|
||||
res, err := exec(ctx, tx, sqlResourceUpdate, sqlResourceRequest{
|
||||
SQLTemplate: sqltemplate.New(b.sqlDialect),
|
||||
WriteEvent: event,
|
||||
GUID: guid,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("update into resource: %w", err)
|
||||
}
|
||||
|
||||
count, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return fmt.Errorf("update into resource: %w", err)
|
||||
}
|
||||
if count == 0 {
|
||||
return fmt.Errorf("no rows affected")
|
||||
}
|
||||
|
||||
// 2. Insert into resource history
|
||||
if _, err := exec(ctx, tx, sqlResourceHistoryInsert, sqlResourceRequest{
|
||||
SQLTemplate: sqltemplate.New(b.sqlDialect),
|
||||
WriteEvent: event,
|
||||
GUID: guid,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("insert into resource history: %w", err)
|
||||
}
|
||||
|
||||
// 3. TODO: Rebuild the whole folder tree structure if we're creating a folder
|
||||
|
||||
// 4. Atomically increpement resource version for this kind
|
||||
rv, err := resourceVersionAtomicInc(ctx, tx, b.sqlDialect, event.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newVersion = rv
|
||||
|
||||
// 5. Update the RV in both resource and resource_history
|
||||
if _, err = exec(ctx, tx, sqlResourceHistoryUpdateRV, sqlResourceUpdateRVRequest{
|
||||
SQLTemplate: sqltemplate.New(b.sqlDialect),
|
||||
GUID: guid,
|
||||
ResourceVersion: newVersion,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("update history rv: %w", err)
|
||||
}
|
||||
if _, err = exec(ctx, tx, sqlResourceUpdateRV, sqlResourceUpdateRVRequest{
|
||||
SQLTemplate: sqltemplate.New(b.sqlDialect),
|
||||
GUID: guid,
|
||||
ResourceVersion: newVersion,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("update resource rv: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return newVersion, err
|
||||
}
|
||||
|
||||
func (b *backend) delete(ctx context.Context, event resource.WriteEvent) (int64, error) {
|
||||
ctx, span := b.tracer.Start(ctx, trace_prefix+"Delete")
|
||||
defer span.End()
|
||||
var newVersion int64
|
||||
guid := uuid.New().String()
|
||||
|
||||
err := b.sqlDB.WithTx(ctx, ReadCommitted, func(ctx context.Context, tx db.Tx) error {
|
||||
// TODO: Set the Labels
|
||||
|
||||
// 1. delete from resource
|
||||
res, err := exec(ctx, tx, sqlResourceDelete, sqlResourceRequest{
|
||||
SQLTemplate: sqltemplate.New(b.sqlDialect),
|
||||
WriteEvent: event,
|
||||
GUID: guid,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete resource: %w", err)
|
||||
}
|
||||
count, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete resource: %w", err)
|
||||
}
|
||||
if count == 0 {
|
||||
return fmt.Errorf("no rows affected")
|
||||
}
|
||||
|
||||
// 2. Add event to resource history
|
||||
if _, err := exec(ctx, tx, sqlResourceHistoryInsert, sqlResourceRequest{
|
||||
SQLTemplate: sqltemplate.New(b.sqlDialect),
|
||||
WriteEvent: event,
|
||||
GUID: guid,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("insert into resource history: %w", err)
|
||||
}
|
||||
|
||||
// 3. TODO: Rebuild the whole folder tree structure if we're creating a folder
|
||||
|
||||
// 4. Atomically increpement resource version for this kind
|
||||
newVersion, err = resourceVersionAtomicInc(ctx, tx, b.sqlDialect, event.Key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 5. Update the RV in resource_history
|
||||
if _, err = exec(ctx, tx, sqlResourceHistoryUpdateRV, sqlResourceUpdateRVRequest{
|
||||
SQLTemplate: sqltemplate.New(b.sqlDialect),
|
||||
GUID: guid,
|
||||
ResourceVersion: newVersion,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("update history rv: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
return newVersion, err
|
||||
}
|
||||
|
||||
func (b *backend) Read(ctx context.Context, req *resource.ReadRequest) (*resource.ReadResponse, error) {
|
||||
_, span := b.tracer.Start(ctx, trace_prefix+".Read")
|
||||
defer span.End()
|
||||
|
||||
// TODO: validate key ?
|
||||
if err := b.Init(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
readReq := sqlResourceReadRequest{
|
||||
SQLTemplate: sqltemplate.New(b.sqlDialect),
|
||||
Request: req,
|
||||
readResponse: new(readResponse),
|
||||
}
|
||||
|
||||
sr := sqlResourceRead
|
||||
if req.ResourceVersion > 0 {
|
||||
// read a specific version
|
||||
sr = sqlResourceHistoryRead
|
||||
}
|
||||
|
||||
res, err := queryRow(ctx, b.sqlDB, sr, readReq)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, resource.ErrNotFound
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("get resource version: %w", err)
|
||||
}
|
||||
|
||||
return &res.ReadResponse, nil
|
||||
}
|
||||
|
||||
func (b *backend) PrepareList(ctx context.Context, req *resource.ListRequest) (*resource.ListResponse, error) {
|
||||
_, span := b.tracer.Start(ctx, trace_prefix+"List")
|
||||
defer span.End()
|
||||
|
||||
// TODO: think about how to handler VersionMatch. We should be able to use latest for the first page (only).
|
||||
|
||||
if req.ResourceVersion > 0 || req.NextPageToken != "" {
|
||||
return b.listAtRevision(ctx, req)
|
||||
}
|
||||
return b.listLatest(ctx, req)
|
||||
}
|
||||
|
||||
// listLatest fetches the resources from the resource table.
|
||||
func (b *backend) listLatest(ctx context.Context, req *resource.ListRequest) (*resource.ListResponse, error) {
|
||||
out := &resource.ListResponse{
|
||||
Items: []*resource.ResourceWrapper{}, // TODO: we could pre-allocate the capacity if we estimate the number of items
|
||||
ResourceVersion: 0,
|
||||
}
|
||||
|
||||
err := b.sqlDB.WithTx(ctx, ReadCommitted, func(ctx context.Context, tx db.Tx) error {
|
||||
var err error
|
||||
|
||||
// TODO: Here the lastest RV might be lower than the actual latest RV
|
||||
// because delete events are not included in the resource table.
|
||||
out.ResourceVersion, err = fetchLatestRV(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Fetch one extra row for Limit
|
||||
lim := req.Limit
|
||||
if req.Limit > 0 {
|
||||
req.Limit++
|
||||
}
|
||||
listReq := sqlResourceListRequest{
|
||||
SQLTemplate: sqltemplate.New(b.sqlDialect),
|
||||
Request: req,
|
||||
Response: new(resource.ResourceWrapper),
|
||||
}
|
||||
query, err := sqltemplate.Execute(sqlResourceList, listReq)
|
||||
if err != nil {
|
||||
return fmt.Errorf("execute SQL template to list resources: %w", err)
|
||||
}
|
||||
|
||||
rows, err := tx.QueryContext(ctx, query, listReq.GetArgs()...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list latest resources: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
for i := int64(1); rows.Next(); i++ {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if err := rows.Scan(listReq.GetScanDest()...); err != nil {
|
||||
return fmt.Errorf("scan row #%d: %w", i, err)
|
||||
}
|
||||
|
||||
if lim > 0 && i > lim {
|
||||
continueToken := &ContinueToken{ResourceVersion: out.ResourceVersion, StartOffset: lim}
|
||||
out.NextPageToken = continueToken.String()
|
||||
break
|
||||
}
|
||||
out.Items = append(out.Items, &resource.ResourceWrapper{
|
||||
ResourceVersion: listReq.Response.ResourceVersion,
|
||||
Value: listReq.Response.Value,
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return out, err
|
||||
}
|
||||
|
||||
// listAtRevision fetches the resources from the resource_history table at a specific revision.
|
||||
func (b *backend) listAtRevision(ctx context.Context, req *resource.ListRequest) (*resource.ListResponse, error) {
|
||||
// Get the RV
|
||||
rv := req.ResourceVersion
|
||||
offset := int64(0)
|
||||
if req.NextPageToken != "" {
|
||||
continueToken, err := GetContinueToken(req.NextPageToken)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get continue token: %w", err)
|
||||
}
|
||||
rv = continueToken.ResourceVersion
|
||||
offset = continueToken.StartOffset
|
||||
}
|
||||
|
||||
out := &resource.ListResponse{
|
||||
Items: []*resource.ResourceWrapper{}, // TODO: we could pre-allocate the capacity if we estimate the number of items
|
||||
ResourceVersion: rv,
|
||||
}
|
||||
|
||||
err := b.sqlDB.WithTx(ctx, ReadCommitted, func(ctx context.Context, tx db.Tx) error {
|
||||
var err error
|
||||
|
||||
// Fetch one extra row for Limit
|
||||
lim := req.Limit
|
||||
if lim > 0 {
|
||||
req.Limit++
|
||||
}
|
||||
listReq := sqlResourceHistoryListRequest{
|
||||
SQLTemplate: sqltemplate.New(b.sqlDialect),
|
||||
Request: &historyListRequest{
|
||||
ResourceVersion: rv,
|
||||
Limit: req.Limit,
|
||||
Offset: offset,
|
||||
Options: req.Options,
|
||||
},
|
||||
Response: new(resource.ResourceWrapper),
|
||||
}
|
||||
query, err := sqltemplate.Execute(sqlResourceHistoryList, listReq)
|
||||
if err != nil {
|
||||
return fmt.Errorf("execute SQL template to list resources at revision: %w", err)
|
||||
}
|
||||
rows, err := tx.QueryContext(ctx, query, listReq.GetArgs()...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list resources at revision: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
for i := int64(1); rows.Next(); i++ {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if err := rows.Scan(listReq.GetScanDest()...); err != nil {
|
||||
return fmt.Errorf("scan row #%d: %w", i, err)
|
||||
}
|
||||
|
||||
if lim > 0 && i > lim {
|
||||
continueToken := &ContinueToken{ResourceVersion: out.ResourceVersion, StartOffset: offset + lim}
|
||||
out.NextPageToken = continueToken.String()
|
||||
break
|
||||
}
|
||||
out.Items = append(out.Items, &resource.ResourceWrapper{
|
||||
ResourceVersion: listReq.Response.ResourceVersion,
|
||||
Value: listReq.Response.Value,
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (b *backend) WatchWriteEvents(ctx context.Context) (<-chan *resource.WrittenEvent, error) {
|
||||
if err := b.Init(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Fetch the lastest RV
|
||||
since, err := fetchLatestRV(ctx, b.sqlDB)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Start the poller
|
||||
stream := make(chan *resource.WrittenEvent)
|
||||
go b.poller(ctx, since, stream)
|
||||
return stream, nil
|
||||
}
|
||||
|
||||
func (b *backend) poller(ctx context.Context, since int64, stream chan<- *resource.WrittenEvent) {
|
||||
var err error
|
||||
|
||||
interval := 100 * time.Millisecond // TODO make this configurable
|
||||
t := time.NewTicker(interval)
|
||||
defer close(stream)
|
||||
defer t.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-b.ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
since, err = b.poll(ctx, since, stream)
|
||||
if err != nil {
|
||||
b.log.Error("watch error", "err", err)
|
||||
}
|
||||
t.Reset(interval)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// fetchLatestRV returns the current maxium RV in the resource table
|
||||
func fetchLatestRV(ctx context.Context, db db.ContextExecer) (int64, error) {
|
||||
// Fetch the lastest RV
|
||||
rows, err := db.QueryContext(ctx, `SELECT COALESCE(max("resource_version"), 0) FROM "resource";`)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("fetch latest rv: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
if rows.Next() {
|
||||
rv := new(int64)
|
||||
if err := rows.Scan(&rv); err != nil {
|
||||
return 0, fmt.Errorf("scan since resource version: %w", err)
|
||||
}
|
||||
return *rv, nil
|
||||
}
|
||||
return 0, fmt.Errorf("no rows")
|
||||
}
|
||||
|
||||
func (b *backend) poll(ctx context.Context, since int64, stream chan<- *resource.WrittenEvent) (int64, error) {
|
||||
ctx, span := b.tracer.Start(ctx, trace_prefix+"poll")
|
||||
defer span.End()
|
||||
|
||||
pollReq := sqlResourceHistoryPollRequest{
|
||||
SQLTemplate: sqltemplate.New(b.sqlDialect),
|
||||
SinceResourceVersion: since,
|
||||
Response: new(historyPollResponse),
|
||||
}
|
||||
query, err := sqltemplate.Execute(sqlResourceHistoryPoll, pollReq)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("execute SQL template to poll for resource history: %w", err)
|
||||
}
|
||||
rows, err := b.sqlDB.QueryContext(ctx, query, pollReq.GetArgs()...)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("poll for resource history: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
next := since
|
||||
for i := 1; rows.Next(); i++ {
|
||||
// check if the context is done
|
||||
if ctx.Err() != nil {
|
||||
return 0, ctx.Err()
|
||||
}
|
||||
if err := rows.Scan(pollReq.GetScanDest()...); err != nil {
|
||||
return 0, fmt.Errorf("scan row #%d polling for resource history: %w", i, err)
|
||||
}
|
||||
resp := pollReq.Response
|
||||
next = resp.ResourceVersion
|
||||
|
||||
stream <- &resource.WrittenEvent{
|
||||
WriteEvent: resource.WriteEvent{
|
||||
Value: resp.Value,
|
||||
Key: &resource.ResourceKey{
|
||||
Namespace: resp.Key.Namespace,
|
||||
Group: resp.Key.Group,
|
||||
Resource: resp.Key.Resource,
|
||||
Name: resp.Key.Name,
|
||||
},
|
||||
Type: resource.WatchEvent_Type(resp.Action),
|
||||
},
|
||||
ResourceVersion: resp.ResourceVersion,
|
||||
// Timestamp: , // TODO: add timestamp
|
||||
}
|
||||
}
|
||||
return next, nil
|
||||
}
|
||||
|
||||
// resourceVersionAtomicInc atomically increases the version of a kind within a
|
||||
// transaction.
|
||||
// TODO: Ideally we should attempt to update the RV in the resource and resource_history tables
|
||||
// in a single roundtrip. This would reduce the latency of the operation, and also increase the
|
||||
// throughput of the system. This is a good candidate for a future optimization.
|
||||
func resourceVersionAtomicInc(ctx context.Context, x db.ContextExecer, d sqltemplate.Dialect, key *resource.ResourceKey) (newVersion int64, err error) {
|
||||
// TODO: refactor this code to run in a multi-statement transaction in order to minimise the number of roundtrips.
|
||||
// 1 Lock the row for update
|
||||
req := sqlResourceVersionRequest{
|
||||
SQLTemplate: sqltemplate.New(d),
|
||||
Group: key.Group,
|
||||
Resource: key.Resource,
|
||||
resourceVersion: new(resourceVersion),
|
||||
}
|
||||
rv, err := queryRow(ctx, x, sqlResourceVersionGet, req)
|
||||
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
// if there wasn't a row associated with the given resource, we create one with
|
||||
// version 1
|
||||
if _, err = exec(ctx, x, sqlResourceVersionInsert, sqlResourceVersionRequest{
|
||||
SQLTemplate: sqltemplate.New(d),
|
||||
Group: key.Group,
|
||||
Resource: key.Resource,
|
||||
}); err != nil {
|
||||
return 0, fmt.Errorf("insert into resource_version: %w", err)
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("increase resource version: %w", err)
|
||||
}
|
||||
nextRV := rv.ResourceVersion + 1
|
||||
// 2. Increment the resource version
|
||||
res, err := exec(ctx, x, sqlResourceVersionInc, sqlResourceVersionRequest{
|
||||
SQLTemplate: sqltemplate.New(d),
|
||||
Group: key.Group,
|
||||
Resource: key.Resource,
|
||||
resourceVersion: &resourceVersion{
|
||||
ResourceVersion: nextRV,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("increase resource version: %w", err)
|
||||
}
|
||||
|
||||
if count, err := res.RowsAffected(); err != nil || count == 0 {
|
||||
return 0, fmt.Errorf("increase resource version did not affect any rows: %w", err)
|
||||
}
|
||||
|
||||
// 3. Retun the incremended value
|
||||
return nextRV, nil
|
||||
}
|
||||
|
||||
// exec uses `req` as input for a non-data returning query generated with
|
||||
// `tmpl`, and executed in `x`.
|
||||
func exec(ctx context.Context, x db.ContextExecer, tmpl *template.Template, req sqltemplate.SQLTemplateIface) (sql.Result, error) {
|
||||
if err := req.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("exec: invalid request for template %q: %w",
|
||||
tmpl.Name(), err)
|
||||
}
|
||||
|
||||
rawQuery, err := sqltemplate.Execute(tmpl, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("execute template: %w", err)
|
||||
}
|
||||
query := sqltemplate.FormatSQL(rawQuery)
|
||||
|
||||
res, err := x.ExecContext(ctx, query, req.GetArgs()...)
|
||||
if err != nil {
|
||||
return nil, SQLError{
|
||||
Err: err,
|
||||
CallType: "Exec",
|
||||
TemplateName: tmpl.Name(),
|
||||
arguments: req.GetArgs(),
|
||||
Query: query,
|
||||
RawQuery: rawQuery,
|
||||
}
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// queryRow uses `req` as input and output for a single-row returning query
|
||||
// generated with `tmpl`, and executed in `x`.
|
||||
func queryRow[T any](ctx context.Context, x db.ContextExecer, tmpl *template.Template, req sqltemplate.WithResults[T]) (T, error) {
|
||||
var zero T
|
||||
|
||||
if err := req.Validate(); err != nil {
|
||||
return zero, fmt.Errorf("query: invalid request for template %q: %w",
|
||||
tmpl.Name(), err)
|
||||
}
|
||||
|
||||
rawQuery, err := sqltemplate.Execute(tmpl, req)
|
||||
if err != nil {
|
||||
return zero, fmt.Errorf("execute template: %w", err)
|
||||
}
|
||||
query := sqltemplate.FormatSQL(rawQuery)
|
||||
|
||||
row := x.QueryRowContext(ctx, query, req.GetArgs()...)
|
||||
if err := row.Err(); err != nil {
|
||||
return zero, SQLError{
|
||||
Err: err,
|
||||
CallType: "QueryRow",
|
||||
TemplateName: tmpl.Name(),
|
||||
arguments: req.GetArgs(),
|
||||
ScanDest: req.GetScanDest(),
|
||||
Query: query,
|
||||
RawQuery: rawQuery,
|
||||
}
|
||||
}
|
||||
|
||||
return scanRow(row, req)
|
||||
}
|
||||
|
||||
type scanner interface {
|
||||
Scan(dest ...any) error
|
||||
}
|
||||
|
||||
// scanRow is used on *sql.Row and *sql.Rows, and is factored out here not to
|
||||
// improving code reuse, but rather for ease of testing.
|
||||
func scanRow[T any](sc scanner, req sqltemplate.WithResults[T]) (zero T, err error) {
|
||||
if err = sc.Scan(req.GetScanDest()...); err != nil {
|
||||
return zero, fmt.Errorf("row scan: %w", err)
|
||||
}
|
||||
|
||||
res, err := req.Results()
|
||||
if err != nil {
|
||||
return zero, fmt.Errorf("row results: %w", err)
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
254
pkg/storage/unified/sql/backend_test.go
Normal file
254
pkg/storage/unified/sql/backend_test.go
Normal file
@ -0,0 +1,254 @@
|
||||
package sql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl"
|
||||
"github.com/grafana/grafana/pkg/tests/testsuite"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
testsuite.Run(m)
|
||||
}
|
||||
|
||||
func TestBackendHappyPath(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dbstore := db.InitTestDB(t)
|
||||
|
||||
rdb, err := dbimpl.ProvideResourceDB(dbstore, setting.NewCfg(), featuremgmt.WithFeatures(featuremgmt.FlagUnifiedStorage), nil)
|
||||
assert.NoError(t, err)
|
||||
store, err := NewBackendStore(backendOptions{
|
||||
DB: rdb,
|
||||
})
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, store)
|
||||
|
||||
stream, err := store.WatchWriteEvents(ctx)
|
||||
assert.NoError(t, err)
|
||||
|
||||
t.Run("Add 3 resources", func(t *testing.T) {
|
||||
rv, err := writeEvent(ctx, store, "item1", resource.WatchEvent_ADDED)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(1), rv)
|
||||
|
||||
rv, err = writeEvent(ctx, store, "item2", resource.WatchEvent_ADDED)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(2), rv)
|
||||
|
||||
rv, err = writeEvent(ctx, store, "item3", resource.WatchEvent_ADDED)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(3), rv)
|
||||
})
|
||||
|
||||
t.Run("Update item2", func(t *testing.T) {
|
||||
rv, err := writeEvent(ctx, store, "item2", resource.WatchEvent_MODIFIED)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(4), rv)
|
||||
})
|
||||
|
||||
t.Run("Delete item1", func(t *testing.T) {
|
||||
rv, err := writeEvent(ctx, store, "item1", resource.WatchEvent_DELETED)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(5), rv)
|
||||
})
|
||||
|
||||
t.Run("Read latest item 2", func(t *testing.T) {
|
||||
resp, err := store.Read(ctx, &resource.ReadRequest{Key: resourceKey("item2")})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(4), resp.ResourceVersion)
|
||||
assert.Equal(t, "item2 MODIFIED", string(resp.Value))
|
||||
})
|
||||
|
||||
t.Run("Read early verion of item2", func(t *testing.T) {
|
||||
resp, err := store.Read(ctx, &resource.ReadRequest{
|
||||
Key: resourceKey("item2"),
|
||||
ResourceVersion: 3, // item2 was created at rv=2 and updated at rv=4
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(2), resp.ResourceVersion)
|
||||
assert.Equal(t, "item2 ADDED", string(resp.Value))
|
||||
})
|
||||
|
||||
t.Run("PrepareList latest", func(t *testing.T) {
|
||||
resp, err := store.PrepareList(ctx, &resource.ListRequest{})
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, resp.Items, 2)
|
||||
assert.Equal(t, "item2 MODIFIED", string(resp.Items[0].Value))
|
||||
assert.Equal(t, "item3 ADDED", string(resp.Items[1].Value))
|
||||
assert.Equal(t, int64(4), resp.ResourceVersion)
|
||||
})
|
||||
|
||||
t.Run("Watch events", func(t *testing.T) {
|
||||
event := <-stream
|
||||
assert.Equal(t, "item1", event.Key.Name)
|
||||
assert.Equal(t, int64(1), event.ResourceVersion)
|
||||
assert.Equal(t, resource.WatchEvent_ADDED, event.Type)
|
||||
event = <-stream
|
||||
assert.Equal(t, "item2", event.Key.Name)
|
||||
assert.Equal(t, int64(2), event.ResourceVersion)
|
||||
assert.Equal(t, resource.WatchEvent_ADDED, event.Type)
|
||||
|
||||
event = <-stream
|
||||
assert.Equal(t, "item3", event.Key.Name)
|
||||
assert.Equal(t, int64(3), event.ResourceVersion)
|
||||
assert.Equal(t, resource.WatchEvent_ADDED, event.Type)
|
||||
|
||||
event = <-stream
|
||||
assert.Equal(t, "item2", event.Key.Name)
|
||||
assert.Equal(t, int64(4), event.ResourceVersion)
|
||||
assert.Equal(t, resource.WatchEvent_MODIFIED, event.Type)
|
||||
|
||||
event = <-stream
|
||||
assert.Equal(t, "item1", event.Key.Name)
|
||||
assert.Equal(t, int64(5), event.ResourceVersion)
|
||||
assert.Equal(t, resource.WatchEvent_DELETED, event.Type)
|
||||
})
|
||||
}
|
||||
|
||||
func TestBackendWatchWriteEventsFromLastest(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dbstore := db.InitTestDB(t)
|
||||
|
||||
rdb, err := dbimpl.ProvideResourceDB(dbstore, setting.NewCfg(), featuremgmt.WithFeatures(featuremgmt.FlagUnifiedStorage), nil)
|
||||
assert.NoError(t, err)
|
||||
store, err := NewBackendStore(backendOptions{
|
||||
DB: rdb,
|
||||
})
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, store)
|
||||
|
||||
// Create a few resources before initing the watch
|
||||
_, err = writeEvent(ctx, store, "item1", resource.WatchEvent_ADDED)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Start the watch
|
||||
stream, err := store.WatchWriteEvents(ctx)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Create one more event
|
||||
_, err = writeEvent(ctx, store, "item2", resource.WatchEvent_ADDED)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "item2", (<-stream).Key.Name)
|
||||
}
|
||||
|
||||
func TestBackendPrepareList(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dbstore := db.InitTestDB(t)
|
||||
|
||||
rdb, err := dbimpl.ProvideResourceDB(dbstore, setting.NewCfg(), featuremgmt.WithFeatures(featuremgmt.FlagUnifiedStorage), nil)
|
||||
assert.NoError(t, err)
|
||||
store, err := NewBackendStore(backendOptions{
|
||||
DB: rdb,
|
||||
})
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, store)
|
||||
|
||||
// Create a few resources before initing the watch
|
||||
_, _ = writeEvent(ctx, store, "item1", resource.WatchEvent_ADDED) // rv=1
|
||||
_, _ = writeEvent(ctx, store, "item2", resource.WatchEvent_ADDED) // rv=2 - will be modified at rv=6
|
||||
_, _ = writeEvent(ctx, store, "item3", resource.WatchEvent_ADDED) // rv=3 - will be deleted at rv=7
|
||||
_, _ = writeEvent(ctx, store, "item4", resource.WatchEvent_ADDED) // rv=4
|
||||
_, _ = writeEvent(ctx, store, "item5", resource.WatchEvent_ADDED) // rv=5
|
||||
_, _ = writeEvent(ctx, store, "item2", resource.WatchEvent_MODIFIED) // rv=6
|
||||
_, _ = writeEvent(ctx, store, "item3", resource.WatchEvent_DELETED) // rv=7
|
||||
_, _ = writeEvent(ctx, store, "item6", resource.WatchEvent_ADDED) // rv=8
|
||||
t.Run("fetch all latest", func(t *testing.T) {
|
||||
res, err := store.PrepareList(ctx, &resource.ListRequest{})
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, res.Items, 5)
|
||||
assert.Empty(t, res.NextPageToken)
|
||||
})
|
||||
|
||||
t.Run("list latest first page ", func(t *testing.T) {
|
||||
res, err := store.PrepareList(ctx, &resource.ListRequest{
|
||||
Limit: 3,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, res.Items, 3)
|
||||
continueToken, err := GetContinueToken(res.NextPageToken)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(8), continueToken.ResourceVersion)
|
||||
assert.Equal(t, int64(3), continueToken.StartOffset)
|
||||
})
|
||||
|
||||
t.Run("list at revision", func(t *testing.T) {
|
||||
res, err := store.PrepareList(ctx, &resource.ListRequest{
|
||||
ResourceVersion: 4,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, res.Items, 4)
|
||||
assert.Equal(t, "item1 ADDED", string(res.Items[0].Value))
|
||||
assert.Equal(t, "item2 ADDED", string(res.Items[1].Value))
|
||||
assert.Equal(t, "item3 ADDED", string(res.Items[2].Value))
|
||||
assert.Equal(t, "item4 ADDED", string(res.Items[3].Value))
|
||||
assert.Empty(t, res.NextPageToken)
|
||||
})
|
||||
|
||||
t.Run("fetch first page at revision with limit", func(t *testing.T) {
|
||||
res, err := store.PrepareList(ctx, &resource.ListRequest{
|
||||
Limit: 3,
|
||||
ResourceVersion: 7,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, res.Items, 3)
|
||||
assert.Equal(t, "item1 ADDED", string(res.Items[0].Value))
|
||||
assert.Equal(t, "item4 ADDED", string(res.Items[1].Value))
|
||||
assert.Equal(t, "item5 ADDED", string(res.Items[2].Value))
|
||||
|
||||
continueToken, err := GetContinueToken(res.NextPageToken)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(7), continueToken.ResourceVersion)
|
||||
assert.Equal(t, int64(3), continueToken.StartOffset)
|
||||
})
|
||||
|
||||
t.Run("fetch second page at revision", func(t *testing.T) {
|
||||
continueToken := &ContinueToken{
|
||||
ResourceVersion: 8,
|
||||
StartOffset: 2,
|
||||
}
|
||||
res, err := store.PrepareList(ctx, &resource.ListRequest{
|
||||
NextPageToken: continueToken.String(),
|
||||
Limit: 2,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, res.Items, 2)
|
||||
assert.Equal(t, "item5 ADDED", string(res.Items[0].Value))
|
||||
assert.Equal(t, "item2 MODIFIED", string(res.Items[1].Value))
|
||||
|
||||
continueToken, err = GetContinueToken(res.NextPageToken)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, int64(8), continueToken.ResourceVersion)
|
||||
assert.Equal(t, int64(4), continueToken.StartOffset)
|
||||
})
|
||||
}
|
||||
|
||||
func writeEvent(ctx context.Context, store *backend, name string, action resource.WatchEvent_Type) (int64, error) {
|
||||
return store.WriteEvent(ctx, resource.WriteEvent{
|
||||
Type: action,
|
||||
Value: []byte(name + " " + resource.WatchEvent_Type_name[int32(action)]),
|
||||
Key: &resource.ResourceKey{
|
||||
Namespace: "namespace",
|
||||
Group: "group",
|
||||
Resource: "resource",
|
||||
Name: name,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func resourceKey(name string) *resource.ResourceKey {
|
||||
return &resource.ResourceKey{
|
||||
Namespace: "namespace",
|
||||
Group: "group",
|
||||
Resource: "resource",
|
||||
Name: name,
|
||||
}
|
||||
}
|
32
pkg/storage/unified/sql/continue.go
Normal file
32
pkg/storage/unified/sql/continue.go
Normal file
@ -0,0 +1,32 @@
|
||||
package sql
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type ContinueToken struct {
|
||||
StartOffset int64 `json:"o"`
|
||||
ResourceVersion int64 `json:"v"`
|
||||
}
|
||||
|
||||
func (c *ContinueToken) String() string {
|
||||
b, _ := json.Marshal(c)
|
||||
return base64.StdEncoding.EncodeToString(b)
|
||||
}
|
||||
|
||||
func GetContinueToken(token string) (*ContinueToken, error) {
|
||||
continueVal, err := base64.StdEncoding.DecodeString(token)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error decoding continue token")
|
||||
}
|
||||
|
||||
t := &ContinueToken{}
|
||||
err = json.Unmarshal(continueVal, t)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return t, nil
|
||||
}
|
7
pkg/storage/unified/sql/data/resource_delete.sql
Normal file
7
pkg/storage/unified/sql/data/resource_delete.sql
Normal file
@ -0,0 +1,7 @@
|
||||
DELETE FROM {{ .Ident "resource" }}
|
||||
WHERE 1 = 1
|
||||
AND {{ .Ident "namespace" }} = {{ .Arg .WriteEvent.Key.Namespace }}
|
||||
AND {{ .Ident "group" }} = {{ .Arg .WriteEvent.Key.Group }}
|
||||
AND {{ .Ident "resource" }} = {{ .Arg .WriteEvent.Key.Resource }}
|
||||
AND {{ .Ident "name" }} = {{ .Arg .WriteEvent.Key.Name }}
|
||||
;
|
23
pkg/storage/unified/sql/data/resource_history_insert.sql
Normal file
23
pkg/storage/unified/sql/data/resource_history_insert.sql
Normal file
@ -0,0 +1,23 @@
|
||||
INSERT INTO {{ .Ident "resource_history" }}
|
||||
(
|
||||
{{ .Ident "guid" }},
|
||||
{{ .Ident "group" }},
|
||||
{{ .Ident "resource" }},
|
||||
{{ .Ident "namespace" }},
|
||||
{{ .Ident "name" }},
|
||||
|
||||
{{ .Ident "value" }},
|
||||
{{ .Ident "action" }}
|
||||
)
|
||||
|
||||
VALUES (
|
||||
{{ .Arg .GUID }},
|
||||
{{ .Arg .WriteEvent.Key.Group }},
|
||||
{{ .Arg .WriteEvent.Key.Resource }},
|
||||
{{ .Arg .WriteEvent.Key.Namespace }},
|
||||
{{ .Arg .WriteEvent.Key.Name }},
|
||||
|
||||
{{ .Arg .WriteEvent.Value }},
|
||||
{{ .Arg .WriteEvent.Type }}
|
||||
)
|
||||
;
|
32
pkg/storage/unified/sql/data/resource_history_list.sql
Normal file
32
pkg/storage/unified/sql/data/resource_history_list.sql
Normal file
@ -0,0 +1,32 @@
|
||||
SELECT
|
||||
kv.{{ .Ident "resource_version" | .Into .Response.ResourceVersion }},
|
||||
{{ .Ident "value" | .Into .Response.Value }}
|
||||
FROM {{ .Ident "resource_history" }} as kv
|
||||
JOIN (
|
||||
SELECT {{ .Ident "guid" }}, max({{ .Ident "resource_version" }}) AS {{ .Ident "resource_version" }}
|
||||
FROM {{ .Ident "resource_history" }} AS mkv
|
||||
WHERE 1 = 1
|
||||
AND {{ .Ident "resource_version" }} <= {{ .Arg .Request.ResourceVersion }}
|
||||
{{ if and .Request.Options .Request.Options.Key }}
|
||||
{{ if .Request.Options.Key.Namespace }}
|
||||
AND {{ .Ident "namespace" }} = {{ .Arg .Request.Options.Key.Namespace }}
|
||||
{{ end }}
|
||||
{{ if .Request.Options.Key.Group }}
|
||||
AND {{ .Ident "group" }} = {{ .Arg .Request.Options.Key.Group }}
|
||||
{{ end }}
|
||||
{{ if .Request.Options.Key.Resource }}
|
||||
AND {{ .Ident "resource" }} = {{ .Arg .Request.Options.Key.Resource }}
|
||||
{{ end }}
|
||||
{{ if .Request.Options.Key.Name }}
|
||||
AND {{ .Ident "name" }} = {{ .Arg .Request.Options.Key.Name }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
GROUP BY mkv.{{ .Ident "namespace" }}, mkv.{{ .Ident "group" }}, mkv.{{ .Ident "resource" }}, mkv.{{ .Ident "name" }}
|
||||
) AS maxkv
|
||||
ON maxkv.{{ .Ident "guid" }} = kv.{{ .Ident "guid" }}
|
||||
WHERE kv.{{ .Ident "action" }} != 3
|
||||
ORDER BY kv.{{ .Ident "resource_version" }} ASC
|
||||
{{ if (gt .Request.Limit 0) }}
|
||||
LIMIT {{ .Arg .Request.Offset }}, {{ .Arg .Request.Limit }}
|
||||
{{ end }}
|
||||
;
|
12
pkg/storage/unified/sql/data/resource_history_poll.sql
Normal file
12
pkg/storage/unified/sql/data/resource_history_poll.sql
Normal file
@ -0,0 +1,12 @@
|
||||
SELECT
|
||||
{{ .Ident "resource_version" | .Into .Response.ResourceVersion }},
|
||||
{{ .Ident "namespace" | .Into .Response.Key.Namespace }},
|
||||
{{ .Ident "group" | .Into .Response.Key.Group }},
|
||||
{{ .Ident "resource" | .Into .Response.Key.Resource }},
|
||||
{{ .Ident "name" | .Into .Response.Key.Name }},
|
||||
{{ .Ident "value" | .Into .Response.Value }},
|
||||
{{ .Ident "action" | .Into .Response.Action }}
|
||||
|
||||
FROM {{ .Ident "resource_history" }}
|
||||
WHERE {{ .Ident "resource_version" }} > {{ .Arg .SinceResourceVersion }}
|
||||
;
|
17
pkg/storage/unified/sql/data/resource_history_read.sql
Normal file
17
pkg/storage/unified/sql/data/resource_history_read.sql
Normal file
@ -0,0 +1,17 @@
|
||||
SELECT
|
||||
{{ .Ident "resource_version" | .Into .ResourceVersion }},
|
||||
{{ .Ident "value" | .Into .Value }}
|
||||
|
||||
FROM {{ .Ident "resource_history" }}
|
||||
|
||||
WHERE 1 = 1
|
||||
AND {{ .Ident "namespace" }} = {{ .Arg .Request.Key.Namespace }}
|
||||
AND {{ .Ident "group" }} = {{ .Arg .Request.Key.Group }}
|
||||
AND {{ .Ident "resource" }} = {{ .Arg .Request.Key.Resource }}
|
||||
AND {{ .Ident "name" }} = {{ .Arg .Request.Key.Name }}
|
||||
{{ if gt .Request.ResourceVersion 0 }}
|
||||
AND {{ .Ident "resource_version" }} <= {{ .Arg .Request.ResourceVersion }}
|
||||
{{ end }}
|
||||
ORDER BY {{ .Ident "resource_version" }} DESC
|
||||
LIMIT 1
|
||||
;
|
@ -0,0 +1,4 @@
|
||||
UPDATE {{ .Ident "resource_history" }}
|
||||
SET {{ .Ident "resource_version" }} = {{ .Arg .ResourceVersion }}
|
||||
WHERE {{ .Ident "guid" }} = {{ .Arg .GUID }}
|
||||
;
|
23
pkg/storage/unified/sql/data/resource_insert.sql
Normal file
23
pkg/storage/unified/sql/data/resource_insert.sql
Normal file
@ -0,0 +1,23 @@
|
||||
INSERT INTO {{ .Ident "resource" }}
|
||||
|
||||
(
|
||||
{{ .Ident "guid" }},
|
||||
{{ .Ident "group" }},
|
||||
{{ .Ident "resource" }},
|
||||
{{ .Ident "namespace" }},
|
||||
{{ .Ident "name" }},
|
||||
|
||||
{{ .Ident "value" }},
|
||||
{{ .Ident "action" }}
|
||||
)
|
||||
VALUES (
|
||||
{{ .Arg .GUID }},
|
||||
{{ .Arg .WriteEvent.Key.Group }},
|
||||
{{ .Arg .WriteEvent.Key.Resource }},
|
||||
{{ .Arg .WriteEvent.Key.Namespace }},
|
||||
{{ .Arg .WriteEvent.Key.Name }},
|
||||
|
||||
{{ .Arg .WriteEvent.Value }},
|
||||
{{ .Arg .WriteEvent.Type }}
|
||||
)
|
||||
;
|
24
pkg/storage/unified/sql/data/resource_list.sql
Normal file
24
pkg/storage/unified/sql/data/resource_list.sql
Normal file
@ -0,0 +1,24 @@
|
||||
SELECT
|
||||
{{ .Ident "resource_version" | .Into .Response.ResourceVersion }},
|
||||
{{ .Ident "value" | .Into .Response.Value }}
|
||||
FROM {{ .Ident "resource" }}
|
||||
WHERE 1 = 1
|
||||
{{ if and .Request.Options .Request.Options.Key }}
|
||||
{{ if .Request.Options.Key.Namespace }}
|
||||
AND {{ .Ident "namespace" }} = {{ .Arg .Request.Options.Key.Namespace }}
|
||||
{{ end }}
|
||||
{{ if .Request.Options.Key.Group }}
|
||||
AND {{ .Ident "group" }} = {{ .Arg .Request.Options.Key.Group }}
|
||||
{{ end }}
|
||||
{{ if .Request.Options.Key.Resource }}
|
||||
AND {{ .Ident "resource" }} = {{ .Arg .Request.Options.Key.Resource }}
|
||||
{{ end }}
|
||||
{{ if .Request.Options.Key.Name }}
|
||||
AND {{ .Ident "name" }} = {{ .Arg .Request.Options.Key.Name }}
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
ORDER BY {{ .Ident "resource_version" }} DESC
|
||||
{{ if (gt .Request.Limit 0) }}
|
||||
LIMIT {{ .Arg .Request.Limit }}
|
||||
{{ end }}
|
||||
;
|
10
pkg/storage/unified/sql/data/resource_read.sql
Normal file
10
pkg/storage/unified/sql/data/resource_read.sql
Normal file
@ -0,0 +1,10 @@
|
||||
SELECT
|
||||
{{ .Ident "resource_version" | .Into .ResourceVersion }},
|
||||
{{ .Ident "value" | .Into .Value }}
|
||||
FROM {{ .Ident "resource" }}
|
||||
WHERE 1 = 1
|
||||
AND {{ .Ident "namespace" }} = {{ .Arg .Request.Key.Namespace }}
|
||||
AND {{ .Ident "group" }} = {{ .Arg .Request.Key.Group }}
|
||||
AND {{ .Ident "resource" }} = {{ .Arg .Request.Key.Resource }}
|
||||
AND {{ .Ident "name" }} = {{ .Arg .Request.Key.Name }}
|
||||
;
|
11
pkg/storage/unified/sql/data/resource_update.sql
Normal file
11
pkg/storage/unified/sql/data/resource_update.sql
Normal file
@ -0,0 +1,11 @@
|
||||
UPDATE {{ .Ident "resource" }}
|
||||
SET
|
||||
{{ .Ident "guid" }} = {{ .Arg .GUID }},
|
||||
{{ .Ident "value" }} = {{ .Arg .WriteEvent.Value }},
|
||||
{{ .Ident "action" }} = {{ .Arg .WriteEvent.Type }}
|
||||
WHERE 1 = 1
|
||||
AND {{ .Ident "group" }} = {{ .Arg .WriteEvent.Key.Group }}
|
||||
AND {{ .Ident "resource" }} = {{ .Arg .WriteEvent.Key.Resource }}
|
||||
AND {{ .Ident "namespace" }} = {{ .Arg .WriteEvent.Key.Namespace }}
|
||||
AND {{ .Ident "name" }} = {{ .Arg .WriteEvent.Key.Name }}
|
||||
;
|
4
pkg/storage/unified/sql/data/resource_update_rv.sql
Normal file
4
pkg/storage/unified/sql/data/resource_update_rv.sql
Normal file
@ -0,0 +1,4 @@
|
||||
UPDATE {{ .Ident "resource" }}
|
||||
SET {{ .Ident "resource_version" }} = {{ .Arg .ResourceVersion }}
|
||||
WHERE {{ .Ident "guid" }} = {{ .Arg .GUID }}
|
||||
;
|
8
pkg/storage/unified/sql/data/resource_version_get.sql
Normal file
8
pkg/storage/unified/sql/data/resource_version_get.sql
Normal file
@ -0,0 +1,8 @@
|
||||
SELECT
|
||||
{{ .Ident "resource_version" | .Into .ResourceVersion }}
|
||||
FROM {{ .Ident "resource_version" }}
|
||||
WHERE 1 = 1
|
||||
AND {{ .Ident "group" }} = {{ .Arg .Group }}
|
||||
AND {{ .Ident "resource" }} = {{ .Arg .Resource }}
|
||||
{{ .SelectFor "UPDATE" }}
|
||||
;
|
7
pkg/storage/unified/sql/data/resource_version_inc.sql
Normal file
7
pkg/storage/unified/sql/data/resource_version_inc.sql
Normal file
@ -0,0 +1,7 @@
|
||||
UPDATE {{ .Ident "resource_version" }}
|
||||
SET
|
||||
{{ .Ident "resource_version" }} = {{ .Arg .ResourceVersion}}
|
||||
WHERE 1 = 1
|
||||
AND {{ .Ident "group" }} = {{ .Arg .Group }}
|
||||
AND {{ .Ident "resource" }} = {{ .Arg .Resource }}
|
||||
;
|
13
pkg/storage/unified/sql/data/resource_version_insert.sql
Normal file
13
pkg/storage/unified/sql/data/resource_version_insert.sql
Normal file
@ -0,0 +1,13 @@
|
||||
INSERT INTO {{ .Ident "resource_version" }}
|
||||
(
|
||||
{{ .Ident "group" }},
|
||||
{{ .Ident "resource" }},
|
||||
{{ .Ident "resource_version" }}
|
||||
)
|
||||
|
||||
VALUES (
|
||||
{{ .Arg .Group }},
|
||||
{{ .Arg .Resource }},
|
||||
1
|
||||
)
|
||||
;
|
59
pkg/storage/unified/sql/db/dbimpl/db.go
Normal file
59
pkg/storage/unified/sql/db/dbimpl/db.go
Normal file
@ -0,0 +1,59 @@
|
||||
package dbimpl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
resourcedb "github.com/grafana/grafana/pkg/storage/unified/sql/db"
|
||||
)
|
||||
|
||||
func NewDB(d *sql.DB, driverName string) resourcedb.DB {
|
||||
return sqldb{
|
||||
DB: d,
|
||||
driverName: driverName,
|
||||
}
|
||||
}
|
||||
|
||||
type sqldb struct {
|
||||
*sql.DB
|
||||
driverName string
|
||||
}
|
||||
|
||||
func (d sqldb) DriverName() string {
|
||||
return d.driverName
|
||||
}
|
||||
|
||||
func (d sqldb) BeginTx(ctx context.Context, opts *sql.TxOptions) (resourcedb.Tx, error) {
|
||||
t, err := d.DB.BeginTx(ctx, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return tx{
|
||||
Tx: t,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d sqldb) WithTx(ctx context.Context, opts *sql.TxOptions, f resourcedb.TxFunc) error {
|
||||
t, err := d.BeginTx(ctx, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin tx: %w", err)
|
||||
}
|
||||
|
||||
if err := f(ctx, t); err != nil {
|
||||
if rollbackErr := t.Rollback(); rollbackErr != nil {
|
||||
return fmt.Errorf("tx err: %w; rollback err: %w", err, rollbackErr)
|
||||
}
|
||||
return fmt.Errorf("tx err: %w", err)
|
||||
}
|
||||
|
||||
if err = t.Commit(); err != nil {
|
||||
return fmt.Errorf("commit err: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type tx struct {
|
||||
*sql.Tx
|
||||
}
|
105
pkg/storage/unified/sql/db/dbimpl/dbEngine.go
Normal file
105
pkg/storage/unified/sql/db/dbimpl/dbEngine.go
Normal file
@ -0,0 +1,105 @@
|
||||
package dbimpl
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
"xorm.io/xorm"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/tracing"
|
||||
"github.com/grafana/grafana/pkg/services/store/entity/db"
|
||||
)
|
||||
|
||||
func getEngineMySQL(getter *sectionGetter, _ tracing.Tracer) (*xorm.Engine, error) {
|
||||
config := mysql.NewConfig()
|
||||
config.User = getter.String("db_user")
|
||||
config.Passwd = getter.String("db_pass")
|
||||
config.Net = "tcp"
|
||||
config.Addr = getter.String("db_host")
|
||||
config.DBName = getter.String("db_name")
|
||||
config.Params = map[string]string{
|
||||
// See: https://dev.mysql.com/doc/refman/en/sql-mode.html
|
||||
"@@SESSION.sql_mode": "ANSI",
|
||||
}
|
||||
config.Collation = "utf8mb4_unicode_ci"
|
||||
config.Loc = time.UTC
|
||||
config.AllowNativePasswords = true
|
||||
config.ClientFoundRows = true
|
||||
|
||||
// TODO: do we want to support these?
|
||||
// config.ServerPubKey = getter.String("db_server_pub_key")
|
||||
// config.TLSConfig = getter.String("db_tls_config_name")
|
||||
|
||||
if err := getter.Err(); err != nil {
|
||||
return nil, fmt.Errorf("config error: %w", err)
|
||||
}
|
||||
|
||||
if strings.HasPrefix(config.Addr, "/") {
|
||||
config.Net = "unix"
|
||||
}
|
||||
|
||||
// FIXME: get rid of xorm
|
||||
engine, err := xorm.NewEngine(db.DriverMySQL, config.FormatDSN())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open database: %w", err)
|
||||
}
|
||||
|
||||
engine.SetMaxOpenConns(0)
|
||||
engine.SetMaxIdleConns(2)
|
||||
engine.SetConnMaxLifetime(4 * time.Hour)
|
||||
|
||||
return engine, nil
|
||||
}
|
||||
|
||||
func getEnginePostgres(getter *sectionGetter, _ tracing.Tracer) (*xorm.Engine, error) {
|
||||
dsnKV := map[string]string{
|
||||
"user": getter.String("db_user"),
|
||||
"password": getter.String("db_pass"),
|
||||
"dbname": getter.String("db_name"),
|
||||
"sslmode": cmp.Or(getter.String("db_sslmode"), "disable"),
|
||||
}
|
||||
|
||||
// TODO: probably interesting:
|
||||
// "passfile", "statement_timeout", "lock_timeout", "connect_timeout"
|
||||
|
||||
// TODO: for CockroachDB, we probably need to use the following:
|
||||
// dsnKV["options"] = "-c enable_experimental_alter_column_type_general=true"
|
||||
// Or otherwise specify it as:
|
||||
// dsnKV["enable_experimental_alter_column_type_general"] = "true"
|
||||
|
||||
// TODO: do we want to support these options in the DSN as well?
|
||||
// "sslkey", "sslcert", "sslrootcert", "sslpassword", "sslsni", "krbspn",
|
||||
// "krbsrvname", "target_session_attrs", "service", "servicefile"
|
||||
|
||||
// More on Postgres connection string parameters:
|
||||
// https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING
|
||||
|
||||
hostport := getter.String("db_host")
|
||||
|
||||
if err := getter.Err(); err != nil {
|
||||
return nil, fmt.Errorf("config error: %w", err)
|
||||
}
|
||||
|
||||
host, port, err := splitHostPortDefault(hostport, "127.0.0.1", "5432")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid db_host: %w", err)
|
||||
}
|
||||
dsnKV["host"] = host
|
||||
dsnKV["port"] = port
|
||||
|
||||
dsn, err := MakeDSN(dsnKV)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error building DSN: %w", err)
|
||||
}
|
||||
|
||||
// FIXME: get rid of xorm
|
||||
engine, err := xorm.NewEngine(db.DriverPostgres, dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open database: %w", err)
|
||||
}
|
||||
|
||||
return engine, nil
|
||||
}
|
92
pkg/storage/unified/sql/db/dbimpl/dbEngine_test.go
Normal file
92
pkg/storage/unified/sql/db/dbimpl/dbEngine_test.go
Normal file
@ -0,0 +1,92 @@
|
||||
package dbimpl
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGetEngineMySQLFromConfig(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("happy path", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
getter := newTestSectionGetter(map[string]string{
|
||||
"db_type": "mysql",
|
||||
"db_host": "/var/run/mysql.socket",
|
||||
"db_name": "grafana",
|
||||
"db_user": "user",
|
||||
"db_password": "password",
|
||||
})
|
||||
engine, err := getEngineMySQL(getter, nil)
|
||||
assert.NotNil(t, engine)
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("invalid string", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
getter := newTestSectionGetter(map[string]string{
|
||||
"db_type": "mysql",
|
||||
"db_host": "/var/run/mysql.socket",
|
||||
"db_name": string(invalidUTF8ByteSequence),
|
||||
"db_user": "user",
|
||||
"db_password": "password",
|
||||
})
|
||||
engine, err := getEngineMySQL(getter, nil)
|
||||
assert.Nil(t, engine)
|
||||
assert.Error(t, err)
|
||||
assert.ErrorIs(t, err, ErrInvalidUTF8Sequence)
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetEnginePostgresFromConfig(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("happy path", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
getter := newTestSectionGetter(map[string]string{
|
||||
"db_type": "mysql",
|
||||
"db_host": "localhost",
|
||||
"db_name": "grafana",
|
||||
"db_user": "user",
|
||||
"db_password": "password",
|
||||
})
|
||||
engine, err := getEnginePostgres(getter, nil)
|
||||
|
||||
assert.NotNil(t, engine)
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("invalid string", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
getter := newTestSectionGetter(map[string]string{
|
||||
"db_type": "mysql",
|
||||
"db_host": string(invalidUTF8ByteSequence),
|
||||
"db_name": "grafana",
|
||||
"db_user": "user",
|
||||
"db_password": "password",
|
||||
})
|
||||
engine, err := getEnginePostgres(getter, nil)
|
||||
|
||||
assert.Nil(t, engine)
|
||||
assert.Error(t, err)
|
||||
assert.ErrorIs(t, err, ErrInvalidUTF8Sequence)
|
||||
})
|
||||
|
||||
t.Run("invalid hostport", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
getter := newTestSectionGetter(map[string]string{
|
||||
"db_type": "mysql",
|
||||
"db_host": "1:1:1",
|
||||
"db_name": "grafana",
|
||||
"db_user": "user",
|
||||
"db_password": "password",
|
||||
})
|
||||
engine, err := getEnginePostgres(getter, nil)
|
||||
|
||||
assert.Nil(t, engine)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
154
pkg/storage/unified/sql/db/dbimpl/db_test.go
Normal file
154
pkg/storage/unified/sql/db/dbimpl/db_test.go
Normal file
@ -0,0 +1,154 @@
|
||||
package dbimpl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sqlmock "github.com/DATA-DOG/go-sqlmock"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
resourcedb "github.com/grafana/grafana/pkg/storage/unified/sql/db"
|
||||
)
|
||||
|
||||
func newCtx(t *testing.T) context.Context {
|
||||
t.Helper()
|
||||
|
||||
d, ok := t.Deadline()
|
||||
if !ok {
|
||||
// provide a default timeout for tests
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
|
||||
return ctx
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithDeadline(context.Background(), d)
|
||||
t.Cleanup(cancel)
|
||||
|
||||
return ctx
|
||||
}
|
||||
|
||||
var errTest = errors.New("because of reasons")
|
||||
|
||||
const driverName = "sqlmock"
|
||||
|
||||
func TestDB_BeginTx(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("happy path", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sqldb, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
db := NewDB(sqldb, driverName)
|
||||
require.Equal(t, driverName, db.DriverName())
|
||||
|
||||
mock.ExpectBegin()
|
||||
tx, err := db.BeginTx(newCtx(t), nil)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, tx)
|
||||
})
|
||||
|
||||
t.Run("fail begin", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sqldb, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
db := NewDB(sqldb, "sqlmock")
|
||||
|
||||
mock.ExpectBegin().WillReturnError(errTest)
|
||||
tx, err := db.BeginTx(newCtx(t), nil)
|
||||
|
||||
require.Nil(t, tx)
|
||||
require.Error(t, err)
|
||||
require.ErrorIs(t, err, errTest)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDB_WithTx(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
newTxFunc := func(err error) resourcedb.TxFunc {
|
||||
return func(context.Context, resourcedb.Tx) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("happy path", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sqldb, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
db := NewDB(sqldb, "sqlmock")
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectCommit()
|
||||
err = db.WithTx(newCtx(t), nil, newTxFunc(nil))
|
||||
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("fail begin", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sqldb, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
db := NewDB(sqldb, "sqlmock")
|
||||
|
||||
mock.ExpectBegin().WillReturnError(errTest)
|
||||
err = db.WithTx(newCtx(t), nil, newTxFunc(nil))
|
||||
|
||||
require.Error(t, err)
|
||||
require.ErrorIs(t, err, errTest)
|
||||
})
|
||||
|
||||
t.Run("fail tx", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sqldb, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
db := NewDB(sqldb, "sqlmock")
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectRollback()
|
||||
err = db.WithTx(newCtx(t), nil, newTxFunc(errTest))
|
||||
|
||||
require.Error(t, err)
|
||||
require.ErrorIs(t, err, errTest)
|
||||
})
|
||||
|
||||
t.Run("fail tx; fail rollback", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sqldb, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
db := NewDB(sqldb, "sqlmock")
|
||||
errTest2 := errors.New("yet another err")
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectRollback().WillReturnError(errTest)
|
||||
err = db.WithTx(newCtx(t), nil, newTxFunc(errTest2))
|
||||
|
||||
require.Error(t, err)
|
||||
require.ErrorIs(t, err, errTest)
|
||||
require.ErrorIs(t, err, errTest2)
|
||||
})
|
||||
|
||||
t.Run("fail commit", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sqldb, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
db := NewDB(sqldb, "sqlmock")
|
||||
|
||||
mock.ExpectBegin()
|
||||
mock.ExpectCommit().WillReturnError(errTest)
|
||||
err = db.WithTx(newCtx(t), nil, newTxFunc(nil))
|
||||
|
||||
require.Error(t, err)
|
||||
require.ErrorIs(t, err, errTest)
|
||||
})
|
||||
}
|
166
pkg/storage/unified/sql/db/dbimpl/dbimpl.go
Normal file
166
pkg/storage/unified/sql/db/dbimpl/dbimpl.go
Normal file
@ -0,0 +1,166 @@
|
||||
package dbimpl
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/dlmiddlecote/sqlstats"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"xorm.io/xorm"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/infra/tracing"
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/session"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
resourcedb "github.com/grafana/grafana/pkg/storage/unified/sql/db"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/db/migrations"
|
||||
)
|
||||
|
||||
var _ resourcedb.ResourceDBInterface = (*ResourceDB)(nil)
|
||||
|
||||
func ProvideResourceDB(db db.DB, cfg *setting.Cfg, features featuremgmt.FeatureToggles, tracer tracing.Tracer) (*ResourceDB, error) {
|
||||
return &ResourceDB{
|
||||
db: db,
|
||||
cfg: cfg,
|
||||
features: features,
|
||||
log: log.New("entity-db"),
|
||||
tracer: tracer,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type ResourceDB struct {
|
||||
once sync.Once
|
||||
onceErr error
|
||||
|
||||
db db.DB
|
||||
features featuremgmt.FeatureToggles
|
||||
engine *xorm.Engine
|
||||
cfg *setting.Cfg
|
||||
log log.Logger
|
||||
tracer tracing.Tracer
|
||||
}
|
||||
|
||||
func (db *ResourceDB) Init() error {
|
||||
db.once.Do(func() {
|
||||
db.onceErr = db.init()
|
||||
})
|
||||
|
||||
return db.onceErr
|
||||
}
|
||||
|
||||
func (db *ResourceDB) GetEngine() (*xorm.Engine, error) {
|
||||
if err := db.Init(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return db.engine, db.onceErr
|
||||
}
|
||||
|
||||
func (db *ResourceDB) init() error {
|
||||
if db.engine != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var engine *xorm.Engine
|
||||
var err error
|
||||
|
||||
// TODO: This should be renamed resource_api
|
||||
getter := §ionGetter{
|
||||
DynamicSection: db.cfg.SectionWithEnvOverrides("resource_api"),
|
||||
}
|
||||
|
||||
dbType := getter.Key("db_type").MustString("")
|
||||
|
||||
// if explicit connection settings are provided, use them
|
||||
if dbType != "" {
|
||||
if dbType == "postgres" {
|
||||
engine, err = getEnginePostgres(getter, db.tracer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// FIXME: this config option is cockroachdb-specific, it's not supported by postgres
|
||||
// FIXME: this only sets this option for the session that we get
|
||||
// from the pool right now. A *sql.DB is a pool of connections,
|
||||
// there is no guarantee that the session where this is run will be
|
||||
// the same where we need to change the type of a column
|
||||
_, err = engine.Exec("SET SESSION enable_experimental_alter_column_type_general=true")
|
||||
if err != nil {
|
||||
db.log.Error("error connecting to postgres", "msg", err.Error())
|
||||
// FIXME: return nil, err
|
||||
}
|
||||
} else if dbType == "mysql" {
|
||||
engine, err = getEngineMySQL(getter, db.tracer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = engine.Ping(); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// TODO: sqlite support
|
||||
return fmt.Errorf("invalid db type specified: %s", dbType)
|
||||
}
|
||||
|
||||
// register sql stat metrics
|
||||
if err := prometheus.Register(sqlstats.NewStatsCollector("unified_storage", engine.DB().DB)); err != nil {
|
||||
db.log.Warn("Failed to register unified storage sql stats collector", "error", err)
|
||||
}
|
||||
|
||||
// configure sql logging
|
||||
debugSQL := getter.Key("log_queries").MustBool(false)
|
||||
if !debugSQL {
|
||||
engine.SetLogger(&xorm.DiscardLogger{})
|
||||
} else {
|
||||
// add stack to database calls to be able to see what repository initiated queries. Top 7 items from the stack as they are likely in the xorm library.
|
||||
// engine.SetLogger(sqlstore.NewXormLogger(log.LvlInfo, log.WithSuffix(log.New("sqlstore.xorm"), log.CallerContextKey, log.StackCaller(log.DefaultCallerDepth))))
|
||||
engine.ShowSQL(true)
|
||||
engine.ShowExecTime(true)
|
||||
}
|
||||
|
||||
// otherwise, try to use the grafana db connection
|
||||
} else {
|
||||
if db.db == nil {
|
||||
return fmt.Errorf("no db connection provided")
|
||||
}
|
||||
|
||||
engine = db.db.GetEngine()
|
||||
}
|
||||
|
||||
db.engine = engine
|
||||
|
||||
if err := migrations.MigrateResourceStore(engine, db.cfg, db.features); err != nil {
|
||||
db.engine = nil
|
||||
return fmt.Errorf("run migrations: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *ResourceDB) GetSession() (*session.SessionDB, error) {
|
||||
engine, err := db.GetEngine()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return session.GetSession(sqlx.NewDb(engine.DB().DB, engine.DriverName())), nil
|
||||
}
|
||||
|
||||
func (db *ResourceDB) GetCfg() *setting.Cfg {
|
||||
return db.cfg
|
||||
}
|
||||
|
||||
func (db *ResourceDB) GetDB() (resourcedb.DB, error) {
|
||||
engine, err := db.GetEngine()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ret := NewDB(engine.DB().DB, engine.Dialect().DriverName())
|
||||
|
||||
return ret, nil
|
||||
}
|
111
pkg/storage/unified/sql/db/dbimpl/util.go
Normal file
111
pkg/storage/unified/sql/db/dbimpl/util.go
Normal file
@ -0,0 +1,111 @@
|
||||
package dbimpl
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidUTF8Sequence = errors.New("invalid UTF-8 sequence")
|
||||
)
|
||||
|
||||
type sectionGetter struct {
|
||||
*setting.DynamicSection
|
||||
err error
|
||||
}
|
||||
|
||||
func (g *sectionGetter) Err() error {
|
||||
return g.err
|
||||
}
|
||||
|
||||
func (g *sectionGetter) String(key string) string {
|
||||
v := g.DynamicSection.Key(key).MustString("")
|
||||
if !utf8.ValidString(v) {
|
||||
g.err = fmt.Errorf("value for key %q: %w", key, ErrInvalidUTF8Sequence)
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
return v
|
||||
}
|
||||
|
||||
// MakeDSN creates a DSN from the given key/value pair. It validates the strings
|
||||
// form valid UTF-8 sequences and escapes values if needed.
|
||||
func MakeDSN(m map[string]string) (string, error) {
|
||||
b := new(strings.Builder)
|
||||
|
||||
ks := keys(m)
|
||||
sort.Strings(ks) // provide deterministic behaviour
|
||||
for _, k := range ks {
|
||||
v := m[k]
|
||||
if !utf8.ValidString(v) {
|
||||
return "", fmt.Errorf("value for DSN key %q: %w", k,
|
||||
ErrInvalidUTF8Sequence)
|
||||
}
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if b.Len() > 0 {
|
||||
_ = b.WriteByte(' ')
|
||||
}
|
||||
_, _ = b.WriteString(k)
|
||||
_ = b.WriteByte('=')
|
||||
writeDSNValue(b, v)
|
||||
}
|
||||
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
func keys(m map[string]string) []string {
|
||||
ret := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
ret = append(ret, k)
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
func writeDSNValue(b *strings.Builder, v string) {
|
||||
numq := strings.Count(v, `'`)
|
||||
numb := strings.Count(v, `\`)
|
||||
if numq+numb == 0 && v != "" {
|
||||
b.WriteString(v)
|
||||
|
||||
return
|
||||
}
|
||||
b.Grow(2 + numq + numb + len(v))
|
||||
|
||||
_ = b.WriteByte('\'')
|
||||
for _, r := range v {
|
||||
if r == '\\' || r == '\'' {
|
||||
_ = b.WriteByte('\\')
|
||||
}
|
||||
_, _ = b.WriteRune(r)
|
||||
}
|
||||
_ = b.WriteByte('\'')
|
||||
}
|
||||
|
||||
// splitHostPortDefault is similar to net.SplitHostPort, but will also accept a
|
||||
// specification with no port and apply the default port instead. It also
|
||||
// applies the given defaults if the results are empty strings.
|
||||
func splitHostPortDefault(hostport, defaultHost, defaultPort string) (string, string, error) {
|
||||
host, port, err := net.SplitHostPort(hostport)
|
||||
if err != nil {
|
||||
// try appending the port
|
||||
host, port, err = net.SplitHostPort(hostport + ":" + defaultPort)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("invalid hostport: %q", hostport)
|
||||
}
|
||||
}
|
||||
host = cmp.Or(host, defaultHost)
|
||||
port = cmp.Or(port, defaultPort)
|
||||
|
||||
return host, port, nil
|
||||
}
|
108
pkg/storage/unified/sql/db/dbimpl/util_test.go
Normal file
108
pkg/storage/unified/sql/db/dbimpl/util_test.go
Normal file
@ -0,0 +1,108 @@
|
||||
package dbimpl
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var invalidUTF8ByteSequence = []byte{0xff, 0xfe, 0xfd}
|
||||
|
||||
func setSectionKeyValues(section *setting.DynamicSection, m map[string]string) {
|
||||
for k, v := range m {
|
||||
section.Key(k).SetValue(v)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestSectionGetter(m map[string]string) *sectionGetter {
|
||||
section := setting.NewCfg().SectionWithEnvOverrides("entity_api")
|
||||
setSectionKeyValues(section, m)
|
||||
|
||||
return §ionGetter{
|
||||
DynamicSection: section,
|
||||
}
|
||||
}
|
||||
|
||||
func TestSectionGetter(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
key = "the key"
|
||||
val = string(invalidUTF8ByteSequence)
|
||||
)
|
||||
|
||||
g := newTestSectionGetter(map[string]string{
|
||||
key: val,
|
||||
})
|
||||
|
||||
v := g.String("whatever")
|
||||
require.Empty(t, v)
|
||||
require.NoError(t, g.Err())
|
||||
|
||||
v = g.String(key)
|
||||
require.Empty(t, v)
|
||||
require.Error(t, g.Err())
|
||||
require.ErrorIs(t, g.Err(), ErrInvalidUTF8Sequence)
|
||||
}
|
||||
|
||||
func TestMakeDSN(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, err := MakeDSN(map[string]string{
|
||||
"db_name": string(invalidUTF8ByteSequence),
|
||||
})
|
||||
require.Empty(t, s)
|
||||
require.Error(t, err)
|
||||
require.ErrorIs(t, err, ErrInvalidUTF8Sequence)
|
||||
|
||||
s, err = MakeDSN(map[string]string{
|
||||
"skip": "",
|
||||
"user": `shou'ld esc\ape`,
|
||||
"pass": "noescape",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, `pass=noescape user='shou\'ld esc\\ape'`, s)
|
||||
}
|
||||
|
||||
func TestSplitHostPort(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
testCases := []struct {
|
||||
hostport string
|
||||
defaultHost string
|
||||
defaultPort string
|
||||
fails bool
|
||||
|
||||
host string
|
||||
port string
|
||||
}{
|
||||
{hostport: "192.168.0.140:456", defaultHost: "", defaultPort: "", host: "192.168.0.140", port: "456"},
|
||||
{hostport: "192.168.0.140", defaultHost: "", defaultPort: "123", host: "192.168.0.140", port: "123"},
|
||||
{hostport: "[::1]:456", defaultHost: "", defaultPort: "", host: "::1", port: "456"},
|
||||
{hostport: "[::1]", defaultHost: "", defaultPort: "123", host: "::1", port: "123"},
|
||||
{hostport: ":456", defaultHost: "1.2.3.4", defaultPort: "", host: "1.2.3.4", port: "456"},
|
||||
{hostport: "xyz.rds.amazonaws.com", defaultHost: "", defaultPort: "123", host: "xyz.rds.amazonaws.com", port: "123"},
|
||||
{hostport: "xyz.rds.amazonaws.com:123", defaultHost: "", defaultPort: "", host: "xyz.rds.amazonaws.com", port: "123"},
|
||||
{hostport: "", defaultHost: "localhost", defaultPort: "1433", host: "localhost", port: "1433"},
|
||||
{hostport: "1:1:1", fails: true},
|
||||
}
|
||||
|
||||
for i, tc := range testCases {
|
||||
t.Run(fmt.Sprintf("test index #%d", i), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
host, port, err := splitHostPortDefault(tc.hostport, tc.defaultHost, tc.defaultPort)
|
||||
if tc.fails {
|
||||
require.Error(t, err)
|
||||
require.Empty(t, host)
|
||||
require.Empty(t, port)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.host, host)
|
||||
require.Equal(t, tc.port, port)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
24
pkg/storage/unified/sql/db/migrations/migrator.go
Normal file
24
pkg/storage/unified/sql/db/migrations/migrator.go
Normal file
@ -0,0 +1,24 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"xorm.io/xorm"
|
||||
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
)
|
||||
|
||||
func MigrateResourceStore(engine *xorm.Engine, cfg *setting.Cfg, features featuremgmt.FeatureToggles) error {
|
||||
// Skip if feature flag is not enabled
|
||||
if !features.IsEnabledGlobally(featuremgmt.FlagUnifiedStorage) {
|
||||
return nil
|
||||
}
|
||||
|
||||
mg := migrator.NewScopedMigrator(engine, cfg, "resource")
|
||||
mg.AddCreateMigration()
|
||||
|
||||
initResourceTables(mg)
|
||||
|
||||
// since it's a new feature enable migration locking by default
|
||||
return mg.Start(true, 0)
|
||||
}
|
101
pkg/storage/unified/sql/db/migrations/resource_mig.go
Normal file
101
pkg/storage/unified/sql/db/migrations/resource_mig.go
Normal file
@ -0,0 +1,101 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
|
||||
)
|
||||
|
||||
func initResourceTables(mg *migrator.Migrator) string {
|
||||
marker := "Initialize resource tables"
|
||||
mg.AddMigration(marker, &migrator.RawSQLMigration{})
|
||||
|
||||
tables := []migrator.Table{}
|
||||
tables = append(tables, migrator.Table{
|
||||
Name: "resource",
|
||||
Columns: []*migrator.Column{
|
||||
// primary identifier
|
||||
{Name: "guid", Type: migrator.DB_NVarchar, Length: 36, Nullable: false, IsPrimaryKey: true},
|
||||
|
||||
{Name: "resource_version", Type: migrator.DB_BigInt, Nullable: true},
|
||||
|
||||
// K8s Identity group+(version)+namespace+resource+name
|
||||
{Name: "group", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
|
||||
{Name: "resource", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
|
||||
{Name: "namespace", Type: migrator.DB_NVarchar, Length: 63, Nullable: false},
|
||||
{Name: "name", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
|
||||
{Name: "value", Type: migrator.DB_LongText, Nullable: true},
|
||||
{Name: "action", Type: migrator.DB_Int, Nullable: false}, // 1: create, 2: update, 3: delete
|
||||
|
||||
// Hashed label set
|
||||
{Name: "label_set", Type: migrator.DB_NVarchar, Length: 64, Nullable: true}, // null is no labels
|
||||
},
|
||||
Indices: []*migrator.Index{
|
||||
{Cols: []string{"namespace", "group", "resource", "name"}, Type: migrator.UniqueIndex},
|
||||
},
|
||||
})
|
||||
|
||||
tables = append(tables, migrator.Table{
|
||||
Name: "resource_history",
|
||||
Columns: []*migrator.Column{
|
||||
// primary identifier
|
||||
{Name: "guid", Type: migrator.DB_NVarchar, Length: 36, Nullable: false, IsPrimaryKey: true},
|
||||
{Name: "resource_version", Type: migrator.DB_BigInt, Nullable: true},
|
||||
|
||||
// K8s Identity group+(version)+namespace+resource+name
|
||||
{Name: "group", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
|
||||
{Name: "resource", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
|
||||
{Name: "namespace", Type: migrator.DB_NVarchar, Length: 63, Nullable: false},
|
||||
{Name: "name", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
|
||||
{Name: "value", Type: migrator.DB_LongText, Nullable: true},
|
||||
{Name: "action", Type: migrator.DB_Int, Nullable: false}, // 1: create, 2: update, 3: delete
|
||||
|
||||
// Hashed label set
|
||||
{Name: "label_set", Type: migrator.DB_NVarchar, Length: 64, Nullable: true}, // null is no labels
|
||||
},
|
||||
Indices: []*migrator.Index{
|
||||
{
|
||||
Cols: []string{"namespace", "group", "resource", "name", "resource_version"},
|
||||
Type: migrator.UniqueIndex,
|
||||
Name: "UQE_resource_history_namespace_group_name_version",
|
||||
},
|
||||
// index to support watch poller
|
||||
{Cols: []string{"resource_version"}, Type: migrator.IndexType},
|
||||
},
|
||||
})
|
||||
|
||||
// tables = append(tables, migrator.Table{
|
||||
// Name: "resource_label_set",
|
||||
// Columns: []*migrator.Column{
|
||||
// {Name: "label_set", Type: migrator.DB_NVarchar, Length: 64, Nullable: false},
|
||||
// {Name: "label", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
|
||||
// {Name: "value", Type: migrator.DB_Text, Nullable: false},
|
||||
// },
|
||||
// Indices: []*migrator.Index{
|
||||
// {Cols: []string{"label_set", "label"}, Type: migrator.UniqueIndex},
|
||||
// },
|
||||
// })
|
||||
|
||||
tables = append(tables, migrator.Table{
|
||||
Name: "resource_version",
|
||||
Columns: []*migrator.Column{
|
||||
{Name: "group", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
|
||||
{Name: "resource", Type: migrator.DB_NVarchar, Length: 190, Nullable: false},
|
||||
{Name: "resource_version", Type: migrator.DB_BigInt, Nullable: false},
|
||||
},
|
||||
Indices: []*migrator.Index{
|
||||
{Cols: []string{"group", "resource"}, Type: migrator.UniqueIndex},
|
||||
},
|
||||
})
|
||||
|
||||
// Initialize all tables
|
||||
for t := range tables {
|
||||
mg.AddMigration("drop table "+tables[t].Name, migrator.NewDropTableMigration(tables[t].Name))
|
||||
mg.AddMigration("create table "+tables[t].Name, migrator.NewAddTableMigration(tables[t]))
|
||||
for i := range tables[t].Indices {
|
||||
mg.AddMigration(fmt.Sprintf("create table %s, index: %d", tables[t].Name, i), migrator.NewAddIndexMigration(tables[t], tables[t].Indices[i]))
|
||||
}
|
||||
}
|
||||
|
||||
return marker
|
||||
}
|
71
pkg/storage/unified/sql/db/service.go
Executable file
71
pkg/storage/unified/sql/db/service.go
Executable file
@ -0,0 +1,71 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
|
||||
"xorm.io/xorm"
|
||||
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore/session"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
)
|
||||
|
||||
const (
|
||||
DriverPostgres = "postgres"
|
||||
DriverMySQL = "mysql"
|
||||
DriverSQLite = "sqlite"
|
||||
DriverSQLite3 = "sqlite3"
|
||||
)
|
||||
|
||||
// ResourceDBInterface provides access to a database capable of supporting the
|
||||
// Entity Server.
|
||||
type ResourceDBInterface interface {
|
||||
Init() error
|
||||
GetCfg() *setting.Cfg
|
||||
GetDB() (DB, error)
|
||||
|
||||
// TODO: deprecate.
|
||||
GetSession() (*session.SessionDB, error)
|
||||
GetEngine() (*xorm.Engine, error)
|
||||
}
|
||||
|
||||
// DB is a thin abstraction on *sql.DB to allow mocking to provide better unit
|
||||
// testing. We purposefully hide database operation methods that would use
|
||||
// context.Background().
|
||||
type DB interface {
|
||||
ContextExecer
|
||||
BeginTx(context.Context, *sql.TxOptions) (Tx, error)
|
||||
WithTx(context.Context, *sql.TxOptions, TxFunc) error
|
||||
PingContext(context.Context) error
|
||||
Stats() sql.DBStats
|
||||
DriverName() string
|
||||
}
|
||||
|
||||
// TxFunc is a function that executes with access to a transaction. The context
|
||||
// it receives is the same context used to create the transaction, and is
|
||||
// provided so that a general prupose TxFunc is able to retrieve information
|
||||
// from that context, and derive other contexts that may be used to run database
|
||||
// operation methods accepting a context. A derived context can be used to
|
||||
// request a specific database operation to take no more than a specific
|
||||
// fraction of the remaining timeout of the transaction context, or to enrich
|
||||
// the downstream observability layer with relevant information regarding the
|
||||
// specific operation being carried out.
|
||||
type TxFunc = func(context.Context, Tx) error
|
||||
|
||||
// Tx is a thin abstraction on *sql.Tx to allow mocking to provide better unit
|
||||
// testing. We allow database operation methods that do not take a
|
||||
// context.Context here since a Tx can only be obtained with DB.BeginTx, which
|
||||
// already takes a context.Context.
|
||||
type Tx interface {
|
||||
ContextExecer
|
||||
Commit() error
|
||||
Rollback() error
|
||||
}
|
||||
|
||||
// ContextExecer is a set of database operation methods that take
|
||||
// context.Context.
|
||||
type ContextExecer interface {
|
||||
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
|
||||
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
|
||||
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
|
||||
}
|
191
pkg/storage/unified/sql/queries.go
Normal file
191
pkg/storage/unified/sql/queries.go
Normal file
@ -0,0 +1,191 @@
|
||||
package sql
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"embed"
|
||||
"fmt"
|
||||
"text/template"
|
||||
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
)
|
||||
|
||||
// Templates setup.
|
||||
var (
|
||||
//go:embed data/*.sql
|
||||
sqlTemplatesFS embed.FS
|
||||
|
||||
sqlTemplates = template.Must(template.New("sql").ParseFS(sqlTemplatesFS, `data/*.sql`))
|
||||
)
|
||||
|
||||
func mustTemplate(filename string) *template.Template {
|
||||
if t := sqlTemplates.Lookup(filename); t != nil {
|
||||
return t
|
||||
}
|
||||
panic(fmt.Sprintf("template file not found: %s", filename))
|
||||
}
|
||||
|
||||
// Templates.
|
||||
var (
|
||||
sqlResourceDelete = mustTemplate("resource_delete.sql")
|
||||
sqlResourceInsert = mustTemplate("resource_insert.sql")
|
||||
sqlResourceUpdate = mustTemplate("resource_update.sql")
|
||||
sqlResourceRead = mustTemplate("resource_read.sql")
|
||||
sqlResourceList = mustTemplate("resource_list.sql")
|
||||
sqlResourceHistoryList = mustTemplate("resource_history_list.sql")
|
||||
sqlResourceUpdateRV = mustTemplate("resource_update_rv.sql")
|
||||
sqlResourceHistoryRead = mustTemplate("resource_history_read.sql")
|
||||
sqlResourceHistoryUpdateRV = mustTemplate("resource_history_update_rv.sql")
|
||||
sqlResourceHistoryInsert = mustTemplate("resource_history_insert.sql")
|
||||
sqlResourceHistoryPoll = mustTemplate("resource_history_poll.sql")
|
||||
|
||||
// sqlResourceLabelsInsert = mustTemplate("resource_labels_insert.sql")
|
||||
sqlResourceVersionGet = mustTemplate("resource_version_get.sql")
|
||||
sqlResourceVersionInc = mustTemplate("resource_version_inc.sql")
|
||||
sqlResourceVersionInsert = mustTemplate("resource_version_insert.sql")
|
||||
)
|
||||
|
||||
// TxOptions.
|
||||
var (
|
||||
ReadCommitted = &sql.TxOptions{
|
||||
Isolation: sql.LevelReadCommitted,
|
||||
}
|
||||
ReadCommittedRO = &sql.TxOptions{
|
||||
Isolation: sql.LevelReadCommitted,
|
||||
ReadOnly: true,
|
||||
}
|
||||
)
|
||||
|
||||
// SQLError is an error returned by the database, which includes additionally
|
||||
// debugging information about what was sent to the database.
|
||||
type SQLError struct {
|
||||
Err error
|
||||
CallType string // either Query, QueryRow or Exec
|
||||
TemplateName string
|
||||
Query string
|
||||
RawQuery string
|
||||
ScanDest []any
|
||||
|
||||
// potentially regulated information is not exported and only directly
|
||||
// available for local testing and local debugging purposes, making sure it
|
||||
// is never marshaled to JSON or any other serialization.
|
||||
|
||||
arguments []any
|
||||
}
|
||||
|
||||
func (e SQLError) Unwrap() error {
|
||||
return e.Err
|
||||
}
|
||||
|
||||
func (e SQLError) Error() string {
|
||||
return fmt.Sprintf("%s: %s with %d input arguments and %d output "+
|
||||
"destination arguments: %v", e.TemplateName, e.CallType,
|
||||
len(e.arguments), len(e.ScanDest), e.Err)
|
||||
}
|
||||
|
||||
type sqlResourceRequest struct {
|
||||
*sqltemplate.SQLTemplate
|
||||
GUID string
|
||||
WriteEvent resource.WriteEvent
|
||||
}
|
||||
|
||||
func (r sqlResourceRequest) Validate() error {
|
||||
return nil // TODO
|
||||
}
|
||||
|
||||
type historyPollResponse struct {
|
||||
Key resource.ResourceKey
|
||||
ResourceVersion int64
|
||||
Value []byte
|
||||
Action int
|
||||
}
|
||||
|
||||
func (r *historyPollResponse) Results() (*historyPollResponse, error) {
|
||||
return r, nil
|
||||
}
|
||||
|
||||
type sqlResourceHistoryPollRequest struct {
|
||||
*sqltemplate.SQLTemplate
|
||||
SinceResourceVersion int64
|
||||
Response *historyPollResponse
|
||||
}
|
||||
|
||||
func (r sqlResourceHistoryPollRequest) Validate() error {
|
||||
return nil // TODO
|
||||
}
|
||||
|
||||
// sqlResourceReadRequest can be used to retrieve a row fromthe "resource" tables.
|
||||
|
||||
type readResponse struct {
|
||||
resource.ReadResponse
|
||||
}
|
||||
|
||||
func (r *readResponse) Results() (*readResponse, error) {
|
||||
return r, nil
|
||||
}
|
||||
|
||||
type sqlResourceReadRequest struct {
|
||||
*sqltemplate.SQLTemplate
|
||||
Request *resource.ReadRequest
|
||||
*readResponse
|
||||
}
|
||||
|
||||
func (r sqlResourceReadRequest) Validate() error {
|
||||
return nil // TODO
|
||||
}
|
||||
|
||||
// List
|
||||
type sqlResourceListRequest struct {
|
||||
*sqltemplate.SQLTemplate
|
||||
Request *resource.ListRequest
|
||||
Response *resource.ResourceWrapper
|
||||
}
|
||||
|
||||
func (r sqlResourceListRequest) Validate() error {
|
||||
return nil // TODO
|
||||
}
|
||||
|
||||
type historyListRequest struct {
|
||||
ResourceVersion, Limit, Offset int64
|
||||
Options *resource.ListOptions
|
||||
}
|
||||
type sqlResourceHistoryListRequest struct {
|
||||
*sqltemplate.SQLTemplate
|
||||
Request *historyListRequest
|
||||
Response *resource.ResourceWrapper
|
||||
}
|
||||
|
||||
func (r sqlResourceHistoryListRequest) Validate() error {
|
||||
return nil // TODO
|
||||
}
|
||||
|
||||
// update RV
|
||||
|
||||
type sqlResourceUpdateRVRequest struct {
|
||||
*sqltemplate.SQLTemplate
|
||||
GUID string
|
||||
ResourceVersion int64
|
||||
}
|
||||
|
||||
func (r sqlResourceUpdateRVRequest) Validate() error {
|
||||
return nil // TODO
|
||||
}
|
||||
|
||||
// resource_version table requests.
|
||||
type resourceVersion struct {
|
||||
ResourceVersion int64
|
||||
}
|
||||
|
||||
func (r *resourceVersion) Results() (*resourceVersion, error) {
|
||||
return r, nil
|
||||
}
|
||||
|
||||
type sqlResourceVersionRequest struct {
|
||||
*sqltemplate.SQLTemplate
|
||||
Group, Resource string
|
||||
*resourceVersion
|
||||
}
|
||||
|
||||
func (r sqlResourceVersionRequest) Validate() error {
|
||||
return nil // TODO
|
||||
}
|
364
pkg/storage/unified/sql/queries_test.go
Normal file
364
pkg/storage/unified/sql/queries_test.go
Normal file
@ -0,0 +1,364 @@
|
||||
package sql
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"errors"
|
||||
"testing"
|
||||
"text/template"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
)
|
||||
|
||||
// debug is meant to provide greater debugging detail about certain errors. The
|
||||
// returned error will either provide more detailed information or be the same
|
||||
// original error, suitable only for local debugging. The details provided are
|
||||
// not meant to be logged, since they could include PII or otherwise
|
||||
// sensitive/confidential information. These information should only be used for
|
||||
// local debugging with fake or otherwise non-regulated information.
|
||||
func debug(err error) error {
|
||||
var d interface{ Debug() string }
|
||||
if errors.As(err, &d) {
|
||||
return errors.New(d.Debug())
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
var _ = debug // silence the `unused` linter
|
||||
|
||||
//go:embed testdata/*
|
||||
var testdataFS embed.FS
|
||||
|
||||
func testdata(t *testing.T, filename string) []byte {
|
||||
t.Helper()
|
||||
b, err := testdataFS.ReadFile(`testdata/` + filename)
|
||||
require.NoError(t, err)
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
func TestQueries(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Each template has one or more test cases, each identified with a
|
||||
// descriptive name (e.g. "happy path", "error twiddling the frobb"). Each
|
||||
// of them will test that for the same input data they must produce a result
|
||||
// that will depend on the Dialect. Expected queries should be defined in
|
||||
// separate files in the testdata directory. This improves the testing
|
||||
// experience by separating test data from test code, since mixing both
|
||||
// tends to make it more difficult to reason about what is being done,
|
||||
// especially as we want testing code to scale and make it easy to add
|
||||
// tests.
|
||||
type (
|
||||
// type aliases to make code more semantic and self-documenting
|
||||
resultSQLFilename = string
|
||||
dialects = []sqltemplate.Dialect
|
||||
expected map[resultSQLFilename]dialects
|
||||
|
||||
testCase = struct {
|
||||
Name string
|
||||
|
||||
// Data should be the struct passed to the template.
|
||||
Data sqltemplate.SQLTemplateIface
|
||||
|
||||
// Expected maps the filename containing the expected result query
|
||||
// to the list of dialects that would produce it. For simple
|
||||
// queries, it is possible that more than one dialect produce the
|
||||
// same output. The filename is expected to be in the `testdata`
|
||||
// directory.
|
||||
Expected expected
|
||||
}
|
||||
)
|
||||
|
||||
// Define tests cases. Most templates are trivial and testing that they
|
||||
// generate correct code for a single Dialect is fine, since the one thing
|
||||
// that always changes is how SQL placeholder arguments are passed (most
|
||||
// Dialects use `?` while PostgreSQL uses `$1`, `$2`, etc.), and that is
|
||||
// something that should be tested in the Dialect implementation instead of
|
||||
// here. We will ask to have at least one test per SQL template, and we will
|
||||
// lean to test MySQL. Templates containing branching (conditionals, loops,
|
||||
// etc.) should be exercised at least once in each of their branches.
|
||||
//
|
||||
// NOTE: in the Data field, make sure to have pointers populated to simulate
|
||||
// data is set as it would be in a real request. The data being correctly
|
||||
// populated in each case should be tested in integration tests, where the
|
||||
// data will actually flow to and from a real database. In this tests we
|
||||
// only care about producing the correct SQL.
|
||||
testCases := map[*template.Template][]*testCase{
|
||||
sqlResourceDelete: {
|
||||
{
|
||||
Name: "single path",
|
||||
Data: &sqlResourceRequest{
|
||||
SQLTemplate: new(sqltemplate.SQLTemplate),
|
||||
WriteEvent: resource.WriteEvent{
|
||||
Key: &resource.ResourceKey{},
|
||||
},
|
||||
},
|
||||
Expected: expected{
|
||||
"resource_delete_mysql_sqlite.sql": dialects{
|
||||
sqltemplate.MySQL,
|
||||
sqltemplate.SQLite,
|
||||
},
|
||||
"resource_delete_postgres.sql": dialects{
|
||||
sqltemplate.PostgreSQL,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
sqlResourceInsert: {
|
||||
{
|
||||
Name: "insert into resource",
|
||||
Data: &sqlResourceRequest{
|
||||
SQLTemplate: new(sqltemplate.SQLTemplate),
|
||||
WriteEvent: resource.WriteEvent{
|
||||
Key: &resource.ResourceKey{},
|
||||
},
|
||||
},
|
||||
Expected: expected{
|
||||
"resource_insert_mysql_sqlite.sql": dialects{
|
||||
sqltemplate.MySQL,
|
||||
sqltemplate.SQLite,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
sqlResourceUpdate: {
|
||||
{
|
||||
Name: "single path",
|
||||
Data: &sqlResourceRequest{
|
||||
SQLTemplate: new(sqltemplate.SQLTemplate),
|
||||
WriteEvent: resource.WriteEvent{
|
||||
Key: &resource.ResourceKey{},
|
||||
},
|
||||
},
|
||||
Expected: expected{
|
||||
"resource_update_mysql_sqlite.sql": dialects{
|
||||
sqltemplate.MySQL,
|
||||
sqltemplate.SQLite,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
sqlResourceRead: {
|
||||
{
|
||||
Name: "without resource version",
|
||||
Data: &sqlResourceReadRequest{
|
||||
SQLTemplate: new(sqltemplate.SQLTemplate),
|
||||
Request: &resource.ReadRequest{
|
||||
Key: &resource.ResourceKey{},
|
||||
},
|
||||
readResponse: new(readResponse),
|
||||
},
|
||||
Expected: expected{
|
||||
"resource_read_mysql_sqlite.sql": dialects{
|
||||
sqltemplate.MySQL,
|
||||
sqltemplate.SQLite,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
sqlResourceList: {
|
||||
{
|
||||
Name: "filter on namespace",
|
||||
Data: &sqlResourceListRequest{
|
||||
SQLTemplate: new(sqltemplate.SQLTemplate),
|
||||
Request: &resource.ListRequest{
|
||||
Limit: 10,
|
||||
Options: &resource.ListOptions{
|
||||
Key: &resource.ResourceKey{
|
||||
Namespace: "ns",
|
||||
},
|
||||
},
|
||||
},
|
||||
Response: new(resource.ResourceWrapper),
|
||||
},
|
||||
Expected: expected{
|
||||
"resource_list_mysql_sqlite.sql": dialects{
|
||||
sqltemplate.MySQL,
|
||||
sqltemplate.SQLite,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
sqlResourceHistoryList: {
|
||||
{
|
||||
Name: "single path",
|
||||
Data: &sqlResourceHistoryListRequest{
|
||||
SQLTemplate: new(sqltemplate.SQLTemplate),
|
||||
Request: &historyListRequest{
|
||||
Limit: 10,
|
||||
Options: &resource.ListOptions{
|
||||
Key: &resource.ResourceKey{
|
||||
Namespace: "ns",
|
||||
},
|
||||
},
|
||||
},
|
||||
Response: new(resource.ResourceWrapper),
|
||||
},
|
||||
Expected: expected{
|
||||
"resource_history_list_mysql_sqlite.sql": dialects{
|
||||
sqltemplate.MySQL,
|
||||
sqltemplate.SQLite,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
sqlResourceUpdateRV: {
|
||||
{
|
||||
Name: "single path",
|
||||
Data: &sqlResourceUpdateRVRequest{
|
||||
SQLTemplate: new(sqltemplate.SQLTemplate),
|
||||
},
|
||||
Expected: expected{
|
||||
"resource_update_rv_mysql_sqlite.sql": dialects{
|
||||
sqltemplate.MySQL,
|
||||
sqltemplate.SQLite,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
sqlResourceHistoryRead: {
|
||||
{
|
||||
Name: "single path",
|
||||
Data: &sqlResourceReadRequest{
|
||||
SQLTemplate: new(sqltemplate.SQLTemplate),
|
||||
Request: &resource.ReadRequest{
|
||||
ResourceVersion: 123,
|
||||
Key: &resource.ResourceKey{},
|
||||
},
|
||||
readResponse: new(readResponse),
|
||||
},
|
||||
Expected: expected{
|
||||
"resource_history_read_mysql_sqlite.sql": dialects{
|
||||
sqltemplate.MySQL,
|
||||
sqltemplate.SQLite,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
sqlResourceHistoryUpdateRV: {
|
||||
{
|
||||
Name: "single path",
|
||||
Data: &sqlResourceUpdateRVRequest{
|
||||
SQLTemplate: new(sqltemplate.SQLTemplate),
|
||||
},
|
||||
Expected: expected{
|
||||
"resource_history_update_rv_mysql_sqlite.sql": dialects{
|
||||
sqltemplate.MySQL,
|
||||
sqltemplate.SQLite,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
sqlResourceHistoryInsert: {
|
||||
{
|
||||
Name: "insert into resource_history",
|
||||
Data: &sqlResourceRequest{
|
||||
SQLTemplate: new(sqltemplate.SQLTemplate),
|
||||
WriteEvent: resource.WriteEvent{
|
||||
Key: &resource.ResourceKey{},
|
||||
},
|
||||
},
|
||||
Expected: expected{
|
||||
"resource_history_insert_mysql_sqlite.sql": dialects{
|
||||
sqltemplate.MySQL,
|
||||
sqltemplate.SQLite,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
sqlResourceVersionGet: {
|
||||
{
|
||||
Name: "single path",
|
||||
Data: &sqlResourceVersionRequest{
|
||||
SQLTemplate: new(sqltemplate.SQLTemplate),
|
||||
resourceVersion: new(resourceVersion),
|
||||
},
|
||||
Expected: expected{
|
||||
"resource_version_get_mysql.sql": dialects{
|
||||
sqltemplate.MySQL,
|
||||
},
|
||||
"resource_version_get_sqlite.sql": dialects{
|
||||
sqltemplate.SQLite,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
sqlResourceVersionInc: {
|
||||
{
|
||||
Name: "increment resource version",
|
||||
Data: &sqlResourceVersionRequest{
|
||||
SQLTemplate: new(sqltemplate.SQLTemplate),
|
||||
resourceVersion: &resourceVersion{
|
||||
ResourceVersion: 123,
|
||||
},
|
||||
},
|
||||
Expected: expected{
|
||||
"resource_version_inc_mysql_sqlite.sql": dialects{
|
||||
sqltemplate.MySQL,
|
||||
sqltemplate.SQLite,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
sqlResourceVersionInsert: {
|
||||
{
|
||||
Name: "single path",
|
||||
Data: &sqlResourceVersionRequest{
|
||||
SQLTemplate: new(sqltemplate.SQLTemplate),
|
||||
},
|
||||
Expected: expected{
|
||||
"resource_version_insert_mysql_sqlite.sql": dialects{
|
||||
sqltemplate.MySQL,
|
||||
sqltemplate.SQLite,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Execute test cases
|
||||
for tmpl, tcs := range testCases {
|
||||
t.Run(tmpl.Name(), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.Name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for filename, ds := range tc.Expected {
|
||||
t.Run(filename, func(t *testing.T) {
|
||||
// not parallel because we're sharing tc.Data, not
|
||||
// worth it deep cloning
|
||||
|
||||
rawQuery := string(testdata(t, filename))
|
||||
expectedQuery := sqltemplate.FormatSQL(rawQuery)
|
||||
|
||||
for _, d := range ds {
|
||||
t.Run(d.Name(), func(t *testing.T) {
|
||||
// not parallel for the same reason
|
||||
|
||||
tc.Data.SetDialect(d)
|
||||
err := tc.Data.Validate()
|
||||
require.NoError(t, err)
|
||||
got, err := sqltemplate.Execute(tmpl, tc.Data)
|
||||
require.NoError(t, err)
|
||||
got = sqltemplate.FormatSQL(got)
|
||||
require.Equal(t, expectedQuery, got)
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
31
pkg/storage/unified/sql/server.go
Normal file
31
pkg/storage/unified/sql/server.go
Normal file
@ -0,0 +1,31 @@
|
||||
package sql
|
||||
|
||||
import (
|
||||
"github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/infra/tracing"
|
||||
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/sql/db/dbimpl"
|
||||
)
|
||||
|
||||
// Creates a ResourceServer
|
||||
func ProvideResourceServer(db db.DB, cfg *setting.Cfg, features featuremgmt.FeatureToggles, tracer tracing.Tracer) (resource.ResourceServer, error) {
|
||||
opts := resource.ResourceServerOptions{
|
||||
Tracer: tracer,
|
||||
}
|
||||
|
||||
eDB, err := dbimpl.ProvideResourceDB(db, cfg, features, tracer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
store, err := NewBackendStore(backendOptions{DB: eDB, Tracer: tracer})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opts.Backend = store
|
||||
opts.Diagnostics = store
|
||||
opts.Lifecycle = store
|
||||
|
||||
return resource.NewResourceServer(opts)
|
||||
}
|
@ -7,7 +7,7 @@ import (
|
||||
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
|
||||
sqltemplate "github.com/grafana/grafana/pkg/services/store/entity/sqlstash/sqltemplate"
|
||||
sqltemplate "github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
)
|
||||
|
||||
// SQLTemplateIface is an autogenerated mock type for the SQLTemplateIface type
|
@ -7,7 +7,7 @@ import (
|
||||
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
|
||||
sqltemplate "github.com/grafana/grafana/pkg/services/store/entity/sqlstash/sqltemplate"
|
||||
sqltemplate "github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
|
||||
)
|
||||
|
||||
// WithResults is an autogenerated mock type for the WithResults type
|
1
pkg/storage/unified/sql/testdata/resource_delete_mysql_sqlite.sql
vendored
Normal file
1
pkg/storage/unified/sql/testdata/resource_delete_mysql_sqlite.sql
vendored
Normal file
@ -0,0 +1 @@
|
||||
DELETE FROM "resource" WHERE 1 = 1 AND "namespace" = ? AND "group" = ? AND "resource" = ? AND "name" = ?;
|
1
pkg/storage/unified/sql/testdata/resource_delete_postgres.sql
vendored
Normal file
1
pkg/storage/unified/sql/testdata/resource_delete_postgres.sql
vendored
Normal file
@ -0,0 +1 @@
|
||||
DELETE FROM "resource" WHERE 1 = 1 AND "namespace" = $1 AND "group" = $2 AND "resource" = $3 AND "name" = $4;
|
3
pkg/storage/unified/sql/testdata/resource_history_insert_mysql_sqlite.sql
vendored
Normal file
3
pkg/storage/unified/sql/testdata/resource_history_insert_mysql_sqlite.sql
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
INSERT INTO "resource_history"
|
||||
("guid", "group", "resource", "namespace", "name", "value", "action")
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?);
|
12
pkg/storage/unified/sql/testdata/resource_history_list_mysql_sqlite.sql
vendored
Normal file
12
pkg/storage/unified/sql/testdata/resource_history_list_mysql_sqlite.sql
vendored
Normal file
@ -0,0 +1,12 @@
|
||||
SELECT kv."resource_version", "value"
|
||||
FROM "resource_history" as kv
|
||||
JOIN (
|
||||
SELECT "guid", max("resource_version") AS "resource_version"
|
||||
FROM "resource_history" AS mkv
|
||||
WHERE 1 = 1 AND "resource_version" <= ? AND "namespace" = ?
|
||||
GROUP BY mkv."namespace", mkv."group", mkv."resource", mkv."name"
|
||||
) AS maxkv ON maxkv."guid" = kv."guid"
|
||||
WHERE kv."action" != 3
|
||||
ORDER BY kv."resource_version" ASC
|
||||
LIMIT ?, ?
|
||||
;
|
6
pkg/storage/unified/sql/testdata/resource_history_read_mysql_sqlite.sql
vendored
Normal file
6
pkg/storage/unified/sql/testdata/resource_history_read_mysql_sqlite.sql
vendored
Normal file
@ -0,0 +1,6 @@
|
||||
SELECT "resource_version", "value"
|
||||
FROM "resource_history"
|
||||
WHERE 1 = 1 AND "namespace" = ? AND "group" = ? AND "resource" = ? AND "name" = ? AND "resource_version" <= ?
|
||||
ORDER BY "resource_version" DESC
|
||||
LIMIT 1
|
||||
;
|
3
pkg/storage/unified/sql/testdata/resource_history_update_rv_mysql_sqlite.sql
vendored
Normal file
3
pkg/storage/unified/sql/testdata/resource_history_update_rv_mysql_sqlite.sql
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
UPDATE "resource_history" SET "resource_version" = ?
|
||||
WHERE "guid" = ?
|
||||
;
|
4
pkg/storage/unified/sql/testdata/resource_insert_mysql_sqlite.sql
vendored
Normal file
4
pkg/storage/unified/sql/testdata/resource_insert_mysql_sqlite.sql
vendored
Normal file
@ -0,0 +1,4 @@
|
||||
INSERT INTO "resource"
|
||||
("guid", "group", "resource", "namespace", "name", "value", "action")
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
;
|
6
pkg/storage/unified/sql/testdata/resource_list_mysql_sqlite.sql
vendored
Normal file
6
pkg/storage/unified/sql/testdata/resource_list_mysql_sqlite.sql
vendored
Normal file
@ -0,0 +1,6 @@
|
||||
SELECT "resource_version", "value"
|
||||
FROM "resource"
|
||||
WHERE 1 = 1 AND "namespace" = ?
|
||||
ORDER BY "resource_version" DESC
|
||||
LIMIT ?
|
||||
;
|
4
pkg/storage/unified/sql/testdata/resource_read_mysql_sqlite.sql
vendored
Normal file
4
pkg/storage/unified/sql/testdata/resource_read_mysql_sqlite.sql
vendored
Normal file
@ -0,0 +1,4 @@
|
||||
SELECT "resource_version", "value"
|
||||
FROM "resource"
|
||||
WHERE 1 = 1 AND "namespace" = ? AND "group" = ? AND "resource" = ? AND "name" = ?
|
||||
;
|
4
pkg/storage/unified/sql/testdata/resource_update_mysql_sqlite.sql
vendored
Normal file
4
pkg/storage/unified/sql/testdata/resource_update_mysql_sqlite.sql
vendored
Normal file
@ -0,0 +1,4 @@
|
||||
UPDATE "resource" SET "guid" = ?, "value" = ?, "action" = ?
|
||||
WHERE 1 =1 AND "group" = ? AND "resource" = ? AND "namespace" = ? AND "name" = ?
|
||||
;
|
||||
|
4
pkg/storage/unified/sql/testdata/resource_update_rv_mysql_sqlite.sql
vendored
Normal file
4
pkg/storage/unified/sql/testdata/resource_update_rv_mysql_sqlite.sql
vendored
Normal file
@ -0,0 +1,4 @@
|
||||
UPDATE "resource" SET "resource_version" = ?
|
||||
WHERE "guid" = ?
|
||||
;
|
||||
|
4
pkg/storage/unified/sql/testdata/resource_version_get_mysql.sql
vendored
Normal file
4
pkg/storage/unified/sql/testdata/resource_version_get_mysql.sql
vendored
Normal file
@ -0,0 +1,4 @@
|
||||
SELECT "resource_version"
|
||||
FROM "resource_version"
|
||||
WHERE 1 = 1 AND "group" = ? AND "resource" = ?
|
||||
FOR UPDATE;
|
4
pkg/storage/unified/sql/testdata/resource_version_get_sqlite.sql
vendored
Normal file
4
pkg/storage/unified/sql/testdata/resource_version_get_sqlite.sql
vendored
Normal file
@ -0,0 +1,4 @@
|
||||
SELECT "resource_version"
|
||||
FROM "resource_version"
|
||||
WHERE 1 = 1 AND "group" = ? AND "resource" = ?
|
||||
;
|
4
pkg/storage/unified/sql/testdata/resource_version_inc_mysql_sqlite.sql
vendored
Normal file
4
pkg/storage/unified/sql/testdata/resource_version_inc_mysql_sqlite.sql
vendored
Normal file
@ -0,0 +1,4 @@
|
||||
UPDATE "resource_version"
|
||||
SET "resource_version" = ?
|
||||
WHERE 1 = 1 AND "group" = ? AND "resource" = ?
|
||||
;
|
3
pkg/storage/unified/sql/testdata/resource_version_insert_mysql_sqlite.sql
vendored
Normal file
3
pkg/storage/unified/sql/testdata/resource_version_insert_mysql_sqlite.sql
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
INSERT INTO "resource_version"
|
||||
("group", "resource", "resource_version")
|
||||
VALUES (?, ?, 1);
|
Loading…
Reference in New Issue
Block a user