Storage: Show history+trash using the list command (#99009)

Co-authored-by: Stephanie Hingtgen <stephanie.hingtgen@grafana.com>
This commit is contained in:
Ryan McKinley 2025-01-17 15:54:25 +03:00 committed by GitHub
parent 67252dfa46
commit 356b32008b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
54 changed files with 1325 additions and 724 deletions

View File

@ -17,6 +17,18 @@ import (
"k8s.io/apimachinery/pkg/types"
)
// LabelKeyGetHistory is used to select object history for an given resource
const LabelKeyGetHistory = "grafana.app/get-history"
// LabelKeyGetTrash is used to list objects that have been (soft) deleted
const LabelKeyGetTrash = "grafana.app/get-trash"
// AnnoKeyKubectlLastAppliedConfig is the annotation kubectl writes with the entire previous config
const AnnoKeyKubectlLastAppliedConfig = "kubectl.kubernetes.io/last-applied-configuration"
// DeletedGeneration is set on Resources that have been (soft) deleted
const DeletedGeneration = int64(-999)
// Annotation keys
const AnnoKeyCreatedBy = "grafana.app/createdBy"

View File

@ -24,22 +24,25 @@ SELECT
WHERE dashboard.is_folder = false
AND dashboard.org_id = {{ .Arg .Query.OrgID }}
{{ if .Query.UseHistoryTable }}
{{ if .Query.UID }}
AND dashboard.uid = {{ .Arg .Query.UID }}
{{ end }}
{{ if .Query.Version }}
AND dashboard_version.version = {{ .Arg .Query.Version }}
AND dashboard_version.version = {{ .Arg .Query.Version }}
{{ else if .Query.LastID }}
AND dashboard_version.version < {{ .Arg .Query.LastID }}
AND dashboard_version.version < {{ .Arg .Query.LastID }}
{{ end }}
ORDER BY dashboard_version.version DESC
{{ else }}
{{ if .Query.UID }}
AND dashboard.uid = {{ .Arg .Query.UID }}
AND dashboard.uid = {{ .Arg .Query.UID }}
{{ else if .Query.LastID }}
AND dashboard.id > {{ .Arg .Query.LastID }}
AND dashboard.id > {{ .Arg .Query.LastID }}
{{ end }}
{{ if .Query.GetTrash }}
AND dashboard.deleted IS NOT NULL
AND dashboard.deleted IS NOT NULL
{{ else if .Query.LastID }}
AND dashboard.deleted IS NULL
AND dashboard.deleted IS NULL
{{ end }}
ORDER BY dashboard.id DESC
{{ end }}

View File

@ -98,8 +98,10 @@ func (a *dashboardSqlAccess) getRows(ctx context.Context, sql *legacysql.LegacyD
return nil, fmt.Errorf("execute template %q: %w", tmpl.Name(), err)
}
q := rawQuery
// q = sqltemplate.RemoveEmptyLines(rawQuery)
// fmt.Printf(">>%s [%+v]", q, req.GetArgs())
// if true {
// pretty := sqltemplate.RemoveEmptyLines(rawQuery)
// fmt.Printf("DASHBOARD QUERY: %s [%+v] // %+v\n", pretty, req.GetArgs(), query)
// }
rows, err := sql.DB.GetSqlxSession().Query(ctx, q, req.GetArgs()...)
if err != nil {
@ -267,6 +269,7 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows) (*dashboardRow, error) {
if deleted.Valid {
meta.SetDeletionTimestamp(ptr.To(metav1.NewTime(deleted.Time)))
meta.SetGeneration(utils.DeletedGeneration)
}
if message.String != "" {

View File

@ -192,6 +192,16 @@ func (a *dashboardSqlAccess) ListIterator(ctx context.Context, req *resource.Lis
return 0, err
}
switch req.Source {
case resource.ListRequest_HISTORY:
query.GetHistory = true
query.UID = req.Options.Key.Name
case resource.ListRequest_TRASH:
query.GetTrash = true
case resource.ListRequest_STORE:
// normal
}
listRV, err := sql.GetResourceVersion(ctx, "dashboard", "updated")
if err != nil {
return 0, err
@ -250,87 +260,6 @@ func (a *dashboardSqlAccess) Search(ctx context.Context, req *resource.ResourceS
return nil, fmt.Errorf("not yet (filter)")
}
/**
func (a *dashboardSqlAccess) History(ctx context.Context, req *resource.HistoryRequest) (*resource.HistoryResponse, error) {
info, err := claims.ParseNamespace(req.Key.Namespace)
if err == nil {
err = isDashboardKey(req.Key, false)
}
if err != nil {
return nil, err
}
token, err := readContinueToken(req.NextPageToken)
if err != nil {
return nil, err
}
if token.orgId > 0 && token.orgId != info.OrgID {
return nil, fmt.Errorf("token and orgID mismatch")
}
limit := int(req.Limit)
if limit < 1 {
limit = 15
}
query := &DashboardQuery{
OrgID: info.OrgID,
Limit: limit + 1,
LastID: token.id,
UID: req.Key.Name,
}
if req.ShowDeleted {
query.GetTrash = true
} else {
query.GetHistory = true
}
sql, err := a.sql(ctx)
if err != nil {
return nil, err
}
rows, err := a.getRows(ctx, sql, query)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
list := &resource.HistoryResponse{}
for rows.Next() {
if rows.err != nil || rows.row == nil {
return list, err
}
row := rows.row
partial := &metav1.PartialObjectMetadata{
ObjectMeta: row.Dash.ObjectMeta,
}
partial.UID = "" // it is not useful/helpful/accurate and just confusing now
val, err := json.Marshal(partial)
if err != nil {
return list, err
}
if len(list.Items) >= limit {
// if query.Requirements.Folder != nil {
// row.token.folder = *query.Requirements.Folder
// }
row.token.id = getVersionFromRV(row.RV) // Use the version as the increment
list.NextPageToken = row.token.String() // will skip this one but start here next time
return list, err
}
list.Items = append(list.Items, &resource.ResourceMeta{
ResourceVersion: row.RV,
PartialObjectMeta: val,
Size: int32(len(rows.Value())),
Hash: "??", // hash the full?
})
}
return list, err
}
**/
func (a *dashboardSqlAccess) ListRepositoryObjects(ctx context.Context, req *resource.ListRepositoryObjectsRequest) (*resource.ListRepositoryObjectsResponse, error) {
return nil, fmt.Errorf("not implemented")
}

View File

@ -15,6 +15,6 @@ SELECT
LEFT OUTER JOIN `grafana`.`user` as updated_user ON dashboard.updated_by = updated_user.id
WHERE dashboard.is_folder = false
AND dashboard.org_id = 2
AND dashboard.id > 22
AND dashboard.deleted IS NULL
AND dashboard.id > 22
AND dashboard.deleted IS NULL
ORDER BY dashboard.id DESC

View File

@ -15,5 +15,5 @@ SELECT
LEFT OUTER JOIN `grafana`.`user` as updated_user ON dashboard.updated_by = updated_user.id
WHERE dashboard.is_folder = false
AND dashboard.org_id = 2
AND dashboard.uid = 'UUU'
AND dashboard.uid = 'UUU'
ORDER BY dashboard.id DESC

View File

@ -16,5 +16,6 @@ SELECT
LEFT OUTER JOIN `grafana`.`user` as updated_user ON dashboard.updated_by = updated_user.id
WHERE dashboard.is_folder = false
AND dashboard.org_id = 2
AND dashboard_version.version = 3
AND dashboard.uid = 'UUU'
AND dashboard_version.version = 3
ORDER BY dashboard_version.version DESC

View File

@ -15,6 +15,6 @@ SELECT
LEFT OUTER JOIN `grafana`.`user` as updated_user ON dashboard.updated_by = updated_user.id
WHERE dashboard.is_folder = false
AND dashboard.org_id = 2
AND dashboard.uid = 'UUU'
AND dashboard.deleted IS NULL
AND dashboard.uid = 'UUU'
AND dashboard.deleted IS NULL
ORDER BY dashboard.id DESC

View File

@ -15,6 +15,6 @@ SELECT
LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard.updated_by = updated_user.id
WHERE dashboard.is_folder = false
AND dashboard.org_id = 2
AND dashboard.id > 22
AND dashboard.deleted IS NULL
AND dashboard.id > 22
AND dashboard.deleted IS NULL
ORDER BY dashboard.id DESC

View File

@ -15,5 +15,5 @@ SELECT
LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard.updated_by = updated_user.id
WHERE dashboard.is_folder = false
AND dashboard.org_id = 2
AND dashboard.uid = 'UUU'
AND dashboard.uid = 'UUU'
ORDER BY dashboard.id DESC

View File

@ -16,5 +16,6 @@ SELECT
LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard.updated_by = updated_user.id
WHERE dashboard.is_folder = false
AND dashboard.org_id = 2
AND dashboard_version.version = 3
AND dashboard.uid = 'UUU'
AND dashboard_version.version = 3
ORDER BY dashboard_version.version DESC

View File

@ -15,6 +15,6 @@ SELECT
LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard.updated_by = updated_user.id
WHERE dashboard.is_folder = false
AND dashboard.org_id = 2
AND dashboard.uid = 'UUU'
AND dashboard.deleted IS NULL
AND dashboard.uid = 'UUU'
AND dashboard.deleted IS NULL
ORDER BY dashboard.id DESC

View File

@ -15,6 +15,6 @@ SELECT
LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard.updated_by = updated_user.id
WHERE dashboard.is_folder = false
AND dashboard.org_id = 2
AND dashboard.id > 22
AND dashboard.deleted IS NULL
AND dashboard.id > 22
AND dashboard.deleted IS NULL
ORDER BY dashboard.id DESC

View File

@ -15,5 +15,5 @@ SELECT
LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard.updated_by = updated_user.id
WHERE dashboard.is_folder = false
AND dashboard.org_id = 2
AND dashboard.uid = 'UUU'
AND dashboard.uid = 'UUU'
ORDER BY dashboard.id DESC

View File

@ -16,5 +16,6 @@ SELECT
LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard.updated_by = updated_user.id
WHERE dashboard.is_folder = false
AND dashboard.org_id = 2
AND dashboard_version.version = 3
AND dashboard.uid = 'UUU'
AND dashboard_version.version = 3
ORDER BY dashboard_version.version DESC

View File

@ -15,6 +15,6 @@ SELECT
LEFT OUTER JOIN "grafana"."user" as updated_user ON dashboard.updated_by = updated_user.id
WHERE dashboard.is_folder = false
AND dashboard.org_id = 2
AND dashboard.uid = 'UUU'
AND dashboard.deleted IS NULL
AND dashboard.uid = 'UUU'
AND dashboard.deleted IS NULL
ORDER BY dashboard.id DESC

View File

@ -14,8 +14,11 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/selection"
"k8s.io/apiserver/pkg/storage"
"github.com/grafana/grafana/pkg/apimachinery/utils"
grafanaregistry "github.com/grafana/grafana/pkg/apiserver/registry/generic"
"github.com/grafana/grafana/pkg/storage/unified/resource"
)
@ -39,6 +42,38 @@ func toListRequest(k *resource.ResourceKey, opts storage.ListOptions) (*resource
for _, r := range requirements {
v := r.Key()
// Parse the history request from labels
if v == utils.LabelKeyGetHistory || v == utils.LabelKeyGetTrash {
if len(requirements) != 1 {
return nil, predicate, apierrors.NewBadRequest("single label supported with: " + v)
}
if !opts.Predicate.Field.Empty() {
return nil, predicate, apierrors.NewBadRequest("field selector not supported with: " + v)
}
if r.Operator() != selection.Equals {
return nil, predicate, apierrors.NewBadRequest("only = operator supported with: " + v)
}
vals := r.Values().List()
if len(vals) != 1 {
return nil, predicate, apierrors.NewBadRequest("expecting single value for: " + v)
}
if v == utils.LabelKeyGetTrash {
req.Source = resource.ListRequest_TRASH
if vals[0] != "true" {
return nil, predicate, apierrors.NewBadRequest("expecting true for: " + v)
}
} else {
req.Source = resource.ListRequest_HISTORY
req.Options.Key.Name = vals[0]
}
req.Options.Labels = nil
req.Options.Fields = nil
return req, storage.Everything, nil
}
req.Options.Labels = append(req.Options.Labels, &resource.Requirement{
Key: v,
Operator: string(r.Operator()),

View File

@ -21,6 +21,8 @@ import (
_ "gocloud.dev/blob/memblob"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"github.com/grafana/grafana/pkg/apimachinery/utils"
)
type CDKBackendOptions struct {
@ -192,7 +194,7 @@ func (s *cdkBackend) ReadResource(ctx context.Context, req *ReadRequest) *Backen
err = nil
}
}
if err == nil && isDeletedMarker(raw) {
if err == nil && isDeletedValue(raw) {
raw = nil
}
if raw == nil {
@ -206,11 +208,11 @@ func (s *cdkBackend) ReadResource(ctx context.Context, req *ReadRequest) *Backen
}
}
func isDeletedMarker(raw []byte) bool {
if bytes.Contains(raw, []byte(`"DeletedMarker"`)) {
func isDeletedValue(raw []byte) bool {
if bytes.Contains(raw, []byte(`"generation":-999`)) {
tmp := &unstructured.Unstructured{}
err := tmp.UnmarshalJSON(raw)
if err == nil && tmp.GetKind() == "DeletedMarker" {
if err == nil && tmp.GetGeneration() == utils.DeletedGeneration {
return true
}
}
@ -218,6 +220,10 @@ func isDeletedMarker(raw []byte) bool {
}
func (s *cdkBackend) ListIterator(ctx context.Context, req *ListRequest, cb func(ListIterator) error) (int64, error) {
if req.Source != ListRequest_STORE {
return 0, fmt.Errorf("listing from history not supported in CDK backend")
}
resources, err := buildTree(ctx, s, req.Options.Key)
if err != nil {
return 0, err
@ -286,7 +292,7 @@ func (c *cdkListIterator) Next() bool {
c.err = err
return false
}
if !isDeletedMarker(raw) {
if !isDeletedValue(raw) {
c.currentRV = latest.rv
c.currentKey = latest.key
c.currentVal = raw

View File

@ -1,37 +0,0 @@
package resource
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
)
// This object is written when an object is deleted
type DeletedMarker struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DeletedMarker) DeepCopyInto(out *DeletedMarker) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeletedMarker.
func (in *DeletedMarker) DeepCopy() *DeletedMarker {
if in == nil {
return nil
}
out := new(DeletedMarker)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *DeletedMarker) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}

File diff suppressed because it is too large Load Diff

View File

@ -225,6 +225,12 @@ enum ResourceVersionMatch {
}
message ListRequest {
enum Source {
STORE = 0; // the standard place
HISTORY = 1;
TRASH = 2;
}
// Starting from the requested page (other query parameters must match!)
string next_page_token = 1;
@ -240,6 +246,9 @@ message ListRequest {
// Filtering
ListOptions options = 5;
// Select values from history or trash
Source source = 6;
}
message ListResponse {

View File

@ -371,6 +371,13 @@ func (s *server) newEvent(ctx context.Context, user claims.AuthInfo, key *Resour
s.log.Error("object must not include a resource version", "key", key)
}
// Make sure the command labels are not saved
for k := range obj.GetLabels() {
if k == utils.LabelKeyGetHistory || k == utils.LabelKeyGetTrash {
return nil, NewBadRequestError("can not save label: " + k)
}
}
check := authz.CheckRequest{
Verb: utils.VerbCreate,
Group: key.Group,
@ -612,7 +619,7 @@ func (s *server) Delete(ctx context.Context, req *DeleteRequest) (*DeleteRespons
if !ok {
return nil, apierrors.NewBadRequest("unable to get user")
}
marker := &DeletedMarker{}
marker := &unstructured.Unstructured{}
err = json.Unmarshal(latest.Value, marker)
if err != nil {
return nil, apierrors.NewBadRequest(
@ -627,12 +634,9 @@ func (s *server) Delete(ctx context.Context, req *DeleteRequest) (*DeleteRespons
obj.SetManagedFields(nil)
obj.SetFinalizers(nil)
obj.SetUpdatedBy(requester.GetUID())
marker.TypeMeta = metav1.TypeMeta{
Kind: "DeletedMarker",
APIVersion: "common.grafana.app/v0alpha1", // ?? or can we stick this in common?
}
marker.Annotations["RestoreResourceVersion"] = fmt.Sprintf("%d", event.PreviousRV)
event.Value, err = json.Marshal(marker)
obj.SetGeneration(utils.DeletedGeneration)
obj.SetAnnotation(utils.AnnoKeyKubectlLastAppliedConfig, "") // clears it
event.Value, err = marker.MarshalJSON()
if err != nil {
return nil, apierrors.NewBadRequest(
fmt.Sprintf("unable creating deletion marker, %v", err))
@ -693,6 +697,15 @@ func (s *server) List(ctx context.Context, req *ListRequest) (*ListResponse, err
ctx, span := s.tracer.Start(ctx, "storage_server.List")
defer span.End()
// The history + trash queries do not yet support additional filters
if req.Source != ListRequest_STORE {
if len(req.Options.Fields) > 0 || len(req.Options.Labels) > 0 {
return &ListResponse{
Error: NewBadRequestError("unexpected field/label selector for history query"),
}, nil
}
}
user, ok := claims.From(ctx)
if !ok || user == nil {
return &ListResponse{
@ -702,6 +715,13 @@ func (s *server) List(ctx context.Context, req *ListRequest) (*ListResponse, err
}}, nil
}
// Do not allow label query for trash/history
for _, v := range req.Options.Labels {
if v.Key == utils.LabelKeyGetHistory || v.Key == utils.LabelKeyGetTrash {
return &ListResponse{Error: NewBadRequestError("history and trash must be requested as source")}, nil
}
}
if req.Limit < 1 {
req.Limit = 50 // default max 50 items in a page
}

View File

@ -119,14 +119,27 @@ func TestSimpleServer(t *testing.T) {
obj.SetAnnotation("test", "hello")
obj.SetUpdatedTimestampMillis(now)
obj.SetUpdatedBy(testUserA.GetUID())
obj.SetLabels(map[string]string{
utils.LabelKeyGetTrash: "", // should not be allowed to save this!
})
raw, err = json.Marshal(tmp)
require.NoError(t, err)
updated, err := server.Update(ctx, &UpdateRequest{
Key: key,
Value: raw,
ResourceVersion: created.ResourceVersion})
require.NoError(t, err)
require.Equal(t, int32(400), updated.Error.Code) // bad request
// remove the invalid labels
obj.SetLabels(nil)
raw, err = json.Marshal(tmp)
require.NoError(t, err)
updated, err = server.Update(ctx, &UpdateRequest{
Key: key,
Value: raw,
ResourceVersion: created.ResourceVersion})
require.NoError(t, err)
require.Nil(t, updated.Error)
require.True(t, updated.ResourceVersion > created.ResourceVersion)

View File

@ -34,9 +34,10 @@ type Backend interface {
}
type BackendOptions struct {
DBProvider db.DBProvider
Tracer trace.Tracer
PollingInterval time.Duration
DBProvider db.DBProvider
Tracer trace.Tracer
PollingInterval time.Duration
SkipDataMigration bool
}
func NewBackend(opts BackendOptions) (Backend, error) {
@ -53,12 +54,13 @@ func NewBackend(opts BackendOptions) (Backend, error) {
pollingInterval = defaultPollingInterval
}
return &backend{
done: ctx.Done(),
cancel: cancel,
log: log.New("sql-resource-server"),
tracer: opts.Tracer,
dbProvider: opts.DBProvider,
pollingInterval: pollingInterval,
done: ctx.Done(),
cancel: cancel,
log: log.New("sql-resource-server"),
tracer: opts.Tracer,
dbProvider: opts.DBProvider,
pollingInterval: pollingInterval,
skipDataMigration: opts.SkipDataMigration,
}, nil
}
@ -74,9 +76,10 @@ type backend struct {
tracer trace.Tracer
// database
dbProvider db.DBProvider
db db.DB
dialect sqltemplate.Dialect
dbProvider db.DBProvider
db db.DB
dialect sqltemplate.Dialect
skipDataMigration bool
// watch streaming
//stream chan *resource.WatchEvent
@ -103,6 +106,12 @@ func (b *backend) initLocked(ctx context.Context) error {
return fmt.Errorf("no dialect for driver %q", driverName)
}
// Process any data manipulation migrations
err = b.runStartupDataMigrations(ctx)
if err != nil {
return err
}
return b.db.PingContext(ctx)
}
@ -477,13 +486,17 @@ func (b *backend) ReadResource(ctx context.Context, req *resource.ReadRequest) *
}
func (b *backend) ListIterator(ctx context.Context, req *resource.ListRequest, cb func(resource.ListIterator) error) (int64, error) {
_, span := b.tracer.Start(ctx, tracePrefix+"List")
ctx, span := b.tracer.Start(ctx, tracePrefix+"List")
defer span.End()
if req.Options == nil || req.Options.Key.Group == "" || req.Options.Key.Resource == "" {
return 0, fmt.Errorf("missing group or resource")
}
if req.Source != resource.ListRequest_STORE {
return b.getHistory(ctx, req, cb)
}
// TODO: think about how to handler VersionMatch. We should be able to use latest for the first page (only).
// TODO: add support for RemainingItemCount
@ -647,6 +660,48 @@ func (b *backend) listAtRevision(ctx context.Context, req *resource.ListRequest,
return iter.listRV, err
}
// listLatest fetches the resources from the resource table.
func (b *backend) getHistory(ctx context.Context, req *resource.ListRequest, cb func(resource.ListIterator) error) (int64, error) {
listReq := sqlGetHistoryRequest{
SQLTemplate: sqltemplate.New(b.dialect),
Key: req.Options.Key,
Trash: req.Source == resource.ListRequest_TRASH,
}
iter := &listIter{}
if req.NextPageToken != "" {
continueToken, err := GetContinueToken(req.NextPageToken)
if err != nil {
return 0, fmt.Errorf("get continue token: %w", err)
}
listReq.StartRV = continueToken.ResourceVersion
}
err := b.db.WithTx(ctx, ReadCommittedRO, func(ctx context.Context, tx db.Tx) error {
var err error
iter.listRV, err = fetchLatestRV(ctx, tx, b.dialect, req.Options.Key.Group, req.Options.Key.Resource)
if err != nil {
return err
}
rows, err := dbutil.QueryRows(ctx, tx, sqlResourceHistoryGet, listReq)
if rows != nil {
defer func() {
if err := rows.Close(); err != nil {
b.log.Warn("listLatest error closing rows", "error", err)
}
}()
}
if err != nil {
return err
}
iter.rows = rows
return cb(iter)
})
return iter.listRV, err
}
func (b *backend) WatchWriteEvents(ctx context.Context) (<-chan *resource.WrittenEvent, error) {
// Get the latest RV
since, err := b.listLatestRVs(ctx)

View File

@ -62,7 +62,10 @@ func setupBackendTest(t *testing.T) (testBackend, context.Context) {
ctx := testutil.NewDefaultTestContext(t)
dbp := test.NewDBProviderMatchWords(t)
b, err := NewBackend(BackendOptions{DBProvider: dbp})
b, err := NewBackend(BackendOptions{
DBProvider: dbp,
SkipDataMigration: true, // Calling migrations makes startup SQL calls (avoid the mock)
})
require.NoError(t, err)
require.NotNil(t, b)
@ -109,7 +112,7 @@ func TestBackend_Init(t *testing.T) {
ctx := testutil.NewDefaultTestContext(t)
dbp := test.NewDBProviderWithPing(t)
b, err := NewBackend(BackendOptions{DBProvider: dbp})
b, err := NewBackend(BackendOptions{DBProvider: dbp, SkipDataMigration: true})
require.NoError(t, err)
require.NotNil(t, b)
@ -166,7 +169,7 @@ func TestBackend_Init(t *testing.T) {
ctx := testutil.NewDefaultTestContext(t)
dbp := test.NewDBProviderWithPing(t)
b, err := NewBackend(BackendOptions{DBProvider: dbp})
b, err := NewBackend(BackendOptions{DBProvider: dbp, SkipDataMigration: true})
require.NoError(t, err)
require.NotNil(t, dbp.DB)
@ -182,7 +185,7 @@ func TestBackend_IsHealthy(t *testing.T) {
ctx := testutil.NewDefaultTestContext(t)
dbp := test.NewDBProviderWithPing(t)
b, err := NewBackend(BackendOptions{DBProvider: dbp})
b, err := NewBackend(BackendOptions{DBProvider: dbp, SkipDataMigration: true})
require.NoError(t, err)
require.NotNil(t, dbp.DB)

View File

@ -0,0 +1,9 @@
SELECT
{{ .Ident "guid" }},
{{ .Ident "value" }},
{{ .Ident "group" }},
{{ .Ident "resource" }},
{{ .Ident "previous_resource_version" }}
FROM {{ .Ident "resource_history" }}
WHERE {{ .Ident "action" }} = 3
AND {{ .Ident "value" }} LIKE {{ .Arg .MarkerQuery }};

View File

@ -0,0 +1,5 @@
SELECT {{ .Ident "value" }}
FROM {{ .Ident "resource_history" }}
WHERE {{ .Ident "group" }} = {{ .Arg .Group }}
AND {{ .Ident "resource" }} = {{ .Arg .Resource }}
AND {{ .Ident "resource_version" }} = {{ .Arg .RV }};

View File

@ -0,0 +1,4 @@
UPDATE {{ .Ident "resource_history" }}
SET {{ .Ident "value" }} = {{ .Arg .Value }}
WHERE {{ .Ident "guid" }} = {{ .Arg .GUID }}
;

View File

@ -0,0 +1,4 @@
DELETE FROM {{ .Ident "resource_history" }}
WHERE 1 = 1
AND {{ .Ident "guid" }} = {{ .Arg .GUID }}

View File

@ -0,0 +1,21 @@
SELECT
{{ .Ident "resource_version" }},
{{ .Ident "namespace" }},
{{ .Ident "name" }},
{{ .Ident "folder" }},
{{ .Ident "value" }}
FROM {{ .Ident "resource_history" }}
WHERE 1 = 1
AND {{ .Ident "namespace" }} = {{ .Arg .Key.Namespace }}
AND {{ .Ident "group" }} = {{ .Arg .Key.Group }}
AND {{ .Ident "resource" }} = {{ .Arg .Key.Resource }}
{{ if .Key.Name }}
AND {{ .Ident "name" }} = {{ .Arg .Key.Name }}
{{ end }}
{{ if .Trash }}
AND {{ .Ident "action" }} = 3
{{ end }}
{{ if (gt .StartRV 0) }}
AND {{ .Ident "resource_version" }} > {{ .Arg .StartRV }}
{{ end }}
ORDER BY resource_version DESC

View File

@ -0,0 +1,133 @@
package sql
import (
"context"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/storage/unified/sql/db"
"github.com/grafana/grafana/pkg/storage/unified/sql/dbutil"
"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
)
// This runs functions before the server is returned as healthy
func (b *backend) runStartupDataMigrations(ctx context.Context) error {
if b.skipDataMigration {
return nil
}
type migrateRow struct {
GUID string
Marker *unstructured.Unstructured
Group string
Resource string
PreviousRV int64
}
// Migrate DeletedMarker to regular resource
err := b.db.WithTx(ctx, ReadCommitted, func(ctx context.Context, tx db.Tx) error {
req := &sqlMigrationQueryRequest{
SQLTemplate: sqltemplate.New(b.dialect),
MarkerQuery: `{"kind":"DeletedMarker"%`,
}
// 1. Find rows with the existing deletion marker
rows, err := dbutil.QueryRows(ctx, tx, sqlMigratorGetDeletionMarkers, req)
if err != nil {
return err
}
migrateRows := make([]migrateRow, 0)
for rows.Next() {
item := migrateRow{Marker: &unstructured.Unstructured{}}
err = rows.Scan(&item.GUID, &req.Value, &item.Group, &item.Resource, &item.PreviousRV)
if err != nil {
return err
}
err = item.Marker.UnmarshalJSON([]byte(req.Value))
if err != nil {
return err
}
migrateRows = append(migrateRows, item)
}
err = rows.Close()
if err != nil {
return err
}
for _, item := range migrateRows {
// 2. Load the previous value referenced by that marker
req := &sqlMigrationQueryRequest{
SQLTemplate: sqltemplate.New(b.dialect),
Group: item.Group,
Resource: item.Resource,
RV: item.PreviousRV,
GUID: item.GUID,
}
rows, err = dbutil.QueryRows(ctx, tx, sqlMigratorGetValueFromRV, req)
if err != nil {
return err
}
if rows.Next() {
err = rows.Scan(&req.Value)
if err != nil {
return err
}
}
err = rows.Close()
if err != nil {
return err
}
req.Reset()
if len(req.Value) > 0 {
previous := &unstructured.Unstructured{}
err = previous.UnmarshalJSON([]byte(req.Value))
if err != nil {
return err
}
// 3. Prepare a new payload
metaMarker, _ := utils.MetaAccessor(item.Marker)
metaPrev, _ := utils.MetaAccessor(previous)
metaPrev.SetDeletionTimestamp(metaMarker.GetDeletionTimestamp())
metaPrev.SetFinalizers(nil)
metaPrev.SetManagedFields(nil)
metaPrev.SetGeneration(utils.DeletedGeneration)
metaPrev.SetAnnotation(utils.AnnoKeyKubectlLastAppliedConfig, "") // clears it
ts, _ := metaMarker.GetUpdatedTimestamp()
if ts != nil {
metaPrev.SetUpdatedTimestamp(ts)
}
buff, err := previous.MarshalJSON()
if err != nil {
return err
}
req.Value = string(buff)
// 4. Update the SQL row with this new value
b.log.Info("Migrating DeletedMarker", "guid", req.GUID, "group", req.Group, "resource", req.Resource)
_, err = dbutil.Exec(ctx, tx, sqlMigratorUpdateValueWithGUID, req)
if err != nil {
return err
}
} else {
// 5. If the previous version is missing, we delete it -- there is nothing to help us restore anyway
b.log.Warn("Removing orphan deletion marker", "guid", req.GUID, "group", req.Group, "resource", req.Resource)
_, err = dbutil.Exec(ctx, tx, sqlResourceHistoryDelete, &sqlResourceHistoryDeleteRequest{
SQLTemplate: sqltemplate.New(b.dialect),
GUID: req.GUID,
})
if err != nil {
return err
}
}
}
return nil
})
return err
}

View File

@ -42,6 +42,8 @@ var (
sqlResoureceHistoryUpdateUid = mustTemplate("resource_history_update_uid.sql")
sqlResourceHistoryInsert = mustTemplate("resource_history_insert.sql")
sqlResourceHistoryPoll = mustTemplate("resource_history_poll.sql")
sqlResourceHistoryGet = mustTemplate("resource_history_get.sql")
sqlResourceHistoryDelete = mustTemplate("resource_history_delete.sql")
// sqlResourceLabelsInsert = mustTemplate("resource_labels_insert.sql")
sqlResourceVersionGet = mustTemplate("resource_version_get.sql")
@ -51,6 +53,10 @@ var (
sqlResourceBlobInsert = mustTemplate("resource_blob_insert.sql")
sqlResourceBlobQuery = mustTemplate("resource_blob_query.sql")
sqlMigratorGetDeletionMarkers = mustTemplate("migrator_get_deletion_markers.sql")
sqlMigratorGetValueFromRV = mustTemplate("migrator_get_value_from_rv.sql")
sqlMigratorUpdateValueWithGUID = mustTemplate("migrator_update_value_with_guid.sql")
)
// TxOptions.
@ -197,6 +203,27 @@ func (r sqlResourceHistoryListRequest) Results() (*resource.ResourceWrapper, err
}, nil
}
type sqlResourceHistoryDeleteRequest struct {
sqltemplate.SQLTemplate
GUID string
// TODO, add other constraints
}
func (r *sqlResourceHistoryDeleteRequest) Validate() error {
return nil // TODO
}
type sqlGetHistoryRequest struct {
sqltemplate.SQLTemplate
Key *resource.ResourceKey
Trash bool // only deleted items
StartRV int64 // from NextPageToken
}
func (r sqlGetHistoryRequest) Validate() error {
return nil // TODO
}
// update resource history
type sqlResourceHistoryUpdateRequest struct {
@ -303,3 +330,19 @@ func (r *sqlResourceVersionListRequest) Results() (*groupResourceVersion, error)
x := *r.groupResourceVersion
return &x, nil
}
// This holds all the variables used in migration queries
type sqlMigrationQueryRequest struct {
sqltemplate.SQLTemplate
MarkerQuery string //
Group string
Resource string
RV int64
GUID string
Value string
}
func (r sqlMigrationQueryRequest) Validate() error {
return nil // TODO
}

View File

@ -207,6 +207,46 @@ func TestUnifiedStorageQueries(t *testing.T) {
},
},
sqlResourceHistoryGet: {
{
Name: "read object history",
Data: &sqlGetHistoryRequest{
SQLTemplate: mocks.NewTestingSQLTemplate(),
Key: &resource.ResourceKey{
Namespace: "nn",
Group: "gg",
Resource: "rr",
Name: "name",
},
},
},
{
Name: "read trash",
Data: &sqlGetHistoryRequest{
SQLTemplate: mocks.NewTestingSQLTemplate(),
Key: &resource.ResourceKey{
Namespace: "nn",
Group: "gg",
Resource: "rr",
},
Trash: true,
},
},
{
Name: "read trash second page",
Data: &sqlGetHistoryRequest{
SQLTemplate: mocks.NewTestingSQLTemplate(),
Key: &resource.ResourceKey{
Namespace: "nn",
Group: "gg",
Resource: "rr",
},
Trash: true,
StartRV: 123456,
},
},
},
sqlResourceVersionGet: {
{
Name: "single path",
@ -317,5 +357,44 @@ func TestUnifiedStorageQueries(t *testing.T) {
},
},
},
sqlResourceHistoryDelete: {
{
Name: "guid",
Data: &sqlResourceHistoryDeleteRequest{
SQLTemplate: mocks.NewTestingSQLTemplate(),
GUID: `xxxx`,
},
},
},
sqlMigratorGetDeletionMarkers: {
{
Name: "list",
Data: &sqlMigrationQueryRequest{
SQLTemplate: mocks.NewTestingSQLTemplate(),
MarkerQuery: `{"kind":"DeletedMarker"%`,
},
},
},
sqlMigratorGetValueFromRV: {
{
Name: "get",
Data: &sqlMigrationQueryRequest{
SQLTemplate: mocks.NewTestingSQLTemplate(),
Group: "ggg",
Resource: "rrr",
RV: 1234,
},
},
},
sqlMigratorUpdateValueWithGUID: {
{
Name: "update",
Data: &sqlMigrationQueryRequest{
SQLTemplate: mocks.NewTestingSQLTemplate(),
GUID: "ggggg",
Value: "{new value}",
},
},
},
}})
}

View File

@ -0,0 +1,9 @@
SELECT
`guid`,
`value`,
`group`,
`resource`,
`previous_resource_version`
FROM `resource_history`
WHERE `action` = 3
AND `value` LIKE '{"kind":"DeletedMarker"%';

View File

@ -0,0 +1,5 @@
SELECT `value`
FROM `resource_history`
WHERE `group` = 'ggg'
AND `resource` = 'rrr'
AND `resource_version` = 1234;

View File

@ -0,0 +1,4 @@
UPDATE `resource_history`
SET `value` = '{new value}'
WHERE `guid` = 'ggggg'
;

View File

@ -0,0 +1,3 @@
DELETE FROM `resource_history`
WHERE 1 = 1
AND `guid` = 'xxxx'

View File

@ -0,0 +1,13 @@
SELECT
`resource_version`,
`namespace`,
`name`,
`folder`,
`value`
FROM `resource_history`
WHERE 1 = 1
AND `namespace` = 'nn'
AND `group` = 'gg'
AND `resource` = 'rr'
AND `name` = 'name'
ORDER BY resource_version DESC

View File

@ -0,0 +1,14 @@
SELECT
`resource_version`,
`namespace`,
`name`,
`folder`,
`value`
FROM `resource_history`
WHERE 1 = 1
AND `namespace` = 'nn'
AND `group` = 'gg'
AND `resource` = 'rr'
AND `action` = 3
AND `resource_version` > 123456
ORDER BY resource_version DESC

View File

@ -0,0 +1,13 @@
SELECT
`resource_version`,
`namespace`,
`name`,
`folder`,
`value`
FROM `resource_history`
WHERE 1 = 1
AND `namespace` = 'nn'
AND `group` = 'gg'
AND `resource` = 'rr'
AND `action` = 3
ORDER BY resource_version DESC

View File

@ -0,0 +1,9 @@
SELECT
"guid",
"value",
"group",
"resource",
"previous_resource_version"
FROM "resource_history"
WHERE "action" = 3
AND "value" LIKE '{"kind":"DeletedMarker"%';

View File

@ -0,0 +1,5 @@
SELECT "value"
FROM "resource_history"
WHERE "group" = 'ggg'
AND "resource" = 'rrr'
AND "resource_version" = 1234;

View File

@ -0,0 +1,4 @@
UPDATE "resource_history"
SET "value" = '{new value}'
WHERE "guid" = 'ggggg'
;

View File

@ -0,0 +1,3 @@
DELETE FROM "resource_history"
WHERE 1 = 1
AND "guid" = 'xxxx'

View File

@ -0,0 +1,13 @@
SELECT
"resource_version",
"namespace",
"name",
"folder",
"value"
FROM "resource_history"
WHERE 1 = 1
AND "namespace" = 'nn'
AND "group" = 'gg'
AND "resource" = 'rr'
AND "name" = 'name'
ORDER BY resource_version DESC

View File

@ -0,0 +1,14 @@
SELECT
"resource_version",
"namespace",
"name",
"folder",
"value"
FROM "resource_history"
WHERE 1 = 1
AND "namespace" = 'nn'
AND "group" = 'gg'
AND "resource" = 'rr'
AND "action" = 3
AND "resource_version" > 123456
ORDER BY resource_version DESC

View File

@ -0,0 +1,13 @@
SELECT
"resource_version",
"namespace",
"name",
"folder",
"value"
FROM "resource_history"
WHERE 1 = 1
AND "namespace" = 'nn'
AND "group" = 'gg'
AND "resource" = 'rr'
AND "action" = 3
ORDER BY resource_version DESC

View File

@ -0,0 +1,9 @@
SELECT
"guid",
"value",
"group",
"resource",
"previous_resource_version"
FROM "resource_history"
WHERE "action" = 3
AND "value" LIKE '{"kind":"DeletedMarker"%';

View File

@ -0,0 +1,5 @@
SELECT "value"
FROM "resource_history"
WHERE "group" = 'ggg'
AND "resource" = 'rrr'
AND "resource_version" = 1234;

View File

@ -0,0 +1,4 @@
UPDATE "resource_history"
SET "value" = '{new value}'
WHERE "guid" = 'ggggg'
;

View File

@ -0,0 +1,3 @@
DELETE FROM "resource_history"
WHERE 1 = 1
AND "guid" = 'xxxx'

View File

@ -0,0 +1,13 @@
SELECT
"resource_version",
"namespace",
"name",
"folder",
"value"
FROM "resource_history"
WHERE 1 = 1
AND "namespace" = 'nn'
AND "group" = 'gg'
AND "resource" = 'rr'
AND "name" = 'name'
ORDER BY resource_version DESC

View File

@ -0,0 +1,14 @@
SELECT
"resource_version",
"namespace",
"name",
"folder",
"value"
FROM "resource_history"
WHERE 1 = 1
AND "namespace" = 'nn'
AND "group" = 'gg'
AND "resource" = 'rr'
AND "action" = 3
AND "resource_version" > 123456
ORDER BY resource_version DESC

View File

@ -0,0 +1,13 @@
SELECT
"resource_version",
"namespace",
"name",
"folder",
"value"
FROM "resource_history"
WHERE 1 = 1
AND "namespace" = 'nn'
AND "group" = 'gg'
AND "resource" = 'rr'
AND "action" = 3
ORDER BY resource_version DESC