IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2024-11-28 03:34:15 -06:00
Code Issues Packages Projects Releases Wiki Activity

Storage: Cleanup object history API (#56215)

Browse Source
This commit is contained in:
Ryan McKinley 2022-10-04 11:57:26 -07:00 committed by GitHub
parent 7715672fb3
commit 4fc9b9aa35
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 658 additions and 411 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.betterer.results
View File

@ -6030,11 +6030,9 @@ exports[`better eslint`] = {
[0, 0, 0, "Unexpected any. Specify a different type.", "11"],
[0, 0, 0, "Unexpected any. Specify a different type.", "12"],
[0, 0, 0, "Unexpected any. Specify a different type.", "13"],
[0, 0, 0, "Unexpected any. Specify a different type.", "14"],
[0, 0, 0, "Do not use any type assertions.", "15"],
[0, 0, 0, "Unexpected any. Specify a different type.", "16"],
[0, 0, 0, "Unexpected any. Specify a different type.", "17"],
[0, 0, 0, "Unexpected any. Specify a different type.", "18"]
[0, 0, 0, "Do not use any type assertions.", "14"],
[0, 0, 0, "Unexpected any. Specify a different type.", "15"],
[0, 0, 0, "Unexpected any. Specify a different type.", "16"]
],
"public/app/plugins/datasource/elasticsearch/components/AddRemove.tsx:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"]

100
pkg/services/store/object/dummy/dummy_server.go
View File

@ -17,9 +17,15 @@ import (
"github.com/grafana/grafana/pkg/setting"
)
type ObjectVersionWithBody struct {
*object.ObjectVersionInfo `json:"info,omitempty"`
Body []byte `json:"body,omitempty"`
}
type RawObjectWithHistory struct {
*object.RawObject `json:"rawObject,omitempty"`
History []*object.RawObject `json:"history,omitempty"`
History []*ObjectVersionWithBody `json:"history,omitempty"`
}
var (
@ -74,13 +80,26 @@ func (i dummyObjectServer) findObject(ctx context.Context, uid string, kind stri
getLatestVersion := version == ""
if getLatestVersion {
objVersion := obj.History[len(obj.History)-1]
return obj, objVersion, nil
return obj, obj.RawObject, nil
}
for _, objVersion := range obj.History {
if objVersion.Version == version {
return obj, objVersion, nil
copy := &object.RawObject{
UID: obj.UID,
Kind: obj.Kind,
Created: obj.Created,
CreatedBy: obj.CreatedBy,
Modified: objVersion.Modified,
ModifiedBy: objVersion.ModifiedBy,
ETag: objVersion.ETag,
Version: objVersion.Version,
// Body is added from the dummy server cache (it does not exist in ObjectVersionInfo)
Body: objVersion.Body,
}
return obj, copy, nil
}
}
@ -125,7 +144,7 @@ func createContentsHash(contents []byte) string {
}
func (i dummyObjectServer) update(ctx context.Context, r *object.WriteObjectRequest, namespace string) (*object.WriteObjectResponse, error) {
var updated *object.RawObject
rsp := &object.WriteObjectResponse{}
updatedCount, err := i.collection.Update(ctx, namespace, func(i *RawObjectWithHistory) (bool, *RawObjectWithHistory, error) {
match := i.UID == r.UID && i.Kind == r.Kind
@ -144,7 +163,7 @@ func (i dummyObjectServer) update(ctx context.Context, r *object.WriteObjectRequ
modifier := userFromContext(ctx)
updated = &object.RawObject{
updated := &object.RawObject{
UID: r.UID,
Kind: r.Kind,
Created: i.Created,
@ -158,12 +177,32 @@ func (i dummyObjectServer) update(ctx context.Context, r *object.WriteObjectRequ
ETag: createContentsHash(r.Body),
Body: r.Body,
Version: fmt.Sprintf("%d", prevVersion+1),
Comment: r.Comment,
}
versionInfo := &ObjectVersionWithBody{
Body: r.Body,
ObjectVersionInfo: &object.ObjectVersionInfo{
Version: updated.Version,
Modified: updated.Modified,
ModifiedBy: updated.ModifiedBy,
Size: updated.Size,
ETag: updated.ETag,
Comment: r.Comment,
},
}
rsp.Object = versionInfo.ObjectVersionInfo
rsp.Status = object.WriteObjectResponse_MODIFIED
// When saving, it must be different than the head version
if i.ETag == updated.ETag {
versionInfo.ObjectVersionInfo.Version = i.Version
rsp.Status = object.WriteObjectResponse_UNCHANGED
return false, nil, nil
}
return true, &RawObjectWithHistory{
RawObject: updated,
History: append(i.History, updated),
History: append(i.History, versionInfo),
}, nil
})
@ -171,14 +210,11 @@ func (i dummyObjectServer) update(ctx context.Context, r *object.WriteObjectRequ
return nil, err
}
if updatedCount == 0 {
if updatedCount == 0 && rsp.Object == nil {
return nil, fmt.Errorf("could not find object with uid %s and kind %s", r.UID, r.Kind)
}
return &object.WriteObjectResponse{
Error: nil,
Object: updated,
}, nil
return rsp, nil
}
func (i dummyObjectServer) insert(ctx context.Context, r *object.WriteObjectRequest, namespace string) (*object.WriteObjectResponse, error) {
@ -200,11 +236,23 @@ func (i dummyObjectServer) insert(ctx context.Context, r *object.WriteObjectRequ
ETag: createContentsHash(r.Body),
Body: r.Body,
Version: fmt.Sprintf("%d", 1),
Comment: r.Comment,
}
info := &object.ObjectVersionInfo{
Version: rawObj.Version,
Modified: rawObj.Modified,
ModifiedBy: rawObj.ModifiedBy,
Size: rawObj.Size,
ETag: rawObj.ETag,
Comment: r.Comment,
}
newObj := &RawObjectWithHistory{
RawObject: rawObj,
History: []*object.RawObject{rawObj},
History: []*ObjectVersionWithBody{{
ObjectVersionInfo: info,
Body: r.Body,
}},
}
err := i.collection.Insert(ctx, namespace, newObj)
@ -214,13 +262,17 @@ func (i dummyObjectServer) insert(ctx context.Context, r *object.WriteObjectRequ
return &object.WriteObjectResponse{
Error: nil,
Object: newObj.RawObject,
Object: info,
Status: object.WriteObjectResponse_CREATED,
}, nil
}
func (i dummyObjectServer) Write(ctx context.Context, r *object.WriteObjectRequest) (*object.WriteObjectResponse, error) {
namespace := namespaceFromUID(r.UID)
obj, err := i.collection.FindFirst(ctx, namespace, func(i *RawObjectWithHistory) (bool, error) {
if i == nil || r == nil {
return false, nil
}
return i.UID == r.UID, nil
})
if err != nil {
@ -263,15 +315,15 @@ func (i dummyObjectServer) History(ctx context.Context, r *object.ObjectHistoryR
return nil, err
}
if obj == nil {
return &object.ObjectHistoryResponse{
Object: nil,
}, nil
rsp := &object.ObjectHistoryResponse{}
if obj != nil {
// Return the most recent versions first
// Better? save them in this order?
for i := len(obj.History) - 1; i >= 0; i-- {
rsp.Versions = append(rsp.Versions, obj.History[i].ObjectVersionInfo)
}
}
return &object.ObjectHistoryResponse{
Object: obj.History,
}, nil
return rsp, nil
}
func (i dummyObjectServer) Search(ctx context.Context, r *object.ObjectSearchRequest) (*object.ObjectSearchResponse, error) {

12
pkg/services/store/object/http.go
View File

@ -60,6 +60,14 @@ func parseRequestParams(req *http.Request) (uid string, kind string, params map[
uid = path
kind = "?"
}
// Read parameters that are encoded in the URL
vals := req.URL.Query()
for k, v := range vals {
if len(v) > 0 {
params[k] = v[0]
}
}
return
}
@ -151,7 +159,7 @@ func (s *httpObjectStore) doWriteObject(c *models.ReqContext) response.Response
Kind: kind,
Body: b,
Comment: params["comment"],
PreviousVersion: params["previous"],
PreviousVersion: params["previousVersion"],
})
if err != nil {
return response.Error(500, "?", err)
@ -164,7 +172,7 @@ func (s *httpObjectStore) doDeleteObject(c *models.ReqContext) response.Response
rsp, err := s.store.Delete(c.Req.Context(), &DeleteObjectRequest{
UID: uid,
Kind: kind,
PreviousVersion: params["previous"],
PreviousVersion: params["previousVersion"],
})
if err != nil {
return response.Error(500, "?", err)

101
pkg/services/store/object/json.go
View File

@ -1,101 +0,0 @@
package object
import (
"encoding/json"
"unsafe"
jsoniter "github.com/json-iterator/go"
)
func init() { //nolint:gochecknoinits
//jsoniter.RegisterTypeEncoder("object.ReadObjectResponse", &readObjectResponseCodec{})
jsoniter.RegisterTypeEncoder("object.RawObject", &rawObjectCodec{})
}
// Unlike the standard JSON marshal, this will write bytes as JSON when it can
type rawObjectCodec struct{}
// Custom marshal for RawObject (if JSON body)
func (obj *RawObject) MarshalJSON() ([]byte, error) {
var json = jsoniter.ConfigCompatibleWithStandardLibrary
return json.Marshal(obj)
}
func (codec *rawObjectCodec) IsEmpty(ptr unsafe.Pointer) bool {
f := (*RawObject)(ptr)
return f.UID == "" && f.Body == nil
}
func (codec *rawObjectCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) {
obj := (*RawObject)(ptr)
stream.WriteObjectStart()
stream.WriteObjectField("UID")
stream.WriteString(obj.UID)
if obj.Kind != "" {
stream.WriteMore()
stream.WriteObjectField("kind")
stream.WriteString(obj.Kind)
}
if obj.Created > 0 {
stream.WriteMore()
stream.WriteObjectField("created")
stream.WriteInt64(obj.Created)
}
if obj.CreatedBy != nil {
stream.WriteMore()
stream.WriteObjectField("createdBy")
stream.WriteVal(obj.CreatedBy)
}
if obj.Modified > 0 {
stream.WriteMore()
stream.WriteObjectField("modified")
stream.WriteInt64(obj.Modified)
}
if obj.ModifiedBy != nil {
stream.WriteMore()
stream.WriteObjectField("modifiedBy")
stream.WriteVal(obj.ModifiedBy)
}
if obj.Size > 0 {
stream.WriteMore()
stream.WriteObjectField("size")
stream.WriteInt64(obj.Size)
}
if obj.ETag != "" {
stream.WriteMore()
stream.WriteObjectField("etag")
stream.WriteString(obj.ETag)
}
if obj.Version != "" {
stream.WriteMore()
stream.WriteObjectField("version")
stream.WriteString(obj.Version)
}
// The one real difference (encodes JSON things directly)
if obj.Body != nil {
stream.WriteMore()
stream.WriteObjectField("body")
if json.Valid(obj.Body) {
stream.WriteRaw(string(obj.Body)) // works for strings
} else {
stream.WriteString("// link to raw bytes //")
//stream.WriteVal(obj.Body)
}
}
if obj.SyncSrc != "" {
stream.WriteMore()
stream.WriteObjectField("syncSrc")
stream.WriteString(obj.SyncSrc)
}
if obj.SyncTime > 0 {
stream.WriteMore()
stream.WriteObjectField("syncTime")
stream.WriteInt64(obj.SyncTime)
}
stream.WriteObjectEnd()
}

675
pkg/services/store/object/object.pb.go
View File

File diff suppressed because it is too large Load Diff

47
pkg/services/store/object/object.proto
View File

@ -47,20 +47,15 @@ message RawObject {
// NOTE: currently managed by the dashboard+dashboard_version tables
string version = 10;
// optional "save" or "commit" message
//
// NOTE: currently managed by the dashboard_version table, and will be returned from a "history" command
string comment = 11;
// Location (path/repo/etc) that defines the canonocal form
//
// NOTE: currently managed by the dashboard_provisioning table
string sync_src = 12;
string sync_src = 11;
// Time in epoch milliseconds that the object was last synced with an external system (provisioning/git)
//
// NOTE: currently managed by the dashboard_provisioning table
int64 sync_time = 13;
int64 sync_time = 12;
}
// Report error while working with objects
@ -76,6 +71,29 @@ message ObjectErrorInfo {
bytes details_json = 3;
}
// This is a subset of RawObject that does not include body or sync info
message ObjectVersionInfo {
// The version will change when the object is saved. It is not necessarily sortable
string version = 1;
// Time in epoch milliseconds that the object was modified
int64 modified = 2;
// Who modified the object
UserInfo modified_by = 3;
// Content Length
int64 size = 4;
// MD5 digest of the body
string ETag = 5;
// optional "save" or "commit" message
//
// NOTE: currently managed by the dashboard_version table, and will be returned from a "history" command
string comment = 6;
}
//-----------------------------------------------
// Get request/response
//-----------------------------------------------
@ -143,10 +161,21 @@ message WriteObjectResponse {
ObjectErrorInfo error = 1;
// Object details with the body removed
RawObject object = 2;
ObjectVersionInfo object = 2;
// Object summary as JSON
bytes summary_json = 3;
// Status code
Status status = 4;
// Status enumeration
enum Status {
ERROR = 0;
CREATED = 1;
MODIFIED = 2;
UNCHANGED = 3;
}
}
//-----------------------------------------------
@ -188,7 +217,7 @@ message ObjectHistoryRequest {
message ObjectHistoryResponse {
// Object metadata without the raw bytes
repeated RawObject object = 1;
repeated ObjectVersionInfo versions = 1;
// More results exist... pass this in the next request
string next_page_token = 2;

2
pkg/services/store/object/object_grpc.pb.go
View File

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc v3.21.5
// - protoc v3.21.7
// source: object.proto
package object

124
pkg/services/store/object/tests/server_integration_test.go
View File

@ -29,12 +29,23 @@ type rawObjectMatcher struct {
modifiedBy *object.UserInfo
body []byte
version *string
}
type objectVersionMatcher struct {
modifiedRange []time.Time
modifiedBy *object.UserInfo
version *string
etag *string
comment *string
}
func userInfoMatches(expected *object.UserInfo, actual *object.UserInfo) (bool, string) {
var mismatches []string
if actual == nil && expected != nil {
return true, "Missing user info"
}
if expected.Id != actual.Id {
mismatches = append(mismatches, fmt.Sprintf("expected ID %d, actual ID: %d", expected.Id, actual.Id))
}
@ -52,6 +63,8 @@ func timestampInRange(ts int64, tsRange []time.Time) bool {
func requireObjectMatch(t *testing.T, obj *object.RawObject, m rawObjectMatcher) {
t.Helper()
require.NotNil(t, obj)
mismatches := ""
if m.uid != nil && *m.uid != obj.UID {
mismatches += fmt.Sprintf("expected UID: %s, actual UID: %s\n", *m.uid, obj.UID)
@ -66,7 +79,7 @@ func requireObjectMatch(t *testing.T, obj *object.RawObject, m rawObjectMatcher)
}
if len(m.modifiedRange) == 2 && !timestampInRange(obj.Modified, m.modifiedRange) {
mismatches += fmt.Sprintf("expected createdBy range: [from %s to %s], actual created: %s\n", m.createdRange[0], m.createdRange[1], time.Unix(obj.Created, 0))
mismatches += fmt.Sprintf("expected createdBy range: [from %s to %s], actual created: %s\n", m.modifiedRange[0], m.modifiedRange[1], time.Unix(obj.Modified, 0))
}
if m.createdBy != nil {
@ -97,8 +110,30 @@ func requireObjectMatch(t *testing.T, obj *object.RawObject, m rawObjectMatcher)
mismatches += fmt.Sprintf("expected version: %s, actual version: %s\n", *m.version, obj.Version)
}
if m.comment != nil && *m.comment != obj.Comment {
mismatches += fmt.Sprintf("expected comment: %s, actual comment: %s\n", *m.comment, obj.Comment)
require.True(t, len(mismatches) == 0, mismatches)
}
func requireVersionMatch(t *testing.T, obj *object.ObjectVersionInfo, m objectVersionMatcher) {
t.Helper()
mismatches := ""
if m.etag != nil && *m.etag != obj.ETag {
mismatches += fmt.Sprintf("expected etag: %s, actual etag: %s\n", *m.etag, obj.ETag)
}
if len(m.modifiedRange) == 2 && !timestampInRange(obj.Modified, m.modifiedRange) {
mismatches += fmt.Sprintf("expected createdBy range: [from %s to %s], actual created: %s\n", m.modifiedRange[0], m.modifiedRange[1], time.Unix(obj.Modified, 0))
}
if m.modifiedBy != nil {
userInfoMatches, msg := userInfoMatches(m.modifiedBy, obj.ModifiedBy)
if !userInfoMatches {
mismatches += fmt.Sprintf("modifiedBy: %s\n", msg)
}
}
if m.version != nil && *m.version != obj.Version {
mismatches += fmt.Sprintf("expected version: %s, actual version: %s\n", *m.version, obj.Version)
}
require.True(t, len(mismatches) == 0, mismatches)
@ -144,18 +179,13 @@ func TestObjectServer(t *testing.T) {
writeResp, err := testCtx.client.Write(ctx, writeReq)
require.NoError(t, err)
objectMatcher := rawObjectMatcher{
uid: &uid,
kind: &kind,
createdRange: []time.Time{before, time.Now()},
versionMatcher := objectVersionMatcher{
modifiedRange: []time.Time{before, time.Now()},
createdBy: fakeUser,
modifiedBy: fakeUser,
body: body,
version: &firstVersion,
comment: &writeReq.Comment,
}
requireObjectMatch(t, writeResp.Object, objectMatcher)
requireVersionMatch(t, writeResp.Object, versionMatcher)
readResp, err := testCtx.client.Read(ctx, &object.ReadObjectRequest{
UID: uid,
@ -165,7 +195,18 @@ func TestObjectServer(t *testing.T) {
})
require.NoError(t, err)
require.Nil(t, readResp.SummaryJson)
requireObjectMatch(t, writeResp.Object, objectMatcher)
objectMatcher := rawObjectMatcher{
uid: &uid,
kind: &kind,
createdRange: []time.Time{before, time.Now()},
modifiedRange: []time.Time{before, time.Now()},
createdBy: fakeUser,
modifiedBy: fakeUser,
body: body,
version: &firstVersion,
}
requireObjectMatch(t, readResp.Object, objectMatcher)
deleteResp, err := testCtx.client.Delete(ctx, &object.DeleteObjectRequest{
UID: uid,
@ -195,6 +236,7 @@ func TestObjectServer(t *testing.T) {
}
writeResp1, err := testCtx.client.Write(ctx, writeReq1)
require.NoError(t, err)
require.Equal(t, object.WriteObjectResponse_CREATED, writeResp1.Status)
body2 := []byte("{\"name\":\"John2\"}")
@ -208,6 +250,14 @@ func TestObjectServer(t *testing.T) {
require.NoError(t, err)
require.NotEqual(t, writeResp1.Object.Version, writeResp2.Object.Version)
// Duplicate write (no change)
writeDupRsp, err := testCtx.client.Write(ctx, writeReq2)
require.NoError(t, err)
require.Nil(t, writeDupRsp.Error)
require.Equal(t, object.WriteObjectResponse_UNCHANGED, writeDupRsp.Status)
require.Equal(t, writeResp2.Object.Version, writeDupRsp.Object.Version)
require.Equal(t, writeResp2.Object.ETag, writeDupRsp.Object.ETag)
body3 := []byte("{\"name\":\"John3\"}")
writeReq3 := &object.WriteObjectRequest{
UID: uid,
@ -228,7 +278,6 @@ func TestObjectServer(t *testing.T) {
modifiedBy: fakeUser,
body: body3,
version: &writeResp3.Object.Version,
comment: &writeReq3.Comment,
}
readRespLatest, err := testCtx.client.Read(ctx, &object.ReadObjectRequest{
UID: uid,
@ -259,7 +308,6 @@ func TestObjectServer(t *testing.T) {
modifiedBy: fakeUser,
body: body,
version: &firstVersion,
comment: &writeReq1.Comment,
})
history, err := testCtx.client.History(ctx, &object.ObjectHistoryRequest{
@ -267,11 +315,11 @@ func TestObjectServer(t *testing.T) {
Kind: kind,
})
require.NoError(t, err)
require.Equal(t, []*object.RawObject{
writeResp1.Object,
writeResp2.Object,
require.Equal(t, []*object.ObjectVersionInfo{
writeResp3.Object,
}, history.Object)
writeResp2.Object,
writeResp1.Object,
}, history.Versions)
deleteResp, err := testCtx.client.Delete(ctx, &object.DeleteObjectRequest{
UID: uid,
@ -316,19 +364,47 @@ func TestObjectServer(t *testing.T) {
require.NoError(t, err)
search, err := testCtx.client.Search(ctx, &object.ObjectSearchRequest{
Kind: []string{kind, kind2},
Kind: []string{kind, kind2},
WithBody: false,
})
require.NoError(t, err)
require.Equal(t, []*object.RawObject{
w1.Object, w2.Object, w3.Object, w4.Object,
}, search.Results)
require.NotNil(t, search)
uids := make([]string, 0, len(search.Results))
kinds := make([]string, 0, len(search.Results))
version := make([]string, 0, len(search.Results))
for _, res := range search.Results {
uids = append(uids, res.UID)
kinds = append(kinds, res.Kind)
version = append(version, res.Version)
}
require.Equal(t, []string{"my-test-entity", "uid2", "uid3", "uid4"}, uids)
require.Equal(t, []string{"dashboard", "dashboard", "kind2", "kind2"}, kinds)
require.Equal(t, []string{
w1.Object.Version,
w2.Object.Version,
w3.Object.Version,
w4.Object.Version,
}, version)
// Again with only one kind
searchKind1, err := testCtx.client.Search(ctx, &object.ObjectSearchRequest{
Kind: []string{kind},
})
require.NoError(t, err)
require.Equal(t, []*object.RawObject{
w1.Object, w2.Object,
}, searchKind1.Results)
uids = make([]string, 0, len(searchKind1.Results))
kinds = make([]string, 0, len(searchKind1.Results))
version = make([]string, 0, len(searchKind1.Results))
for _, res := range searchKind1.Results {
uids = append(uids, res.UID)
kinds = append(kinds, res.Kind)
version = append(version, res.Version)
}
require.Equal(t, []string{"my-test-entity", "uid2"}, uids)
require.Equal(t, []string{"dashboard", "dashboard"}, kinds)
require.Equal(t, []string{
w1.Object.Version,
w2.Object.Version,
}, version)
})
}