ObjectStore: Write json as json when possible (#56433)

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -66,6 +66,7 @@ export interface FeatureToggles {
   internationalization?: boolean;
   topnav?: boolean;
   grpcServer?: boolean;
+  objectStore?: boolean;
   traceqlEditor?: boolean;
   redshiftAsyncQueryDataSupport?: boolean;
   athenaAsyncQueryDataSupport?: boolean;

pkg/services/featuremgmt/registry.go

@@ -277,7 +277,14 @@ var (
 			Description:     "Run GRPC server",
 			State:           FeatureStateAlpha,
 			RequiresDevMode: true,
-		}, {
+		},
+		{
+			Name:            "objectStore",
+			Description:     "SQL based object store",
+			State:           FeatureStateAlpha,
+			RequiresDevMode: true,
+		},
+		{
 			Name:        "traceqlEditor",
 			Description: "Show the TraceQL editor in the explore page",
 			State:       FeatureStateAlpha,

pkg/services/featuremgmt/toggles_gen.go

@@ -207,6 +207,10 @@ const (
 	// Run GRPC server
 	FlagGrpcServer = "grpcServer"
 
+	// FlagObjectStore
+	// SQL based object store
+	FlagObjectStore = "objectStore"
+
 	// FlagTraceqlEditor
 	// Show the TraceQL editor in the explore page
 	FlagTraceqlEditor = "traceqlEditor"

pkg/services/store/object/dummy/dummy_server.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/md5"
 	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"strconv"
@@ -23,13 +24,14 @@ type ObjectVersionWithBody struct {
 }
 
 type RawObjectWithHistory struct {
-	*object.RawObject `json:"rawObject,omitempty"`
-	History           []*ObjectVersionWithBody `json:"history,omitempty"`
+	Object  *object.RawObject        `json:"object,omitempty"`
+	Summary *object.ObjectSummary    `json:"summary,omitempty"`
+	History []*ObjectVersionWithBody `json:"history,omitempty"`
 }
 
 var (
 	// increment when RawObject changes
-	rawObjectVersion = 3
+	rawObjectVersion = 6
 )
 
 func ProvideDummyObjectServer(cfg *setting.Cfg, grpcServerProvider grpcserver.Provider) object.ObjectStoreServer {
@@ -57,7 +59,7 @@ func (i dummyObjectServer) findObject(ctx context.Context, uid string, kind stri
 	}
 
 	obj, err := i.collection.FindFirst(ctx, namespaceFromUID(uid), func(i *RawObjectWithHistory) (bool, error) {
-		return i.UID == uid && i.Kind == kind, nil
+		return i.Object.UID == uid && i.Object.Kind == kind, nil
 	})
 
 	if err != nil {
@@ -70,16 +72,16 @@ func (i dummyObjectServer) findObject(ctx context.Context, uid string, kind stri
 
 	getLatestVersion := version == ""
 	if getLatestVersion {
-		return obj, obj.RawObject, nil
+		return obj, obj.Object, nil
 	}
 
 	for _, objVersion := range obj.History {
 		if objVersion.Version == version {
 			copy := &object.RawObject{
-				UID:       obj.UID,
-				Kind:      obj.Kind,
-				Created:   obj.Created,
-				CreatedBy: obj.CreatedBy,
+				UID:       obj.Object.UID,
+				Kind:      obj.Object.Kind,
+				Created:   obj.Object.Created,
+				CreatedBy: obj.Object.CreatedBy,
 				Updated:   objVersion.Updated,
 				UpdatedBy: objVersion.UpdatedBy,
 				ETag:      objVersion.ETag,
@@ -109,10 +111,21 @@ func (i dummyObjectServer) Read(ctx context.Context, r *object.ReadObjectRequest
 		}, nil
 	}
 
-	return &object.ReadObjectResponse{
-		Object:      objVersion,
-		SummaryJson: nil,
-	}, nil
+	rsp := &object.ReadObjectResponse{
+		Object: objVersion,
+	}
+	if r.WithSummary {
+		summary, _, e2 := object.GetSafeSaveObject(&object.WriteObjectRequest{
+			UID:  r.UID,
+			Kind: r.Kind,
+			Body: objVersion.Body,
+		})
+		if e2 != nil {
+			return nil, e2
+		}
+		rsp.SummaryJson, err = json.Marshal(summary)
+	}
+	return rsp, err
 }
 
 func (i dummyObjectServer) BatchRead(ctx context.Context, batchR *object.BatchReadObjectRequest) (*object.BatchReadObjectResponse, error) {
@@ -137,16 +150,16 @@ func (i dummyObjectServer) update(ctx context.Context, r *object.WriteObjectRequ
 	rsp := &object.WriteObjectResponse{}
 
 	updatedCount, err := i.collection.Update(ctx, namespace, func(i *RawObjectWithHistory) (bool, *RawObjectWithHistory, error) {
-		match := i.UID == r.UID && i.Kind == r.Kind
+		match := i.Object.UID == r.UID && i.Object.Kind == r.Kind
 		if !match {
 			return false, nil, nil
 		}
 
-		if r.PreviousVersion != "" && i.Version != r.PreviousVersion {
-			return false, nil, fmt.Errorf("expected the previous version to be %s, but was %s", r.PreviousVersion, i.Version)
+		if r.PreviousVersion != "" && i.Object.Version != r.PreviousVersion {
+			return false, nil, fmt.Errorf("expected the previous version to be %s, but was %s", r.PreviousVersion, i.Object.Version)
 		}
 
-		prevVersion, err := strconv.Atoi(i.Version)
+		prevVersion, err := strconv.Atoi(i.Object.Version)
 		if err != nil {
 			return false, nil, err
 		}
@@ -156,8 +169,8 @@ func (i dummyObjectServer) update(ctx context.Context, r *object.WriteObjectRequ
 		updated := &object.RawObject{
 			UID:       r.UID,
 			Kind:      r.Kind,
-			Created:   i.Created,
-			CreatedBy: i.CreatedBy,
+			Created:   i.Object.Created,
+			CreatedBy: i.Object.CreatedBy,
 			Updated:   time.Now().Unix(),
 			UpdatedBy: object.GetUserIDString(modifier),
 			Size:      int64(len(r.Body)),
@@ -181,15 +194,15 @@ func (i dummyObjectServer) update(ctx context.Context, r *object.WriteObjectRequ
 		rsp.Status = object.WriteObjectResponse_UPDATED
 
 		// When saving, it must be different than the head version
-		if i.ETag == updated.ETag {
-			versionInfo.ObjectVersionInfo.Version = i.Version
+		if i.Object.ETag == updated.ETag {
+			versionInfo.ObjectVersionInfo.Version = i.Object.Version
 			rsp.Status = object.WriteObjectResponse_UNCHANGED
 			return false, nil, nil
 		}
 
 		return true, &RawObjectWithHistory{
-			RawObject: updated,
-			History:   append(i.History, versionInfo),
+			Object:  updated,
+			History: append(i.History, versionInfo),
 		}, nil
 	})
 
@@ -229,7 +242,7 @@ func (i dummyObjectServer) insert(ctx context.Context, r *object.WriteObjectRequ
 	}
 
 	newObj := &RawObjectWithHistory{
-		RawObject: rawObj,
+		Object: rawObj,
 		History: []*ObjectVersionWithBody{{
 			ObjectVersionInfo: info,
 			Body:              r.Body,
@@ -254,7 +267,7 @@ func (i dummyObjectServer) Write(ctx context.Context, r *object.WriteObjectReque
 		if i == nil || r == nil {
 			return false, nil
 		}
-		return i.UID == r.UID, nil
+		return i.Object.UID == r.UID, nil
 	})
 	if err != nil {
 		return nil, err
@@ -269,10 +282,10 @@ func (i dummyObjectServer) Write(ctx context.Context, r *object.WriteObjectReque
 
 func (i dummyObjectServer) Delete(ctx context.Context, r *object.DeleteObjectRequest) (*object.DeleteObjectResponse, error) {
 	_, err := i.collection.Delete(ctx, namespaceFromUID(r.UID), func(i *RawObjectWithHistory) (bool, error) {
-		match := i.UID == r.UID && i.Kind == r.Kind
+		match := i.Object.UID == r.UID && i.Object.Kind == r.Kind
 		if match {
-			if r.PreviousVersion != "" && i.Version != r.PreviousVersion {
-				return false, fmt.Errorf("expected the previous version to be %s, but was %s", r.PreviousVersion, i.Version)
+			if r.PreviousVersion != "" && i.Object.Version != r.PreviousVersion {
+				return false, fmt.Errorf("expected the previous version to be %s, but was %s", r.PreviousVersion, i.Object.Version)
 			}
 
 			return true, nil
@@ -319,7 +332,7 @@ func (i dummyObjectServer) Search(ctx context.Context, r *object.ObjectSearchReq
 	// TODO more filters
 	objects, err := i.collection.Find(ctx, namespaceFromUID("TODO"), func(i *RawObjectWithHistory) (bool, error) {
 		if len(r.Kind) != 0 {
-			if _, ok := kindMap[i.Kind]; !ok {
+			if _, ok := kindMap[i.Object.Kind]; !ok {
 				return false, nil
 			}
 		}
@@ -332,13 +345,13 @@ func (i dummyObjectServer) Search(ctx context.Context, r *object.ObjectSearchReq
 	searchResults := make([]*object.ObjectSearchResult, 0)
 	for _, o := range objects {
 		searchResults = append(searchResults, &object.ObjectSearchResult{
-			UID:       o.UID,
-			Kind:      o.Kind,
-			Version:   o.Version,
-			Updated:   o.Updated,
-			UpdatedBy: o.UpdatedBy,
+			UID:       o.Object.UID,
+			Kind:      o.Object.Kind,
+			Version:   o.Object.Version,
+			Updated:   o.Object.Updated,
+			UpdatedBy: o.Object.UpdatedBy,
 			Name:      "? name from summary",
-			Body:      o.Body,
+			Body:      o.Object.Body,
 		})
 	}
 

pkg/services/store/object/dummy/dummy_server_test.go

@@ -0,0 +1,69 @@
+package objectdummyserver
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/grafana/grafana/pkg/services/store/object"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRawEncoders(t *testing.T) {
+	body, err := json.Marshal(map[string]interface{}{
+		"hello": "world",
+		"field": 1.23,
+	})
+	require.NoError(t, err)
+
+	raw := &ObjectVersionWithBody{
+		&object.ObjectVersionInfo{
+			Version: "A",
+		},
+		body,
+	}
+
+	b, err := json.Marshal(raw)
+	require.NoError(t, err)
+
+	str := string(b)
+	fmt.Printf("expect: %s", str)
+	require.JSONEq(t, `{"info":{"version":"A"},"body":"eyJmaWVsZCI6MS4yMywiaGVsbG8iOiJ3b3JsZCJ9"}`, str)
+
+	copy := &ObjectVersionWithBody{}
+	err = json.Unmarshal(b, copy)
+	require.NoError(t, err)
+}
+
+func TestRawObjectWithHistory(t *testing.T) {
+	body, err := json.Marshal(map[string]interface{}{
+		"hello": "world",
+		"field": 1.23,
+	})
+	require.NoError(t, err)
+
+	raw := &RawObjectWithHistory{
+		Object: &object.RawObject{
+			Version: "A",
+			Body:    body,
+		},
+		History: make([]*ObjectVersionWithBody, 0),
+	}
+	raw.History = append(raw.History, &ObjectVersionWithBody{
+		&object.ObjectVersionInfo{
+			Version: "B",
+		},
+		body,
+	})
+
+	b, err := json.Marshal(raw)
+	require.NoError(t, err)
+
+	str := string(b)
+	fmt.Printf("expect: %s", str)
+	require.JSONEq(t, `{"object":{"UID":"","version":"A","body":{"field":1.23,"hello":"world"}},"history":[{"info":{"version":"B"},"body":"eyJmaWVsZCI6MS4yMywiaGVsbG8iOiJ3b3JsZCJ9"}]}`, str)
+
+	copy := &ObjectVersionWithBody{}
+	err = json.Unmarshal(b, copy)
+	require.NoError(t, err)
+}

pkg/services/store/object/http.go

@@ -76,9 +76,9 @@ func (s *httpObjectStore) doGetObject(c *models.ReqContext) response.Response {
 	rsp, err := s.store.Read(c.Req.Context(), &ReadObjectRequest{
 		UID:         uid,
 		Kind:        kind,
-		Version:     params["version"], // ?version = XYZ
-		WithBody:    true,              // ?? allow false?
-		WithSummary: true,              // ?? allow false?
+		Version:     params["version"],           // ?version = XYZ
+		WithBody:    params["body"] != "false",   // default to true
+		WithSummary: params["summary"] == "true", // default to false
 	})
 	if err != nil {
 		return response.Error(500, "error fetching object", err)
@@ -200,5 +200,16 @@ func (s *httpObjectStore) doListFolder(c *models.ReqContext) response.Response {
 }
 
 func (s *httpObjectStore) doSearch(c *models.ReqContext) response.Response {
-	return response.JSON(501, "Not implemented yet")
+	req := &ObjectSearchRequest{
+		WithBody:   true,
+		WithLabels: true,
+		WithFields: true,
+		// TODO!!!
+	}
+
+	rsp, err := s.store.Search(c.Req.Context(), req)
+	if err != nil {
+		return response.Error(500, "?", err)
+	}
+	return response.JSON(200, rsp)
 }

pkg/services/store/object/json.go

@@ -0,0 +1,312 @@
+package object
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"unsafe"
+
+	jsoniter "github.com/json-iterator/go"
+)
+
+func init() { //nolint:gochecknoinits
+	jsoniter.RegisterTypeEncoder("object.ObjectSearchResult", &searchResultCodec{})
+	jsoniter.RegisterTypeEncoder("object.WriteObjectResponse", &writeResponseCodec{})
+	jsoniter.RegisterTypeEncoder("object.ReadObjectResponse", &readResponseCodec{})
+
+	jsoniter.RegisterTypeEncoder("object.RawObject", &rawObjectCodec{})
+	jsoniter.RegisterTypeDecoder("object.RawObject", &rawObjectCodec{})
+}
+
+func writeRawJson(stream *jsoniter.Stream, val []byte) {
+	if json.Valid(val) {
+		_, _ = stream.Write(val)
+	} else {
+		stream.WriteString(string(val))
+	}
+}
+
+// Unlike the standard JSON marshal, this will write bytes as JSON when it can
+type rawObjectCodec struct{}
+
+func (obj *RawObject) MarshalJSON() ([]byte, error) {
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	return json.Marshal(obj)
+}
+
+// UnmarshalJSON will read JSON into a RawObject
+func (obj *RawObject) UnmarshalJSON(b []byte) error {
+	if obj == nil {
+		return fmt.Errorf("unexpected nil for raw objcet")
+	}
+	iter := jsoniter.ParseBytes(jsoniter.ConfigDefault, b)
+	readRawObject(iter, obj)
+	return iter.Error
+}
+
+func (codec *rawObjectCodec) IsEmpty(ptr unsafe.Pointer) bool {
+	f := (*RawObject)(ptr)
+	return f.UID == "" && f.Body == nil
+}
+
+func (codec *rawObjectCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) {
+	obj := (*RawObject)(ptr)
+	stream.WriteObjectStart()
+	stream.WriteObjectField("UID")
+	stream.WriteString(obj.UID)
+
+	if obj.Kind != "" {
+		stream.WriteMore()
+		stream.WriteObjectField("kind")
+		stream.WriteString(obj.Kind)
+	}
+	if obj.Version != "" {
+		stream.WriteMore()
+		stream.WriteObjectField("version")
+		stream.WriteString(obj.Version)
+	}
+	if obj.Created > 0 {
+		stream.WriteMore()
+		stream.WriteObjectField("created")
+		stream.WriteInt64(obj.Created)
+	}
+	if obj.Updated > 0 {
+		stream.WriteMore()
+		stream.WriteObjectField("updated")
+		stream.WriteInt64(obj.Updated)
+	}
+	if obj.CreatedBy != "" {
+		stream.WriteMore()
+		stream.WriteObjectField("createdBy")
+		stream.WriteString(obj.CreatedBy)
+	}
+	if obj.UpdatedBy != "" {
+		stream.WriteMore()
+		stream.WriteObjectField("updatedBy")
+		stream.WriteString(obj.UpdatedBy)
+	}
+	if obj.Body != nil {
+		stream.WriteMore()
+		if json.Valid(obj.Body) {
+			stream.WriteObjectField("body")
+			stream.WriteRaw(string(obj.Body)) // works for strings
+		} else {
+			sEnc := base64.StdEncoding.EncodeToString(obj.Body)
+			stream.WriteObjectField("body_base64")
+			stream.WriteString(sEnc) // works for strings
+		}
+	}
+	if obj.ETag != "" {
+		stream.WriteMore()
+		stream.WriteObjectField("etag")
+		stream.WriteString(obj.ETag)
+	}
+	if obj.Size > 0 {
+		stream.WriteMore()
+		stream.WriteObjectField("size")
+		stream.WriteInt64(obj.Size)
+	}
+
+	if obj.Sync != nil {
+		stream.WriteMore()
+		stream.WriteObjectField("sync")
+		stream.WriteVal(obj.Sync)
+	}
+
+	stream.WriteObjectEnd()
+}
+
+func (codec *rawObjectCodec) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) {
+	*(*RawObject)(ptr) = RawObject{}
+	raw := (*RawObject)(ptr)
+	readRawObject(iter, raw)
+}
+
+func readRawObject(iter *jsoniter.Iterator, raw *RawObject) {
+	for l1Field := iter.ReadObject(); l1Field != ""; l1Field = iter.ReadObject() {
+		switch l1Field {
+		case "UID":
+			raw.UID = iter.ReadString()
+		case "kind":
+			raw.Kind = iter.ReadString()
+		case "updated":
+			raw.Updated = iter.ReadInt64()
+		case "updatedBy":
+			raw.UpdatedBy = iter.ReadString()
+		case "created":
+			raw.Created = iter.ReadInt64()
+		case "createdBy":
+			raw.CreatedBy = iter.ReadString()
+		case "size":
+			raw.Size = iter.ReadInt64()
+		case "etag":
+			raw.ETag = iter.ReadString()
+		case "version":
+			raw.Version = iter.ReadString()
+		case "sync":
+			raw.Sync = &RawObjectSyncInfo{}
+			iter.ReadVal(raw.Sync)
+
+		case "body":
+			var val interface{}
+			iter.ReadVal(&val) // ??? is there a smarter way to just keep the underlying bytes without read+marshal
+			body, err := json.Marshal(val)
+			if err != nil {
+				iter.ReportError("raw object", "error creating json from body")
+				return
+			}
+			raw.Body = body
+
+		case "body_base64":
+			val := iter.ReadString()
+			body, err := base64.StdEncoding.DecodeString(val)
+			if err != nil {
+				iter.ReportError("raw object", "error decoding base64 body")
+				return
+			}
+			raw.Body = body
+
+		default:
+			iter.ReportError("raw object", "unexpected field: "+l1Field)
+			return
+		}
+	}
+}
+
+// Unlike the standard JSON marshal, this will write bytes as JSON when it can
+type readResponseCodec struct{}
+
+func (obj *ReadObjectResponse) MarshalJSON() ([]byte, error) {
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	return json.Marshal(obj)
+}
+
+func (codec *readResponseCodec) IsEmpty(ptr unsafe.Pointer) bool {
+	f := (*ReadObjectResponse)(ptr)
+	return f == nil
+}
+
+func (codec *readResponseCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) {
+	obj := (*ReadObjectResponse)(ptr)
+	stream.WriteObjectStart()
+	stream.WriteObjectField("object")
+	stream.WriteVal(obj.Object)
+
+	if len(obj.SummaryJson) > 0 {
+		stream.WriteMore()
+		stream.WriteObjectField("summary")
+		writeRawJson(stream, obj.SummaryJson)
+	}
+
+	stream.WriteObjectEnd()
+}
+
+// Unlike the standard JSON marshal, this will write bytes as JSON when it can
+type searchResultCodec struct{}
+
+func (obj *ObjectSearchResult) MarshalJSON() ([]byte, error) {
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	return json.Marshal(obj)
+}
+
+func (codec *searchResultCodec) IsEmpty(ptr unsafe.Pointer) bool {
+	f := (*ObjectSearchResult)(ptr)
+	return f.UID == "" && f.Body == nil
+}
+
+func (codec *searchResultCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) {
+	obj := (*ObjectSearchResult)(ptr)
+	stream.WriteObjectStart()
+	stream.WriteObjectField("UID")
+	stream.WriteString(obj.UID)
+
+	if obj.Kind != "" {
+		stream.WriteMore()
+		stream.WriteObjectField("kind")
+		stream.WriteString(obj.Kind)
+	}
+	if obj.Name != "" {
+		stream.WriteMore()
+		stream.WriteObjectField("name")
+		stream.WriteString(obj.Name)
+	}
+	if obj.Description != "" {
+		stream.WriteMore()
+		stream.WriteObjectField("description")
+		stream.WriteString(obj.Description)
+	}
+	if obj.Updated > 0 {
+		stream.WriteMore()
+		stream.WriteObjectField("updated")
+		stream.WriteInt64(obj.Updated)
+	}
+	if obj.UpdatedBy != "" {
+		stream.WriteMore()
+		stream.WriteObjectField("updatedBy")
+		stream.WriteVal(obj.UpdatedBy)
+	}
+	if obj.Body != nil {
+		stream.WriteMore()
+		if json.Valid(obj.Body) {
+			stream.WriteObjectField("body")
+			_, _ = stream.Write(obj.Body) // works for strings
+		} else {
+			stream.WriteObjectField("body_base64")
+			stream.WriteVal(obj.Body) // works for strings
+		}
+	}
+	if obj.Labels != nil {
+		stream.WriteMore()
+		stream.WriteObjectField("labels")
+		stream.WriteVal(obj.Labels)
+	}
+	if obj.ErrorJson != nil {
+		stream.WriteMore()
+		stream.WriteObjectField("error")
+		writeRawJson(stream, obj.ErrorJson)
+	}
+	if obj.FieldsJson != nil {
+		stream.WriteMore()
+		stream.WriteObjectField("fields")
+		writeRawJson(stream, obj.FieldsJson)
+	}
+
+	stream.WriteObjectEnd()
+}
+
+// Unlike the standard JSON marshal, this will write bytes as JSON when it can
+type writeResponseCodec struct{}
+
+func (obj *WriteObjectResponse) MarshalJSON() ([]byte, error) {
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	return json.Marshal(obj)
+}
+
+func (codec *writeResponseCodec) IsEmpty(ptr unsafe.Pointer) bool {
+	f := (*WriteObjectResponse)(ptr)
+	return f == nil
+}
+
+func (codec *writeResponseCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) {
+	obj := (*WriteObjectResponse)(ptr)
+	stream.WriteObjectStart()
+	stream.WriteObjectField("status")
+	stream.WriteString(obj.Status.String())
+
+	if obj.Error != nil {
+		stream.WriteMore()
+		stream.WriteObjectField("error")
+		stream.WriteVal(obj.Error)
+	}
+	if obj.Object != nil {
+		stream.WriteMore()
+		stream.WriteObjectField("object")
+		stream.WriteVal(obj.Object)
+	}
+	if len(obj.SummaryJson) > 0 {
+		stream.WriteMore()
+		stream.WriteObjectField("summary")
+		writeRawJson(stream, obj.SummaryJson)
+	}
+	stream.WriteObjectEnd()
+}

pkg/services/store/object/json_test.go

@@ -0,0 +1,34 @@
+package object
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestRawEncoders(t *testing.T) {
+	body, err := json.Marshal(map[string]interface{}{
+		"hello": "world",
+		"field": 1.23,
+	})
+	require.NoError(t, err)
+
+	raw := &RawObject{
+		UID:     "a",
+		Kind:    "b",
+		Version: "c",
+		ETag:    "d",
+		Body:    body,
+	}
+
+	b, err := json.MarshalIndent(raw, "", "  ")
+	require.NoError(t, err)
+
+	str := string(b)
+	require.JSONEq(t, `{"UID":"a","kind":"b","version":"c","body":{"field":1.23,"hello":"world"},"etag":"d"}`, str)
+
+	copy := &RawObject{}
+	err = json.Unmarshal(b, copy)
+	require.NoError(t, err)
+}

pkg/services/store/object/kind.go

@@ -1,5 +1,10 @@
 package object
 
+import (
+	"fmt"
+	"time"
+)
+
 // NOTE this is just a temporary registry/list so we can use constants
 // TODO replace with codegen from kind schema system
 
@@ -8,3 +13,50 @@ const StandardKindFolder = "folder"
 const StandardKindPanel = "panel"         // types: heatmap, timeseries, table, ...
 const StandardKindDataSource = "ds"       // types: influx, prometheus, test, ...
 const StandardKindTransform = "transform" // types: joinByField, pivot, organizeFields, ...
+
+// This is a stub -- it will soon lookup in a registry of known "kinds"
+// Each kind will be able to define:
+//  1. sanitize/normalize function  (ie get safe bytes)
+//  2. SummaryProvier
+func GetSafeSaveObject(r *WriteObjectRequest) (*ObjectSummary, []byte, error) {
+	summary := &ObjectSummary{
+		Name:        fmt.Sprintf("hello: %s", r.Kind),
+		Description: fmt.Sprintf("Wrote at %s", time.Now().Local().String()),
+		Labels: map[string]string{
+			"hello": "world",
+			"tag1":  "",
+			"tag2":  "",
+		},
+		Fields: map[string]interface{}{
+			"field1": "a string",
+			"field2": 1.224,
+			"field4": true,
+		},
+		Error:  nil, // ignore for now
+		Nested: nil, // ignore for now
+		References: []*ExternalReference{
+			{
+				Kind: "ds",
+				Type: "influx",
+				UID:  "xyz",
+			},
+			{
+				Kind: "panel",
+				Type: "heatmap",
+			},
+			{
+				Kind: "panel",
+				Type: "timeseries",
+			},
+		},
+	}
+
+	if summary.UID != "" && r.UID != summary.UID {
+		return nil, nil, fmt.Errorf("internal UID mismatch")
+	}
+	if summary.Kind != "" && r.Kind != summary.Kind {
+		return nil, nil, fmt.Errorf("internal Kind mismatch")
+	}
+
+	return summary, r.Body, nil
+}

pkg/services/store/object/object.proto

@@ -38,15 +38,16 @@ message RawObject {
   // NOTE: currently managed by the dashboard+dashboard_version tables
   string version = 10;
 
-  // Location (path/repo/etc) that defines the canonocal form
-  //
+  // External location info
+  RawObjectSyncInfo sync = 11;
+}
+
+message RawObjectSyncInfo {
   // NOTE: currently managed by the dashboard_provisioning table
-  string sync_src = 11;
+  string source = 11;
 
   // Time in epoch milliseconds that the object was last synced with an external system (provisioning/git)
-  //
-  // NOTE: currently managed by the dashboard_provisioning table
-  int64 sync_time = 12;
+  int64 time = 12;
 }
 
 // Report error while working with objects
@@ -286,10 +287,10 @@ message ObjectSearchResult {
   map<string,string> labels = 9;
 
   // Optionally include extracted JSON
-  string fields_json = 10;
+  bytes fields_json = 10;
 
   // ObjectErrorInfo in json
-  string error_json = 11;
+  bytes error_json = 11;
 }
 
 message ObjectSearchResponse {