Storage: dualwriter update implementation (#85844)

* Add update methods for the dual writer

* improve errors

* [WIP] add tests for the update method

* Move example package to its own package so it can be used by the rest package. Finish tests

* Add codeowners

* Use Pod as a dummy resource

* :int

* Lint

* [REVIEW] rename var

* [REVIEW] don't rely on legacy storage at all in mode4

* Update pkg/apiserver/rest/dualwriter_mode2.go

Co-authored-by: Arati R. <33031346+suntala@users.noreply.github.com>

* [REVIEW] improve comment

* Update pkg/apiserver/rest/dualwriter_mode1.go

Co-authored-by: Dan Cech <dcech@grafana.com>

* [REVIEW] improve mode3

* Lint

* Move test files

* Update pkg/apiserver/rest/dualwriter_mode2.go

Co-authored-by: Arati R. <33031346+suntala@users.noreply.github.com>

* Lint

* Update pkg/apiserver/rest/dualwriter_mode4_test.go

Co-authored-by: Arati R. <33031346+suntala@users.noreply.github.com>

* Fix error

* Lint

* Update pkg/apiserver/rest/dualwriter_mode2.go

Co-authored-by: Arati R. <33031346+suntala@users.noreply.github.com>

* Don't set the flag to true as updatedObj creates an object in case it's not found

* Lint

* Lint

* Add tests on update

* Lint

---------

Co-authored-by: Arati R. <33031346+suntala@users.noreply.github.com>
Co-authored-by: Dan Cech <dcech@grafana.com>

pkg/apiserver/rest/dualwriter.go

@@ -70,6 +70,7 @@ var errDualWriterCreaterMissing = errors.New("legacy storage rest.Creater is mis
 var errDualWriterListerMissing = errors.New("legacy storage rest.Lister is missing")
 var errDualWriterDeleterMissing = errors.New("legacy storage rest.GracefulDeleter is missing")
 var errDualWriterCollectionDeleterMissing = errors.New("legacy storage rest.CollectionDeleter is missing")
+var errDualWriterUpdaterMissing = errors.New("legacy storage rest.Updater is missing")
 
 type DualWriterMode int
 

pkg/apiserver/rest/dualwriter_mode1.go

@@ -64,3 +64,13 @@ func (d *DualWriterMode1) DeleteCollection(ctx context.Context, deleteValidation
 
 	return legacy.DeleteCollection(ctx, deleteValidation, options, listOptions)
 }
+
+func (d *DualWriterMode1) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
+	legacy, ok := d.Legacy.(rest.Updater)
+	if !ok {
+		klog.FromContext(ctx).Error(errDualWriterUpdaterMissing, "legacy storage rest.Updater is missing")
+		return nil, false, errDualWriterUpdaterMissing
+	}
+
+	return legacy.Update(ctx, name, objInfo, createValidation, updateValidation, forceAllowCreate, options)
+}

pkg/apiserver/rest/dualwriter_mode1_test.go

@@ -16,6 +16,7 @@ import (
 const kind = "dummy"
 
 var exampleObj = &example.Pod{TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "1"}, Spec: example.PodSpec{}, Status: example.PodStatus{}}
+var exampleObjDifferentRV = &example.Pod{TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{Name: "foo", ResourceVersion: "3"}, Spec: example.PodSpec{}, Status: example.PodStatus{}}
 var anotherObj = &example.Pod{TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{Name: "bar", ResourceVersion: "2"}, Spec: example.PodSpec{}, Status: example.PodStatus{}}
 var failingObj = &example.Pod{TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{Name: "object-fail", ResourceVersion: "2"}, Spec: example.PodSpec{}, Status: example.PodStatus{}}
 
@@ -300,3 +301,65 @@ func TestMode1_DeleteCollection(t *testing.T) {
 		assert.NotEqual(t, obj, anotherObj)
 	}
 }
+
+func TestMode1_Update(t *testing.T) {
+	type testCase struct {
+		name           string
+		input          string
+		setupLegacyFn  func(m *mock.Mock, input string)
+		setupStorageFn func(m *mock.Mock, input string)
+		wantErr        bool
+	}
+	tests :=
+		[]testCase{
+			{
+				name:  "update an object in legacy",
+				input: "foo",
+				setupLegacyFn: func(m *mock.Mock, input string) {
+					m.On("Update", context.Background(), input, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(exampleObj, false, nil)
+				},
+				setupStorageFn: func(m *mock.Mock, input string) {
+					m.On("Update", context.Background(), input, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(anotherObj, false, nil)
+				},
+			},
+			{
+				name:  "error updating an object in legacy",
+				input: "object-fail",
+				setupLegacyFn: func(m *mock.Mock, input string) {
+					m.On("Update", context.Background(), input, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, false, errors.New("error"))
+				},
+				setupStorageFn: func(m *mock.Mock, input string) {
+					m.On("Update", context.Background(), input, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(anotherObj, false, nil)
+				},
+				wantErr: true,
+			},
+		}
+
+	for _, tt := range tests {
+		l := (LegacyStorage)(nil)
+		s := (Storage)(nil)
+		m := &mock.Mock{}
+
+		ls := legacyStoreMock{m, l}
+		us := storageMock{m, s}
+
+		if tt.setupLegacyFn != nil {
+			tt.setupLegacyFn(m, tt.input)
+		}
+		if tt.setupStorageFn != nil {
+			tt.setupStorageFn(m, tt.input)
+		}
+
+		dw := SelectDualWriter(Mode1, ls, us)
+
+		obj, _, err := dw.Update(context.Background(), tt.input, UpdatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{})
+
+		if tt.wantErr {
+			assert.Error(t, err)
+			continue
+		}
+
+		assert.Equal(t, obj, exampleObj)
+		assert.NotEqual(t, obj, anotherObj)
+	}
+}

pkg/apiserver/rest/dualwriter_mode2.go

@@ -35,21 +35,23 @@ func (d *DualWriterMode2) Create(ctx context.Context, obj runtime.Object, create
 		return created, err
 	}
 
-	c, err := enrichObject(obj, created)
+	accessorCreated, err := meta.Accessor(created)
 	if err != nil {
 		return created, err
 	}
 
-	accessor, err := meta.Accessor(c)
+	accessorOld, err := meta.Accessor(obj)
 	if err != nil {
 		return created, err
 	}
 
+	enrichObject(accessorOld, accessorCreated)
+
 	// create method expects an empty resource version
-	accessor.SetResourceVersion("")
-	accessor.SetUID("")
+	accessorCreated.SetResourceVersion("")
+	accessorCreated.SetUID("")
 
-	rsp, err := d.Storage.Create(ctx, c, createValidation, options)
+	rsp, err := d.Storage.Create(ctx, created, createValidation, options)
 	if err != nil {
 		klog.FromContext(ctx).Error(err, "unable to create object in Storage", "mode", 2)
 	}
@@ -144,6 +146,16 @@ func (d *DualWriterMode2) DeleteCollection(ctx context.Context, deleteValidation
 	return res, err
 }
 
+func enrichObject(accessorO, accessorC metav1.Object) {
+	accessorC.SetLabels(accessorO.GetLabels())
+
+	ac := accessorC.GetAnnotations()
+	for k, v := range accessorO.GetAnnotations() {
+		ac[k] = v
+	}
+	accessorC.SetAnnotations(ac)
+}
+
 func (d *DualWriterMode2) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
 	legacy, ok := d.Legacy.(rest.GracefulDeleter)
 	if !ok {
@@ -168,23 +180,64 @@ func (d *DualWriterMode2) Delete(ctx context.Context, name string, deleteValidat
 	return deletedLS, async, err
 }
 
-func enrichObject(orig, copy runtime.Object) (runtime.Object, error) {
-	accessorC, err := meta.Accessor(copy)
+// Update overrides the generic behavior of the Storage and writes first to the legacy storage and then to storage.
+func (d *DualWriterMode2) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
+	legacy, ok := d.Legacy.(rest.Updater)
+	if !ok {
+		return nil, false, errDualWriterUpdaterMissing
+	}
+
+	var notFound bool
+
+	// get old and new (updated) object so they can be stored in legacy store
+	old, err := d.Storage.Get(ctx, name, &metav1.GetOptions{})
 	if err != nil {
-		return nil, err
+		if !apierrors.IsNotFound(err) {
+			klog.FromContext(ctx).Error(err, "could not get object", "mode", Mode2)
+			return nil, false, err
+		}
+		klog.FromContext(ctx).Error(err, "object not found for update, creating one", "mode", Mode2)
+		notFound = true
 	}
-	accessorO, err := meta.Accessor(orig)
+
+	// obj can be populated in case it's found or empty in case it's not found
+	updated, err := objInfo.UpdatedObject(ctx, old)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
-	accessorC.SetLabels(accessorO.GetLabels())
-
-	ac := accessorC.GetAnnotations()
-	for k, v := range accessorO.GetAnnotations() {
-		ac[k] = v
+	obj, created, err := legacy.Update(ctx, name, &updateWrapper{upstream: objInfo, updated: updated}, createValidation, updateValidation, forceAllowCreate, options)
+	if err != nil {
+		klog.FromContext(ctx).Error(err, "could not update in legacy storage", "mode", Mode2)
+		return obj, created, err
 	}
-	accessorC.SetAnnotations(ac)
 
-	return copy, nil
+	if notFound {
+		return d.Storage.Update(ctx, name, objInfo, createValidation, updateValidation, forceAllowCreate, options)
+	}
+
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return nil, false, err
+	}
+
+	// only if object exists
+	accessorOld, err := meta.Accessor(old)
+	if err != nil {
+		return nil, false, err
+	}
+
+	enrichObject(accessorOld, accessor)
+
+	// keep the same UID and resource_version
+	accessor.SetResourceVersion(accessorOld.GetResourceVersion())
+	accessor.SetUID(accessorOld.GetUID())
+	objInfo = &updateWrapper{
+		upstream: objInfo,
+		updated:  obj,
+	}
+
+	// TODO: relies on GuaranteedUpdate creating the object if
+	// it doesn't exist: https://github.com/grafana/grafana/pull/85206
+	return d.Storage.Update(ctx, name, objInfo, createValidation, updateValidation, forceAllowCreate, options)
 }

pkg/apiserver/rest/dualwriter_mode2_test.go

@@ -368,7 +368,111 @@ func TestMode2_DeleteCollection(t *testing.T) {
 			continue
 		}
 
-		us.AssertNotCalled(t, "DeleteCollection", context.Background(), tt.input, func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{})
+		assert.Equal(t, obj, exampleObj)
+		assert.NotEqual(t, obj, anotherObj)
+	}
+}
+
+func TestMode2_Update(t *testing.T) {
+	type testCase struct {
+		name           string
+		input          string
+		setupLegacyFn  func(m *mock.Mock, input string)
+		setupStorageFn func(m *mock.Mock, input string)
+		setupGetFn     func(m *mock.Mock, input string)
+		wantErr        bool
+	}
+	tests :=
+		[]testCase{
+			{
+				name:  "update an object in both stores",
+				input: "foo",
+				setupLegacyFn: func(m *mock.Mock, input string) {
+					m.On("Update", context.Background(), input, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(exampleObj, false, nil)
+				},
+				setupStorageFn: func(m *mock.Mock, input string) {
+					m.On("Update", context.Background(), input, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(exampleObj, false, nil)
+				},
+				setupGetFn: func(m *mock.Mock, input string) {
+					m.On("Get", context.Background(), input, mock.Anything).Return(exampleObjDifferentRV, nil)
+				},
+			},
+			{
+				name:  "object is not found in storage",
+				input: "not-found",
+				setupLegacyFn: func(m *mock.Mock, input string) {
+					m.On("Update", context.Background(), input, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(exampleObj, false, nil)
+				},
+				setupStorageFn: func(m *mock.Mock, input string) {
+					m.On("Update", context.Background(), input, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(exampleObj, false, nil)
+				},
+				setupGetFn: func(m *mock.Mock, input string) {
+					m.On("Get", context.Background(), input, mock.Anything).Return(nil, apierrors.NewNotFound(schema.GroupResource{Group: "", Resource: "pods"}, "not found"))
+				},
+			},
+			{
+				name:  "error finding object storage",
+				input: "object-fail",
+				setupGetFn: func(m *mock.Mock, input string) {
+					m.On("Get", context.Background(), input, mock.Anything).Return(nil, errors.New("error"))
+				},
+				wantErr: true,
+			},
+			{
+				name:  "error updating legacy store",
+				input: "object-fail",
+				setupLegacyFn: func(m *mock.Mock, input string) {
+					m.On("Update", context.Background(), input, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, false, errors.New("error"))
+				},
+				setupGetFn: func(m *mock.Mock, input string) {
+					m.On("Get", context.Background(), input, mock.Anything).Return(exampleObjDifferentRV, nil)
+				},
+				wantErr: true,
+			},
+			{
+				name:  "error updating storage",
+				input: "object-fail",
+				setupLegacyFn: func(m *mock.Mock, input string) {
+					m.On("Update", context.Background(), input, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(exampleObj, false, nil)
+				},
+				setupStorageFn: func(m *mock.Mock, input string) {
+					m.On("Update", context.Background(), input, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, false, errors.New("error"))
+				},
+				setupGetFn: func(m *mock.Mock, input string) {
+					m.On("Get", context.Background(), input, mock.Anything).Return(exampleObj, nil)
+				},
+				wantErr: true,
+			},
+		}
+
+	for _, tt := range tests {
+		l := (LegacyStorage)(nil)
+		s := (Storage)(nil)
+		m := &mock.Mock{}
+
+		ls := legacyStoreMock{m, l}
+		us := storageMock{m, s}
+
+		if tt.setupGetFn != nil {
+			tt.setupGetFn(m, tt.input)
+		}
+
+		if tt.setupLegacyFn != nil {
+			tt.setupLegacyFn(m, tt.input)
+		}
+		if tt.setupStorageFn != nil {
+			tt.setupStorageFn(m, tt.input)
+		}
+
+		dw := SelectDualWriter(Mode2, ls, us)
+
+		obj, _, err := dw.Update(context.Background(), tt.input, UpdatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{})
+
+		if tt.wantErr {
+			assert.Error(t, err)
+			continue
+		}
+
 		assert.Equal(t, obj, exampleObj)
 		assert.NotEqual(t, obj, anotherObj)
 	}

pkg/apiserver/rest/dualwriter_mode3.go

@@ -69,6 +69,44 @@ func (d *DualWriterMode3) Delete(ctx context.Context, name string, deleteValidat
 	return deleted, async, err
 }
 
+// Update overrides the behavior of the generic DualWriter and writes first to Storage and then to LegacyStorage.
+func (d *DualWriterMode3) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
+	old, err := d.Storage.Get(ctx, name, &metav1.GetOptions{})
+	if err != nil {
+		return nil, false, err
+	}
+
+	updated, err := objInfo.UpdatedObject(ctx, old)
+	if err != nil {
+		return nil, false, err
+	}
+	objInfo = &updateWrapper{
+		upstream: objInfo,
+		updated:  updated,
+	}
+
+	obj, created, err := d.Storage.Update(ctx, name, objInfo, createValidation, updateValidation, forceAllowCreate, options)
+	if err != nil {
+		klog.FromContext(ctx).Error(err, "could not write to US", "mode", Mode3)
+		return obj, created, err
+	}
+
+	legacy, ok := d.Legacy.(rest.Updater)
+	if !ok {
+		klog.FromContext(ctx).Error(errDualWriterUpdaterMissing, "legacy storage update not implemented")
+		return obj, created, err
+	}
+
+	_, _, errLeg := legacy.Update(ctx, name, &updateWrapper{
+		upstream: objInfo,
+		updated:  obj,
+	}, createValidation, updateValidation, forceAllowCreate, options)
+	if errLeg != nil {
+		klog.FromContext(ctx).Error(errLeg, "could not update object in legacy store", "mode", Mode3)
+	}
+	return obj, created, err
+}
+
 // DeleteCollection overrides the behavior of the generic DualWriter and deletes from both LegacyStorage and Storage.
 func (d *DualWriterMode3) DeleteCollection(ctx context.Context, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions, listOptions *metainternalversion.ListOptions) (runtime.Object, error) {
 	legacy, ok := d.Legacy.(rest.CollectionDeleter)

pkg/apiserver/rest/dualwriter_mode3_test.go

@@ -8,6 +8,7 @@ import (
 	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/apis/example"
 )
 
 func TestMode3(t *testing.T) {
@@ -53,4 +54,19 @@ func TestMode3(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, 1, lsSpy.Counts("LegacyStorage.DeleteCollection"))
 	assert.Equal(t, 1, sSpy.Counts("Storage.DeleteCollection"))
+
+	// Update: it should update in both storages
+	dummy := &example.Pod{}
+	uoi := UpdatedObjInfoObj{}
+	_, err = uoi.UpdatedObject(context.Background(), dummy)
+	assert.NoError(t, err)
+
+	var validateObjFn = func(ctx context.Context, obj runtime.Object) error { return nil }
+	var validateObjUpdateFn = func(ctx context.Context, obj, old runtime.Object) error { return nil }
+
+	_, _, err = dw.Update(context.Background(), kind, uoi, validateObjFn, validateObjUpdateFn, false, &metav1.UpdateOptions{})
+	assert.NoError(t, err)
+	assert.Equal(t, 1, lsSpy.Counts("LegacyStorage.Update"))
+	assert.Equal(t, 1, sSpy.Counts("Storage.Update"))
+	assert.NoError(t, err)
 }

pkg/apiserver/rest/dualwriter_mode4.go

@@ -39,3 +39,8 @@ func (d *DualWriterMode4) Delete(ctx context.Context, name string, deleteValidat
 func (d *DualWriterMode4) DeleteCollection(ctx context.Context, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions, listOptions *metainternalversion.ListOptions) (runtime.Object, error) {
 	return d.Storage.DeleteCollection(ctx, deleteValidation, options, listOptions)
 }
+
+// Update overrides the generic behavior of the Storage and writes only to US.
+func (d *DualWriterMode4) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
+	return d.Storage.Update(ctx, name, objInfo, createValidation, updateValidation, forceAllowCreate, options)
+}

pkg/apiserver/rest/dualwriter_mode4_test.go

@@ -8,6 +8,7 @@ import (
 	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/apis/example"
 )
 
 func TestMode4(t *testing.T) {
@@ -53,4 +54,19 @@ func TestMode4(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, 0, lsSpy.Counts("LegacyStorage.DeleteCollection"))
 	assert.Equal(t, 1, sSpy.Counts("Storage.DeleteCollection"))
+
+	// Update: it should update only in Storage
+	dummy := &example.Pod{}
+	uoi := UpdatedObjInfoObj{}
+	_, err = uoi.UpdatedObject(context.Background(), dummy)
+	assert.NoError(t, err)
+
+	var validateObjFn = func(ctx context.Context, obj runtime.Object) error { return nil }
+	var validateObjUpdateFn = func(ctx context.Context, obj, old runtime.Object) error { return nil }
+
+	_, _, err = dw.Update(context.Background(), kind, uoi, validateObjFn, validateObjUpdateFn, false, &metav1.UpdateOptions{})
+	assert.NoError(t, err)
+	assert.Equal(t, 0, lsSpy.Counts("LegacyStorage.Update"))
+	assert.Equal(t, 1, sSpy.Counts("Storage.Update"))
+	assert.NoError(t, err)
 }

pkg/apiserver/rest/spy_client.go

@@ -8,6 +8,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/apis/example"
 	"k8s.io/apiserver/pkg/registry/rest"
 	"k8s.io/klog/v2"
 )
@@ -65,7 +66,7 @@ func (c *spyStorageClient) Create(ctx context.Context, obj runtime.Object, valit
 func (c *spyStorageClient) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
 	c.spy.record("Storage.Get")
 	klog.Info("method: Storage.Get")
-	return nil, nil
+	return &example.Pod{}, nil
 }
 
 func (c *spyStorageClient) List(ctx context.Context, options *metainternalversion.ListOptions) (runtime.Object, error) {
@@ -89,10 +90,18 @@ func (c *spyStorageClient) List(ctx context.Context, options *metainternalversio
 	return &dummyList{Items: []dummyObject{i1, i2}}, nil
 }
 
+type UpdatedObjInfoObj struct{}
+
+func (u UpdatedObjInfoObj) UpdatedObject(ctx context.Context, oldObj runtime.Object) (newObj runtime.Object, err error) {
+	return &example.Pod{}, nil
+}
+
+func (u UpdatedObjInfoObj) Preconditions() *metav1.Preconditions { return &metav1.Preconditions{} }
+
 func (c *spyStorageClient) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
 	c.spy.record("Storage.Update")
 	klog.Info("method: Storage.Update")
-	return nil, false, nil
+	return &example.Pod{}, false, nil
 }
 
 func (c *spyStorageClient) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) {
@@ -149,7 +158,7 @@ func (c *spyLegacyStorageClient) Create(ctx context.Context, obj runtime.Object,
 func (c *spyLegacyStorageClient) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
 	c.spy.record("LegacyStorage.Get")
 	klog.Info("method: LegacyStorage.Get")
-	return nil, nil
+	return &example.Pod{}, nil
 }
 
 func (c *spyLegacyStorageClient) NewList() runtime.Object {
@@ -181,7 +190,7 @@ func (c *spyLegacyStorageClient) List(ctx context.Context, options *metainternal
 func (c *spyLegacyStorageClient) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
 	c.spy.record("LegacyStorage.Update")
 	klog.Info("method: LegacyStorage.Update")
-	return nil, false, nil
+	return &example.Pod{}, false, nil
 }
 
 func (c *spyLegacyStorageClient) Delete(ctx context.Context, name string, deleteValidation rest.ValidateObjectFunc, options *metav1.DeleteOptions) (runtime.Object, bool, error) {

pkg/apiserver/rest/storage_mocks_test.go

@@ -52,6 +52,9 @@ func (m legacyStoreMock) List(ctx context.Context, options *metainternalversion.
 
 func (m legacyStoreMock) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
 	args := m.Called(ctx, name, objInfo, createValidation, updateValidation, forceAllowCreate, options)
+	if name == "object-fail" {
+		return nil, false, args.Error(2)
+	}
 	return args.Get(0).(runtime.Object), args.Bool(1), args.Error(2)
 }
 
@@ -80,6 +83,9 @@ func (m storageMock) Get(ctx context.Context, name string, options *metav1.GetOp
 	if name == "object-fail" {
 		return nil, args.Error(1)
 	}
+	if name == "not-found" {
+		return nil, args.Error(1)
+	}
 	return args.Get(0).(runtime.Object), args.Error(1)
 }
 
@@ -106,6 +112,9 @@ func (m storageMock) List(ctx context.Context, options *metainternalversion.List
 
 func (m storageMock) Update(ctx context.Context, name string, objInfo rest.UpdatedObjectInfo, createValidation rest.ValidateObjectFunc, updateValidation rest.ValidateObjectUpdateFunc, forceAllowCreate bool, options *metav1.UpdateOptions) (runtime.Object, bool, error) {
 	args := m.Called(ctx, name, objInfo, createValidation, updateValidation, forceAllowCreate, options)
+	if name == "object-fail" {
+		return nil, false, args.Error(2)
+	}
 	return args.Get(0).(runtime.Object), args.Bool(1), args.Error(2)
 }