GH-12936 Add KVListWithOptions function to plugin.Helpers (#13576)

Automatic Merge

plugin/helpers.go

@@ -56,6 +56,11 @@ type Helpers interface {
 	// Minimum server version: 5.6
 	KVSetWithExpiryJSON(key string, value interface{}, expireInSeconds int64) error
 
+	// KVListWithOptions returns all keys that match the given options.  If no options are provided then all keys are returned.
+	//
+	// Minimum server version: 5.6
+	KVListWithOptions(options ...KVListOption) ([]string, error)
+
 	// CheckRequiredServerConfiguration checks if the server is configured according to
 	// plugin requirements.
 	//

plugin/helpers_kv.go

@@ -5,6 +5,7 @@ package plugin
 
 import (
 	"encoding/json"
+	"strings"
 
 	"github.com/pkg/errors"
 )
@@ -129,3 +130,93 @@ func (p *HelpersImpl) KVSetWithExpiryJSON(key string, value interface{}, expireI
 
 	return nil
 }
+
+type kvListOptions struct {
+	checkers []func(key string) (keep bool, err error)
+}
+
+func (o *kvListOptions) checkAll(key string) (keep bool, err error) {
+	for _, check := range o.checkers {
+		keep, err := check(key)
+		if err != nil {
+			return false, err
+		}
+		if !keep {
+			return false, nil
+		}
+	}
+
+	// key made it through all checkers
+	return true, nil
+}
+
+// KVListOption represents a single input option for KVListWithOptions
+type KVListOption func(*kvListOptions)
+
+// WithPrefix only return keys that start with the given string.
+func WithPrefix(prefix string) KVListOption {
+	return WithChecker(func(key string) (keep bool, err error) {
+		return strings.HasPrefix(key, prefix), nil
+	})
+}
+
+// WithChecker allows for a custom filter function to determine which keys to return.
+// Returning true will keep the key and false will filter it out. Returning an error
+// will halt KVListWithOptions immediately and pass the error up (with no other results).
+func WithChecker(f func(key string) (keep bool, err error)) KVListOption {
+	return func(args *kvListOptions) {
+		args.checkers = append(args.checkers, f)
+	}
+}
+
+// kvListPerPage is the number of keys KVListWithOptions gets per request
+const kvListPerPage = 100
+
+// KVListWithOptions implements Helpers.KVListWithOptions.
+func (p *HelpersImpl) KVListWithOptions(options ...KVListOption) ([]string, error) {
+	err := p.ensureServerVersion("5.6.0")
+	if err != nil {
+		return nil, err
+	}
+	// convert functional options into args struct
+	args := &kvListOptions{}
+	for _, opt := range options {
+		opt(args)
+	}
+	ret := make([]string, 0)
+
+	// get our keys a batch at a time, filter out the ones we don't want based on our args
+	// any errors will hault the whole process and return the error raw
+	for i := 0; ; i++ {
+		keys, appErr := p.API.KVList(i, kvListPerPage)
+		if appErr != nil {
+			return nil, appErr
+		}
+
+		if len(args.checkers) == 0 {
+			// no checkers, just append the whole block at once
+			ret = append(ret, keys...)
+		} else {
+			// we have a filter, so check each key, all checkers must say key
+			// for us to keep a key
+			for _, key := range keys {
+				keep, err := args.checkAll(key)
+				if err != nil {
+					return nil, err
+				}
+				if !keep {
+					continue
+				}
+
+				// didn't get filtered out, add to our return
+				ret = append(ret, key)
+			}
+		}
+
+		if len(keys) < kvListPerPage {
+			break
+		}
+	}
+
+	return ret, nil
+}

plugin/helpers_kv_test.go

@@ -4,6 +4,7 @@
 package plugin_test
 
 import (
+	"strconv"
 	"testing"
 
 	"github.com/mattermost/mattermost-server/v5/model"
@@ -395,3 +396,243 @@ func TestKVSetWithExpiryJSON(t *testing.T) {
 		assert.NoError(t, err)
 	})
 }
+
+func TestKVListWithOptions(t *testing.T) {
+	t.Run("incompatible server version", func(t *testing.T) {
+		api := &plugintest.API{}
+		api.On("GetServerVersion").Return("5.1.0")
+
+		p := &plugin.HelpersImpl{API: api}
+
+		keys, err := p.KVListWithOptions()
+
+		api.AssertExpectations(t)
+		assert.Nil(t, keys)
+		assert.Error(t, err)
+		assert.Equal(t, "incompatible server version for plugin, minimum required version: 5.6.0, current version: 5.1.0", err.Error())
+	})
+
+	t.Run("KVList error", func(t *testing.T) {
+		p := &plugin.HelpersImpl{}
+
+		api := &plugintest.API{}
+		api.On("GetServerVersion").Return("5.6.0")
+		api.On("KVList", 0, 100).Return([]string{}, &model.AppError{})
+		p.API = api
+
+		keys, err := p.KVListWithOptions()
+		api.AssertExpectations(t)
+		assert.Empty(t, keys)
+		assert.Error(t, err)
+	})
+
+	t.Run("No keys", func(t *testing.T) {
+		p := &plugin.HelpersImpl{}
+
+		api := &plugintest.API{}
+		api.On("GetServerVersion").Return("5.6.0")
+		api.On("KVList", 0, 100).Return(nil, nil)
+		p.API = api
+
+		keys, err := p.KVListWithOptions()
+		api.AssertExpectations(t)
+		assert.Empty(t, keys)
+		assert.Nil(t, err)
+	})
+
+	t.Run("Basic Success, one page", func(t *testing.T) {
+		p := &plugin.HelpersImpl{}
+
+		api := &plugintest.API{}
+		api.On("GetServerVersion").Return("5.6.0")
+		api.On("KVList", 0, 100).Return([]string{"key1", "key2"}, nil)
+		p.API = api
+
+		keys, err := p.KVListWithOptions()
+		api.AssertExpectations(t)
+		assert.ElementsMatch(t, keys, []string{"key1", "key2"})
+		assert.Nil(t, err)
+	})
+
+	t.Run("Basic Success, two page", func(t *testing.T) {
+		p := &plugin.HelpersImpl{}
+
+		api := &plugintest.API{}
+		api.On("GetServerVersion").Return("5.6.0")
+		api.On("KVList", 0, 100).Return(getKeys(100), nil)
+		api.On("KVList", 1, 100).Return([]string{"key100"}, nil)
+		p.API = api
+
+		keys, err := p.KVListWithOptions()
+		api.AssertExpectations(t)
+		assert.ElementsMatch(t, keys, getKeys(101))
+		assert.Nil(t, err)
+	})
+
+	t.Run("error on second page", func(t *testing.T) {
+		p := &plugin.HelpersImpl{}
+
+		api := &plugintest.API{}
+		api.On("GetServerVersion").Return("5.6.0")
+		api.On("KVList", 0, 100).Return(getKeys(100), nil)
+		api.On("KVList", 1, 100).Return([]string{"key100"}, &model.AppError{})
+		p.API = api
+
+		keys, err := p.KVListWithOptions()
+		api.AssertExpectations(t)
+		assert.Empty(t, keys)
+		assert.Error(t, err)
+	})
+
+	t.Run("success, two page, filter prefix, one", func(t *testing.T) {
+		p := &plugin.HelpersImpl{}
+
+		api := &plugintest.API{}
+		api.On("GetServerVersion").Return("5.6.0")
+		api.On("KVList", 0, 100).Return(getKeys(100), nil)
+		api.On("KVList", 1, 100).Return([]string{"key100"}, nil)
+		p.API = api
+
+		keys, err := p.KVListWithOptions(plugin.WithPrefix("key99"))
+		api.AssertExpectations(t)
+		assert.ElementsMatch(t, keys, []string{"key99"})
+		assert.Nil(t, err)
+	})
+
+	t.Run("success, two page, filter prefix, all", func(t *testing.T) {
+		p := &plugin.HelpersImpl{}
+
+		api := &plugintest.API{}
+		api.On("GetServerVersion").Return("5.6.0")
+		api.On("KVList", 0, 100).Return(getKeys(100), nil)
+		api.On("KVList", 1, 100).Return([]string{"key100"}, nil)
+		p.API = api
+
+		keys, err := p.KVListWithOptions(plugin.WithPrefix("notkey"))
+		api.AssertExpectations(t)
+		assert.Empty(t, keys)
+		assert.Nil(t, err)
+	})
+
+	t.Run("success, two page, filter prefix, none", func(t *testing.T) {
+		p := &plugin.HelpersImpl{}
+
+		api := &plugintest.API{}
+		api.On("GetServerVersion").Return("5.6.0")
+		api.On("KVList", 0, 100).Return(getKeys(100), nil)
+		api.On("KVList", 1, 100).Return([]string{"key100"}, nil)
+		p.API = api
+
+		keys, err := p.KVListWithOptions(plugin.WithPrefix("key"))
+		api.AssertExpectations(t)
+		assert.ElementsMatch(t, keys, getKeys(101))
+		assert.Nil(t, err)
+	})
+
+	t.Run("success, two page, checker func, one", func(t *testing.T) {
+		p := &plugin.HelpersImpl{}
+
+		api := &plugintest.API{}
+		api.On("GetServerVersion").Return("5.6.0")
+		api.On("KVList", 0, 100).Return(getKeys(100), nil)
+		api.On("KVList", 1, 100).Return([]string{"key100"}, nil)
+		p.API = api
+
+		check := func(key string) (bool, error) {
+			if key == "key1" {
+				return true, nil
+			}
+			return false, nil
+		}
+
+		keys, err := p.KVListWithOptions(plugin.WithChecker(check))
+		api.AssertExpectations(t)
+		assert.ElementsMatch(t, keys, []string{"key1"})
+		assert.Nil(t, err)
+	})
+
+	t.Run("success, two page, checker func, all", func(t *testing.T) {
+		p := &plugin.HelpersImpl{}
+
+		api := &plugintest.API{}
+		api.On("GetServerVersion").Return("5.6.0")
+		api.On("KVList", 0, 100).Return(getKeys(100), nil)
+		api.On("KVList", 1, 100).Return([]string{"key100"}, nil)
+		p.API = api
+
+		check := func(key string) (bool, error) {
+			return false, nil
+		}
+
+		keys, err := p.KVListWithOptions(plugin.WithChecker(check))
+		api.AssertExpectations(t)
+		assert.Empty(t, keys)
+		assert.Nil(t, err)
+	})
+
+	t.Run("success, two page, checker func, none", func(t *testing.T) {
+		p := &plugin.HelpersImpl{}
+
+		api := &plugintest.API{}
+		api.On("GetServerVersion").Return("5.6.0")
+		api.On("KVList", 0, 100).Return(getKeys(100), nil)
+		api.On("KVList", 1, 100).Return([]string{"key100"}, nil)
+		p.API = api
+
+		check := func(key string) (bool, error) {
+			return true, nil
+		}
+
+		keys, err := p.KVListWithOptions(plugin.WithChecker(check))
+		api.AssertExpectations(t)
+		assert.ElementsMatch(t, keys, getKeys(101))
+		assert.Nil(t, err)
+	})
+
+	t.Run("error, checker func", func(t *testing.T) {
+		p := &plugin.HelpersImpl{}
+
+		api := &plugintest.API{}
+		api.On("GetServerVersion").Return("5.6.0")
+		api.On("KVList", 0, 100).Return([]string{"key1"}, nil)
+		p.API = api
+
+		check := func(key string) (bool, error) {
+			return true, &model.AppError{}
+		}
+
+		keys, err := p.KVListWithOptions(plugin.WithChecker(check))
+		api.AssertExpectations(t)
+		assert.Empty(t, keys)
+		assert.Error(t, err)
+	})
+
+	t.Run("success, filter and checker func, partial on both", func(t *testing.T) {
+		p := &plugin.HelpersImpl{}
+
+		api := &plugintest.API{}
+		api.On("GetServerVersion").Return("5.6.0")
+		api.On("KVList", 0, 100).Return([]string{"key1", "key2", "notkey3", "key4", "key5"}, nil)
+		p.API = api
+
+		check := func(key string) (bool, error) {
+			if key == "key1" || key == "key5" {
+				return false, nil
+			}
+			return true, nil
+		}
+
+		keys, err := p.KVListWithOptions(plugin.WithPrefix("key"), plugin.WithChecker(check))
+		api.AssertExpectations(t)
+		assert.ElementsMatch(t, keys, []string{"key2", "key4"})
+		assert.Nil(t, err)
+	})
+}
+
+func getKeys(count int) []string {
+	ret := make([]string, count)
+	for i := 0; i < count; i++ {
+		ret[i] = "key" + strconv.Itoa(i)
+	}
+	return ret
+}

plugin/plugintest/helpers.go

@@ -150,6 +150,35 @@ func (_m *Helpers) KVGetJSON(key string, value interface{}) (bool, error) {
 	return r0, r1
 }
 
+// KVListWithOptions provides a mock function with given fields: options
+func (_m *Helpers) KVListWithOptions(options ...plugin.KVListOption) ([]string, error) {
+	_va := make([]interface{}, len(options))
+	for _i := range options {
+		_va[_i] = options[_i]
+	}
+	var _ca []interface{}
+	_ca = append(_ca, _va...)
+	ret := _m.Called(_ca...)
+
+	var r0 []string
+	if rf, ok := ret.Get(0).(func(...plugin.KVListOption) []string); ok {
+		r0 = rf(options...)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]string)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(...plugin.KVListOption) error); ok {
+		r1 = rf(options...)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // KVSetJSON provides a mock function with given fields: key, value
 func (_m *Helpers) KVSetJSON(key string, value interface{}) error {
 	ret := _m.Called(key, value)