IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2025-02-25 18:55:24 -06:00
Code Issues Packages Projects Releases Wiki Activity

MM-20104 -Migrate channelCache cache from store/sqlstore/channel_store.go to th… (#13254)

Browse Source 
* Migrate channelCache cache from store/sqlstore/channel_store.go to the new store/localcachelayer

* fix import for new path

* remove unnecessary code, refactor cache constant

* resolve conflicts

* fix missing variable and import after merging

* refactor code, fix current issue

Co-authored-by: mattermod <mattermod@users.noreply.github.com>
This commit is contained in:
Vladimir Lebedev
2019-12-20 11:04:58 +03:00
committed by Agniva De Sarker
parent 722924b910
commit e4481aae21
5 changed files with 106 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
store/localcachelayer/channel_layer.go
View File

@@ -37,15 +37,25 @@ func (s *LocalCacheChannelStore) handleClusterInvalidateChannelGuestCounts(msg *
}
}
func (s *LocalCacheChannelStore) handleClusterInvalidateChannelById(msg *model.ClusterMessage) {
if msg.Data == CLEAR_CACHE_MESSAGE_DATA {
s.rootStore.channelByIdCache.Purge()
} else {
s.rootStore.channelByIdCache.Remove(msg.Data)
}
}
func (s LocalCacheChannelStore) ClearCaches() {
s.rootStore.doClearCacheCluster(s.rootStore.channelMemberCountsCache)
s.rootStore.doClearCacheCluster(s.rootStore.channelPinnedPostCountsCache)
s.rootStore.doClearCacheCluster(s.rootStore.channelGuestCountCache)
s.rootStore.doClearCacheCluster(s.rootStore.channelByIdCache)
s.ChannelStore.ClearCaches()
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter("Channel Pinned Post Counts - Purge")
s.rootStore.metrics.IncrementMemCacheInvalidationCounter("Channel Member Counts - Purge")
s.rootStore.metrics.IncrementMemCacheInvalidationCounter("Channel Guest Count - Purge")
s.rootStore.metrics.IncrementMemCacheInvalidationCounter("Channel - Purge")
}
}
@@ -70,6 +80,13 @@ func (s LocalCacheChannelStore) InvalidateGuestCount(channelId string) {
}
}
func (s LocalCacheChannelStore) InvalidateChannel(channelId string) {
s.rootStore.doInvalidateCacheCluster(s.rootStore.channelByIdCache, channelId)
if s.rootStore.metrics != nil {
s.rootStore.metrics.IncrementMemCacheInvalidationCounter("Channel - Remove by ChannelId")
}
}
func (s LocalCacheChannelStore) GetMemberCount(channelId string, allowFromCache bool) (int64, *model.AppError) {
if allowFromCache {
if count := s.rootStore.doStandardReadCache(s.rootStore.channelMemberCountsCache, channelId); count != nil {
@@ -132,3 +149,21 @@ func (s LocalCacheChannelStore) GetPinnedPostCount(channelId string, allowFromCa
return count, nil
}
func (s LocalCacheChannelStore) Get(id string, allowFromCache bool) (*model.Channel, *model.AppError) {
if allowFromCache {
if cacheItem := s.rootStore.doStandardReadCache(s.rootStore.channelByIdCache, id); cacheItem != nil {
ch := cacheItem.(*model.Channel).DeepCopy()
return ch, nil
}
}
ch, err := s.ChannelStore.Get(id, allowFromCache)
if allowFromCache && err == nil {
s.rootStore.doStandardAddToCache(s.rootStore.channelByIdCache, id, ch)
}
return ch, err
}

61
store/localcachelayer/channel_layer_test.go
View File

@@ -4,6 +4,7 @@
package localcachelayer
import (
"github.com/mattermost/mattermost-server/v5/model"
"testing"
"github.com/stretchr/testify/assert"
@@ -214,3 +215,63 @@ func TestChannelStoreGuestCountCache(t *testing.T) {
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "GetGuestCount", 2)
})
}
func TestChannelStoreChannel(t *testing.T) {
channelId := "channel1"
fakeChannel := model.Channel{Id: channelId}
t.Run("first call by id not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore()
cachedStore := NewLocalCacheLayer(mockStore, nil, nil)
channel, err := cachedStore.Channel().Get(channelId, true)
require.Nil(t, err)
assert.Equal(t, channel, &fakeChannel)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 1)
channel, err = cachedStore.Channel().Get(channelId, true)
require.Nil(t, err)
assert.Equal(t, channel, &fakeChannel)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 1)
})
t.Run("first call not cached, second force no cached", func(t *testing.T) {
mockStore := getMockStore()
cachedStore := NewLocalCacheLayer(mockStore, nil, nil)
cachedStore.Channel().Get(channelId, true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Channel().Get(channelId, false)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first call force no cached, second not cached, third cached", func(t *testing.T) {
mockStore := getMockStore()
cachedStore := NewLocalCacheLayer(mockStore, nil, nil)
cachedStore.Channel().Get(channelId, false)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Channel().Get(channelId, true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 2)
cachedStore.Channel().Get(channelId, true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first call not cached, clear cache, second call not cached", func(t *testing.T) {
mockStore := getMockStore()
cachedStore := NewLocalCacheLayer(mockStore, nil, nil)
cachedStore.Channel().Get(channelId, true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Channel().ClearCaches()
cachedStore.Channel().Get(channelId, true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first call not cached, invalidate cache, second call not cached", func(t *testing.T) {
mockStore := getMockStore()
cachedStore := NewLocalCacheLayer(mockStore, nil, nil)
cachedStore.Channel().Get(channelId, true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Channel().InvalidateChannel(channelId)
cachedStore.Channel().Get(channelId, true)
mockStore.Channel().(*mocks.ChannelStore).AssertNumberOfCalls(t, "Get", 2)
})
}

6
store/localcachelayer/layer.go
View File

@@ -48,6 +48,8 @@ const (
TEAM_CACHE_SEC = 30 * 60
CLEAR_CACHE_MESSAGE_DATA = ""
CHANNEL_CACHE_SEC = 15 * 60 // 15 mins
)
type LocalCacheStore struct {
@@ -67,6 +69,7 @@ type LocalCacheStore struct {
channelMemberCountsCache *utils.Cache
channelGuestCountCache *utils.Cache
channelPinnedPostCountsCache *utils.Cache
channelByIdCache *utils.Cache
webhook LocalCacheWebhookStore
webhookCache *utils.Cache
post LocalCachePostStore
@@ -98,6 +101,7 @@ func NewLocalCacheLayer(baseStore store.Store, metrics einterfaces.MetricsInterf
localCacheStore.channelPinnedPostCountsCache = utils.NewLruWithParams(CHANNEL_PINNEDPOSTS_COUNTS_CACHE_SIZE, "ChannelPinnedPostsCounts", CHANNEL_PINNEDPOSTS_COUNTS_CACHE_SEC, model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_PINNEDPOSTS_COUNTS)
localCacheStore.channelMemberCountsCache = utils.NewLruWithParams(CHANNEL_MEMBERS_COUNTS_CACHE_SIZE, "ChannelMemberCounts", CHANNEL_MEMBERS_COUNTS_CACHE_SEC, model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_MEMBER_COUNTS)
localCacheStore.channelGuestCountCache = utils.NewLruWithParams(CHANNEL_GUEST_COUNT_CACHE_SIZE, "ChannelGuestsCount", CHANNEL_GUEST_COUNT_CACHE_SEC, model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_GUEST_COUNT)
localCacheStore.channelByIdCache = utils.NewLruWithParams(model.CHANNEL_CACHE_SIZE, "channelById", CHANNEL_CACHE_SEC, model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL)
localCacheStore.channel = LocalCacheChannelStore{ChannelStore: baseStore.Channel(), rootStore: &localCacheStore}
localCacheStore.lastPostTimeCache = utils.NewLruWithParams(LAST_POST_TIME_CACHE_SIZE, "LastPostTime", LAST_POST_TIME_CACHE_SEC, model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_LAST_POST_TIME)
localCacheStore.postLastPostsCache = utils.NewLruWithParams(LAST_POSTS_CACHE_SIZE, "LastPost", LAST_POSTS_CACHE_SEC, model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_LAST_POSTS)
@@ -118,6 +122,7 @@ func NewLocalCacheLayer(baseStore store.Store, metrics einterfaces.MetricsInterf
cluster.RegisterClusterMessageHandler(model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_PINNEDPOSTS_COUNTS, localCacheStore.channel.handleClusterInvalidateChannelPinnedPostCount)
cluster.RegisterClusterMessageHandler(model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_MEMBER_COUNTS, localCacheStore.channel.handleClusterInvalidateChannelMemberCounts)
cluster.RegisterClusterMessageHandler(model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_GUEST_COUNT, localCacheStore.channel.handleClusterInvalidateChannelGuestCounts)
cluster.RegisterClusterMessageHandler(model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL, localCacheStore.channel.handleClusterInvalidateChannelById)
cluster.RegisterClusterMessageHandler(model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_LAST_POSTS, localCacheStore.post.handleClusterInvalidateLastPosts)
cluster.RegisterClusterMessageHandler(model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_PROFILE_BY_IDS, localCacheStore.user.handleClusterInvalidateScheme)
cluster.RegisterClusterMessageHandler(model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_TEAMS, localCacheStore.team.handleClusterInvalidateTeam)
@@ -217,6 +222,7 @@ func (s *LocalCacheStore) Invalidate() {
s.doClearCacheCluster(s.channelMemberCountsCache)
s.doClearCacheCluster(s.channelPinnedPostCountsCache)
s.doClearCacheCluster(s.channelGuestCountCache)
s.doClearCacheCluster(s.channelByIdCache)
s.doClearCacheCluster(s.postLastPostsCache)
s.doClearCacheCluster(s.lastPostTimeCache)
s.doClearCacheCluster(s.userProfileByIdsCache)

4
store/localcachelayer/main_test.go
View File

@@ -60,12 +60,16 @@ func getMockStore() *mocks.Store {
mockCount := int64(10)
mockGuestCount := int64(12)
channelId := "channel1"
fakeChannelId := model.Channel{Id: channelId}
mockChannelStore := mocks.ChannelStore{}
mockChannelStore.On("ClearCaches").Return()
mockChannelStore.On("GetMemberCount", "id", true).Return(mockCount, nil)
mockChannelStore.On("GetMemberCount", "id", false).Return(mockCount, nil)
mockChannelStore.On("GetGuestCount", "id", true).Return(mockGuestCount, nil)
mockChannelStore.On("GetGuestCount", "id", false).Return(mockGuestCount, nil)
mockChannelStore.On("Get", channelId, true).Return(&fakeChannelId, nil)
mockChannelStore.On("Get", channelId, false).Return(&fakeChannelId, nil)
mockStore.On("Channel").Return(&mockChannelStore)
mockPinnedPostsCount := int64(10)

22
store/sqlstore/channel_store.go
View File

@@ -276,19 +276,16 @@ type publicChannel struct {
var allChannelMembersForUserCache = utils.NewLru(ALL_CHANNEL_MEMBERS_FOR_USER_CACHE_SIZE)
var allChannelMembersNotifyPropsForChannelCache = utils.NewLru(ALL_CHANNEL_MEMBERS_NOTIFY_PROPS_FOR_CHANNEL_CACHE_SIZE)
var channelCache = utils.NewLru(model.CHANNEL_CACHE_SIZE)
var channelByNameCache = utils.NewLru(model.CHANNEL_CACHE_SIZE)
func (s SqlChannelStore) ClearCaches() {
allChannelMembersForUserCache.Purge()
allChannelMembersNotifyPropsForChannelCache.Purge()
channelCache.Purge()
channelByNameCache.Purge()
if s.metrics != nil {
s.metrics.IncrementMemCacheInvalidationCounter("All Channel Members for User - Purge")
s.metrics.IncrementMemCacheInvalidationCounter("All Channel Members Notify Props for Channel - Purge")
s.metrics.IncrementMemCacheInvalidationCounter("Channel - Purge")
s.metrics.IncrementMemCacheInvalidationCounter("Channel By Name - Purge")
}
}
@@ -661,10 +658,6 @@ func (s SqlChannelStore) GetChannelUnread(channelId, userId string) (*model.Chan
}
func (s SqlChannelStore) InvalidateChannel(id string) {
channelCache.Remove(id)
if s.metrics != nil {
s.metrics.IncrementMemCacheInvalidationCounter("Channel - Remove by ChannelId")
}
}
func (s SqlChannelStore) InvalidateChannelByName(teamId, name string) {
@@ -705,20 +698,6 @@ func (s SqlChannelStore) get(id string, master bool, allowFromCache bool) (*mode
db = s.GetReplica()
}
if allowFromCache {
if cacheItem, ok := channelCache.Get(id); ok {
if s.metrics != nil {
s.metrics.IncrementMemCacheHitCounter("Channel")
}
ch := cacheItem.(*model.Channel).DeepCopy()
return ch, nil
}
}
if s.metrics != nil {
s.metrics.IncrementMemCacheMissCounter("Channel")
}
obj, err := db.Get(model.Channel{}, id)
if err != nil {
return nil, model.NewAppError("SqlChannelStore.Get", "store.sql_channel.get.find.app_error", nil, "id="+id+", "+err.Error(), http.StatusInternalServerError)
@@ -729,7 +708,6 @@ func (s SqlChannelStore) get(id string, master bool, allowFromCache bool) (*mode
}
ch := obj.(*model.Channel)
channelCache.AddWithExpiresInSecs(id, ch, CHANNEL_CACHE_SEC)
return ch, nil
}