Files
mattermost/store/local_cache_supplier.go

135 lines
3.8 KiB
Go
Raw Normal View History

// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package store
import (
"context"
2017-09-06 23:05:10 -07:00
"github.com/mattermost/mattermost-server/einterfaces"
"github.com/mattermost/mattermost-server/model"
"github.com/mattermost/mattermost-server/utils"
)
const (
REACTION_CACHE_SIZE = 20000
REACTION_CACHE_SEC = 30 * 60
ROLE_CACHE_SIZE = 20000
ROLE_CACHE_SEC = 30 * 60
SCHEME_CACHE_SIZE = 20000
SCHEME_CACHE_SEC = 30 * 60
Ldap groups phase1 (#9752) * Initial models, API, app, and persistence of groups and group syncing. * Consistent letter casing in ldif. * Moves group-specific migrations into func. * Adds API endpoint to retrieve LDAP groups (and associated MM groups) one tree level at a time. * Adds mattermost group id to SCIMGroup (if available). * Splits user and group creation so that memberOf works. Returns users from ldap interface. * Updates method name. * Returns users IDs instead of User. * Removes non-essential group data. * MM-11807: Add GroupFilter to LDAP config. (#9513) * MM-11807: Add GroupFilter to LDAP config. * Add diagnostic. * Adds new config option for using 'memberOf' overlay. * Adds API endpoint to link a group. * Removes debug statements. * Adds unlink group API endpoint. * Fix to LDAP API. Adds API method to client4 and app. * Adds some missing app methods. Renames API unexported func. * Fixes link/unlink API path to accept valid DNs. * Allow any character for DN portion of path. * Switches from DN to objectGUID or entryUUID as the remote identifier linking LDAP groups to MM groups. * Formatting. * Formatting. * Setting group name field to an ID for phase 1. * Adds an LDAP config field to Setting up configuration for local LDAP. * Changes to LDAP and GroupStore interfaces. * Draft of nesting groups in API response. * Removes unnecessary tree models. * Updates group membershipt create store method to also restore. * Adds new config to test config. * Accept AD format length. * Switches to SetUniqueTogether method. * Updates revert. * Tweaks to syncing queries . * Updates query for pending team and channel memberships. * Removes old GroupSyncableScanner usage. Some formatting and renaming. * Fixes bug setting syncable type in selecting paged. * Adds tests for syncables populator. * Only add users to teams and channels that are not deleted. * Renames method. * Updates test LDAP setup. * Removes memberof config stuff. * Renames. * Updates test data. * Fix for gofmt. * Adds missing license. * Adds missing teardowns. * Test fix. * Adds a cycle to the groups test data. * Changes API to return flat list. * Removes some unused interface and app methods. * Returns empty braces if results are empty. * Adds more LDAP test data. * Fix for test data error. * Adds error. * Moves test groups. * Adds OU for load test data. * Moves load test ou creation to load data. * Adds a new bool flag to SCIMGroups. * Removes SCIMGroup completely. * Removes FULL JOIN because it is not supported in MySQL. * Adds tests for sync queries; renames constant. * Bad merge fix. * Vet fix. * Returning OK on delete ldap group link * Removes foreign key constraints. * Adding total to the ldap getAllGroups api endpoint * Adds get group members page. * Removes pagination from groups syncables list API. * Adding syncable check now that foreign key constraint is removes. * Joins teams and channels to group syncables. * Adds group member count. * Adding GetAllChannels and SearchAllChannels for system admins only * Fix. * Test fix from pagination removal. * Orders groupmembers by createat. * Fixing search of all channels * Test fix after removing pagination. * JSON syntax error fix. * Changing tests (for now) pending investigation. * Adding GetAllChannels and SearchAllChannels tests for the store * Adding GetAllChannels and SearchAllChannels API tests * Omit empty JSON values of group syncables. * Fixing GetAllChannels and SearchAllChannels tests * Fixing GetAllChannels and SearchAllChannels store tests * Fixing GetAllChannels api tests * Adds 'LDAP groups' feature flag. (#9861) * Migrate new client functions to idiomatic error handling * Test fixes. * Simplification of groups api (#9860) * Simplification of groups api * Fixing RequireSyncableType * Test fix. * Update api4/group.go Co-Authored-By: mkraft <martinkraft@gmail.com> * Update api4/group.go Co-Authored-By: mkraft <martinkraft@gmail.com> * Update api4/group.go Co-Authored-By: mkraft <martinkraft@gmail.com> * Update api4/group.go Co-Authored-By: mkraft <martinkraft@gmail.com> * Update api4/group.go Co-Authored-By: mkraft <martinkraft@gmail.com> * Update api4/group.go Co-Authored-By: mkraft <martinkraft@gmail.com> * Update api4/group.go Co-Authored-By: mkraft <martinkraft@gmail.com> * Update api4/group.go Co-Authored-By: mkraft <martinkraft@gmail.com> * Update api4/group.go Co-Authored-By: mkraft <martinkraft@gmail.com> * Update api4/group.go Co-Authored-By: mkraft <martinkraft@gmail.com> * Fix copy/paste error. * Fix copy/paste error. * Adds missing return, changes to correct HTTP status code. * Adds missing return, changes status codes. * Check for license. * Renames variable for new signature. * Adds client method to get a group. * Adds client method and tests for PatchGroup. * Adds more API tests. * Adds groups API tests. * Adds client method and tests for getting group syncables. * Adds tests for patching group teams and channels. * Update to translations. * Removes test. * Fix incorrect conditional. * Removes unnecessary nil check. * Removes unnecessary return. * Updates comment, removes unused variable. * Uses consistent JSON unmarshal pattern. * Uses consistent JSON unmarshal pattern. * Moves const block. * Switches 'already linked' from error to success response. * Removes commented-out code. * Switched to status ok. * Add parens for readability. * Fix copy/paste error. * Unexport some structs. * Removes repeated validity check. * Return without attempting commit if there's a rollback. * Fix incorrect HTTP status code. * Update store/sqlstore/group_supplier.go Co-Authored-By: mkraft <martinkraft@gmail.com> * Adds utility methods for going from groupsyncable to groupteam and groupchannel. * Fixing george suggestions (#9911) * Test fix. * Adds QA data to VC with visualization. * Fixes typo in graph image. * Update display name when re-linking in case it has changed in LDAP. * Adds ability to configure group display name and unique identifier. (#9923) * Adds ability to configure group display name and unique identifier. * Adds some configs to confi-ldap make command. * Fix for move of session. * Exposes method for use by SAML package. * Switches GroupSyncableType from int to string. * Update Jenkins build files. * Removes unused variable assignment. * Removes old unnecessary early return. * Removes unnecessary variable. * Moves param parsing before license and permissions checks. * Removes old code. * Compares agains underlying error rather than error id. * Switches tests to assertions. * Adds more assertions. * Adds missing return. * Adds space after comma for added legibility. * Moves a view model to the api package. * Unexports method. * Uses id validator function. * Fix docker-compose flag. * Typo fix. * Moves index creation to supplier. * Removes bad merge. * Renames parameter. * Re-adds space. * Removes unnecessary transaction. * Escapes the Groups table name with backticks because it is a reserved keyword. * Fix roles cache bug * Removing unnecesiary deserializing function * Switches table name rather than custom SQL everywhere for Postgres without backticks. * Removes redundant check for sql.ErrNoRows. * Removes redundant check for sql.ErrNoRows. * Removes data integrity check and redundant nil conditional. * Removes redundant check for sql.ErrNoRows. * Removes unnecessary query. * Removes ID length validation from persistence tier. * Makes some supplier methods idempotent. * Removes some empty switch defaults. * Renames Group Type field to Source. * Fix for mistaken field name change. * Uses IsValidId function. * Removes comment. * Changes json key name. * Removes test because no longer validating user. * Moves model state validation to app layer. * Don't create Groups.CanLeave column until phase 2. * Removes state validation until properties are used in phase 2. * Removes duplicated check. * Removes state validation until properties are used in phase 2. * Removes some tests until phase 2. * Comment-out a bunch of test related to CanLeave. * Extra unmarshal validation check. Removes more code for CanLeave. * Removes tests for CanLeave. * Explict error msg. * Rewrite queries. * Changes index name. Adds index. * Removes assertion. * Adds experimental feature flag.
2019-01-10 15:17:31 -05:00
GROUP_CACHE_SIZE = 20000
GROUP_CACHE_SEC = 30 * 60
CLEAR_CACHE_MESSAGE_DATA = ""
)
type LocalCacheSupplier struct {
next LayeredStoreSupplier
reactionCache *utils.Cache
roleCache *utils.Cache
schemeCache *utils.Cache
2017-09-14 12:01:44 -05:00
metrics einterfaces.MetricsInterface
cluster einterfaces.ClusterInterface
}
// Caching Interface
type ObjectCache interface {
AddWithExpiresInSecs(key, value interface{}, expireAtSecs int64)
AddWithDefaultExpires(key, value interface{})
Purge()
Get(key interface{}) (value interface{}, ok bool)
Remove(key interface{})
Len() int
Name() string
GetInvalidateClusterEvent() string
}
2017-09-14 12:01:44 -05:00
func NewLocalCacheSupplier(metrics einterfaces.MetricsInterface, cluster einterfaces.ClusterInterface) *LocalCacheSupplier {
supplier := &LocalCacheSupplier{
reactionCache: utils.NewLruWithParams(REACTION_CACHE_SIZE, "Reaction", REACTION_CACHE_SEC, model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_REACTIONS),
roleCache: utils.NewLruWithParams(ROLE_CACHE_SIZE, "Role", ROLE_CACHE_SEC, model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_ROLES),
schemeCache: utils.NewLruWithParams(SCHEME_CACHE_SIZE, "Scheme", SCHEME_CACHE_SEC, model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_SCHEMES),
2017-09-14 12:01:44 -05:00
metrics: metrics,
cluster: cluster,
}
2017-09-21 04:13:34 -05:00
if cluster != nil {
cluster.RegisterClusterMessageHandler(model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_REACTIONS, supplier.handleClusterInvalidateReaction)
cluster.RegisterClusterMessageHandler(model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_ROLES, supplier.handleClusterInvalidateRole)
}
2017-09-21 04:13:34 -05:00
return supplier
}
func (s *LocalCacheSupplier) SetChainNext(next LayeredStoreSupplier) {
s.next = next
}
func (s *LocalCacheSupplier) Next() LayeredStoreSupplier {
return s.next
}
func (s *LocalCacheSupplier) doStandardReadCache(ctx context.Context, cache ObjectCache, key string, hints ...LayeredStoreHint) *LayeredStoreSupplierResult {
if hintsContains(hints, LSH_NO_CACHE) {
2017-09-14 12:01:44 -05:00
if s.metrics != nil {
s.metrics.IncrementMemCacheMissCounter(cache.Name())
}
return nil
}
if cacheItem, ok := cache.Get(key); ok {
2017-09-14 12:01:44 -05:00
if s.metrics != nil {
s.metrics.IncrementMemCacheHitCounter(cache.Name())
}
result := NewSupplierResult()
result.Data = cacheItem
return result
}
2017-09-14 12:01:44 -05:00
if s.metrics != nil {
s.metrics.IncrementMemCacheMissCounter(cache.Name())
}
return nil
}
func (s *LocalCacheSupplier) doStandardAddToCache(ctx context.Context, cache ObjectCache, key string, result *LayeredStoreSupplierResult, hints ...LayeredStoreHint) {
if result.Err == nil && result.Data != nil {
cache.AddWithDefaultExpires(key, result.Data)
}
}
func (s *LocalCacheSupplier) doInvalidateCacheCluster(cache ObjectCache, key string) {
cache.Remove(key)
2017-09-14 12:01:44 -05:00
if s.cluster != nil {
msg := &model.ClusterMessage{
Event: cache.GetInvalidateClusterEvent(),
SendType: model.CLUSTER_SEND_BEST_EFFORT,
Data: key,
}
2017-09-14 12:01:44 -05:00
s.cluster.SendClusterMessage(msg)
}
}
func (s *LocalCacheSupplier) doClearCacheCluster(cache ObjectCache) {
cache.Purge()
2017-09-14 12:01:44 -05:00
if s.cluster != nil {
msg := &model.ClusterMessage{
Event: cache.GetInvalidateClusterEvent(),
SendType: model.CLUSTER_SEND_BEST_EFFORT,
Data: CLEAR_CACHE_MESSAGE_DATA,
}
2017-09-14 12:01:44 -05:00
s.cluster.SendClusterMessage(msg)
}
}
func (s *LocalCacheSupplier) Invalidate() {
s.doClearCacheCluster(s.reactionCache)
s.doClearCacheCluster(s.roleCache)
s.doClearCacheCluster(s.schemeCache)
}