IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2026-08-26 21:27:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

Skip intensive stat DB queries when more than a set number of users on the system (#4876)

Browse Source
This commit is contained in:
Joram Wilander
2016-12-22 17:00:05 -03:00
committed by enahum
parent 768fe6bec3
commit a793eb8651
4 changed files with 174 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/admin.go
+75 -21
View File
@@ -367,6 +367,19 @@ func getAnalytics(c *Context, w http.ResponseWriter, r *http.Request) {
teamId := params["id"]
name := params["name"]
skipIntensiveQueries := false
var systemUserCount int64
if r := <-Srv.Store.User().AnalyticsUniqueUserCount(""); r.Err != nil {
c.Err = r.Err
return
} else {
systemUserCount = r.Data.(int64)
if systemUserCount > int64(*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics) {
l4g.Debug("More than %v users on the system, intensive queries skipped", *utils.Cfg.AnalyticsSettings.MaxUsersForStatistics)
skipIntensiveQueries = true
}
}
if name == "standard" {
var rows model.AnalyticsRows = make([]*model.AnalyticsRow, 8)
rows[0] = &model.AnalyticsRow{"channel_open_count", 0}
@@ -380,10 +393,18 @@ func getAnalytics(c *Context, w http.ResponseWriter, r *http.Request) {
openChan := Srv.Store.Channel().AnalyticsTypeCount(teamId, model.CHANNEL_OPEN)
privateChan := Srv.Store.Channel().AnalyticsTypeCount(teamId, model.CHANNEL_PRIVATE)
postChan := Srv.Store.Post().AnalyticsPostCount(teamId, false, false)
userChan := Srv.Store.User().AnalyticsUniqueUserCount(teamId)
teamChan := Srv.Store.Team().AnalyticsTeamCount()
var userChan store.StoreChannel
if teamId != "" {
userChan = Srv.Store.User().AnalyticsUniqueUserCount(teamId)
}
var postChan store.StoreChannel
if !skipIntensiveQueries {
postChan = Srv.Store.Post().AnalyticsPostCount(teamId, false, false)
}
if r := <-openChan; r.Err != nil {
c.Err = r.Err
return
@@ -398,18 +419,26 @@ func getAnalytics(c *Context, w http.ResponseWriter, r *http.Request) {
rows[1].Value = float64(r.Data.(int64))
}
if r := <-postChan; r.Err != nil {
c.Err = r.Err
return
if postChan == nil {
rows[2].Value = -1
} else {
rows[2].Value = float64(r.Data.(int64))
if r := <-postChan; r.Err != nil {
c.Err = r.Err
return
} else {
rows[2].Value = float64(r.Data.(int64))
}
}
if r := <-userChan; r.Err != nil {
c.Err = r.Err
return
if userChan == nil {
rows[3].Value = float64(systemUserCount)
} else {
rows[3].Value = float64(r.Data.(int64))
if r := <-userChan; r.Err != nil {
c.Err = r.Err
return
} else {
rows[3].Value = float64(r.Data.(int64))
}
}
if r := <-teamChan; r.Err != nil {
@@ -449,6 +478,12 @@ func getAnalytics(c *Context, w http.ResponseWriter, r *http.Request) {
w.Write([]byte(rows.ToJson()))
} else if name == "post_counts_day" {
if skipIntensiveQueries {
rows := model.AnalyticsRows{&model.AnalyticsRow{"", -1}}
w.Write([]byte(rows.ToJson()))
return
}
if r := <-Srv.Store.Post().AnalyticsPostCountsByDay(teamId); r.Err != nil {
c.Err = r.Err
return
@@ -456,6 +491,12 @@ func getAnalytics(c *Context, w http.ResponseWriter, r *http.Request) {
w.Write([]byte(r.Data.(model.AnalyticsRows).ToJson()))
}
} else if name == "user_counts_with_posts_day" {
if skipIntensiveQueries {
rows := model.AnalyticsRows{&model.AnalyticsRow{"", -1}}
w.Write([]byte(rows.ToJson()))
return
}
if r := <-Srv.Store.Post().AnalyticsUserCountsWithPostsByDay(teamId); r.Err != nil {
c.Err = r.Err
return
@@ -471,25 +512,38 @@ func getAnalytics(c *Context, w http.ResponseWriter, r *http.Request) {
rows[4] = &model.AnalyticsRow{"command_count", 0}
rows[5] = &model.AnalyticsRow{"session_count", 0}
fileChan := Srv.Store.Post().AnalyticsPostCount(teamId, true, false)
hashtagChan := Srv.Store.Post().AnalyticsPostCount(teamId, false, true)
iHookChan := Srv.Store.Webhook().AnalyticsIncomingCount(teamId)
oHookChan := Srv.Store.Webhook().AnalyticsOutgoingCount(teamId)
commandChan := Srv.Store.Command().AnalyticsCommandCount(teamId)
sessionChan := Srv.Store.Session().AnalyticsSessionCount()
if r := <-fileChan; r.Err != nil {
c.Err = r.Err
return
} else {
rows[0].Value = float64(r.Data.(int64))
var fileChan store.StoreChannel
var hashtagChan store.StoreChannel
if !skipIntensiveQueries {
fileChan = Srv.Store.Post().AnalyticsPostCount(teamId, true, false)
hashtagChan = Srv.Store.Post().AnalyticsPostCount(teamId, false, true)
}
if r := <-hashtagChan; r.Err != nil {
c.Err = r.Err
return
if fileChan == nil {
rows[0].Value = -1
} else {
rows[1].Value = float64(r.Data.(int64))
if r := <-fileChan; r.Err != nil {
c.Err = r.Err
return
} else {
rows[0].Value = float64(r.Data.(int64))
}
}
if hashtagChan == nil {
rows[1].Value = -1
} else {
if r := <-hashtagChan; r.Err != nil {
c.Err = r.Err
return
} else {
rows[1].Value = float64(r.Data.(int64))
}
}
if r := <-iHookChan; r.Err != nil {
api/admin_test.go
+86
View File
@@ -192,6 +192,12 @@ func TestGetTeamAnalyticsStandard(t *testing.T) {
t.Fatal("Shouldn't have permissions")
}
maxUsersForStats := *utils.Cfg.AnalyticsSettings.MaxUsersForStatistics
defer func() {
*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = maxUsersForStats
}()
*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = 1000000
if result, err := th.SystemAdminClient.GetTeamAnalytics(th.BasicTeam.Id, "standard"); err != nil {
t.Fatal(err)
} else {
@@ -303,6 +309,24 @@ func TestGetTeamAnalyticsStandard(t *testing.T) {
t.Fatal()
}
}
*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = 1
if result, err := th.SystemAdminClient.GetSystemAnalytics("standard"); err != nil {
t.Fatal(err)
} else {
rows := result.Data.(model.AnalyticsRows)
if rows[2].Name != "post_count" {
t.Log(rows.ToJson())
t.Fatal()
}
if rows[2].Value != -1 {
t.Log(rows.ToJson())
t.Fatal()
}
}
}
func TestGetPostCount(t *testing.T) {
@@ -316,6 +340,12 @@ func TestGetPostCount(t *testing.T) {
t.Fatal("Shouldn't have permissions")
}
maxUsersForStats := *utils.Cfg.AnalyticsSettings.MaxUsersForStatistics
defer func() {
*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = maxUsersForStats
}()
*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = 1000000
if result, err := th.SystemAdminClient.GetTeamAnalytics(th.BasicTeam.Id, "post_counts_day"); err != nil {
t.Fatal(err)
} else {
@@ -326,6 +356,19 @@ func TestGetPostCount(t *testing.T) {
t.Fatal()
}
}
*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = 1
if result, err := th.SystemAdminClient.GetTeamAnalytics(th.BasicTeam.Id, "post_counts_day"); err != nil {
t.Fatal(err)
} else {
rows := result.Data.(model.AnalyticsRows)
if rows[0].Value != -1 {
t.Log(rows.ToJson())
t.Fatal()
}
}
}
func TestUserCountsWithPostsByDay(t *testing.T) {
@@ -339,6 +382,12 @@ func TestUserCountsWithPostsByDay(t *testing.T) {
t.Fatal("Shouldn't have permissions")
}
maxUsersForStats := *utils.Cfg.AnalyticsSettings.MaxUsersForStatistics
defer func() {
*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = maxUsersForStats
}()
*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = 1000000
if result, err := th.SystemAdminClient.GetTeamAnalytics(th.BasicTeam.Id, "user_counts_with_posts_day"); err != nil {
t.Fatal(err)
} else {
@@ -349,6 +398,19 @@ func TestUserCountsWithPostsByDay(t *testing.T) {
t.Fatal()
}
}
*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = 1
if result, err := th.SystemAdminClient.GetTeamAnalytics(th.BasicTeam.Id, "user_counts_with_posts_day"); err != nil {
t.Fatal(err)
} else {
rows := result.Data.(model.AnalyticsRows)
if rows[0].Value != -1 {
t.Log(rows.ToJson())
t.Fatal()
}
}
}
func TestGetTeamAnalyticsExtra(t *testing.T) {
@@ -360,6 +422,12 @@ func TestGetTeamAnalyticsExtra(t *testing.T) {
t.Fatal("Shouldn't have permissions")
}
maxUsersForStats := *utils.Cfg.AnalyticsSettings.MaxUsersForStatistics
defer func() {
*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = maxUsersForStats
}()
*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = 1000000
if result, err := th.SystemAdminClient.GetTeamAnalytics(th.BasicTeam.Id, "extra_counts"); err != nil {
t.Fatal(err)
} else {
@@ -461,6 +529,24 @@ func TestGetTeamAnalyticsExtra(t *testing.T) {
t.Fatal()
}
}
*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = 1
if result, err := th.SystemAdminClient.GetSystemAnalytics("extra_counts"); err != nil {
t.Fatal(err)
} else {
rows := result.Data.(model.AnalyticsRows)
if rows[0].Value != -1 {
t.Log(rows.ToJson())
t.Fatal()
}
if rows[1].Value != -1 {
t.Log(rows.ToJson())
t.Fatal()
}
}
}
func TestAdminResetMfa(t *testing.T) {
config/config.json
+3
View File
@@ -241,6 +241,9 @@
"BlockProfileRate": 0,
"ListenAddress": ":8067"
},
"AnalyticsSettings": {
"MaxUsersForStatistics": 2500
},
"WebrtcSettings": {
"Enable": false,
"GatewayWebsocketUrl": "",
model/config.go
+10
View File
@@ -105,6 +105,10 @@ type MetricsSettings struct {
ListenAddress *string
}
type AnalyticsSettings struct {
MaxUsersForStatistics *int
}
type SSOSettings struct {
Enable bool
Secret string
@@ -345,6 +349,7 @@ type Config struct {
NativeAppSettings NativeAppSettings
ClusterSettings ClusterSettings
MetricsSettings MetricsSettings
AnalyticsSettings AnalyticsSettings
WebrtcSettings WebrtcSettings
}
@@ -845,6 +850,11 @@ func (o *Config) SetDefaults() {
*o.MetricsSettings.Enable = false
}
if o.AnalyticsSettings.MaxUsersForStatistics == nil {
o.AnalyticsSettings.MaxUsersForStatistics = new(int)
*o.AnalyticsSettings.MaxUsersForStatistics = 2500
}
if o.ComplianceSettings.Enable == nil {
o.ComplianceSettings.Enable = new(bool)
*o.ComplianceSettings.Enable = false