MM-8607 Add ability to turn off non-critical services when under load (#13212)

* MM-8607: add ability to turn off non-critical services under load

* server busy invalid param unit tests

* MM-8607: rename server busy endpoints

* MM-8607: handle case where App not initialized

* MM-8607: additional unit test cases per feedback.

* MM-8607: use decorator to check isbusy when adding endpoint route

* MM-8607: rename endpoints, use struct for json

* Update api4/system.go

Fix misspelled log output

Co-Authored-By: Saturnino Abril <saturnino.abril@gmail.com>

* MM-8607: fix i18n order; max seconds for server busy expiry

api4/apitestlib.go

@@ -636,6 +636,11 @@ func CheckInternalErrorStatus(t *testing.T, resp *model.Response) {
 	checkHTTPStatus(t, resp, http.StatusInternalServerError, true)
 }
 
+func CheckServiceUnavailableStatus(t *testing.T, resp *model.Response) {
+	t.Helper()
+	checkHTTPStatus(t, resp, http.StatusServiceUnavailable, true)
+}
+
 func CheckErrorMessage(t *testing.T, resp *model.Response, errorId string) {
 	t.Helper()
 

api4/channel.go

@@ -17,8 +17,8 @@ func (api *API) InitChannel() {
 	api.BaseRoutes.Channels.Handle("", api.ApiSessionRequired(getAllChannels)).Methods("GET")
 	api.BaseRoutes.Channels.Handle("", api.ApiSessionRequired(createChannel)).Methods("POST")
 	api.BaseRoutes.Channels.Handle("/direct", api.ApiSessionRequired(createDirectChannel)).Methods("POST")
-	api.BaseRoutes.Channels.Handle("/search", api.ApiSessionRequired(searchAllChannels)).Methods("POST")
-	api.BaseRoutes.Channels.Handle("/group/search", api.ApiSessionRequired(searchGroupChannels)).Methods("POST")
+	api.BaseRoutes.Channels.Handle("/search", api.ApiSessionRequiredDisableWhenBusy(searchAllChannels)).Methods("POST")
+	api.BaseRoutes.Channels.Handle("/group/search", api.ApiSessionRequiredDisableWhenBusy(searchGroupChannels)).Methods("POST")
 	api.BaseRoutes.Channels.Handle("/group", api.ApiSessionRequired(createGroupChannel)).Methods("POST")
 	api.BaseRoutes.Channels.Handle("/members/{user_id:[A-Za-z0-9]+}/view", api.ApiSessionRequired(viewChannel)).Methods("POST")
 	api.BaseRoutes.Channels.Handle("/{channel_id:[A-Za-z0-9]+}/scheme", api.ApiSessionRequired(updateChannelScheme)).Methods("PUT")
@@ -26,8 +26,8 @@ func (api *API) InitChannel() {
 	api.BaseRoutes.ChannelsForTeam.Handle("", api.ApiSessionRequired(getPublicChannelsForTeam)).Methods("GET")
 	api.BaseRoutes.ChannelsForTeam.Handle("/deleted", api.ApiSessionRequired(getDeletedChannelsForTeam)).Methods("GET")
 	api.BaseRoutes.ChannelsForTeam.Handle("/ids", api.ApiSessionRequired(getPublicChannelsByIdsForTeam)).Methods("POST")
-	api.BaseRoutes.ChannelsForTeam.Handle("/search", api.ApiSessionRequired(searchChannelsForTeam)).Methods("POST")
-	api.BaseRoutes.ChannelsForTeam.Handle("/search_archived", api.ApiSessionRequired(searchArchivedChannelsForTeam)).Methods("POST")
+	api.BaseRoutes.ChannelsForTeam.Handle("/search", api.ApiSessionRequiredDisableWhenBusy(searchChannelsForTeam)).Methods("POST")
+	api.BaseRoutes.ChannelsForTeam.Handle("/search_archived", api.ApiSessionRequiredDisableWhenBusy(searchArchivedChannelsForTeam)).Methods("POST")
 	api.BaseRoutes.ChannelsForTeam.Handle("/autocomplete", api.ApiSessionRequired(autocompleteChannelsForTeam)).Methods("GET")
 	api.BaseRoutes.ChannelsForTeam.Handle("/search_autocomplete", api.ApiSessionRequired(autocompleteChannelsForTeamForSearch)).Methods("GET")
 	api.BaseRoutes.User.Handle("/teams/{team_id:[A-Za-z0-9]+}/channels", api.ApiSessionRequired(getChannelsForTeamForUser)).Methods("GET")

api4/handlers.go

@@ -107,3 +107,23 @@ func (api *API) ApiSessionRequiredTrustRequester(h func(*Context, http.ResponseW
 	return handler
 
 }
+
+// DisableWhenBusy provides a handler for API endpoints which should be disabled when the server is under load,
+// responding with HTTP 503 (Service Unavailable).
+func (api *API) ApiSessionRequiredDisableWhenBusy(h func(*Context, http.ResponseWriter, *http.Request)) http.Handler {
+	handler := &web.Handler{
+		GetGlobalAppOptions: api.GetGlobalAppOptions,
+		HandleFunc:          h,
+		HandlerName:         web.GetHandlerName(h),
+		RequireSession:      true,
+		TrustRequester:      false,
+		RequireMfa:          false,
+		IsStatic:            false,
+		DisableWhenBusy:     true,
+	}
+	if *api.ConfigService.Config().ServiceSettings.WebserverMode == "gzip" {
+		return gziphandler.GzipHandler(handler)
+	}
+	return handler
+
+}

api4/post.go

@@ -25,7 +25,7 @@ func (api *API) InitPost() {
 
 	api.BaseRoutes.ChannelForUser.Handle("/posts/unread", api.ApiSessionRequired(getPostsForChannelAroundLastUnread)).Methods("GET")
 
-	api.BaseRoutes.Team.Handle("/posts/search", api.ApiSessionRequired(searchPosts)).Methods("POST")
+	api.BaseRoutes.Team.Handle("/posts/search", api.ApiSessionRequiredDisableWhenBusy(searchPosts)).Methods("POST")
 	api.BaseRoutes.Post.Handle("", api.ApiSessionRequired(updatePost)).Methods("PUT")
 	api.BaseRoutes.Post.Handle("/patch", api.ApiSessionRequired(patchPost)).Methods("PUT")
 	api.BaseRoutes.PostForUser.Handle("/set_unread", api.ApiSessionRequired(setPostUnread)).Methods("POST")

api4/system.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"net/http"
 	"runtime"
+	"strconv"
 	"time"
 
 	"github.com/mattermost/mattermost-server/mlog"
@@ -16,7 +17,11 @@ import (
 	"github.com/mattermost/mattermost-server/utils"
 )
 
-const REDIRECT_LOCATION_CACHE_SIZE = 10000
+const (
+	REDIRECT_LOCATION_CACHE_SIZE = 10000
+	DEFAULT_SERVER_BUSY_SECONDS  = 3600
+	MAX_SERVER_BUSY_SECONDS      = 86400
+)
 
 var redirectLocationDataCache = utils.NewLru(REDIRECT_LOCATION_CACHE_SIZE)
 
@@ -40,6 +45,10 @@ func (api *API) InitSystem() {
 	api.BaseRoutes.ApiRoot.Handle("/redirect_location", api.ApiSessionRequiredTrustRequester(getRedirectLocation)).Methods("GET")
 
 	api.BaseRoutes.ApiRoot.Handle("/notifications/ack", api.ApiSessionRequired(pushNotificationAck)).Methods("POST")
+
+	api.BaseRoutes.ApiRoot.Handle("/server_busy", api.ApiSessionRequired(setServerBusy)).Methods("POST")
+	api.BaseRoutes.ApiRoot.Handle("/server_busy", api.ApiSessionRequired(getServerBusyExpires)).Methods("GET")
+	api.BaseRoutes.ApiRoot.Handle("/server_busy/clear", api.ApiSessionRequired(clearServerBusy)).Methods("POST")
 }
 
 func getSystemPing(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -444,3 +453,51 @@ func pushNotificationAck(c *Context, w http.ResponseWriter, r *http.Request) {
 
 	ReturnStatusOK(w)
 }
+
+func setServerBusy(c *Context, w http.ResponseWriter, r *http.Request) {
+	if !c.App.SessionHasPermissionTo(c.App.Session, model.PERMISSION_MANAGE_SYSTEM) {
+		c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
+		return
+	}
+
+	// number of seconds to keep server marked busy
+	secs := r.URL.Query().Get("seconds")
+	if secs == "" {
+		secs = strconv.FormatInt(DEFAULT_SERVER_BUSY_SECONDS, 10)
+	}
+
+	i, err := strconv.ParseInt(secs, 10, 64)
+	if err != nil || i <= 0 || i > MAX_SERVER_BUSY_SECONDS {
+		c.SetInvalidUrlParam(fmt.Sprintf("seconds must be 1 - %d", MAX_SERVER_BUSY_SECONDS))
+		return
+	}
+
+	c.App.Srv.Busy.Set(time.Second * time.Duration(i))
+	mlog.Warn("server busy state activated - non-critical services disabled", mlog.Int64("seconds", i))
+	ReturnStatusOK(w)
+}
+
+func clearServerBusy(c *Context, w http.ResponseWriter, r *http.Request) {
+	if !c.App.SessionHasPermissionTo(c.App.Session, model.PERMISSION_MANAGE_SYSTEM) {
+		c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
+		return
+	}
+	c.App.Srv.Busy.Clear()
+	mlog.Info("server busy state cleared - non-critical services enabled")
+	ReturnStatusOK(w)
+}
+
+func getServerBusyExpires(c *Context, w http.ResponseWriter, r *http.Request) {
+	if !c.App.SessionHasPermissionTo(c.App.Session, model.PERMISSION_MANAGE_SYSTEM) {
+		c.SetPermissionError(model.PERMISSION_MANAGE_SYSTEM)
+		return
+	}
+
+	busy := c.App.Srv.Busy
+	sbs := &model.ServerBusyState{
+		Busy:       busy.IsBusy(),
+		Expires:    busy.Expires().Unix(),
+		Expires_ts: busy.Expires().UTC().Format("Mon Jan 2 15:04:05 -0700 MST 2006"),
+	}
+	w.Write([]byte(sbs.ToJson()))
+}

api4/system_test.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/mattermost/mattermost-server/mlog"
 	"github.com/mattermost/mattermost-server/model"
@@ -507,3 +508,117 @@ func TestRedirectLocation(t *testing.T) {
 	_, resp = Client.GetRedirectLocation("", "")
 	CheckUnauthorizedStatus(t, resp)
 }
+
+func TestSetServerBusy(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	const secs = 30
+
+	t.Run("as system user", func(t *testing.T) {
+		ok, resp := th.Client.SetServerBusy(secs)
+		CheckForbiddenStatus(t, resp)
+		require.False(t, ok, "should not set server busy due to no permission")
+		require.False(t, th.App.Srv.Busy.IsBusy(), "server should not be marked busy")
+	})
+
+	t.Run("as system admin", func(t *testing.T) {
+		ok, resp := th.SystemAdminClient.SetServerBusy(secs)
+		CheckNoError(t, resp)
+		require.True(t, ok, "should set server busy successfully")
+		require.True(t, th.App.Srv.Busy.IsBusy(), "server should be marked busy")
+	})
+}
+
+func TestSetServerBusyInvalidParam(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	t.Run("as system admin, invalid param", func(t *testing.T) {
+		params := []int{-1, 0, MAX_SERVER_BUSY_SECONDS + 1}
+		for _, p := range params {
+			ok, resp := th.SystemAdminClient.SetServerBusy(p)
+			CheckBadRequestStatus(t, resp)
+			require.False(t, ok, "should not set server busy due to invalid param ", p)
+			require.False(t, th.App.Srv.Busy.IsBusy(), "server should not be marked busy due to invalid param ", p)
+		}
+	})
+}
+
+func TestClearServerBusy(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	th.App.Srv.Busy.Set(time.Second * 30)
+	t.Run("as system user", func(t *testing.T) {
+		ok, resp := th.Client.ClearServerBusy()
+		CheckForbiddenStatus(t, resp)
+		require.False(t, ok, "should not clear server busy flag due to no permission.")
+		require.True(t, th.App.Srv.Busy.IsBusy(), "server should be marked busy")
+	})
+
+	th.App.Srv.Busy.Set(time.Second * 30)
+	t.Run("as system admin", func(t *testing.T) {
+		ok, resp := th.SystemAdminClient.ClearServerBusy()
+		CheckNoError(t, resp)
+		require.True(t, ok, "should clear server busy flag successfully")
+		require.False(t, th.App.Srv.Busy.IsBusy(), "server should not be marked busy")
+	})
+}
+
+func TestGetServerBusyExpires(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	th.App.Srv.Busy.Set(time.Second * 30)
+
+	t.Run("as system user", func(t *testing.T) {
+		_, resp := th.Client.GetServerBusyExpires()
+		CheckForbiddenStatus(t, resp)
+	})
+
+	t.Run("as system admin", func(t *testing.T) {
+		expires, resp := th.SystemAdminClient.GetServerBusyExpires()
+		CheckNoError(t, resp)
+		require.Greater(t, expires.Unix(), time.Now().Unix())
+	})
+}
+
+func TestServerBusy503(t *testing.T) {
+	th := Setup().InitBasic()
+	defer th.TearDown()
+
+	th.App.Srv.Busy.Set(time.Second * 30)
+
+	t.Run("search users while busy", func(t *testing.T) {
+		us := &model.UserSearch{Term: "test"}
+		_, resp := th.SystemAdminClient.SearchUsers(us)
+		CheckServiceUnavailableStatus(t, resp)
+	})
+
+	t.Run("search teams while busy", func(t *testing.T) {
+		ts := &model.TeamSearch{}
+		_, resp := th.SystemAdminClient.SearchTeams(ts)
+		CheckServiceUnavailableStatus(t, resp)
+	})
+
+	t.Run("search channels while busy", func(t *testing.T) {
+		cs := &model.ChannelSearch{}
+		_, resp := th.SystemAdminClient.SearchChannels("foo", cs)
+		CheckServiceUnavailableStatus(t, resp)
+	})
+
+	t.Run("search archived channels while busy", func(t *testing.T) {
+		cs := &model.ChannelSearch{}
+		_, resp := th.SystemAdminClient.SearchArchivedChannels("foo", cs)
+		CheckServiceUnavailableStatus(t, resp)
+	})
+
+	th.App.Srv.Busy.Clear()
+
+	t.Run("search users while not busy", func(t *testing.T) {
+		us := &model.UserSearch{Term: "test"}
+		_, resp := th.SystemAdminClient.SearchUsers(us)
+		CheckNoError(t, resp)
+	})
+}

api4/team.go

@@ -34,7 +34,7 @@ func (api *API) InitTeam() {
 	api.BaseRoutes.Teams.Handle("", api.ApiSessionRequired(createTeam)).Methods("POST")
 	api.BaseRoutes.Teams.Handle("", api.ApiSessionRequired(getAllTeams)).Methods("GET")
 	api.BaseRoutes.Teams.Handle("/{team_id:[A-Za-z0-9]+}/scheme", api.ApiSessionRequired(updateTeamScheme)).Methods("PUT")
-	api.BaseRoutes.Teams.Handle("/search", api.ApiSessionRequired(searchTeams)).Methods("POST")
+	api.BaseRoutes.Teams.Handle("/search", api.ApiSessionRequiredDisableWhenBusy(searchTeams)).Methods("POST")
 	api.BaseRoutes.TeamsForUser.Handle("", api.ApiSessionRequired(getTeamsForUser)).Methods("GET")
 	api.BaseRoutes.TeamsForUser.Handle("/unread", api.ApiSessionRequired(getTeamsUnreadForUser)).Methods("GET")
 

api4/user.go

@@ -24,7 +24,7 @@ func (api *API) InitUser() {
 	api.BaseRoutes.Users.Handle("", api.ApiSessionRequired(getUsers)).Methods("GET")
 	api.BaseRoutes.Users.Handle("/ids", api.ApiSessionRequired(getUsersByIds)).Methods("POST")
 	api.BaseRoutes.Users.Handle("/usernames", api.ApiSessionRequired(getUsersByNames)).Methods("POST")
-	api.BaseRoutes.Users.Handle("/search", api.ApiSessionRequired(searchUsers)).Methods("POST")
+	api.BaseRoutes.Users.Handle("/search", api.ApiSessionRequiredDisableWhenBusy(searchUsers)).Methods("POST")
 	api.BaseRoutes.Users.Handle("/autocomplete", api.ApiSessionRequired(autocompleteUsers)).Methods("GET")
 	api.BaseRoutes.Users.Handle("/stats", api.ApiSessionRequired(getTotalUsersStats)).Methods("GET")
 	api.BaseRoutes.Users.Handle("/group_channels", api.ApiSessionRequired(getUsersByGroupChannelIds)).Methods("POST")

app/busy.go

@@ -0,0 +1,64 @@
+// Copyright (c) 2019-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package app
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+type Busy struct {
+	busy int32 // protected via atomic for fast IsBusy calls
+
+	mux     sync.RWMutex
+	timer   *time.Timer
+	expires time.Time
+}
+
+// IsBusy returns true if the server has been marked as busy.
+func (b *Busy) IsBusy() bool {
+	if b == nil {
+		return false
+	}
+	return atomic.LoadInt32(&b.busy) != 0
+}
+
+// Set marks the server as busy for dur duration.
+func (b *Busy) Set(dur time.Duration) {
+	b.mux.Lock()
+	defer b.mux.Unlock()
+
+	b.clear()
+	atomic.StoreInt32(&b.busy, 1)
+
+	b.timer = time.AfterFunc(dur, b.Clear)
+	b.expires = time.Now().Add(dur)
+}
+
+// ClearBusy marks the server as not busy.
+func (b *Busy) Clear() {
+	b.mux.Lock()
+	defer b.mux.Unlock()
+	b.clear()
+}
+
+// must hold mutex
+func (b *Busy) clear() {
+	if b.timer != nil {
+		b.timer.Stop() // don't drain timer.C channel for AfterFunc timers.
+	}
+	b.timer = nil
+	b.expires = time.Time{}
+	atomic.StoreInt32(&b.busy, 0)
+}
+
+// Expires returns the expected time that the server
+// will be marked not busy. This expiry can be extended
+// via additional calls to SetBusy.
+func (b *Busy) Expires() time.Time {
+	b.mux.RLock()
+	defer b.mux.RUnlock()
+	return b.expires
+}

app/busy_test.go

@@ -0,0 +1,72 @@
+// Copyright (c) 2019-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package app
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestBusySet(t *testing.T) {
+	busy := &Busy{}
+
+	isNotBusy := func() bool {
+		return !busy.IsBusy()
+	}
+
+	require.False(t, busy.IsBusy())
+
+	busy.Set(time.Millisecond * 100)
+	require.True(t, busy.IsBusy())
+	// should automatically expire after 100ms
+	require.Eventually(t, isNotBusy, time.Second*5, time.Millisecond*20)
+
+	// test set after auto expiry
+	busy.Set(time.Second * 30)
+	require.True(t, busy.IsBusy())
+	expire := busy.Expires()
+	require.Greater(t, expire.Unix(), time.Now().Add(time.Second*10).Unix())
+
+	// test extending existing expiry
+	busy.Set(time.Minute * 5)
+	require.True(t, busy.IsBusy())
+	expire = busy.Expires()
+	require.Greater(t, expire.Unix(), time.Now().Add(time.Minute*2).Unix())
+
+	busy.Clear()
+	require.False(t, busy.IsBusy())
+}
+
+func TestBusyExpires(t *testing.T) {
+	busy := &Busy{}
+
+	isNotBusy := func() bool {
+		return !busy.IsBusy()
+	}
+
+	// get expiry before it is set
+	expire := busy.Expires()
+	// should be time.Time zero value
+	require.Equal(t, time.Time{}.Unix(), expire.Unix())
+
+	// get expiry after it is set
+	busy.Set(time.Minute * 5)
+	expire = busy.Expires()
+	require.Greater(t, expire.Unix(), time.Now().Add(time.Minute*2).Unix())
+
+	// get expiry after clear
+	busy.Clear()
+	expire = busy.Expires()
+	// should be time.Time zero value
+	require.Equal(t, time.Time{}.Unix(), expire.Unix())
+
+	// get expiry after auto-expire
+	busy.Set(time.Millisecond * 100)
+	require.Eventually(t, isNotBusy, time.Second*5, time.Millisecond*20)
+	expire = busy.Expires()
+	// should be time.Time zero value
+	require.Equal(t, time.Time{}.Unix(), expire.Unix())
+}

app/server.go

@@ -53,6 +53,7 @@ type Server struct {
 	Server      *http.Server
 	ListenAddr  *net.TCPAddr
 	RateLimiter *RateLimiter
+	Busy        *Busy
 
 	didFinishListen chan struct{}
 
@@ -473,6 +474,7 @@ func (s *Server) Start() error {
 		s.RateLimiter = rateLimiter
 		handler = rateLimiter.RateLimitHandler(handler)
 	}
+	s.Busy = &Busy{}
 
 	// Creating a logger for logging errors from http.Server at error level
 	errStdLog, err := s.Log.StdLogAt(mlog.LevelError, mlog.String("source", "httpserver"))

app/status.go

@@ -228,6 +228,10 @@ func (a *App) SetStatusOnline(userId string, manual bool) {
 }
 
 func (a *App) BroadcastStatus(status *model.Status) {
+	if a.Srv.Busy.IsBusy() {
+		// this is considered a non-critical service and will be disabled when server busy.
+		return
+	}
 	event := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_STATUS_CHANGE, "", "", status.UserId, nil)
 	event.Add("status", status.Status)
 	event.Add("user_id", status.UserId)

i18n/en.json

@@ -1090,6 +1090,10 @@
     "id": "api.context.permissions.app_error",
     "translation": "You do not have the appropriate permissions"
   },
+  {
+    "id": "api.context.server_busy.app_error",
+    "translation": "Server is busy, non-critical services are temporarily unavailable"
+  },
   {
     "id": "api.context.session_expired.app_error",
     "translation": "Invalid or expired session, please login again."
@@ -2806,6 +2810,10 @@
     "id": "api.websocket_handler.invalid_param.app_error",
     "translation": "Invalid {{.Name}} parameter"
   },
+  {
+    "id": "api.websocket_handler.server_busy.app_error",
+    "translation": "Server is busy, non-critical services are temporarily unavailable"
+  },
   {
     "id": "app.admin.test_email.failure",
     "translation": "Connection unsuccessful: {{.Error}}"

model/client4.go

@@ -420,6 +420,10 @@ func (c *Client4) GetRedirectLocationRoute() string {
 	return fmt.Sprintf("/redirect_location")
 }
 
+func (c *Client4) GetServerBusyRoute() string {
+	return "/server_busy"
+}
+
 func (c *Client4) GetUserTermsOfServiceRoute(userId string) string {
 	return c.GetUserRoute(userId) + "/terms_of_service"
 }
@@ -4622,6 +4626,41 @@ func (c *Client4) GetRedirectLocation(urlParam, etag string) (string, *Response)
 	return MapFromJson(r.Body)["location"], BuildResponse(r)
 }
 
+// SetServerBusy will mark the server as busy, which disables non-critical services for `secs` seconds.
+func (c *Client4) SetServerBusy(secs int) (bool, *Response) {
+	url := fmt.Sprintf("%s?seconds=%d", c.GetServerBusyRoute(), secs)
+	r, err := c.DoApiPost(url, "")
+	if err != nil {
+		return false, BuildErrorResponse(r, err)
+	}
+	defer closeBody(r)
+	return CheckStatusOK(r), BuildResponse(r)
+}
+
+// ClearServerBusy will mark the server as not busy.
+func (c *Client4) ClearServerBusy() (bool, *Response) {
+	r, err := c.DoApiPost(c.GetServerBusyRoute()+"/clear", "")
+	if err != nil {
+		return false, BuildErrorResponse(r, err)
+	}
+	defer closeBody(r)
+	return CheckStatusOK(r), BuildResponse(r)
+}
+
+// GetServerBusyExpires returns the time when a server marked busy
+// will automatically have the flag cleared.
+func (c *Client4) GetServerBusyExpires() (*time.Time, *Response) {
+	r, err := c.DoApiGet(c.GetServerBusyRoute(), "")
+	if err != nil {
+		return nil, BuildErrorResponse(r, err)
+	}
+	defer closeBody(r)
+
+	sbs := ServerBusyStateFromJson(r.Body)
+	expires := time.Unix(sbs.Expires, 0)
+	return &expires, BuildResponse(r)
+}
+
 // RegisterTermsOfServiceAction saves action performed by a user against a specific terms of service.
 func (c *Client4) RegisterTermsOfServiceAction(userId, termsOfServiceId string, accepted bool) (*bool, *Response) {
 	url := c.GetUserTermsOfServiceRoute(userId)

model/system.go

@@ -50,3 +50,20 @@ type SystemECDSAKey struct {
 	Y     *big.Int `json:"y"`
 	D     *big.Int `json:"d,omitempty"`
 }
+
+type ServerBusyState struct {
+	Busy       bool   `json:"busy"`
+	Expires    int64  `json:"expires"`
+	Expires_ts string `json:"expires_ts,omitempty"`
+}
+
+func (sbs *ServerBusyState) ToJson() string {
+	b, _ := json.Marshal(sbs)
+	return string(b)
+}
+
+func ServerBusyStateFromJson(r io.Reader) *ServerBusyState {
+	var sbs *ServerBusyState
+	json.NewDecoder(r).Decode(&sbs)
+	return sbs
+}

model/system_test.go

@@ -6,6 +6,7 @@ package model
 import (
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 )
@@ -17,3 +18,13 @@ func TestSystemJson(t *testing.T) {
 
 	require.Equal(t, "test", result.Name, "ids do not match")
 }
+
+func TestServerBusyJson(t *testing.T) {
+	now := time.Now()
+	sbs := ServerBusyState{Busy: true, Expires: now.Unix()}
+	json := sbs.ToJson()
+	result := ServerBusyStateFromJson(strings.NewReader(json))
+
+	require.Equal(t, sbs.Busy, result.Busy, "busy state does not match")
+	require.Equal(t, sbs.Expires, result.Expires, "expiry does not match")
+}

model/utils.go

@@ -207,7 +207,7 @@ func MapToJson(objmap map[string]string) string {
 	return string(b)
 }
 
-// MapToJson converts a map to a json string
+// MapBoolToJson converts a map to a json string
 func MapBoolToJson(objmap map[string]bool) string {
 	b, _ := json.Marshal(objmap)
 	return string(b)

web/context.go

@@ -165,6 +165,10 @@ func (c *Context) SetInvalidUrlParam(parameter string) {
 	c.Err = NewInvalidUrlParamError(parameter)
 }
 
+func (c *Context) SetServerBusyError() {
+	c.Err = NewServerBusyError()
+}
+
 func (c *Context) HandleEtag(etag string, routeName string, w http.ResponseWriter, r *http.Request) bool {
 	metrics := c.App.Metrics
 	if et := r.Header.Get(model.HEADER_ETAG_CLIENT); len(etag) > 0 {
@@ -193,6 +197,10 @@ func NewInvalidUrlParamError(parameter string) *model.AppError {
 	err := model.NewAppError("Context", "api.context.invalid_url_param.app_error", map[string]interface{}{"Name": parameter}, "", http.StatusBadRequest)
 	return err
 }
+func NewServerBusyError() *model.AppError {
+	err := model.NewAppError("Context", "api.context.server_busy.app_error", nil, "", http.StatusServiceUnavailable)
+	return err
+}
 
 func (c *Context) SetPermissionError(permission *model.Permission) {
 	c.Err = c.App.MakePermissionError(permission)

web/handlers.go

@@ -66,6 +66,7 @@ type Handler struct {
 	TrustRequester      bool
 	RequireMfa          bool
 	IsStatic            bool
+	DisableWhenBusy     bool
 
 	cspShaDirective string
 }
@@ -159,6 +160,10 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		c.MfaRequired()
 	}
 
+	if c.Err == nil && h.DisableWhenBusy && c.App.Srv.Busy.IsBusy() {
+		c.SetServerBusyError()
+	}
+
 	if c.Err == nil {
 		h.HandleFunc(c, w, r)
 	}

wsapi/user.go

@@ -13,6 +13,11 @@ func (api *API) InitUser() {
 }
 
 func (api *API) userTyping(req *model.WebSocketRequest) (map[string]interface{}, *model.AppError) {
+	if api.App.Srv.Busy.IsBusy() {
+		// this is considered a non-critical service and will be disabled when server busy.
+		return nil, NewServerBusyWebSocketError(req.Action)
+	}
+
 	var ok bool
 	var channelId string
 	if channelId, ok = req.Data["channel_id"].(string); !ok || len(channelId) != 26 {

wsapi/websocket_handler.go

@@ -59,3 +59,7 @@ func (wh webSocketHandler) ServeWebSocket(conn *app.WebConn, r *model.WebSocketR
 func NewInvalidWebSocketParamError(action string, name string) *model.AppError {
 	return model.NewAppError("websocket: "+action, "api.websocket_handler.invalid_param.app_error", map[string]interface{}{"Name": name}, "", http.StatusBadRequest)
 }
+
+func NewServerBusyWebSocketError(action string) *model.AppError {
+	return model.NewAppError("websocket: "+action, "api.websocket_handler.server_busy.app_error", nil, "", http.StatusServiceUnavailable)
+}