Admin log table (#21153)

* Reformat Log Output for SysAdmin UI * Restore old code * Update * Backend first pass * PR feedback * Set helpers to private * Lint fix * Fix mocks * Fix another mock * Fix lint again * Fix i18n * Update app/admin.go Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com> * PR feedback * FIx lint issue --------- Co-authored-by: Daniel Schalla <daniel@schalla.me> Co-authored-by: Nevyana Angelova <nevyangelova@Nevyanas-MacBook-Pro.local> Co-authored-by: Devin Binnie <devin.binnie@mattermost.com> Co-authored-by: Devin Binnie <52460000+devinbinnie@users.noreply.github.com> Co-authored-by: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com> Co-authored-by: Mattermod <mattermod@users.noreply.github.com> Co-authored-by: Mattermost Build <build@mattermost.com>
2025-02-25 18:55:24 -06:00 · 2023-02-14 12:39:10 +02:00
parent 5a075e01a5
commit 7275887d14
12 changed files with 268 additions and 24 deletions
--- a/api4/system.go
+++ b/api4/system.go
@@ -48,6 +48,7 @@ func (api *API) InitSystem() {
 	api.BaseRoutes.APIRoot.Handle("/caches/invalidate", api.APISessionRequired(invalidateCaches)).Methods("POST")

 	api.BaseRoutes.APIRoot.Handle("/logs", api.APISessionRequired(getLogs)).Methods("GET")
+	api.BaseRoutes.APIRoot.Handle("/logs/query", api.APISessionRequired(queryLogs)).Methods("POST")
 	api.BaseRoutes.APIRoot.Handle("/logs", api.APIHandler(postLog)).Methods("POST")

 	api.BaseRoutes.APIRoot.Handle("/analytics/old", api.APISessionRequired(getAnalytics)).Methods("GET")
@@ -326,6 +327,52 @@ func invalidateCaches(c *Context, w http.ResponseWriter, r *http.Request) {
 	ReturnStatusOK(w)
 }

+func queryLogs(c *Context, w http.ResponseWriter, r *http.Request) {
+	auditRec := c.MakeAuditRecord("queryLogs", audit.Fail)
+	defer c.LogAuditRec(auditRec)
+
+	if *c.App.Config().ExperimentalSettings.RestrictSystemAdmin {
+		c.Err = model.NewAppError("queryLogs", "api.restricted_system_admin", nil, "", http.StatusForbidden)
+		return
+	}
+
+	if !c.App.SessionHasPermissionTo(*c.AppContext.Session(), model.PermissionGetLogs) {
+		c.SetPermissionError(model.PermissionGetLogs)
+		return
+	}
+
+	var logFilter *model.LogFilter
+	err := json.NewDecoder(r.Body).Decode(&logFilter)
+	if err != nil {
+		c.Err = model.NewAppError("queryLogs", "api.system.logs.invalidFilter", nil, "", http.StatusInternalServerError)
+		return
+	}
+
+	logs, logerr := c.App.QueryLogs(c.Params.Page, c.Params.LogsPerPage, logFilter)
+	if logerr != nil {
+		c.Err = logerr
+		return
+	}
+
+	logsJSON := make(map[string][]interface{})
+	var result interface{}
+	for node, logLines := range logs {
+		for _, log := range logLines {
+			err2 := json.Unmarshal([]byte(log), &result)
+			if err2 == nil {
+				logsJSON[node] = append(logsJSON[node], result)
+			} else {
+				mlog.Warn("Error parsing log line in Server Logs")
+			}
+		}
+	}
+
+	auditRec.AddMeta("page", c.Params.Page)
+	auditRec.AddMeta("logs_per_page", c.Params.LogsPerPage)
+
+	w.Write(model.ToJSON(logsJSON))
+}
+
 func getLogs(c *Context, w http.ResponseWriter, r *http.Request) {
 	auditRec := c.MakeAuditRecord("getLogs", audit.Fail)
 	defer c.LogAuditRec(auditRec)
--- a/app/admin.go
+++ b/app/admin.go
@@ -37,7 +37,7 @@ func (s *Server) GetLogs(page, perPage int) ([]string, *model.AppError) {
 		}
 	}

-	melines, err := s.GetLogsSkipSend(page, perPage)
+	melines, err := s.GetLogsSkipSend(page, perPage, &model.LogFilter{})
 	if err != nil {
 		return nil, err
 	}
@@ -56,16 +56,75 @@ func (s *Server) GetLogs(page, perPage int) ([]string, *model.AppError) {
 	return lines, nil
 }

+func (s *Server) QueryLogs(page, perPage int, logFilter *model.LogFilter) (map[string][]string, *model.AppError) {
+	logData := make(map[string][]string)
+
+	serverName := "default"
+
+	license := s.License()
+	if license != nil && *license.Features.Cluster && s.platform.Cluster() != nil && *s.platform.Config().ClusterSettings.Enable {
+		if info := s.platform.Cluster().GetMyClusterInfo(); info != nil {
+			serverName = info.Hostname
+		} else {
+			mlog.Error("Could not get cluster info")
+		}
+	}
+
+	serverNames := logFilter.ServerNames
+	if len(serverNames) > 0 {
+		for _, nodeName := range serverNames {
+			if nodeName == "default" {
+				AddLocalLogs(logData, s, page, perPage, nodeName, logFilter)
+			}
+		}
+	} else {
+		AddLocalLogs(logData, s, page, perPage, serverName, logFilter)
+	}
+
+	if s.platform.Cluster() != nil && *s.Config().ClusterSettings.Enable {
+		clusterLogs, err := s.platform.Cluster().QueryLogs(page, perPage)
+		if err != nil {
+			return nil, err
+		}
+
+		if clusterLogs != nil && len(serverNames) > 0 {
+			for _, filteredNodeName := range serverNames {
+				logData[filteredNodeName] = clusterLogs[filteredNodeName]
+			}
+		} else {
+			for nodeName, logs := range clusterLogs {
+				logData[nodeName] = logs
+			}
+		}
+	}
+
+	return logData, nil
+}
+
+func AddLocalLogs(logData map[string][]string, s *Server, page, perPage int, serverName string, logFilter *model.LogFilter) *model.AppError {
+	currentServerLogs, err := s.GetLogsSkipSend(page, perPage, logFilter)
+	if err != nil {
+		return err
+	}
+
+	logData[serverName] = currentServerLogs
+	return nil
+}
+
+func (a *App) QueryLogs(page, perPage int, logFilter *model.LogFilter) (map[string][]string, *model.AppError) {
+	return a.Srv().QueryLogs(page, perPage, logFilter)
+}
+
 func (a *App) GetLogs(page, perPage int) ([]string, *model.AppError) {
 	return a.Srv().GetLogs(page, perPage)
 }

-func (s *Server) GetLogsSkipSend(page, perPage int) ([]string, *model.AppError) {
-	return s.platform.GetLogsSkipSend(page, perPage)
+func (s *Server) GetLogsSkipSend(page, perPage int, logFilter *model.LogFilter) ([]string, *model.AppError) {
+	return s.platform.GetLogsSkipSend(page, perPage, logFilter)
 }

-func (a *App) GetLogsSkipSend(page, perPage int) ([]string, *model.AppError) {
-	return a.Srv().GetLogsSkipSend(page, perPage)
+func (a *App) GetLogsSkipSend(page, perPage int, logFilter *model.LogFilter) ([]string, *model.AppError) {
+	return a.Srv().GetLogsSkipSend(page, perPage, logFilter)
 }

 func (a *App) GetClusterStatus() []*model.ClusterInfo {
--- a/app/app_iface.go
+++ b/app/app_iface.go
@@ -682,7 +682,7 @@ type AppIface interface {
 	GetLatestTermsOfService() (*model.TermsOfService, *model.AppError)
 	GetLatestVersion(latestVersionUrl string) (*model.GithubReleaseInfo, *model.AppError)
 	GetLogs(page, perPage int) ([]string, *model.AppError)
-	GetLogsSkipSend(page, perPage int) ([]string, *model.AppError)
+	GetLogsSkipSend(page, perPage int, logFilter *model.LogFilter) ([]string, *model.AppError)
 	GetMemberCountsByGroup(ctx context.Context, channelID string, includeTimezones bool) ([]*model.ChannelMemberCountByGroup, *model.AppError)
 	GetMessageForNotification(post *model.Post, translateFunc i18n.TranslateFunc) string
 	GetMultipleEmojiByName(c request.CTX, names []string) ([]*model.Emoji, *model.AppError)
@@ -961,6 +961,7 @@ type AppIface interface {
 	PublishUserTyping(userID, channelID, parentId string) *model.AppError
 	PurgeBleveIndexes() *model.AppError
 	PurgeElasticsearchIndexes() *model.AppError
+	QueryLogs(page, perPage int, logFilter *model.LogFilter) (map[string][]string, *model.AppError)
 	ReadFile(path string) ([]byte, *model.AppError)
 	RecycleDatabaseConnection()
 	RegenCommandToken(cmd *model.Command) (*model.Command, *model.AppError)
--- a/app/busy_test.go
+++ b/app/busy_test.go
@@ -134,13 +134,18 @@ func (c *ClusterMock) StartInterNodeCommunication() {}
 func (c *ClusterMock) StopInterNodeCommunication()  {}
 func (c *ClusterMock) RegisterClusterMessageHandler(event model.ClusterEvent, crm einterfaces.ClusterMessageHandler) {
 }
-func (c *ClusterMock) GetClusterId() string                                       { return "cluster_mock" }
-func (c *ClusterMock) IsLeader() bool                                             { return false }
-func (c *ClusterMock) GetMyClusterInfo() *model.ClusterInfo                       { return nil }
-func (c *ClusterMock) GetClusterInfos() []*model.ClusterInfo                      { return nil }
-func (c *ClusterMock) NotifyMsg(buf []byte)                                       {}
-func (c *ClusterMock) GetClusterStats() ([]*model.ClusterStats, *model.AppError)  { return nil, nil }
-func (c *ClusterMock) GetLogs(page, perPage int) ([]string, *model.AppError)      { return nil, nil }
+func (c *ClusterMock) GetClusterId() string                                      { return "cluster_mock" }
+func (c *ClusterMock) IsLeader() bool                                            { return false }
+func (c *ClusterMock) GetMyClusterInfo() *model.ClusterInfo                      { return nil }
+func (c *ClusterMock) GetClusterInfos() []*model.ClusterInfo                     { return nil }
+func (c *ClusterMock) NotifyMsg(buf []byte)                                      {}
+func (c *ClusterMock) GetClusterStats() ([]*model.ClusterStats, *model.AppError) { return nil, nil }
+func (c *ClusterMock) GetLogs(page, perPage int) ([]string, *model.AppError) {
+	return nil, nil
+}
+func (c *ClusterMock) QueryLogs(page, perPage int) (map[string][]string, *model.AppError) {
+	return nil, nil
+}
 func (c *ClusterMock) GetPluginStatuses() (model.PluginStatuses, *model.AppError) { return nil, nil }
 func (c *ClusterMock) ConfigChanged(previousConfig *model.Config, newConfig *model.Config, sendToOtherServer bool) *model.AppError {
 	return nil
--- a/app/opentracing/opentracing_layer.go
+++ b/app/opentracing/opentracing_layer.go
@@ -7138,7 +7138,7 @@ func (a *OpenTracingAppLayer) GetLogs(page int, perPage int) ([]string, *model.A
 	return resultVar0, resultVar1
 }

-func (a *OpenTracingAppLayer) GetLogsSkipSend(page int, perPage int) ([]string, *model.AppError) {
+func (a *OpenTracingAppLayer) GetLogsSkipSend(page int, perPage int, logFilter *model.LogFilter) ([]string, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetLogsSkipSend")

@@ -7150,7 +7150,7 @@ func (a *OpenTracingAppLayer) GetLogsSkipSend(page int, perPage int) ([]string,
 	}()

 	defer span.Finish()
-	resultVar0, resultVar1 := a.app.GetLogsSkipSend(page, perPage)
+	resultVar0, resultVar1 := a.app.GetLogsSkipSend(page, perPage, logFilter)

 	if resultVar1 != nil {
 		span.LogFields(spanlog.Error(resultVar1))
@@ -13646,6 +13646,28 @@ func (a *OpenTracingAppLayer) PurgeElasticsearchIndexes() *model.AppError {
 	return resultVar0
 }

+func (a *OpenTracingAppLayer) QueryLogs(page int, perPage int, logFilter *model.LogFilter) (map[string][]string, *model.AppError) {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.QueryLogs")
+
+	a.ctx = newCtx
+	a.app.Srv().Store().SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store().SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0, resultVar1 := a.app.QueryLogs(page, perPage, logFilter)
+
+	if resultVar1 != nil {
+		span.LogFields(spanlog.Error(resultVar1))
+		ext.Error.Set(span, true)
+	}
+
+	return resultVar0, resultVar1
+}
+
 func (a *OpenTracingAppLayer) ReadFile(path string) ([]byte, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ReadFile")
--- a/app/platform/busy_test.go
+++ b/app/platform/busy_test.go
@@ -134,13 +134,16 @@ func (c *ClusterMock) StartInterNodeCommunication() {}
 func (c *ClusterMock) StopInterNodeCommunication()  {}
 func (c *ClusterMock) RegisterClusterMessageHandler(event model.ClusterEvent, crm einterfaces.ClusterMessageHandler) {
 }
-func (c *ClusterMock) GetClusterId() string                                       { return "cluster_mock" }
-func (c *ClusterMock) IsLeader() bool                                             { return false }
-func (c *ClusterMock) GetMyClusterInfo() *model.ClusterInfo                       { return nil }
-func (c *ClusterMock) GetClusterInfos() []*model.ClusterInfo                      { return nil }
-func (c *ClusterMock) NotifyMsg(buf []byte)                                       {}
-func (c *ClusterMock) GetClusterStats() ([]*model.ClusterStats, *model.AppError)  { return nil, nil }
-func (c *ClusterMock) GetLogs(page, perPage int) ([]string, *model.AppError)      { return nil, nil }
+func (c *ClusterMock) GetClusterId() string                                      { return "cluster_mock" }
+func (c *ClusterMock) IsLeader() bool                                            { return false }
+func (c *ClusterMock) GetMyClusterInfo() *model.ClusterInfo                      { return nil }
+func (c *ClusterMock) GetClusterInfos() []*model.ClusterInfo                     { return nil }
+func (c *ClusterMock) NotifyMsg(buf []byte)                                      {}
+func (c *ClusterMock) GetClusterStats() ([]*model.ClusterStats, *model.AppError) { return nil, nil }
+func (c *ClusterMock) GetLogs(page, perPage int) ([]string, *model.AppError)     { return nil, nil }
+func (c *ClusterMock) QueryLogs(page, perPage int) (map[string][]string, *model.AppError) {
+	return nil, nil
+}
 func (c *ClusterMock) GetPluginStatuses() (model.PluginStatuses, *model.AppError) { return nil, nil }
 func (c *ClusterMock) ConfigChanged(previousConfig *model.Config, newConfig *model.Config, sendToOtherServer bool) *model.AppError {
 	return nil
--- a/app/platform/log.go
+++ b/app/platform/log.go
@@ -5,6 +5,7 @@ package platform

 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -123,7 +124,7 @@ func (ps *PlatformService) RemoveUnlicensedLogTargets(license *model.License) {
 	})
 }

-func (ps *PlatformService) GetLogsSkipSend(page, perPage int) ([]string, *model.AppError) {
+func (ps *PlatformService) GetLogsSkipSend(page, perPage int, logFilter *model.LogFilter) ([]string, *model.AppError) {
 	var lines []string

 	if *ps.Config().LogSettings.EnableFile {
@@ -172,7 +173,22 @@ func (ps *PlatformService) GetLogsSkipSend(page, perPage int) ([]string, *model.
 					if err != nil {
 						return nil, model.NewAppError("getLogs", "api.admin.file_read_error", nil, "", http.StatusInternalServerError).Wrap(err)
 					}
-					lines = append(lines, string(line))
+
+					filtered := false
+					var entry *model.LogEntry
+					err = json.Unmarshal(line, &entry)
+					if err != nil {
+						mlog.Debug("Failed to parse line, skipping")
+					} else {
+						filtered = isLogFilteredByLevel(logFilter, entry) || filtered
+						filtered = isLogFilteredByDate(logFilter, entry) || filtered
+					}
+
+					if filtered {
+						lineCount--
+					} else {
+						lines = append(lines, string(line))
+					}
 				}
 				if pos == 0 {
 					break
@@ -194,3 +210,48 @@ func (ps *PlatformService) GetLogsSkipSend(page, perPage int) ([]string, *model.

 	return lines, nil
 }
+
+func isLogFilteredByLevel(logFilter *model.LogFilter, entry *model.LogEntry) bool {
+	logLevels := logFilter.LogLevels
+	if len(logLevels) == 0 {
+		return false
+	}
+
+	for _, level := range logLevels {
+		if entry.Level == level {
+			return false
+		}
+	}
+
+	return true
+}
+
+func isLogFilteredByDate(logFilter *model.LogFilter, entry *model.LogEntry) bool {
+	if logFilter.DateFrom == "" && logFilter.DateTo == "" {
+		return false
+	}
+
+	dateFrom, err := time.Parse("2006-01-02 15:04:05.999 -07:00", logFilter.DateFrom)
+	if err != nil {
+		dateFrom = time.Time{}
+	}
+	dateTo, err := time.Parse("2006-01-02 15:04:05.999 -07:00", logFilter.DateTo)
+	if err != nil {
+		dateTo = time.Now()
+	}
+
+	timestamp, err := time.Parse("2006-01-02 15:04:05.999 -07:00", entry.Timestamp)
+	if err != nil {
+		mlog.Debug("Cannot parse timestamp, skipping")
+		return false
+	}
+
+	if timestamp.Equal(dateFrom) || timestamp.Equal(dateTo) {
+		return false
+	}
+	if timestamp.After(dateFrom) && timestamp.Before(dateTo) {
+		return false
+	}
+
+	return true
+}
--- a/einterfaces/cluster.go
+++ b/einterfaces/cluster.go
@@ -26,6 +26,7 @@ type ClusterInterface interface {
 	NotifyMsg(buf []byte)
 	GetClusterStats() ([]*model.ClusterStats, *model.AppError)
 	GetLogs(page, perPage int) ([]string, *model.AppError)
+	QueryLogs(page, perPage int) (map[string][]string, *model.AppError)
 	GetPluginStatuses() (model.PluginStatuses, *model.AppError)
 	ConfigChanged(previousConfig *model.Config, newConfig *model.Config, sendToOtherServer bool) *model.AppError
 }
--- a/einterfaces/mocks/ClusterInterface.go
+++ b/einterfaces/mocks/ClusterInterface.go
@@ -186,6 +186,31 @@ func (_m *ClusterInterface) NotifyMsg(buf []byte) {
 	_m.Called(buf)
 }

+// QueryLogs provides a mock function with given fields: page, perPage
+func (_m *ClusterInterface) QueryLogs(page int, perPage int) (map[string][]string, *model.AppError) {
+	ret := _m.Called(page, perPage)
+
+	var r0 map[string][]string
+	if rf, ok := ret.Get(0).(func(int, int) map[string][]string); ok {
+		r0 = rf(page, perPage)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(map[string][]string)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(int, int) *model.AppError); ok {
+		r1 = rf(page, perPage)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
 // RegisterClusterMessageHandler provides a mock function with given fields: event, crm
 func (_m *ClusterInterface) RegisterClusterMessageHandler(event model.ClusterEvent, crm einterfaces.ClusterMessageHandler) {
 	_m.Called(event, crm)
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -2947,6 +2947,10 @@
    "id": "api.system.id_loaded.not_available.app_error",
    "translation": "ID Loaded Push Notifications are not configured or supported on this server."
  },
+  {
+    "id": "api.system.logs.invalidFilter",
+    "translation": "Invalid log filter"
+  },
  {
    "id": "api.system.update_notices.clear_failed",
    "translation": "Clearing old product notices failed"
--- a/model/system.go
+++ b/model/system.go
@@ -181,3 +181,15 @@ type AppliedMigration struct {
 	Version int    `json:"version"`
 	Name    string `json:"name"`
 }
+
+type LogFilter struct {
+	ServerNames []string `json:"server_names"`
+	LogLevels   []string `json:"log_levels"`
+	DateFrom    string   `json:"date_from"`
+	DateTo      string   `json:"date_to"`
+}
+
+type LogEntry struct {
+	Timestamp string
+	Level     string
+}
--- a/testlib/cluster.go
+++ b/testlib/cluster.go
@@ -59,6 +59,10 @@ func (c *FakeClusterInterface) GetLogs(page, perPage int) ([]string, *model.AppE
 	return []string{}, nil
 }

+func (c *FakeClusterInterface) QueryLogs(page, perPage int) (map[string][]string, *model.AppError) {
+	return make(map[string][]string), nil
+}
+
 func (c *FakeClusterInterface) ConfigChanged(previousConfig *model.Config, newConfig *model.Config, sendToOtherServer bool) *model.AppError {
 	return nil
 }