IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2025-02-25 18:55:24 -06:00
Code Issues Packages Projects Releases Wiki Activity

Working on refactoring jobs service (#19205)

Browse Source 
* Working on refactoring jobs service

* Making more consistent with the previous existing code

* Remove no longer needed functions

* Making a base PeridicScheduler to use it in most of the schedulers implementations

* Removing accidental complexity from on of the jobs

* Removing accidental complexity from expirynotify

* Fixing compilation from previous commit

* Remove accidental complexity from the export_delete job

* Simplifying the workers by making a reusable worker

* Using simple worker for export_delete job

* Simpliying export process job

* Simpliying extract content job

* Simpliying import delete job

* Simpliying import process job

* Simpliying product noticies job

* Simpliying fix crt channel unreads job (only removing the uneeded register function)

* Simpliying migrations job (only removing the uneeded register function)

* fixup

* Simpliying plugins job (only removing the uneeded register function)

* Simpliying bleve indexing job (only removing the uneeded register function)

* Simpliying resend invitation email job (only removing the uneeded register function)

* Fixing tests

* Simplifying migration tests infrastructure

* Adding missed license to files

* Adding an empty file to imports package to ensure this package exist even without enterprise repo

* Regenerating einterfaces mocks

* Adding missed license to files

* Updating i18n/en.json file

* help fixing enterprise tests compilation

* Adding new DailyScheduler

* Fixing typo and changing the waitTime type for periodic sechduler

* Making the daily scheduler more generic

* Adding comments to clarify not used parameters in interface scheduler interface implementations

* Using merror to handle multiple errors in jobs workers

* Fixing linter errors

* Addressing PR review comments

* Reverting go.tools.mod changes

* Removing the static check for worker type in the model (moving it to the insertion of new jobs

* Moving migrations job to the jobs directory

* Fixing (and improving a bit) tests

* Apply suggestions from code review

Co-authored-by: Doug Lauder <wiggin77@warpmail.net>

* Fixing enterprise tests

* Removing unneeded InitWorkers/InitSchedulers calls

* Fix expirenotify job when error happens

* Fixing govet errors

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Co-authored-by: Doug Lauder <wiggin77@warpmail.net>
This commit is contained in:
Jesús Espino 2022-02-14 18:21:18 +01:00 committed by GitHub
parent 7398451030
commit 2c3e289509
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
71 changed files with 968 additions and 2574 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api4/apitestlib.go
View File

@ -189,8 +189,6 @@ func setupTestHelper(dbStore store.Store, searchEngine *searchengine.Broker, ent
if enterprise {
th.App.Srv().SetLicense(model.NewTestLicense())
th.App.Srv().Jobs.InitWorkers()
th.App.Srv().Jobs.InitSchedulers()
} else {
th.App.Srv().SetLicense(nil)
}

36
api4/job_test.go
View File

@ -19,33 +19,29 @@ func TestCreateJob(t *testing.T) {
defer th.TearDown()
job := &model.Job{
Type: model.JobTypeMessageExport,
Type: model.JobTypeActiveUsers,
Data: map[string]string{
"thing": "stuff",
},
}
_, resp, err := th.SystemManagerClient.CreateJob(job)
require.Error(t, err)
CheckForbiddenStatus(t, resp)
t.Run("valid job as user without permissions", func(t *testing.T) {
_, resp, err := th.SystemManagerClient.CreateJob(job)
require.Error(t, err)
CheckForbiddenStatus(t, resp)
})
received, _, err := th.SystemAdminClient.CreateJob(job)
require.NoError(t, err)
t.Run("valid job as user with permissions", func(t *testing.T) {
received, _, err := th.SystemAdminClient.CreateJob(job)
require.NoError(t, err)
defer th.App.Srv().Store.Job().Delete(received.Id)
})
defer th.App.Srv().Store.Job().Delete(received.Id)
job = &model.Job{
Type: model.NewId(),
}
_, resp, err = th.SystemAdminClient.CreateJob(job)
require.Error(t, err)
CheckBadRequestStatus(t, resp)
job.Type = model.JobTypeElasticsearchPostIndexing
_, resp, err = th.Client.CreateJob(job)
require.Error(t, err)
CheckForbiddenStatus(t, resp)
t.Run("invalid job type as user without permissions", func(t *testing.T) {
_, resp, err := th.SystemAdminClient.CreateJob(&model.Job{Type: model.NewId()})
require.Error(t, err)
CheckBadRequestStatus(t, resp)
})
}
func TestGetJob(t *testing.T) {

4
app/app_iface.go
View File

@ -236,7 +236,7 @@ type AppIface interface {
// NewWebConn returns a new WebConn instance.
NewWebConn(cfg *WebConnConfig) *WebConn
// NotifySessionsExpired is called periodically from the job server to notify any mobile sessions that have expired.
NotifySessionsExpired() *model.AppError
NotifySessionsExpired() error
// OverrideIconURLIfEmoji changes the post icon override URL prop, if it has an emoji icon,
// so that it points to the URL (relative) of the emoji - static if emoji is default, /api if custom.
OverrideIconURLIfEmoji(post *model.Post)
@ -405,7 +405,7 @@ type AppIface interface {
BuildPostReactions(postID string) (*[]ReactionImportData, *model.AppError)
BuildPushNotificationMessage(contentsConfig string, post *model.Post, user *model.User, channel *model.Channel, channelName string, senderName string, explicitMention bool, channelWideMention bool, replyToThreadType string) (*model.PushNotification, *model.AppError)
BuildSamlMetadataObject(idpMetadata []byte) (*model.SamlMetadataResponse, *model.AppError)
BulkExport(writer io.Writer, outPath string, opts BulkExportOpts) *model.AppError
BulkExport(writer io.Writer, outPath string, opts model.BulkExportOpts) *model.AppError
BulkImport(c *request.Context, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun bool, workers int) (*model.AppError, int)
BulkImportWithPath(c *request.Context, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun bool, workers int, importPath string) (*model.AppError, int)
CancelJob(jobId string) *model.AppError

78
app/enterprise.go
View File

@ -6,7 +6,6 @@ package app
import (
"github.com/mattermost/mattermost-server/v6/einterfaces"
ejobs "github.com/mattermost/mattermost-server/v6/einterfaces/jobs"
tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/services/searchengine"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
@ -60,9 +59,9 @@ func RegisterJobsElasticsearchAggregatorInterface(f func(*Server) ejobs.Elastics
jobsElasticsearchAggregatorInterface = f
}
var jobsElasticsearchIndexerInterface func(*Server) tjobs.IndexerJobInterface
var jobsElasticsearchIndexerInterface func(*Server) ejobs.IndexerJobInterface
func RegisterJobsElasticsearchIndexerInterface(f func(*Server) tjobs.IndexerJobInterface) {
func RegisterJobsElasticsearchIndexerInterface(f func(*Server) ejobs.IndexerJobInterface) {
jobsElasticsearchIndexerInterface = f
}
@ -72,85 +71,12 @@ func RegisterJobsLdapSyncInterface(f func(*Server) ejobs.LdapSyncInterface) {
jobsLdapSyncInterface = f
}
var jobsMigrationsInterface func(*Server) tjobs.MigrationsJobInterface
func RegisterJobsMigrationsJobInterface(f func(*Server) tjobs.MigrationsJobInterface) {
jobsMigrationsInterface = f
}
var jobsPluginsInterface func(*Server) tjobs.PluginsJobInterface
func RegisterJobsPluginsJobInterface(f func(*Server) tjobs.PluginsJobInterface) {
jobsPluginsInterface = f
}
var jobsBleveIndexerInterface func(*Server) tjobs.IndexerJobInterface
func RegisterJobsBleveIndexerInterface(f func(*Server) tjobs.IndexerJobInterface) {
jobsBleveIndexerInterface = f
}
var jobsActiveUsersInterface func(*Server) tjobs.ActiveUsersJobInterface
func RegisterJobsActiveUsersInterface(f func(*Server) tjobs.ActiveUsersJobInterface) {
jobsActiveUsersInterface = f
}
var jobsResendInvitationEmailInterface func(*Server) ejobs.ResendInvitationEmailJobInterface
// RegisterJobsResendInvitationEmailInterface is used to register or initialize the jobsResendInvitationEmailInterface
func RegisterJobsResendInvitationEmailInterface(f func(*Server) ejobs.ResendInvitationEmailJobInterface) {
jobsResendInvitationEmailInterface = f
}
var jobsCloudInterface func(*Server) ejobs.CloudJobInterface
func RegisterJobsCloudInterface(f func(*Server) ejobs.CloudJobInterface) {
jobsCloudInterface = f
}
var jobsExpiryNotifyInterface func(*Server) tjobs.ExpiryNotifyJobInterface
func RegisterJobsExpiryNotifyJobInterface(f func(*Server) tjobs.ExpiryNotifyJobInterface) {
jobsExpiryNotifyInterface = f
}
var jobsImportProcessInterface func(*Server) tjobs.ImportProcessInterface
func RegisterJobsImportProcessInterface(f func(*Server) tjobs.ImportProcessInterface) {
jobsImportProcessInterface = f
}
var jobsImportDeleteInterface func(*Server) tjobs.ImportDeleteInterface
func RegisterJobsImportDeleteInterface(f func(*Server) tjobs.ImportDeleteInterface) {
jobsImportDeleteInterface = f
}
var jobsExportProcessInterface func(*Server) tjobs.ExportProcessInterface
func RegisterJobsExportProcessInterface(f func(*Server) tjobs.ExportProcessInterface) {
jobsExportProcessInterface = f
}
var jobsExportDeleteInterface func(*Server) tjobs.ExportDeleteInterface
func RegisterJobsExportDeleteInterface(f func(*Server) tjobs.ExportDeleteInterface) {
jobsExportDeleteInterface = f
}
var jobsExtractContentInterface func(*Server) tjobs.ExtractContentInterface
func RegisterJobsExtractContentInterface(f func(*Server) tjobs.ExtractContentInterface) {
jobsExtractContentInterface = f
}
var productNoticesJobInterface func(*Server) tjobs.ProductNoticesJobInterface
func RegisterProductNoticesJobInterface(f func(*Server) tjobs.ProductNoticesJobInterface) {
productNoticesJobInterface = f
}
var ldapInterface func(*Server) einterfaces.LdapInterface
func RegisterLdapInterface(f func(*Server) einterfaces.LdapInterface) {

2
app/expirynotify.go
View File

@ -16,7 +16,7 @@ const (
)
// NotifySessionsExpired is called periodically from the job server to notify any mobile sessions that have expired.
func (a *App) NotifySessionsExpired() *model.AppError {
func (a *App) NotifySessionsExpired() error {
if *a.Config().EmailSettings.SendPushNotifications {
pushServer := *a.Config().EmailSettings.PushNotificationServer
if license := a.ch.srv.License(); pushServer == model.MHPNS && (license == nil || !*license.Features.MHPNS) {

5
app/expirynotify_test.go
View File

@ -34,7 +34,7 @@ func TestNotifySessionsExpired(t *testing.T) {
err := th.App.NotifySessionsExpired()
// no error, but also no requests sent
require.Nil(t, err)
require.NoError(t, err)
require.Equal(t, 0, handler.numReqs())
})
@ -65,8 +65,7 @@ func TestNotifySessionsExpired(t *testing.T) {
}
err := th.App.NotifySessionsExpired()
require.Nil(t, err)
require.NoError(t, err)
require.Equal(t, 2, handler.numReqs())
expected := []string{"22222", "33333"}

15
app/export.go
View File

@ -20,15 +20,6 @@ import (
"github.com/mattermost/mattermost-server/v6/store"
)
type BulkExportOpts struct {
IncludeAttachments bool
CreateArchive bool
}
// ExportDataDir is the name of the directory were to store additional data
// included with the export (e.g. file attachments).
const ExportDataDir = "data"
// We use this map to identify the exportable preferences.
// Here we link the preference category and name, to the name of the relevant field in the import struct.
var exportablePreferences = map[ComparablePreference]string{{
@ -64,7 +55,7 @@ var exportablePreferences = map[ComparablePreference]string{{
}: "EmailInterval",
}
func (a *App) BulkExport(writer io.Writer, outPath string, opts BulkExportOpts) *model.AppError {
func (a *App) BulkExport(writer io.Writer, outPath string, opts model.BulkExportOpts) *model.AppError {
var zipWr *zip.Writer
if opts.CreateArchive {
var err error
@ -706,7 +697,7 @@ func (a *App) exportFile(outPath, filePath string, zipWr *zip.Writer) *model.App
if zipWr != nil {
wr, err = zipWr.CreateHeader(&zip.FileHeader{
Name: filepath.Join(ExportDataDir, filePath),
Name: filepath.Join(model.ExportDataDir, filePath),
Method: zip.Store,
})
if err != nil {
@ -714,7 +705,7 @@ func (a *App) exportFile(outPath, filePath string, zipWr *zip.Writer) *model.App
nil, "err="+err.Error(), http.StatusInternalServerError)
}
} else {
filePath = filepath.Join(outPath, ExportDataDir, filePath)
filePath = filepath.Join(outPath, model.ExportDataDir, filePath)
if err = os.MkdirAll(filepath.Dir(filePath), 0700); err != nil {
return model.NewAppError("exportFileAttachment", "app.export.export_attachment.mkdirall.error",
nil, "err="+err.Error(), http.StatusInternalServerError)

20
app/export_test.go
View File

@ -179,7 +179,7 @@ func TestExportAllUsers(t *testing.T) {
require.Nil(t, err)
var b bytes.Buffer
err = th1.App.BulkExport(&b, "somePath", BulkExportOpts{})
err = th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{})
require.Nil(t, err)
th2 := Setup(t)
@ -227,7 +227,7 @@ func TestExportDMChannel(t *testing.T) {
th1.CreateDmChannel(th1.BasicUser2)
var b bytes.Buffer
err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{})
err := th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{})
require.Nil(t, err)
channels, nErr := th1.App.Srv().Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
@ -268,7 +268,7 @@ func TestExportDMChannel(t *testing.T) {
th1.App.PermanentDeleteUser(th1.Context, th1.BasicUser)
var b bytes.Buffer
err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{})
err := th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{})
require.Nil(t, err)
th2 := Setup(t).InitBasic()
@ -292,7 +292,7 @@ func TestExportDMChannelToSelf(t *testing.T) {
th1.CreateDmChannel(th1.BasicUser)
var b bytes.Buffer
err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{})
err := th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{})
require.Nil(t, err)
channels, nErr := th1.App.Srv().Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
@ -330,7 +330,7 @@ func TestExportGMChannel(t *testing.T) {
th1.CreateGroupChannel(user1, user2)
var b bytes.Buffer
err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{})
err := th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{})
require.Nil(t, err)
channels, nErr := th1.App.Srv().Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
@ -362,7 +362,7 @@ func TestExportGMandDMChannels(t *testing.T) {
th1.CreateGroupChannel(user1, user2)
var b bytes.Buffer
err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{})
err := th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{})
require.Nil(t, err)
channels, nErr := th1.App.Srv().Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
@ -445,7 +445,7 @@ func TestExportDMandGMPost(t *testing.T) {
assert.Equal(t, 4, len(posts))
var b bytes.Buffer
appErr := th1.App.BulkExport(&b, "somePath", BulkExportOpts{})
appErr := th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{})
require.Nil(t, appErr)
th1.TearDown()
@ -520,7 +520,7 @@ func TestExportPostWithProps(t *testing.T) {
require.NotEmpty(t, posts[1].Props)
var b bytes.Buffer
appErr := th1.App.BulkExport(&b, "somePath", BulkExportOpts{})
appErr := th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{})
require.Nil(t, appErr)
th1.TearDown()
@ -558,7 +558,7 @@ func TestExportDMPostWithSelf(t *testing.T) {
th1.CreatePost(dmChannel)
var b bytes.Buffer
err := th1.App.BulkExport(&b, "somePath", BulkExportOpts{})
err := th1.App.BulkExport(&b, "somePath", model.BulkExportOpts{})
require.Nil(t, err)
posts, nErr := th1.App.Srv().Store.Post().GetDirectPostParentsForExportAfter(1000, "0000000")
@ -622,7 +622,7 @@ func TestBulkExport(t *testing.T) {
require.NoError(t, err)
defer exportFile.Close()
opts := BulkExportOpts{
opts := model.BulkExportOpts{
IncludeAttachments: true,
CreateArchive: true,
}

2
app/import.go
View File

@ -327,7 +327,7 @@ func (a *App) ListImports() ([]string, *model.AppError) {
results := make([]string, 0, len(imports))
for i := 0; i < len(imports); i++ {
filename := filepath.Base(imports[i])
if !strings.HasSuffix(filename, IncompleteUploadSuffix) {
if !strings.HasSuffix(filename, model.IncompleteUploadSuffix) {
results = append(results, filename)
}
}

2
app/import_test.go
View File

@ -489,7 +489,7 @@ func TestImportBulkImportWithAttachments(t *testing.T) {
th.App.UpdateConfig(func(cfg *model.Config) { cfg.TeamSettings.MaxUsersPerTeam = model.NewInt(1000) })
appErr, _ := th.App.BulkImportWithPath(th.Context, jsonFile, importZipReader, false, 1, ExportDataDir)
appErr, _ := th.App.BulkImportWithPath(th.Context, jsonFile, importZipReader, false, 1, model.ExportDataDir)
require.Nil(t, appErr)
adminUser, appErr := th.App.GetUserByUsername("sysadmin")

4
app/opentracing/opentracing_layer.go
View File

@ -973,7 +973,7 @@ func (a *OpenTracingAppLayer) BuildSamlMetadataObject(idpMetadata []byte) (*mode
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) BulkExport(writer io.Writer, outPath string, opts app.BulkExportOpts) *model.AppError {
func (a *OpenTracingAppLayer) BulkExport(writer io.Writer, outPath string, opts model.BulkExportOpts) *model.AppError {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.BulkExport")
@ -11742,7 +11742,7 @@ func (a *OpenTracingAppLayer) NotifyAndSetWarnMetricAck(warnMetricId string, sen
return resultVar0
}
func (a *OpenTracingAppLayer) NotifySessionsExpired() *model.AppError {
func (a *OpenTracingAppLayer) NotifySessionsExpired() error {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.NotifySessionsExpired")

150
app/server.go
View File

@ -45,13 +45,25 @@ import (
"github.com/mattermost/mattermost-server/v6/config"
"github.com/mattermost/mattermost-server/v6/einterfaces"
"github.com/mattermost/mattermost-server/v6/jobs"
"github.com/mattermost/mattermost-server/v6/jobs/active_users"
"github.com/mattermost/mattermost-server/v6/jobs/expirynotify"
"github.com/mattermost/mattermost-server/v6/jobs/export_delete"
"github.com/mattermost/mattermost-server/v6/jobs/export_process"
"github.com/mattermost/mattermost-server/v6/jobs/extract_content"
"github.com/mattermost/mattermost-server/v6/jobs/import_delete"
"github.com/mattermost/mattermost-server/v6/jobs/import_process"
"github.com/mattermost/mattermost-server/v6/jobs/migrations"
"github.com/mattermost/mattermost-server/v6/jobs/product_notices"
"github.com/mattermost/mattermost-server/v6/jobs/resend_invitation_email"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/plugin/scheduler"
"github.com/mattermost/mattermost-server/v6/services/awsmeter"
"github.com/mattermost/mattermost-server/v6/services/cache"
"github.com/mattermost/mattermost-server/v6/services/httpservice"
"github.com/mattermost/mattermost-server/v6/services/remotecluster"
"github.com/mattermost/mattermost-server/v6/services/searchengine"
"github.com/mattermost/mattermost-server/v6/services/searchengine/bleveengine"
"github.com/mattermost/mattermost-server/v6/services/searchengine/bleveengine/indexer"
"github.com/mattermost/mattermost-server/v6/services/sharedchannel"
"github.com/mattermost/mattermost-server/v6/services/telemetry"
"github.com/mattermost/mattermost-server/v6/services/timezones"
@ -1862,72 +1874,110 @@ func (ch *Channels) ClientConfigHash() string {
func (s *Server) initJobs() {
s.Jobs = jobs.NewJobServer(s, s.Store, s.Metrics)
s.Jobs.InitWorkers()
s.Jobs.InitSchedulers()
if jobsDataRetentionJobInterface != nil {
s.Jobs.DataRetentionJob = jobsDataRetentionJobInterface(s)
builder := jobsDataRetentionJobInterface(s)
s.Jobs.RegisterJobType(model.JobTypeDataRetention, builder.MakeWorker(), builder.MakeScheduler())
}
if jobsMessageExportJobInterface != nil {
s.Jobs.MessageExportJob = jobsMessageExportJobInterface(s)
builder := jobsMessageExportJobInterface(s)
s.Jobs.RegisterJobType(model.JobTypeMessageExport, builder.MakeWorker(), builder.MakeScheduler())
}
if jobsElasticsearchAggregatorInterface != nil {
s.Jobs.ElasticsearchAggregator = jobsElasticsearchAggregatorInterface(s)
builder := jobsElasticsearchAggregatorInterface(s)
s.Jobs.RegisterJobType(model.JobTypeElasticsearchPostAggregation, builder.MakeWorker(), builder.MakeScheduler())
}
if jobsElasticsearchIndexerInterface != nil {
s.Jobs.ElasticsearchIndexer = jobsElasticsearchIndexerInterface(s)
}
if jobsBleveIndexerInterface != nil {
s.Jobs.BleveIndexer = jobsBleveIndexerInterface(s)
}
if jobsMigrationsInterface != nil {
s.Jobs.Migrations = jobsMigrationsInterface(s)
builder := jobsElasticsearchIndexerInterface(s)
s.Jobs.RegisterJobType(model.JobTypeElasticsearchPostIndexing, builder.MakeWorker(), nil)
}
if jobsLdapSyncInterface != nil {
s.Jobs.LdapSync = jobsLdapSyncInterface(s)
}
if jobsPluginsInterface != nil {
s.Jobs.Plugins = jobsPluginsInterface(s)
}
if jobsExpiryNotifyInterface != nil {
s.Jobs.ExpiryNotify = jobsExpiryNotifyInterface(s)
}
if productNoticesJobInterface != nil {
s.Jobs.ProductNotices = productNoticesJobInterface(s)
}
if jobsImportProcessInterface != nil {
s.Jobs.ImportProcess = jobsImportProcessInterface(s)
}
if jobsImportDeleteInterface != nil {
s.Jobs.ImportDelete = jobsImportDeleteInterface(s)
}
if jobsExportDeleteInterface != nil {
s.Jobs.ExportDelete = jobsExportDeleteInterface(s)
}
if jobsExportProcessInterface != nil {
s.Jobs.ExportProcess = jobsExportProcessInterface(s)
}
if jobsExportProcessInterface != nil {
s.Jobs.ExportProcess = jobsExportProcessInterface(s)
}
if jobsActiveUsersInterface != nil {
s.Jobs.ActiveUsers = jobsActiveUsersInterface(s)
builder := jobsLdapSyncInterface(s)
s.Jobs.RegisterJobType(model.JobTypeLdapSync, builder.MakeWorker(), builder.MakeScheduler())
}
if jobsCloudInterface != nil {
s.Jobs.Cloud = jobsCloudInterface(s)
builder := jobsCloudInterface(s)
s.Jobs.RegisterJobType(model.JobTypeCloud, builder.MakeWorker(), builder.MakeScheduler())
}
if jobsResendInvitationEmailInterface != nil {
s.Jobs.ResendInvitationEmails = jobsResendInvitationEmailInterface(s)
}
s.Jobs.RegisterJobType(
model.JobTypeBlevePostIndexing,
indexer.MakeWorker(s.Jobs, s.SearchEngine.BleveEngine.(*bleveengine.BleveEngine)),
nil,
)
if jobsExtractContentInterface != nil {
s.Jobs.ExtractContent = jobsExtractContentInterface(s)
}
s.Jobs.RegisterJobType(
model.JobTypeMigrations,
migrations.MakeWorker(s.Jobs, s.Store),
migrations.MakeScheduler(s.Jobs, s.Store),
)
s.Jobs.InitWorkers()
s.Jobs.InitSchedulers()
s.Jobs.RegisterJobType(
model.JobTypePlugins,
scheduler.MakeWorker(s.Jobs, New(ServerConnector(s.Channels()))),
scheduler.MakeScheduler(s.Jobs),
)
s.Jobs.RegisterJobType(
model.JobTypeExpiryNotify,
expirynotify.MakeWorker(s.Jobs, New(ServerConnector(s.Channels())).NotifySessionsExpired),
expirynotify.MakeScheduler(s.Jobs),
)
s.Jobs.RegisterJobType(
model.JobTypeProductNotices,
product_notices.MakeWorker(s.Jobs, New(ServerConnector(s.Channels()))),
product_notices.MakeScheduler(s.Jobs),
)
s.Jobs.RegisterJobType(
model.JobTypeImportProcess,
import_process.MakeWorker(s.Jobs, New(ServerConnector(s.Channels()))),
nil,
)
s.Jobs.RegisterJobType(
model.JobTypeImportDelete,
import_delete.MakeWorker(s.Jobs, New(ServerConnector(s.Channels())), s.Store),
import_delete.MakeScheduler(s.Jobs),
)
s.Jobs.RegisterJobType(
model.JobTypeExportDelete,
export_delete.MakeWorker(s.Jobs, New(ServerConnector(s.Channels()))),
export_delete.MakeScheduler(s.Jobs),
)
s.Jobs.RegisterJobType(
model.JobTypeExportProcess,
export_process.MakeWorker(s.Jobs, New(ServerConnector(s.Channels()))),
nil,
)
s.Jobs.RegisterJobType(
model.JobTypeActiveUsers,
active_users.MakeWorker(s.Jobs, s.Store, func() einterfaces.MetricsInterface { return s.Metrics }),
active_users.MakeScheduler(s.Jobs),
)
s.Jobs.RegisterJobType(
model.JobTypeResendInvitationEmail,
resend_invitation_email.MakeWorker(s.Jobs, New(ServerConnector(s.Channels())), s.Store, s.telemetryService),
nil,
)
s.Jobs.RegisterJobType(
model.JobTypeExtractContent,
extract_content.MakeWorker(s.Jobs, New(ServerConnector(s.Channels())), s.Store),
nil,
)
}
func (s *Server) TelemetryId() string {

3
app/upload.go
View File

@ -20,7 +20,6 @@ import (
)
const minFirstPartSize = 5 * 1024 * 1024 // 5MB
const IncompleteUploadSuffix = ".tmp"
func (a *App) runPluginsHook(c *request.Context, info *model.FileInfo, file io.Reader) *model.AppError {
pluginsEnvironment := a.GetPluginsEnvironment()
@ -195,7 +194,7 @@ func (a *App) UploadData(c *request.Context, us *model.UploadSession, rd io.Read
uploadPath := us.Path
if us.Type == model.UploadTypeImport {
uploadPath += IncompleteUploadSuffix
uploadPath += model.IncompleteUploadSuffix
}
// make sure it's not possible to upload more data than what is expected.

3
cmd/mattermost/commands/export.go
View File

@ -10,7 +10,6 @@ import (
"path/filepath"
"time"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/audit"
"github.com/mattermost/mattermost-server/v6/model"
@ -227,7 +226,7 @@ func bulkExportCmdF(command *cobra.Command, args []string) error {
return err
}
var opts app.BulkExportOpts
var opts model.BulkExportOpts
opts.IncludeAttachments = attachments
opts.CreateArchive = archive
if err := a.BulkExport(fileWriter, filepath.Dir(outPath), opts); err != nil {

2
jobs/interfaces/indexer_interface.go → einterfaces/jobs/indexer_interface.go
View File

@ -1,7 +1,7 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package interfaces
package jobs
import (
"github.com/mattermost/mattermost-server/v6/model"

11
einterfaces/jobs/resend_invitation_email.go
View File

@ -1,11 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package jobs
import "github.com/mattermost/mattermost-server/v6/model"
// ResendInvitationEmailJobInterface defines the interface for the job to resend invitation emails
type ResendInvitationEmailJobInterface interface {
MakeWorker() model.Worker
MakeScheduler() model.Scheduler
}

31
einterfaces/mocks/IndexerJobInterface.go Normal file
View File

@ -0,0 +1,31 @@
// Code generated by mockery v1.0.0. DO NOT EDIT.
// Regenerate this file using `make einterfaces-mocks`.
package mocks
import (
model "github.com/mattermost/mattermost-server/v6/model"
mock "github.com/stretchr/testify/mock"
)
// IndexerJobInterface is an autogenerated mock type for the IndexerJobInterface type
type IndexerJobInterface struct {
mock.Mock
}
// MakeWorker provides a mock function with given fields:
func (_m *IndexerJobInterface) MakeWorker() model.Worker {
ret := _m.Called()
var r0 model.Worker
if rf, ok := ret.Get(0).(func() model.Worker); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(model.Worker)
}
}
return r0
}

12
i18n/en.json
View File

@ -7559,18 +7559,6 @@
"id": "error",
"translation": "Error"
},
{
"id": "extrac_content.worker.do_job.invalid_input.from",
"translation": "Invalid input value 'from'"
},
{
"id": "extrac_content.worker.do_job.invalid_input.to",
"translation": "Invalid input value 'to'"
},
{
"id": "extract_content.worker.do_job.file_info",
"translation": "Failed to get file information for content extraction."
},
{
"id": "group_not_associated_to_synced_team",
"translation": "Group cannot be associated to the channel until it is first associated to the parent group-synced team."

42
imports/placeholder.go
View File

@ -1,42 +1,8 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
// As part of the enterprise code integration machinery a go file is copied to
// this package, this file ensure the existence of the package anyway, allowing
// the rest of the code to import the package when enterprise code is there and
// when is not.
package imports
import (
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
_ "github.com/mattermost/mattermost-server/v6/migrations"
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
_ "github.com/mattermost/mattermost-server/v6/plugin/scheduler"
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
_ "github.com/mattermost/mattermost-server/v6/services/searchengine/bleveengine/indexer"
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
_ "github.com/mattermost/mattermost-server/v6/jobs/expirynotify"
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
_ "github.com/mattermost/mattermost-server/v6/jobs/active_users"
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
_ "github.com/mattermost/mattermost-server/v6/jobs/product_notices"
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
_ "github.com/mattermost/mattermost-server/v6/jobs/import_process"
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
_ "github.com/mattermost/mattermost-server/v6/jobs/import_delete"
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
_ "github.com/mattermost/mattermost-server/v6/jobs/export_process"
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
_ "github.com/mattermost/mattermost-server/v6/jobs/export_delete"
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
_ "github.com/mattermost/mattermost-server/v6/jobs/resend_invitation_email"
// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
_ "github.com/mattermost/mattermost-server/v6/jobs/extract_content"
)

43
jobs/active_users/scheduler.go
View File

@ -6,46 +6,15 @@ package active_users
import (
"time"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/jobs"
"github.com/mattermost/mattermost-server/v6/model"
)
const (
SchedFreqMinutes = 10
)
const schedFreq = 10 * time.Minute
type Scheduler struct {
App *app.App
}
func (m *ActiveUsersJobInterfaceImpl) MakeScheduler() model.Scheduler {
return &Scheduler{m.App}
}
func (scheduler *Scheduler) Name() string {
return JobName + "Scheduler"
}
func (scheduler *Scheduler) JobType() string {
return model.JobTypeActiveUsers
}
func (scheduler *Scheduler) Enabled(cfg *model.Config) bool {
// Only enabled when Metrics are enabled.
return *cfg.MetricsSettings.Enable
}
func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
nextTime := time.Now().Add(SchedFreqMinutes * time.Minute)
return &nextTime
}
func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
data := map[string]string{}
job, err := scheduler.App.Srv().Jobs.CreateJob(model.JobTypeActiveUsers, data)
if err != nil {
return nil, err
func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler {
isEnabled := func(cfg *model.Config) bool {
return *cfg.MetricsSettings.Enable
}
return job, nil
return jobs.NewPeriodicScheduler(jobServer, model.JobTypeActiveUsers, schedFreq, isEnabled)
}

118
jobs/active_users/worker.go
View File

@ -4,117 +4,31 @@
package active_users
import (
"net/http"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/einterfaces"
"github.com/mattermost/mattermost-server/v6/jobs"
tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
"github.com/mattermost/mattermost-server/v6/store"
)
const (
JobName = "ActiveUsers"
)
type Worker struct {
name string
stop chan bool
stopped chan bool
jobs chan model.Job
jobServer *jobs.JobServer
app *app.App
}
func init() {
app.RegisterJobsActiveUsersInterface(func(s *app.Server) tjobs.ActiveUsersJobInterface {
a := app.New(app.ServerConnector(s.Channels()))
return &ActiveUsersJobInterfaceImpl{a}
})
}
type ActiveUsersJobInterfaceImpl struct {
App *app.App
}
func (m *ActiveUsersJobInterfaceImpl) MakeWorker() model.Worker {
worker := Worker{
name: JobName,
stop: make(chan bool, 1),
stopped: make(chan bool, 1),
jobs: make(chan model.Job),
jobServer: m.App.Srv().Jobs,
app: m.App,
func MakeWorker(jobServer *jobs.JobServer, store store.Store, getMetrics func() einterfaces.MetricsInterface) model.Worker {
isEnabled := func(cfg *model.Config) bool {
return *cfg.MetricsSettings.Enable
}
return &worker
}
func (worker *Worker) Run() {
mlog.Debug("Worker started", mlog.String("worker", worker.name))
defer func() {
mlog.Debug("Worker finished", mlog.String("worker", worker.name))
worker.stopped <- true
}()
for {
select {
case <-worker.stop:
mlog.Debug("Worker received stop signal", mlog.String("worker", worker.name))
return
case job := <-worker.jobs:
mlog.Debug("Worker received a new candidate job.", mlog.String("worker", worker.name))
worker.DoJob(&job)
execute := func(job *model.Job) error {
count, err := store.User().Count(model.UserCountOptions{IncludeDeleted: false})
if err != nil {
return err
}
if getMetrics() != nil {
getMetrics().ObserveEnabledUsers(count)
}
return nil
}
}
func (worker *Worker) Stop() {
mlog.Debug("Worker stopping", mlog.String("worker", worker.name))
worker.stop <- true
<-worker.stopped
}
func (worker *Worker) JobChannel() chan<- model.Job {
return worker.jobs
}
func (worker *Worker) DoJob(job *model.Job) {
if claimed, err := worker.jobServer.ClaimJob(job); err != nil {
mlog.Warn("Worker experienced an error while trying to claim job",
mlog.String("worker", worker.name),
mlog.String("job_id", job.Id),
mlog.String("error", err.Error()))
return
} else if !claimed {
return
}
count, err := worker.app.Srv().Store.User().Count(model.UserCountOptions{IncludeDeleted: false})
if err != nil {
mlog.Error("Worker: Failed to get active user count", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
worker.setJobError(job, model.NewAppError("DoJob", "app.user.get_total_users_count.app_error", nil, err.Error(), http.StatusInternalServerError))
return
}
if worker.app.Metrics() != nil {
worker.app.Metrics().ObserveEnabledUsers(count)
}
mlog.Info("Worker: Job is complete", mlog.String("worker", worker.name), mlog.String("job_id", job.Id))
worker.setJobSuccess(job)
}
func (worker *Worker) setJobSuccess(job *model.Job) {
if err := worker.app.Srv().Jobs.SetJobSuccess(job); err != nil {
mlog.Error("Worker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
worker.setJobError(job, err)
}
}
func (worker *Worker) setJobError(job *model.Job, appError *model.AppError) {
if err := worker.app.Srv().Jobs.SetJobError(job, appError); err != nil {
mlog.Error("Worker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
}
worker := jobs.NewSimpleWorker(JobName, jobServer, execute, isEnabled)
return worker
}

72
jobs/base_schedulers.go Normal file
View File

@ -0,0 +1,72 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package jobs
import (
"time"
"github.com/mattermost/mattermost-server/v6/model"
)
type PeriodicScheduler struct {
jobs *JobServer
period time.Duration
jobType string
enabledFunc func(cfg *model.Config) bool
}
func NewPeriodicScheduler(jobs *JobServer, jobType string, period time.Duration, enabledFunc func(cfg *model.Config) bool) *PeriodicScheduler {
return &PeriodicScheduler{
period: period,
jobType: jobType,
enabledFunc: enabledFunc,
jobs: jobs,
}
}
func (scheduler *PeriodicScheduler) Enabled(cfg *model.Config) bool {
return scheduler.enabledFunc(cfg)
}
func (scheduler *PeriodicScheduler) NextScheduleTime(_ *model.Config, _ time.Time /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) *time.Time {
nextTime := time.Now().Add(scheduler.period)
return &nextTime
}
func (scheduler *PeriodicScheduler) ScheduleJob(_ *model.Config /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) (*model.Job, *model.AppError) {
return scheduler.jobs.CreateJob(scheduler.jobType, nil)
}
type DailyScheduler struct {
jobs *JobServer
startTimeFunc func(cfg *model.Config) *time.Time
jobType string
enabledFunc func(cfg *model.Config) bool
}
func NewDailyScheduler(jobs *JobServer, jobType string, startTimeFunc func(cfg *model.Config) *time.Time, enabledFunc func(cfg *model.Config) bool) *DailyScheduler {
return &DailyScheduler{
startTimeFunc: startTimeFunc,
jobType: jobType,
enabledFunc: enabledFunc,
jobs: jobs,
}
}
func (scheduler *DailyScheduler) Enabled(cfg *model.Config) bool {
return scheduler.enabledFunc(cfg)
}
func (scheduler *DailyScheduler) NextScheduleTime(cfg *model.Config, now time.Time /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) *time.Time {
scheduledTime := scheduler.startTimeFunc(cfg)
if scheduledTime == nil {
return nil
}
return GenerateNextStartDateTime(now, *scheduledTime)
}
func (scheduler *DailyScheduler) ScheduleJob(_ *model.Config /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) (*model.Job, *model.AppError) {
return scheduler.jobs.CreateJob(scheduler.jobType, nil)
}

103
jobs/base_workers.go Normal file
View File

@ -0,0 +1,103 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package jobs
import (
"net/http"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
)
type SimpleWorker struct {
name string
stop chan bool
stopped chan bool
jobs chan model.Job
jobServer *JobServer
execute func(job *model.Job) error
isEnabled func(cfg *model.Config) bool
}
func NewSimpleWorker(name string, jobServer *JobServer, execute func(job *model.Job) error, isEnabled func(cfg *model.Config) bool) *SimpleWorker {
worker := SimpleWorker{
name: name,
stop: make(chan bool, 1),
stopped: make(chan bool, 1),
jobs: make(chan model.Job),
jobServer: jobServer,
execute: execute,
isEnabled: isEnabled,
}
return &worker
}
func (worker *SimpleWorker) Run() {
mlog.Debug("Worker started", mlog.String("worker", worker.name))
defer func() {
mlog.Debug("Worker finished", mlog.String("worker", worker.name))
worker.stopped <- true
}()
for {
select {
case <-worker.stop:
mlog.Debug("Worker received stop signal", mlog.String("worker", worker.name))
return
case job := <-worker.jobs:
mlog.Debug("Worker received a new candidate job.", mlog.String("worker", worker.name))
worker.DoJob(&job)
}
}
}
func (worker *SimpleWorker) Stop() {
mlog.Debug("Worker stopping", mlog.String("worker", worker.name))
worker.stop <- true
<-worker.stopped
}
func (worker *SimpleWorker) JobChannel() chan<- model.Job {
return worker.jobs
}
func (worker *SimpleWorker) IsEnabled(cfg *model.Config) bool {
return worker.isEnabled(cfg)
}
func (worker *SimpleWorker) DoJob(job *model.Job) {
if claimed, err := worker.jobServer.ClaimJob(job); err != nil {
mlog.Warn("SimpleWorker experienced an error while trying to claim job",
mlog.String("worker", worker.name),
mlog.String("job_id", job.Id),
mlog.Err(err))
return
} else if !claimed {
return
}
err := worker.execute(job)
if err != nil {
mlog.Error("SimpleWorker: Failed to get active user count", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
worker.setJobError(job, model.NewAppError("DoJob", "app.user.get_total_users_count.app_error", nil, err.Error(), http.StatusInternalServerError))
return
}
mlog.Info("SimpleWorker: Job is complete", mlog.String("worker", worker.name), mlog.String("job_id", job.Id))
worker.setJobSuccess(job)
}
func (worker *SimpleWorker) setJobSuccess(job *model.Job) {
if err := worker.jobServer.SetJobSuccess(job); err != nil {
mlog.Error("SimpleWorker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
worker.setJobError(job, err)
}
}
func (worker *SimpleWorker) setJobError(job *model.Job, appError *model.AppError) {
if err := worker.jobServer.SetJobError(job, appError); err != nil {
mlog.Error("SimpleWorker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
}
}

20
jobs/expirynotify/expirynotify.go
View File

@ -1,20 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package expirynotify
import (
"github.com/mattermost/mattermost-server/v6/app"
tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces"
)
type ExpiryNotifyJobInterfaceImpl struct {
App *app.App
}
func init() {
app.RegisterJobsExpiryNotifyJobInterface(func(s *app.Server) tjobs.ExpiryNotifyJobInterface {
a := app.New(app.ServerConnector(s.Channels()))
return &ExpiryNotifyJobInterfaceImpl{a}
})
}

43
jobs/expirynotify/scheduler.go
View File

@ -6,46 +6,15 @@ package expirynotify
import (
"time"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/jobs"
"github.com/mattermost/mattermost-server/v6/model"
)
const (
SchedFreqMinutes = 10
)
const schedFreq = 10 * time.Minute
type Scheduler struct {
App *app.App
}
func (m *ExpiryNotifyJobInterfaceImpl) MakeScheduler() model.Scheduler {
return &Scheduler{m.App}
}
func (scheduler *Scheduler) Name() string {
return JobName + "Scheduler"
}
func (scheduler *Scheduler) JobType() string {
return model.JobTypeExpiryNotify
}
func (scheduler *Scheduler) Enabled(cfg *model.Config) bool {
// Only enabled when ExtendSessionLengthWithActivity is enabled.
return *cfg.ServiceSettings.ExtendSessionLengthWithActivity
}
func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
nextTime := time.Now().Add(SchedFreqMinutes * time.Minute)
return &nextTime
}
func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
data := map[string]string{}
job, err := scheduler.App.Srv().Jobs.CreateJob(model.JobTypeExpiryNotify, data)
if err != nil {
return nil, err
func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler {
isEnabled := func(cfg *model.Config) bool {
return *cfg.ServiceSettings.ExtendSessionLengthWithActivity
}
return job, nil
return jobs.NewPeriodicScheduler(jobServer, model.JobTypeExpiryNotify, schedFreq, isEnabled)
}

89
jobs/expirynotify/worker.go
View File

@ -4,97 +4,20 @@
package expirynotify
import (
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/jobs"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
)
const (
JobName = "ExpiryNotify"
)
type Worker struct {
name string
stop chan bool
stopped chan bool
jobs chan model.Job
jobServer *jobs.JobServer
app *app.App
}
func (m *ExpiryNotifyJobInterfaceImpl) MakeWorker() model.Worker {
worker := Worker{
name: JobName,
stop: make(chan bool, 1),
stopped: make(chan bool, 1),
jobs: make(chan model.Job),
jobServer: m.App.Srv().Jobs,
app: m.App,
func MakeWorker(jobServer *jobs.JobServer, notifySessionsExpired func() error) model.Worker {
isEnabled := func(cfg *model.Config) bool {
return *cfg.ServiceSettings.ExtendSessionLengthWithActivity
}
return &worker
}
func (worker *Worker) Run() {
mlog.Debug("Worker started", mlog.String("worker", worker.name))
defer func() {
mlog.Debug("Worker finished", mlog.String("worker", worker.name))
worker.stopped <- true
}()
for {
select {
case <-worker.stop:
mlog.Debug("Worker received stop signal", mlog.String("worker", worker.name))
return
case job := <-worker.jobs:
mlog.Debug("Worker received a new candidate job.", mlog.String("worker", worker.name))
worker.DoJob(&job)
}
}
}
func (worker *Worker) Stop() {
mlog.Debug("Worker stopping", mlog.String("worker", worker.name))
worker.stop <- true
<-worker.stopped
}
func (worker *Worker) JobChannel() chan<- model.Job {
return worker.jobs
}
func (worker *Worker) DoJob(job *model.Job) {
if claimed, err := worker.jobServer.ClaimJob(job); err != nil {
mlog.Warn("Worker experienced an error while trying to claim job",
mlog.String("worker", worker.name),
mlog.String("job_id", job.Id),
mlog.String("error", err.Error()))
return
} else if !claimed {
return
}
if err := worker.app.NotifySessionsExpired(); err != nil {
mlog.Error("Worker: Failed to notify clients of expired session", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
worker.setJobError(job, err)
return
}
mlog.Info("Worker: Job is complete", mlog.String("worker", worker.name), mlog.String("job_id", job.Id))
worker.setJobSuccess(job)
}
func (worker *Worker) setJobSuccess(job *model.Job) {
if err := worker.app.Srv().Jobs.SetJobSuccess(job); err != nil {
mlog.Error("Worker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
worker.setJobError(job, err)
}
}
func (worker *Worker) setJobError(job *model.Job, appError *model.AppError) {
if err := worker.app.Srv().Jobs.SetJobError(job, appError); err != nil {
mlog.Error("Worker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
execute := func(job *model.Job) error {
return notifySessionsExpired()
}
return jobs.NewSimpleWorker(JobName, jobServer, execute, isEnabled)
}

43
jobs/export_delete/scheduler.go
View File

@ -6,46 +6,15 @@ package export_delete
import (
"time"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/jobs"
"github.com/mattermost/mattermost-server/v6/model"
)
const (
jobName = "ExportDelete"
schedFrequency = 24 * time.Hour
)
const schedFreq = 24 * time.Hour
type Scheduler struct {
app *app.App
}
func (i *ExportDeleteInterfaceImpl) MakeScheduler() model.Scheduler {
return &Scheduler{i.app}
}
func (scheduler *Scheduler) Name() string {
return jobName + "Scheduler"
}
func (scheduler *Scheduler) JobType() string {
return model.JobTypeExportDelete
}
func (scheduler *Scheduler) Enabled(cfg *model.Config) bool {
return *cfg.ExportSettings.Directory != "" && *cfg.ExportSettings.RetentionDays > 0
}
func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
nextTime := time.Now().Add(schedFrequency)
return &nextTime
}
func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
data := map[string]string{}
job, err := scheduler.app.Srv().Jobs.CreateJob(model.JobTypeExportDelete, data)
if err != nil {
return nil, err
func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler {
isEnabled := func(cfg *model.Config) bool {
return *cfg.ExportSettings.Directory != "" && *cfg.ExportSettings.RetentionDays > 0
}
return job, nil
return jobs.NewPeriodicScheduler(jobServer, model.JobTypeExportDelete, schedFreq, isEnabled)
}

151
jobs/export_delete/worker.go
View File

@ -7,132 +7,61 @@ import (
"path/filepath"
"time"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/jobs"
tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/services/configservice"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
"github.com/wiggin77/merror"
)
func init() {
app.RegisterJobsExportDeleteInterface(func(s *app.Server) tjobs.ExportDeleteInterface {
a := app.New(app.ServerConnector(s.Channels()))
return &ExportDeleteInterfaceImpl{a}
})
const jobName = "ExportDelete"
type AppIface interface {
configservice.ConfigService
ListDirectory(path string) ([]string, *model.AppError)
FileModTime(path string) (time.Time, *model.AppError)
RemoveFile(path string) *model.AppError
}
type ExportDeleteInterfaceImpl struct {
app *app.App
}
type ExportDeleteWorker struct {
name string
stopChan chan struct{}
stoppedChan chan struct{}
jobsChan chan model.Job
jobServer *jobs.JobServer
app *app.App
}
func (i *ExportDeleteInterfaceImpl) MakeWorker() model.Worker {
return &ExportDeleteWorker{
name: "ExportDelete",
stopChan: make(chan struct{}),
stoppedChan: make(chan struct{}),
jobsChan: make(chan model.Job),
jobServer: i.app.Srv().Jobs,
app: i.app,
func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker {
isEnabled := func(cfg *model.Config) bool {
return *cfg.ExportSettings.Directory != "" && *cfg.ExportSettings.RetentionDays > 0
}
}
func (w *ExportDeleteWorker) JobChannel() chan<- model.Job {
return w.jobsChan
}
func (w *ExportDeleteWorker) Run() {
mlog.Debug("Worker started", mlog.String("worker", w.name))
defer func() {
mlog.Debug("Worker finished", mlog.String("worker", w.name))
close(w.stoppedChan)
}()
for {
select {
case <-w.stopChan:
mlog.Debug("Worker received stop signal", mlog.String("worker", w.name))
return
case job := <-w.jobsChan:
mlog.Debug("Worker received a new candidate job.", mlog.String("worker", w.name))
w.doJob(&job)
}
}
}
func (w *ExportDeleteWorker) Stop() {
mlog.Debug("Worker stopping", mlog.String("worker", w.name))
close(w.stopChan)
<-w.stoppedChan
}
func (w *ExportDeleteWorker) doJob(job *model.Job) {
if claimed, err := w.jobServer.ClaimJob(job); err != nil {
mlog.Warn("Worker experienced an error while trying to claim job",
mlog.String("worker", w.name),
mlog.String("job_id", job.Id),
mlog.String("error", err.Error()))
return
} else if !claimed {
return
}
exportPath := *w.app.Config().ExportSettings.Directory
retentionTime := time.Duration(*w.app.Config().ExportSettings.RetentionDays) * 24 * time.Hour
exports, appErr := w.app.ListDirectory(exportPath)
if appErr != nil {
w.setJobError(job, appErr)
return
}
var hasErrs bool
for i := range exports {
filename := filepath.Base(exports[i])
modTime, appErr := w.app.FileModTime(filepath.Join(exportPath, filename))
execute := func(job *model.Job) error {
exportPath := *app.Config().ExportSettings.Directory
retentionTime := time.Duration(*app.Config().ExportSettings.RetentionDays) * 24 * time.Hour
exports, appErr := app.ListDirectory(exportPath)
if appErr != nil {
mlog.Debug("Worker: Failed to get file modification time",
mlog.Err(appErr), mlog.String("export", exports[i]))
hasErrs = true
continue
return appErr
}
if time.Now().After(modTime.Add(retentionTime)) {
// remove file data from storage.
if appErr := w.app.RemoveFile(exports[i]); appErr != nil {
mlog.Debug("Worker: Failed to remove file",
errors := merror.New()
for i := range exports {
filename := filepath.Base(exports[i])
modTime, appErr := app.FileModTime(filepath.Join(exportPath, filename))
if appErr != nil {
mlog.Debug("Worker: Failed to get file modification time",
mlog.Err(appErr), mlog.String("export", exports[i]))
hasErrs = true
errors.Append(appErr)
continue
}
if time.Now().After(modTime.Add(retentionTime)) {
// remove file data from storage.
if appErr := app.RemoveFile(exports[i]); appErr != nil {
mlog.Debug("Worker: Failed to remove file",
mlog.Err(appErr), mlog.String("export", exports[i]))
errors.Append(appErr)
continue
}
}
}
}
if hasErrs {
mlog.Warn("Worker: errors occurred")
}
mlog.Info("Worker: Job is complete", mlog.String("worker", w.name), mlog.String("job_id", job.Id))
w.setJobSuccess(job)
}
func (w *ExportDeleteWorker) setJobSuccess(job *model.Job) {
if err := w.app.Srv().Jobs.SetJobSuccess(job); err != nil {
mlog.Error("Worker: Failed to set success for job", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
w.setJobError(job, err)
}
}
func (w *ExportDeleteWorker) setJobError(job *model.Job, appError *model.AppError) {
if err := w.app.Srv().Jobs.SetJobError(job, appError); err != nil {
mlog.Error("Worker: Failed to set job error", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
if err := errors.ErrorOrNil(); err != nil {
mlog.Warn("Worker: errors occurred", mlog.String("job-name", jobName), mlog.Err(err))
}
return nil
}
worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled)
return worker
}

165
jobs/export_process/worker.go
View File

@ -7,133 +7,58 @@ import (
"io"
"path/filepath"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/jobs"
tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/services/configservice"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
)
func init() {
app.RegisterJobsExportProcessInterface(func(s *app.Server) tjobs.ExportProcessInterface {
a := app.New(app.ServerConnector(s.Channels()))
return &ExportProcessInterfaceImpl{a}
})
const jobName = "ExportProcess"
type AppIface interface {
configservice.ConfigService
WriteFile(fr io.Reader, path string) (int64, *model.AppError)
BulkExport(writer io.Writer, outPath string, opts model.BulkExportOpts) *model.AppError
}
type ExportProcessInterfaceImpl struct {
app *app.App
}
type ExportProcessWorker struct {
name string
stopChan chan struct{}
stoppedChan chan struct{}
jobsChan chan model.Job
jobServer *jobs.JobServer
app *app.App
}
func (i *ExportProcessInterfaceImpl) MakeWorker() model.Worker {
return &ExportProcessWorker{
name: "ExportProcess",
stopChan: make(chan struct{}),
stoppedChan: make(chan struct{}),
jobsChan: make(chan model.Job),
jobServer: i.app.Srv().Jobs,
app: i.app,
}
}
func (w *ExportProcessWorker) JobChannel() chan<- model.Job {
return w.jobsChan
}
func (w *ExportProcessWorker) Run() {
mlog.Debug("Worker started", mlog.String("worker", w.name))
defer func() {
mlog.Debug("Worker finished", mlog.String("worker", w.name))
close(w.stoppedChan)
}()
for {
select {
case <-w.stopChan:
mlog.Debug("Worker received stop signal", mlog.String("worker", w.name))
return
case job := <-w.jobsChan:
mlog.Debug("Worker received a new candidate job.", mlog.String("worker", w.name))
w.doJob(&job)
func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker {
isEnabled := func(cfg *model.Config) bool { return true }
execute := func(job *model.Job) error {
opts := model.BulkExportOpts{
CreateArchive: true,
}
includeAttachments, ok := job.Data["include_attachments"]
if ok && includeAttachments == "true" {
opts.IncludeAttachments = true
}
outPath := *app.Config().ExportSettings.Directory
exportFilename := model.NewId() + "_export.zip"
rd, wr := io.Pipe()
errCh := make(chan *model.AppError, 1)
go func() {
defer close(errCh)
_, appErr := app.WriteFile(rd, filepath.Join(outPath, exportFilename))
errCh <- appErr
}()
appErr := app.BulkExport(wr, outPath, opts)
if err := wr.Close(); err != nil {
mlog.Warn("Worker: error closing writer")
}
if appErr != nil {
return appErr
}
if appErr := <-errCh; appErr != nil {
return appErr
}
return nil
}
}
func (w *ExportProcessWorker) Stop() {
mlog.Debug("Worker stopping", mlog.String("worker", w.name))
close(w.stopChan)
<-w.stoppedChan
}
func (w *ExportProcessWorker) doJob(job *model.Job) {
if claimed, err := w.jobServer.ClaimJob(job); err != nil {
mlog.Warn("Worker experienced an error while trying to claim job",
mlog.String("worker", w.name),
mlog.String("job_id", job.Id),
mlog.String("error", err.Error()))
return
} else if !claimed {
return
}
opts := app.BulkExportOpts{
CreateArchive: true,
}
includeAttachments, ok := job.Data["include_attachments"]
if ok && includeAttachments == "true" {
opts.IncludeAttachments = true
}
outPath := *w.app.Config().ExportSettings.Directory
exportFilename := model.NewId() + "_export.zip"
rd, wr := io.Pipe()
errCh := make(chan *model.AppError, 1)
go func() {
defer close(errCh)
_, appErr := w.app.WriteFile(rd, filepath.Join(outPath, exportFilename))
errCh <- appErr
}()
appErr := w.app.BulkExport(wr, outPath, opts)
if err := wr.Close(); err != nil {
mlog.Warn("Worker: error closing writer")
}
if appErr != nil {
w.setJobError(job, appErr)
return
}
if appErr := <-errCh; appErr != nil {
w.setJobError(job, appErr)
return
}
mlog.Info("Worker: Job is complete", mlog.String("worker", w.name), mlog.String("job_id", job.Id))
w.setJobSuccess(job)
}
func (w *ExportProcessWorker) setJobSuccess(job *model.Job) {
if err := w.app.Srv().Jobs.SetJobSuccess(job); err != nil {
mlog.Error("Worker: Failed to set success for job", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
w.setJobError(job, err)
}
}
func (w *ExportProcessWorker) setJobError(job *model.Job, appError *model.AppError) {
if err := w.app.Srv().Jobs.SetJobError(job, appError); err != nil {
mlog.Error("Worker: Failed to set job error", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
}
worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled)
return worker
}

212
jobs/extract_content/worker.go
View File

@ -4,15 +4,12 @@
package extract_content
import (
"net/http"
"strconv"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/app/request"
"github.com/mattermost/mattermost-server/v6/jobs"
tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
"github.com/mattermost/mattermost-server/v6/store"
)
var ignoredFiles = map[string]bool{
@ -22,155 +19,74 @@ var ignoredFiles = map[string]bool{
"mkv": true,
}
func init() {
app.RegisterJobsExtractContentInterface(func(s *app.Server) tjobs.ExtractContentInterface {
a := app.New(app.ServerConnector(s.Channels()))
return &ExtractContentInterfaceImpl{a}
})
const jobName = "ExtractContent"
type AppIface interface {
ExtractContentFromFileInfo(fileInfo *model.FileInfo) error
}
type ExtractContentInterfaceImpl struct {
app *app.App
}
type ExtractContentWorker struct {
name string
stopChan chan struct{}
stoppedChan chan struct{}
jobsChan chan model.Job
jobServer *jobs.JobServer
app *app.App
appContext *request.Context
}
func (i *ExtractContentInterfaceImpl) MakeWorker() model.Worker {
return &ExtractContentWorker{
name: "ExtractContent",
stopChan: make(chan struct{}),
stoppedChan: make(chan struct{}),
jobsChan: make(chan model.Job),
jobServer: i.app.Srv().Jobs,
app: i.app,
appContext: &request.Context{},
func MakeWorker(jobServer *jobs.JobServer, app AppIface, store store.Store) model.Worker {
isEnabled := func(cfg *model.Config) bool {
return true
}
}
func (w *ExtractContentWorker) JobChannel() chan<- model.Job {
return w.jobsChan
}
func (w *ExtractContentWorker) Run() {
mlog.Debug("Worker started", mlog.String("worker", w.name))
defer func() {
mlog.Debug("Worker finished", mlog.String("worker", w.name))
close(w.stoppedChan)
}()
for {
select {
case <-w.stopChan:
mlog.Debug("Worker received stop signal", mlog.String("worker", w.name))
return
case job := <-w.jobsChan:
mlog.Debug("Worker received a new candidate job.", mlog.String("worker", w.name))
w.doJob(&job)
}
}
}
func (w *ExtractContentWorker) Stop() {
mlog.Debug("Worker stopping", mlog.String("worker", w.name))
close(w.stopChan)
<-w.stoppedChan
}
func (w *ExtractContentWorker) doJob(job *model.Job) {
if claimed, err := w.jobServer.ClaimJob(job); err != nil {
mlog.Warn("Worker experienced an error while trying to claim job",
mlog.String("worker", w.name),
mlog.String("job_id", job.Id),
mlog.String("error", err.Error()))
return
} else if !claimed {
return
}
var err error
var fromTS int64 = 0
var toTS int64 = model.GetMillis()
if fromStr, ok := job.Data["from"]; ok {
if fromTS, err = strconv.ParseInt(fromStr, 10, 64); err != nil {
w.setJobError(job, model.NewAppError("ExtractContentWorker", "extrac_content.worker.do_job.invalid_input.from", nil, "", http.StatusBadRequest))
return
}
fromTS *= 1000
}
if toStr, ok := job.Data["to"]; ok {
if toTS, err = strconv.ParseInt(toStr, 10, 64); err != nil {
w.setJobError(job, model.NewAppError("ExtractContentWorker", "extrac_content.worker.do_job.invalid_input.to", nil, "", http.StatusBadRequest))
return
}
toTS *= 1000
}
var nFiles int
var nErrs int
for {
opts := model.GetFileInfosOptions{
Since: fromTS,
SortBy: model.FileinfoSortByCreated,
IncludeDeleted: false,
}
fileInfos, err := w.app.Srv().Store.FileInfo().GetWithOptions(0, 1000, &opts)
if err != nil {
w.setJobError(job, model.NewAppError("ExtractContentWorker", "extract_content.worker.do_job.file_info", nil, err.Error(), http.StatusInternalServerError))
return
}
if len(fileInfos) == 0 {
break
}
for _, fileInfo := range fileInfos {
if !ignoredFiles[fileInfo.Extension] {
mlog.Debug("extracting file", mlog.String("filename", fileInfo.Name), mlog.String("filepath", fileInfo.Path))
err = w.app.ExtractContentFromFileInfo(fileInfo)
if err != nil {
mlog.Warn("Failed to extract file content", mlog.Err(err), mlog.String("file_info_id", fileInfo.Id))
nErrs++
}
nFiles++
execute := func(job *model.Job) error {
var err error
var fromTS int64 = 0
var toTS int64 = model.GetMillis()
if fromStr, ok := job.Data["from"]; ok {
if fromTS, err = strconv.ParseInt(fromStr, 10, 64); err != nil {
return err
}
fromTS *= 1000
}
lastFileInfo := fileInfos[len(fileInfos)-1]
if lastFileInfo.CreateAt > toTS {
break
if toStr, ok := job.Data["to"]; ok {
if toTS, err = strconv.ParseInt(toStr, 10, 64); err != nil {
return err
}
toTS *= 1000
}
fromTS = lastFileInfo.CreateAt + 1
}
job.Data["errors"] = strconv.Itoa(nErrs)
job.Data["processed"] = strconv.Itoa(nFiles)
w.updateData(job)
mlog.Info("Worker: Job is complete", mlog.String("worker", w.name), mlog.String("job_id", job.Id))
w.setJobSuccess(job)
}
func (w *ExtractContentWorker) setJobSuccess(job *model.Job) {
if err := w.app.Srv().Jobs.SetJobSuccess(job); err != nil {
mlog.Error("Worker: Failed to set success for job", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
w.setJobError(job, err)
}
}
func (w *ExtractContentWorker) setJobError(job *model.Job, appError *model.AppError) {
if err := w.app.Srv().Jobs.SetJobError(job, appError); err != nil {
mlog.Error("Worker: Failed to set job error", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
}
}
func (w *ExtractContentWorker) updateData(job *model.Job) {
if err := w.app.Srv().Jobs.UpdateInProgressJobData(job); err != nil {
mlog.Error("Worker: Failed to update job data", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
var nFiles int
var nErrs int
for {
opts := model.GetFileInfosOptions{
Since: fromTS,
SortBy: model.FileinfoSortByCreated,
IncludeDeleted: false,
}
fileInfos, err := store.FileInfo().GetWithOptions(0, 1000, &opts)
if err != nil {
return err
}
if len(fileInfos) == 0 {
break
}
for _, fileInfo := range fileInfos {
if !ignoredFiles[fileInfo.Extension] {
mlog.Debug("extracting file", mlog.String("filename", fileInfo.Name), mlog.String("filepath", fileInfo.Path))
err = app.ExtractContentFromFileInfo(fileInfo)
if err != nil {
mlog.Warn("Failed to extract file content", mlog.Err(err), mlog.String("file_info_id", fileInfo.Id))
nErrs++
}
nFiles++
}
}
lastFileInfo := fileInfos[len(fileInfos)-1]
if lastFileInfo.CreateAt > toTS {
break
}
fromTS = lastFileInfo.CreateAt + 1
}
job.Data["errors"] = strconv.Itoa(nErrs)
job.Data["processed"] = strconv.Itoa(nFiles)
if err := jobServer.UpdateInProgressJobData(job); err != nil {
mlog.Error("Worker: Failed to update job data", mlog.String("worker", model.JobTypeExtractContent), mlog.String("job_id", job.Id), mlog.Err(err))
}
return nil
}
worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled)
return worker
}

43
jobs/import_delete/scheduler.go
View File

@ -6,46 +6,15 @@ package import_delete
import (
"time"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/jobs"
"github.com/mattermost/mattermost-server/v6/model"
)
const (
jobName = "ImportDelete"
schedFrequency = 24 * time.Hour
)
const schedFreq = 24 * time.Hour
type Scheduler struct {
app *app.App
}
func (i *ImportDeleteInterfaceImpl) MakeScheduler() model.Scheduler {
return &Scheduler{i.app}
}
func (scheduler *Scheduler) Name() string {
return jobName + "Scheduler"
}
func (scheduler *Scheduler) JobType() string {
return model.JobTypeImportDelete
}
func (scheduler *Scheduler) Enabled(cfg *model.Config) bool {
return *cfg.ImportSettings.Directory != "" && *cfg.ImportSettings.RetentionDays > 0
}
func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
nextTime := time.Now().Add(schedFrequency)
return &nextTime
}
func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
data := map[string]string{}
job, err := scheduler.app.Srv().Jobs.CreateJob(model.JobTypeImportDelete, data)
if err != nil {
return nil, err
func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler {
isEnabled := func(cfg *model.Config) bool {
return *cfg.ImportSettings.Directory != "" && *cfg.ImportSettings.RetentionDays > 0
}
return job, nil
return jobs.NewPeriodicScheduler(jobServer, model.JobTypeImportDelete, schedFreq, isEnabled)
}

217
jobs/import_delete/worker.go
View File

@ -8,166 +8,95 @@ import (
"path/filepath"
"time"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/jobs"
tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/services/configservice"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
"github.com/mattermost/mattermost-server/v6/store"
"github.com/wiggin77/merror"
)
func init() {
app.RegisterJobsImportDeleteInterface(func(s *app.Server) tjobs.ImportDeleteInterface {
a := app.New(app.ServerConnector(s.Channels()))
return &ImportDeleteInterfaceImpl{a}
})
const jobName = "ImportDelete"
type AppIface interface {
configservice.ConfigService
ListDirectory(path string) ([]string, *model.AppError)
FileModTime(path string) (time.Time, *model.AppError)
RemoveFile(path string) *model.AppError
}
type ImportDeleteInterfaceImpl struct {
app *app.App
}
type ImportDeleteWorker struct {
name string
stopChan chan struct{}
stoppedChan chan struct{}
jobsChan chan model.Job
jobServer *jobs.JobServer
app *app.App
}
func (i *ImportDeleteInterfaceImpl) MakeWorker() model.Worker {
return &ImportDeleteWorker{
name: "ImportDelete",
stopChan: make(chan struct{}),
stoppedChan: make(chan struct{}),
jobsChan: make(chan model.Job),
jobServer: i.app.Srv().Jobs,
app: i.app,
func MakeWorker(jobServer *jobs.JobServer, app AppIface, s store.Store) model.Worker {
isEnabled := func(cfg *model.Config) bool {
return *cfg.ImportSettings.Directory != "" && *cfg.ImportSettings.RetentionDays > 0
}
}
func (w *ImportDeleteWorker) JobChannel() chan<- model.Job {
return w.jobsChan
}
func (w *ImportDeleteWorker) Run() {
mlog.Debug("Worker started", mlog.String("worker", w.name))
defer func() {
mlog.Debug("Worker finished", mlog.String("worker", w.name))
close(w.stoppedChan)
}()
for {
select {
case <-w.stopChan:
mlog.Debug("Worker received stop signal", mlog.String("worker", w.name))
return
case job := <-w.jobsChan:
mlog.Debug("Worker received a new candidate job.", mlog.String("worker", w.name))
w.doJob(&job)
}
}
}
func (w *ImportDeleteWorker) Stop() {
mlog.Debug("Worker stopping", mlog.String("worker", w.name))
close(w.stopChan)
<-w.stoppedChan
}
func (w *ImportDeleteWorker) doJob(job *model.Job) {
if claimed, err := w.jobServer.ClaimJob(job); err != nil {
mlog.Warn("Worker experienced an error while trying to claim job",
mlog.String("worker", w.name),
mlog.String("job_id", job.Id),
mlog.String("error", err.Error()))
return
} else if !claimed {
return
}
importPath := *w.app.Config().ImportSettings.Directory
retentionTime := time.Duration(*w.app.Config().ImportSettings.RetentionDays) * 24 * time.Hour
imports, appErr := w.app.ListDirectory(importPath)
if appErr != nil {
w.setJobError(job, appErr)
return
}
var hasErrs bool
for i := range imports {
filename := filepath.Base(imports[i])
modTime, appErr := w.app.FileModTime(filepath.Join(importPath, filename))
execute := func(job *model.Job) error {
importPath := *app.Config().ImportSettings.Directory
retentionTime := time.Duration(*app.Config().ImportSettings.RetentionDays) * 24 * time.Hour
imports, appErr := app.ListDirectory(importPath)
if appErr != nil {
mlog.Debug("Worker: Failed to get file modification time",
mlog.Err(appErr), mlog.String("import", imports[i]))
hasErrs = true
continue
return appErr
}
if time.Now().After(modTime.Add(retentionTime)) {
// expected format if uploaded through the API is
// ${uploadID}_${filename}${app.IncompleteUploadSuffix}
minLen := 26 + 1 + len(app.IncompleteUploadSuffix)
// check if it's an incomplete upload and attempt to delete its session.
if len(filename) > minLen && filepath.Ext(filename) == app.IncompleteUploadSuffix {
uploadID := filename[:26]
if storeErr := w.app.Srv().Store.UploadSession().Delete(uploadID); storeErr != nil {
mlog.Debug("Worker: Failed to delete UploadSession",
mlog.Err(storeErr), mlog.String("upload_id", uploadID))
hasErrs = true
continue
}
} else {
// check if fileinfo exists and if so delete it.
filePath := filepath.Join(imports[i])
info, storeErr := w.app.Srv().Store.FileInfo().GetByPath(filePath)
var nfErr *store.ErrNotFound
if storeErr != nil && !errors.As(storeErr, &nfErr) {
mlog.Debug("Worker: Failed to get FileInfo",
mlog.Err(storeErr), mlog.String("path", filePath))
hasErrs = true
continue
} else if storeErr == nil {
if storeErr = w.app.Srv().Store.FileInfo().PermanentDelete(info.Id); storeErr != nil {
mlog.Debug("Worker: Failed to delete FileInfo",
mlog.Err(storeErr), mlog.String("file_id", info.Id))
hasErrs = true
continue
}
}
}
// remove file data from storage.
if appErr := w.app.RemoveFile(imports[i]); appErr != nil {
mlog.Debug("Worker: Failed to remove file",
multipleErrors := merror.New()
for i := range imports {
filename := filepath.Base(imports[i])
modTime, appErr := app.FileModTime(filepath.Join(importPath, filename))
if appErr != nil {
mlog.Debug("Worker: Failed to get file modification time",
mlog.Err(appErr), mlog.String("import", imports[i]))
hasErrs = true
multipleErrors.Append(appErr)
continue
}
if time.Now().After(modTime.Add(retentionTime)) {
// expected format if uploaded through the API is
// ${uploadID}_${filename}${model.IncompleteUploadSuffix}
minLen := 26 + 1 + len(model.IncompleteUploadSuffix)
// check if it's an incomplete upload and attempt to delete its session.
if len(filename) > minLen && filepath.Ext(filename) == model.IncompleteUploadSuffix {
uploadID := filename[:26]
if storeErr := s.UploadSession().Delete(uploadID); storeErr != nil {
mlog.Debug("Worker: Failed to delete UploadSession",
mlog.Err(storeErr), mlog.String("upload_id", uploadID))
multipleErrors.Append(storeErr)
continue
}
} else {
// check if fileinfo exists and if so delete it.
filePath := filepath.Join(imports[i])
info, storeErr := s.FileInfo().GetByPath(filePath)
var nfErr *store.ErrNotFound
if storeErr != nil && !errors.As(storeErr, &nfErr) {
mlog.Debug("Worker: Failed to get FileInfo",
mlog.Err(storeErr), mlog.String("path", filePath))
multipleErrors.Append(storeErr)
continue
} else if storeErr == nil {
if storeErr = s.FileInfo().PermanentDelete(info.Id); storeErr != nil {
mlog.Debug("Worker: Failed to delete FileInfo",
mlog.Err(storeErr), mlog.String("file_id", info.Id))
multipleErrors.Append(storeErr)
continue
}
}
}
// remove file data from storage.
if appErr := app.RemoveFile(imports[i]); appErr != nil {
mlog.Debug("Worker: Failed to remove file",
mlog.Err(appErr), mlog.String("import", imports[i]))
multipleErrors.Append(appErr)
continue
}
}
}
}
if hasErrs {
mlog.Warn("Worker: errors occurred")
}
mlog.Info("Worker: Job is complete", mlog.String("worker", w.name), mlog.String("job_id", job.Id))
w.setJobSuccess(job)
}
func (w *ImportDeleteWorker) setJobSuccess(job *model.Job) {
if err := w.app.Srv().Jobs.SetJobSuccess(job); err != nil {
mlog.Error("Worker: Failed to set success for job", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
w.setJobError(job, err)
}
}
func (w *ImportDeleteWorker) setJobError(job *model.Job, appError *model.AppError) {
if err := w.app.Srv().Jobs.SetJobError(job, appError); err != nil {
mlog.Error("Worker: Failed to set job error", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
if err := multipleErrors.ErrorOrNil(); err != nil {
mlog.Warn("Worker: errors occurred", mlog.String("job-name", jobName), mlog.Err(err))
}
return nil
}
worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled)
return worker
}

235
jobs/import_process/worker.go
View File

@ -12,182 +12,95 @@ import (
"strconv"
"strings"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/app/request"
"github.com/mattermost/mattermost-server/v6/jobs"
tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
"github.com/mattermost/mattermost-server/v6/services/configservice"
"github.com/mattermost/mattermost-server/v6/shared/filestore"
)
func init() {
app.RegisterJobsImportProcessInterface(func(s *app.Server) tjobs.ImportProcessInterface {
a := app.New(app.ServerConnector(s.Channels()))
return &ImportProcessInterfaceImpl{a}
})
const jobName = "ImportProcess"
type AppIface interface {
configservice.ConfigService
RemoveFile(path string) *model.AppError
FileExists(path string) (bool, *model.AppError)
FileSize(path string) (int64, *model.AppError)
FileReader(path string) (filestore.ReadCloseSeeker, *model.AppError)
BulkImportWithPath(c *request.Context, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun bool, workers int, importPath string) (*model.AppError, int)
}
type ImportProcessInterfaceImpl struct {
app *app.App
}
type ImportProcessWorker struct {
name string
stopChan chan struct{}
stoppedChan chan struct{}
jobsChan chan model.Job
jobServer *jobs.JobServer
app *app.App
appContext *request.Context
}
func (i *ImportProcessInterfaceImpl) MakeWorker() model.Worker {
return &ImportProcessWorker{
name: "ImportProcess",
stopChan: make(chan struct{}),
stoppedChan: make(chan struct{}),
jobsChan: make(chan model.Job),
jobServer: i.app.Srv().Jobs,
app: i.app,
appContext: &request.Context{},
func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker {
appContext := &request.Context{}
isEnabled := func(cfg *model.Config) bool {
return true
}
}
func (w *ImportProcessWorker) JobChannel() chan<- model.Job {
return w.jobsChan
}
func (w *ImportProcessWorker) Run() {
mlog.Debug("Worker started", mlog.String("worker", w.name))
defer func() {
mlog.Debug("Worker finished", mlog.String("worker", w.name))
close(w.stoppedChan)
}()
for {
select {
case <-w.stopChan:
mlog.Debug("Worker received stop signal", mlog.String("worker", w.name))
return
case job := <-w.jobsChan:
mlog.Debug("Worker received a new candidate job.", mlog.String("worker", w.name))
w.doJob(&job)
}
}
}
func (w *ImportProcessWorker) Stop() {
mlog.Debug("Worker stopping", mlog.String("worker", w.name))
close(w.stopChan)
<-w.stoppedChan
}
func (w *ImportProcessWorker) doJob(job *model.Job) {
if claimed, err := w.jobServer.ClaimJob(job); err != nil {
mlog.Warn("Worker experienced an error while trying to claim job",
mlog.String("worker", w.name),
mlog.String("job_id", job.Id),
mlog.String("error", err.Error()))
return
} else if !claimed {
return
}
importFileName, ok := job.Data["import_file"]
if !ok {
appError := model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.missing_file", nil, "", http.StatusBadRequest)
w.setJobError(job, appError)
return
}
importFilePath := filepath.Join(*w.app.Config().ImportSettings.Directory, importFileName)
if ok, err := w.app.FileExists(importFilePath); err != nil {
w.setJobError(job, err)
return
} else if !ok {
appError := model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.file_exists", nil, "", http.StatusBadRequest)
w.setJobError(job, appError)
return
}
importFileSize, appErr := w.app.FileSize(importFilePath)
if appErr != nil {
w.setJobError(job, appErr)
return
}
importFile, appErr := w.app.FileReader(importFilePath)
if appErr != nil {
w.setJobError(job, appErr)
return
}
defer importFile.Close()
importZipReader, err := zip.NewReader(importFile.(io.ReaderAt), importFileSize)
if err != nil {
appError := model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, err.Error(), http.StatusInternalServerError)
w.setJobError(job, appError)
return
}
// find JSONL import file.
var jsonFile io.ReadCloser
for _, f := range importZipReader.File {
if filepath.Ext(f.Name) != ".jsonl" {
continue
}
// avoid "zip slip"
if strings.Contains(f.Name, "..") {
appError := model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, "jsonFilePath contains path traversal", http.StatusForbidden)
w.setJobError(job, appError)
return
execute := func(job *model.Job) error {
importFileName, ok := job.Data["import_file"]
if !ok {
return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.missing_file", nil, "", http.StatusBadRequest)
}
jsonFile, err = f.Open()
importFilePath := filepath.Join(*app.Config().ImportSettings.Directory, importFileName)
if ok, err := app.FileExists(importFilePath); err != nil {
return err
} else if !ok {
return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.file_exists", nil, "", http.StatusBadRequest)
}
importFileSize, appErr := app.FileSize(importFilePath)
if appErr != nil {
return appErr
}
importFile, appErr := app.FileReader(importFilePath)
if appErr != nil {
return appErr
}
defer importFile.Close()
importZipReader, err := zip.NewReader(importFile.(io.ReaderAt), importFileSize)
if err != nil {
appError := model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, err.Error(), http.StatusInternalServerError)
w.setJobError(job, appError)
return
return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, err.Error(), http.StatusInternalServerError)
}
defer jsonFile.Close()
break
}
// find JSONL import file.
var jsonFile io.ReadCloser
for _, f := range importZipReader.File {
if filepath.Ext(f.Name) != ".jsonl" {
continue
}
// avoid "zip slip"
if strings.Contains(f.Name, "..") {
return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, "jsonFilePath contains path traversal", http.StatusForbidden)
}
if jsonFile == nil {
appError := model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.missing_jsonl", nil, "jsonFile was nil", http.StatusBadRequest)
w.setJobError(job, appError)
return
}
jsonFile, err = f.Open()
if err != nil {
return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, err.Error(), http.StatusInternalServerError)
}
// do the actual import.
appErr, lineNumber := w.app.BulkImportWithPath(w.appContext, jsonFile, importZipReader, false, runtime.NumCPU(), app.ExportDataDir)
if appErr != nil {
job.Data["line_number"] = strconv.Itoa(lineNumber)
w.setJobError(job, appErr)
return
}
defer jsonFile.Close()
break
}
// remove import file when done.
if appErr := w.app.RemoveFile(importFilePath); appErr != nil {
w.setJobError(job, appErr)
return
}
if jsonFile == nil {
return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.missing_jsonl", nil, "jsonFile was nil", http.StatusBadRequest)
}
mlog.Info("Worker: Job is complete", mlog.String("worker", w.name), mlog.String("job_id", job.Id))
w.setJobSuccess(job)
}
func (w *ImportProcessWorker) setJobSuccess(job *model.Job) {
if err := w.app.Srv().Jobs.SetJobSuccess(job); err != nil {
mlog.Error("Worker: Failed to set success for job", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
w.setJobError(job, err)
}
}
func (w *ImportProcessWorker) setJobError(job *model.Job, appError *model.AppError) {
if err := w.app.Srv().Jobs.SetJobError(job, appError); err != nil {
mlog.Error("Worker: Failed to set job error", mlog.String("worker", w.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
}
// do the actual import.
appErr, lineNumber := app.BulkImportWithPath(appContext, jsonFile, importZipReader, false, runtime.NumCPU(), model.ExportDataDir)
if appErr != nil {
job.Data["line_number"] = strconv.Itoa(lineNumber)
return appErr
}
// remove import file when done.
if appErr := app.RemoveFile(importFilePath); appErr != nil {
return appErr
}
return nil
}
worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled)
return worker
}

13
jobs/interfaces/active_users_interface.go
View File

@ -1,13 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package interfaces
import (
"github.com/mattermost/mattermost-server/v6/model"
)
type ActiveUsersJobInterface interface {
MakeWorker() model.Worker
MakeScheduler() model.Scheduler
}

13
jobs/interfaces/expirynotify_interface.go
View File

@ -1,13 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package interfaces
import (
"github.com/mattermost/mattermost-server/v6/model"
)
type ExpiryNotifyJobInterface interface {
MakeWorker() model.Worker
MakeScheduler() model.Scheduler
}

11
jobs/interfaces/export_delete_interface.go
View File

@ -1,11 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package interfaces
import "github.com/mattermost/mattermost-server/v6/model"
type ExportDeleteInterface interface {
MakeWorker() model.Worker
MakeScheduler() model.Scheduler
}

12
jobs/interfaces/export_process_interface.go
View File

@ -1,12 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package interfaces
import (
"github.com/mattermost/mattermost-server/v6/model"
)
type ExportProcessInterface interface {
MakeWorker() model.Worker
}

12
jobs/interfaces/extract_content_interface.go
View File

@ -1,12 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package interfaces
import (
"github.com/mattermost/mattermost-server/v6/model"
)
type ExtractContentInterface interface {
MakeWorker() model.Worker
}

11
jobs/interfaces/import_delete_interface.go
View File

@ -1,11 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package interfaces
import "github.com/mattermost/mattermost-server/v6/model"
type ImportDeleteInterface interface {
MakeWorker() model.Worker
MakeScheduler() model.Scheduler
}

12
jobs/interfaces/import_process_interface.go
View File

@ -1,12 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package interfaces
import (
"github.com/mattermost/mattermost-server/v6/model"
)
type ImportProcessInterface interface {
MakeWorker() model.Worker
}

13
jobs/interfaces/migrations_interface.go
View File

@ -1,13 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package interfaces
import (
"github.com/mattermost/mattermost-server/v6/model"
)
type MigrationsJobInterface interface {
MakeWorker() model.Worker
MakeScheduler() model.Scheduler
}

13
jobs/interfaces/plugins_interface.go
View File

@ -1,13 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package interfaces
import (
"github.com/mattermost/mattermost-server/v6/model"
)
type PluginsJobInterface interface {
MakeWorker() model.Worker
MakeScheduler() model.Scheduler
}

13
jobs/interfaces/product_notices_interface.go
View File

@ -1,13 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package interfaces
import (
"github.com/mattermost/mattermost-server/v6/model"
)
type ProductNoticesJobInterface interface {
MakeWorker() model.Worker
MakeScheduler() model.Scheduler
}

4
jobs/jobs.go
View File

@ -31,6 +31,10 @@ func (srv *JobServer) CreateJob(jobType string, jobData map[string]string) (*mod
return nil, err
}
if srv.workers.Get(job.Type) == nil {
return nil, model.NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+job.Id, http.StatusBadRequest)
}
if _, err := srv.Store.Job().Save(&job); err != nil {
return nil, model.NewAppError("CreateJob", "app.job.save.app_error", nil, err.Error(), http.StatusInternalServerError)
}

130
jobs/jobs_watcher.go
View File

@ -72,131 +72,11 @@ func (watcher *Watcher) PollAndNotify() {
}
for _, job := range jobs {
if job.Type == model.JobTypeDataRetention {
if watcher.workers.DataRetention != nil {
select {
case watcher.workers.DataRetention.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JobTypeMessageExport {
if watcher.workers.MessageExport != nil {
select {
case watcher.workers.MessageExport.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JobTypeElasticsearchPostIndexing {
if watcher.workers.ElasticsearchIndexing != nil {
select {
case watcher.workers.ElasticsearchIndexing.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JobTypeElasticsearchPostAggregation {
if watcher.workers.ElasticsearchAggregation != nil {
select {
case watcher.workers.ElasticsearchAggregation.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JobTypeBlevePostIndexing {
if watcher.workers.BleveIndexing != nil {
select {
case watcher.workers.BleveIndexing.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JobTypeLdapSync {
if watcher.workers.LdapSync != nil {
select {
case watcher.workers.LdapSync.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JobTypeMigrations {
if watcher.workers.Migrations != nil {
select {
case watcher.workers.Migrations.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JobTypePlugins {
if watcher.workers.Plugins != nil {
select {
case watcher.workers.Plugins.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JobTypeExpiryNotify {
if watcher.workers.ExpiryNotify != nil {
select {
case watcher.workers.ExpiryNotify.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JobTypeProductNotices {
if watcher.workers.ProductNotices != nil {
select {
case watcher.workers.ProductNotices.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JobTypeActiveUsers {
if watcher.workers.ActiveUsers != nil {
select {
case watcher.workers.ActiveUsers.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JobTypeImportProcess {
if watcher.workers.ImportProcess != nil {
select {
case watcher.workers.ImportProcess.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JobTypeImportDelete {
if watcher.workers.ImportDelete != nil {
select {
case watcher.workers.ImportDelete.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JobTypeExportProcess {
if watcher.workers.ExportProcess != nil {
select {
case watcher.workers.ExportProcess.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JobTypeExportDelete {
if watcher.workers.ExportDelete != nil {
select {
case watcher.workers.ExportDelete.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JobTypeCloud {
if watcher.workers.Cloud != nil {
select {
case watcher.workers.Cloud.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JobTypeResendInvitationEmail {
if watcher.workers.ResendInvitationEmail != nil {
select {
case watcher.workers.ResendInvitationEmail.JobChannel() <- *job:
default:
}
}
} else if job.Type == model.JobTypeExtractContent {
if watcher.workers.ExtractContent != nil {
select {
case watcher.workers.ExtractContent.JobChannel() <- *job:
default:
}
worker := watcher.workers.Get(job.Type)
if worker != nil {
select {
case worker.JobChannel() <- *job:
default:
}
}
}

4
migrations/advanced_permissions_phase_2.go → jobs/migrations/advanced_permissions_phase_2.go
View File

@ -71,7 +71,7 @@ func (worker *Worker) runAdvancedPermissionsPhase2Migration(lastDone string) (bo
if progress.CurrentTable == "TeamMembers" {
// Run a TeamMembers migration batch.
result, err := worker.srv.Store.Team().MigrateTeamMembers(progress.LastTeamId, progress.LastUserId)
result, err := worker.store.Team().MigrateTeamMembers(progress.LastTeamId, progress.LastUserId)
if err != nil {
return false, progress.ToJSON(), model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "app.team.migrate_team_members.update.app_error", nil, err.Error(), http.StatusInternalServerError)
}
@ -86,7 +86,7 @@ func (worker *Worker) runAdvancedPermissionsPhase2Migration(lastDone string) (bo
progress.LastUserId = result["UserId"]
} else if progress.CurrentTable == "ChannelMembers" {
// Run a ChannelMembers migration batch.
data, err := worker.srv.Store.Channel().MigrateChannelMembers(progress.LastChannelId, progress.LastUserId)
data, err := worker.store.Channel().MigrateChannelMembers(progress.LastChannelId, progress.LastUserId)
if err != nil {
return false, progress.ToJSON(), model.NewAppError("MigrationsWorker.runAdvancedPermissionsPhase2Migration", "app.channel.migrate_channel_members.select.app_error", nil, err.Error(), http.StatusInternalServerError)
}

32
jobs/migrations/helper_test.go Normal file
View File

@ -0,0 +1,32 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package migrations
import (
"testing"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/store"
)
func Setup(tb testing.TB) store.Store {
store := mainHelper.GetStore()
store.DropAllTables()
return store
}
func deleteAllJobsByTypeAndMigrationKey(store store.Store, jobType string, migrationKey string) {
jobs, err := store.Job().GetAllByType(model.JobTypeMigrations)
if err != nil {
panic(err)
}
for _, job := range jobs {
if key, ok := job.Data[JobDataKeyMigration]; ok && key == migrationKey {
if _, err = store.Job().Delete(job.Id); err != nil {
panic(err)
}
}
}
}

0
migrations/main_test.go → jobs/migrations/main_test.go
View File

12
migrations/migrations.go → jobs/migrations/migrations.go
View File

@ -6,8 +6,6 @@ package migrations
import (
"net/http"
"github.com/mattermost/mattermost-server/v6/app"
tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/store"
)
@ -21,16 +19,6 @@ const (
JobDataKeyMigrationLastDone = "last_done"
)
type MigrationsJobInterfaceImpl struct {
srv *app.Server
}
func init() {
app.RegisterJobsMigrationsJobInterface(func(s *app.Server) tjobs.MigrationsJobInterface {
return &MigrationsJobInterfaceImpl{s}
})
}
func MakeMigrationsList() []string {
return []string{
model.MigrationKeyAdvancedPermissionsPhase2,

25
migrations/migrations_test.go → jobs/migrations/migrations_test.go
View File

@ -16,15 +16,14 @@ func TestGetMigrationState(t *testing.T) {
if testing.Short() {
t.SkipNow()
}
th := Setup(t)
defer th.TearDown()
store := Setup(t)
migrationKey := model.NewId()
th.DeleteAllJobsByTypeAndMigrationKey(model.JobTypeMigrations, migrationKey)
deleteAllJobsByTypeAndMigrationKey(store, model.JobTypeMigrations, migrationKey)
// Test with no job yet.
state, job, err := GetMigrationState(migrationKey, th.App.Srv().Store)
state, job, err := GetMigrationState(migrationKey, store)
assert.Nil(t, err)
assert.Nil(t, job)
assert.Equal(t, "unscheduled", state)
@ -34,15 +33,15 @@ func TestGetMigrationState(t *testing.T) {
Name: migrationKey,
Value: "true",
}
nErr := th.App.Srv().Store.System().Save(&system)
nErr := store.System().Save(&system)
assert.NoError(t, nErr)
state, job, err = GetMigrationState(migrationKey, th.App.Srv().Store)
state, job, err = GetMigrationState(migrationKey, store)
assert.Nil(t, err)
assert.Nil(t, job)
assert.Equal(t, "completed", state)
_, nErr = th.App.Srv().Store.System().PermanentDeleteByName(migrationKey)
_, nErr = store.System().PermanentDeleteByName(migrationKey)
assert.NoError(t, nErr)
// Test with a job scheduled in "pending" state.
@ -56,10 +55,10 @@ func TestGetMigrationState(t *testing.T) {
Type: model.JobTypeMigrations,
}
j1, nErr = th.App.Srv().Store.Job().Save(j1)
j1, nErr = store.Job().Save(j1)
require.NoError(t, nErr)
state, job, err = GetMigrationState(migrationKey, th.App.Srv().Store)
state, job, err = GetMigrationState(migrationKey, store)
assert.Nil(t, err)
assert.Equal(t, j1.Id, job.Id)
assert.Equal(t, "in_progress", state)
@ -75,10 +74,10 @@ func TestGetMigrationState(t *testing.T) {
Type: model.JobTypeMigrations,
}
j2, nErr = th.App.Srv().Store.Job().Save(j2)
j2, nErr = store.Job().Save(j2)
require.NoError(t, nErr)
state, job, err = GetMigrationState(migrationKey, th.App.Srv().Store)
state, job, err = GetMigrationState(migrationKey, store)
assert.Nil(t, err)
assert.Equal(t, j2.Id, job.Id)
assert.Equal(t, "in_progress", state)
@ -94,10 +93,10 @@ func TestGetMigrationState(t *testing.T) {
Type: model.JobTypeMigrations,
}
j3, nErr = th.App.Srv().Store.Job().Save(j3)
j3, nErr = store.Job().Save(j3)
require.NoError(t, nErr)
state, job, err = GetMigrationState(migrationKey, th.App.Srv().Store)
state, job, err = GetMigrationState(migrationKey, store)
assert.Nil(t, err)
assert.Equal(t, j3.Id, job.Id)
assert.Equal(t, "unscheduled", state)

36
migrations/scheduler.go → jobs/migrations/scheduler.go
View File

@ -6,9 +6,10 @@ package migrations
import (
"time"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/jobs"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
"github.com/mattermost/mattermost-server/v6/store"
)
const (
@ -16,20 +17,13 @@ const (
)
type Scheduler struct {
srv *app.Server
jobServer *jobs.JobServer
store store.Store
allMigrationsCompleted bool
}
func (m *MigrationsJobInterfaceImpl) MakeScheduler() model.Scheduler {
return &Scheduler{m.srv, false}
}
func (scheduler *Scheduler) Name() string {
return "MigrationsScheduler"
}
func (scheduler *Scheduler) JobType() string {
return model.JobTypeMigrations
func MakeScheduler(jobServer *jobs.JobServer, store store.Store) model.Scheduler {
return &Scheduler{jobServer, store, false}
}
func (scheduler *Scheduler) Enabled(_ *model.Config) bool {
@ -48,22 +42,22 @@ func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, p
//nolint:unparam
func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
mlog.Debug("Scheduling Job", mlog.String("scheduler", scheduler.Name()))
mlog.Debug("Scheduling Job", mlog.String("scheduler", model.JobTypeMigrations))
// Work through the list of migrations in order. Schedule the first one that isn't done (assuming it isn't in progress already).
for _, key := range MakeMigrationsList() {
state, job, err := GetMigrationState(key, scheduler.srv.Store)
state, job, err := GetMigrationState(key, scheduler.store)
if err != nil {
mlog.Error("Failed to determine status of migration: ", mlog.String("scheduler", scheduler.Name()), mlog.String("migration_key", key), mlog.String("error", err.Error()))
mlog.Error("Failed to determine status of migration: ", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key), mlog.Err(err))
return nil, nil
}
if state == MigrationStateInProgress {
// Check the migration job isn't wedged.
if job != nil && job.LastActivityAt < model.GetMillis()-MigrationJobWedgedTimeoutMilliseconds && job.CreateAt < model.GetMillis()-MigrationJobWedgedTimeoutMilliseconds {
mlog.Warn("Job appears to be wedged. Rescheduling another instance.", mlog.String("scheduler", scheduler.Name()), mlog.String("wedged_job_id", job.Id), mlog.String("migration_key", key))
if err := scheduler.srv.Jobs.SetJobError(job, nil); err != nil {
mlog.Error("Worker: Failed to set job error", mlog.String("scheduler", scheduler.Name()), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
mlog.Warn("Job appears to be wedged. Rescheduling another instance.", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("wedged_job_id", job.Id), mlog.String("migration_key", key))
if err := scheduler.jobServer.SetJobError(job, nil); err != nil {
mlog.Error("Worker: Failed to set job error", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("job_id", job.Id), mlog.Err(err))
}
return scheduler.createJob(key, job)
}
@ -77,7 +71,7 @@ func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, las
}
if state == MigrationStateUnscheduled {
mlog.Debug("Scheduling a new job for migration.", mlog.String("scheduler", scheduler.Name()), mlog.String("migration_key", key))
mlog.Debug("Scheduling a new job for migration.", mlog.String("scheduler", model.JobTypeMigrations), mlog.String("migration_key", key))
return scheduler.createJob(key, job)
}
@ -87,7 +81,7 @@ func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, las
// If we reached here, then there aren't any migrations left to run.
scheduler.allMigrationsCompleted = true
mlog.Debug("All migrations are complete.", mlog.String("scheduler", scheduler.Name()))
mlog.Debug("All migrations are complete.", mlog.String("scheduler", model.JobTypeMigrations))
return nil, nil
}
@ -103,7 +97,7 @@ func (scheduler *Scheduler) createJob(migrationKey string, lastJob *model.Job) (
JobDataKeyMigrationLastDone: lastDone,
}
job, err := scheduler.srv.Jobs.CreateJob(model.JobTypeMigrations, data)
job, err := scheduler.jobServer.CreateJob(model.JobTypeMigrations, data)
if err != nil {
return nil, err
}

26
migrations/worker.go → jobs/migrations/worker.go
View File

@ -8,10 +8,10 @@ import (
"net/http"
"time"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/jobs"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
"github.com/mattermost/mattermost-server/v6/store"
)
const (
@ -24,17 +24,17 @@ type Worker struct {
stopped chan bool
jobs chan model.Job
jobServer *jobs.JobServer
srv *app.Server
store store.Store
}
func (m *MigrationsJobInterfaceImpl) MakeWorker() model.Worker {
func MakeWorker(jobServer *jobs.JobServer, store store.Store) model.Worker {
worker := Worker{
name: "Migrations",
stop: make(chan bool, 1),
stopped: make(chan bool, 1),
jobs: make(chan model.Job),
jobServer: m.srv.Jobs,
srv: m.srv,
jobServer: jobServer,
store: store,
}
return &worker
@ -70,6 +70,10 @@ func (worker *Worker) JobChannel() chan<- model.Job {
return worker.jobs
}
func (worker *Worker) IsEnabled(_ *model.Config) bool {
return true
}
func (worker *Worker) DoJob(job *model.Job) {
if claimed, err := worker.jobServer.ClaimJob(job); err != nil {
mlog.Info("Worker experienced an error while trying to claim job",
@ -83,7 +87,7 @@ func (worker *Worker) DoJob(job *model.Job) {
cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background())
cancelWatcherChan := make(chan interface{}, 1)
go worker.srv.Jobs.CancellationWatcher(cancelCtx, job.Id, cancelWatcherChan)
go worker.jobServer.CancellationWatcher(cancelCtx, job.Id, cancelWatcherChan)
defer cancelCancelWatcher()
@ -111,7 +115,7 @@ func (worker *Worker) DoJob(job *model.Job) {
return
} else {
job.Data[JobDataKeyMigrationLastDone] = progress
if err := worker.srv.Jobs.UpdateInProgressJobData(job); err != nil {
if err := worker.jobServer.UpdateInProgressJobData(job); err != nil {
mlog.Error("Worker: Failed to update migration status data for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
worker.setJobError(job, err)
return
@ -122,20 +126,20 @@ func (worker *Worker) DoJob(job *model.Job) {
}
func (worker *Worker) setJobSuccess(job *model.Job) {
if err := worker.srv.Jobs.SetJobSuccess(job); err != nil {
if err := worker.jobServer.SetJobSuccess(job); err != nil {
mlog.Error("Worker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
worker.setJobError(job, err)
}
}
func (worker *Worker) setJobError(job *model.Job, appError *model.AppError) {
if err := worker.srv.Jobs.SetJobError(job, appError); err != nil {
if err := worker.jobServer.SetJobError(job, appError); err != nil {
mlog.Error("Worker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
}
}
func (worker *Worker) setJobCanceled(job *model.Job) {
if err := worker.srv.Jobs.SetJobCanceled(job); err != nil {
if err := worker.jobServer.SetJobCanceled(job); err != nil {
mlog.Error("Worker: Failed to mark job as canceled", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
}
}
@ -157,7 +161,7 @@ func (worker *Worker) runMigration(key string, lastDone string) (bool, string, *
}
if done {
if nErr := worker.srv.Store.System().Save(&model.System{Name: key, Value: "true"}); nErr != nil {
if nErr := worker.store.System().Save(&model.System{Name: key, Value: "true"}); nErr != nil {
return false, "", model.NewAppError("runMigration", "migrations.system.save.app_error", nil, nErr.Error(), http.StatusInternalServerError)
}
}

20
jobs/product_notices/product_notices.go
View File

@ -1,20 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package product_notices
import (
"github.com/mattermost/mattermost-server/v6/app"
tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces"
)
type ProductNoticesJobInterfaceImpl struct {
App *app.App
}
func init() {
app.RegisterProductNoticesJobInterface(func(s *app.Server) tjobs.ProductNoticesJobInterface {
a := app.New(app.ServerConnector(s.Channels()))
return &ProductNoticesJobInterfaceImpl{a}
})
}

32
jobs/product_notices/scheduler.go
View File

@ -6,29 +6,12 @@ package product_notices
import (
"time"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/jobs"
"github.com/mattermost/mattermost-server/v6/model"
)
type Scheduler struct {
App *app.App
}
func (m *ProductNoticesJobInterfaceImpl) MakeScheduler() model.Scheduler {
return &Scheduler{m.App}
}
func (scheduler *Scheduler) Name() string {
return JobName + "Scheduler"
}
func (scheduler *Scheduler) JobType() string {
return model.JobTypeProductNotices
}
func (scheduler *Scheduler) Enabled(cfg *model.Config) bool {
// Only enabled when ExtendSessionLengthWithActivity is enabled.
return *cfg.AnnouncementSettings.AdminNoticesEnabled || *cfg.AnnouncementSettings.UserNoticesEnabled
*jobs.PeriodicScheduler
}
func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
@ -36,12 +19,9 @@ func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, p
return &nextTime
}
func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
data := map[string]string{}
job, err := scheduler.App.Srv().Jobs.CreateJob(model.JobTypeProductNotices, data)
if err != nil {
return nil, err
func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler {
isEnabled := func(cfg *model.Config) bool {
return *cfg.AnnouncementSettings.AdminNoticesEnabled || *cfg.AnnouncementSettings.UserNoticesEnabled
}
return job, nil
return &Scheduler{PeriodicScheduler: jobs.NewPeriodicScheduler(jobServer, model.JobTypeProductNotices, 0, isEnabled)}
}

95
jobs/product_notices/worker.go
View File

@ -4,97 +4,28 @@
package product_notices
import (
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/jobs"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
)
const (
JobName = "ProductNotices"
)
const jobName = "ProductNotices"
type Worker struct {
name string
stop chan bool
stopped chan bool
jobs chan model.Job
jobServer *jobs.JobServer
app *app.App
type AppIface interface {
UpdateProductNotices() *model.AppError
}
func (m *ProductNoticesJobInterfaceImpl) MakeWorker() model.Worker {
worker := Worker{
name: JobName,
stop: make(chan bool, 1),
stopped: make(chan bool, 1),
jobs: make(chan model.Job),
jobServer: m.App.Srv().Jobs,
app: m.App,
func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker {
isEnabled := func(cfg *model.Config) bool {
return *cfg.AnnouncementSettings.AdminNoticesEnabled || *cfg.AnnouncementSettings.UserNoticesEnabled
}
return &worker
}
func (worker *Worker) Run() {
mlog.Debug("Worker started", mlog.String("worker", worker.name))
defer func() {
mlog.Debug("Worker finished", mlog.String("worker", worker.name))
worker.stopped <- true
}()
for {
select {
case <-worker.stop:
mlog.Debug("Worker received stop signal", mlog.String("worker", worker.name))
return
case job := <-worker.jobs:
mlog.Debug("Worker received a new candidate job.", mlog.String("worker", worker.name))
worker.DoJob(&job)
execute := func(job *model.Job) error {
if err := app.UpdateProductNotices(); err != nil {
mlog.Error("Worker: Failed to fetch product notices", mlog.String("worker", model.JobTypeProductNotices), mlog.String("job_id", job.Id), mlog.Err(err))
return err
}
return nil
}
}
func (worker *Worker) Stop() {
mlog.Debug("Worker stopping", mlog.String("worker", worker.name))
worker.stop <- true
<-worker.stopped
}
func (worker *Worker) JobChannel() chan<- model.Job {
return worker.jobs
}
func (worker *Worker) DoJob(job *model.Job) {
if claimed, err := worker.jobServer.ClaimJob(job); err != nil {
mlog.Warn("Worker experienced an error while trying to claim job",
mlog.String("worker", worker.name),
mlog.String("job_id", job.Id),
mlog.String("error", err.Error()))
return
} else if !claimed {
return
}
if err := worker.app.UpdateProductNotices(); err != nil {
mlog.Error("Worker: Failed to fetch product notices", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
worker.setJobError(job, err)
return
}
mlog.Info("Worker: Job is complete", mlog.String("worker", worker.name), mlog.String("job_id", job.Id))
worker.setJobSuccess(job)
}
func (worker *Worker) setJobSuccess(job *model.Job) {
if err := worker.app.Srv().Jobs.SetJobSuccess(job); err != nil {
mlog.Error("Worker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
worker.setJobError(job, err)
}
}
func (worker *Worker) setJobError(job *model.Job, appError *model.AppError) {
if err := worker.app.Srv().Jobs.SetJobError(job, appError); err != nil {
mlog.Error("Worker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
}
worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled)
return worker
}

19
jobs/resend_invitation_email/resend_invitation_email.go
View File

@ -1,19 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package resend_invitation_email
import (
"github.com/mattermost/mattermost-server/v6/app"
ejobs "github.com/mattermost/mattermost-server/v6/einterfaces/jobs"
)
type ResendInvitationEmailJobInterfaceImpl struct {
App *app.App
}
func init() {
app.RegisterJobsResendInvitationEmailInterface(func(s *app.Server) ejobs.ResendInvitationEmailJobInterface {
a := app.New(app.ServerConnector(s.Channels()))
return &ResendInvitationEmailJobInterfaceImpl{a}
})
}

42
jobs/resend_invitation_email/scheduler.go
View File

@ -1,42 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package resend_invitation_email
import (
"time"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/model"
)
const ResendInvitationEmailJob = "ResendInvitationEmailJob"
type ResendInvitationEmailScheduler struct {
App *app.App
}
func (rse *ResendInvitationEmailJobInterfaceImpl) MakeScheduler() model.Scheduler {
return &ResendInvitationEmailScheduler{rse.App}
}
func (s *ResendInvitationEmailScheduler) Name() string {
return ResendInvitationEmailJob + "Scheduler"
}
func (s *ResendInvitationEmailScheduler) JobType() string {
return model.JobTypeResendInvitationEmail
}
func (s *ResendInvitationEmailScheduler) Enabled(cfg *model.Config) bool {
return *cfg.ServiceSettings.EnableEmailInvitations
}
func (s *ResendInvitationEmailScheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
t := time.Now().Add(5 * time.Second)
return &t
}
func (s *ResendInvitationEmailScheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
// noop because we manually schedule the job in api4.inviteUsersToTeam handler
return nil, nil
}

67
jobs/resend_invitation_email/worker.go
View File

@ -8,8 +8,10 @@ import (
"os"
"strconv"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/jobs"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/services/configservice"
"github.com/mattermost/mattermost-server/v6/services/telemetry"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
"github.com/mattermost/mattermost-server/v6/store"
)
@ -18,21 +20,34 @@ const TwentyFourHoursInMillis int64 = 86400000
const FourtyEightHoursInMillis int64 = 172800000
const SeventyTwoHoursInMillis int64 = 259200000
type ResendInvitationEmailWorker struct {
name string
stop chan bool
stopped chan bool
jobs chan model.Job
App *app.App
type AppIface interface {
configservice.ConfigService
GetUserByEmail(email string) (*model.User, *model.AppError)
GetTeamMembersByIds(teamID string, userIDs []string, restrictions *model.ViewUsersRestrictions) ([]*model.TeamMember, *model.AppError)
InviteNewUsersToTeamGracefully(emailList []string, teamID, senderId string, reminderInterval string) ([]*model.EmailInviteWithError, *model.AppError)
}
func (rse *ResendInvitationEmailJobInterfaceImpl) MakeWorker() model.Worker {
type ResendInvitationEmailWorker struct {
name string
stop chan bool
stopped chan bool
jobs chan model.Job
jobServer *jobs.JobServer
app AppIface
store store.Store
telemetryService *telemetry.TelemetryService
}
func MakeWorker(jobServer *jobs.JobServer, app AppIface, store store.Store, telemetryService *telemetry.TelemetryService) model.Worker {
worker := ResendInvitationEmailWorker{
name: ResendInvitationEmailJob,
stop: make(chan bool, 1),
stopped: make(chan bool, 1),
jobs: make(chan model.Job),
App: rse.App,
name: model.JobTypeResendInvitationEmail,
stop: make(chan bool, 1),
stopped: make(chan bool, 1),
jobs: make(chan model.Job),
jobServer: jobServer,
app: app,
store: store,
telemetryService: telemetryService,
}
return &worker
}
@ -57,6 +72,10 @@ func (rseworker *ResendInvitationEmailWorker) Run() {
}
}
func (rseworker *ResendInvitationEmailWorker) IsEnabled(cfg *model.Config) bool {
return *cfg.ServiceSettings.EnableEmailInvitations
}
func (rseworker *ResendInvitationEmailWorker) Stop() {
mlog.Debug("Worker stopping", mlog.String("worker", rseworker.name))
rseworker.stop <- true
@ -68,7 +87,7 @@ func (rseworker *ResendInvitationEmailWorker) JobChannel() chan<- model.Job {
}
func (rseworker *ResendInvitationEmailWorker) DoJob(job *model.Job) {
resendInviteEmailIntervalFlag := rseworker.App.Config().FeatureFlags.ResendInviteEmailInterval
resendInviteEmailIntervalFlag := rseworker.app.Config().FeatureFlags.ResendInviteEmailInterval
switch resendInviteEmailIntervalFlag {
case "48":
@ -99,7 +118,7 @@ func (rseworker *ResendInvitationEmailWorker) DoJob_24_72(job *model.Job) {
}
func (rseworker *ResendInvitationEmailWorker) Execute(job *model.Job, elapsedTimeSinceSchedule, firstDuration, secondDuration int64, firstDurationTelemetryValue, secondDurationTelemetryValue string) {
systemValue, sysValErr := rseworker.App.Srv().Store.System().GetByName(job.Id)
systemValue, sysValErr := rseworker.store.System().GetByName(job.Id)
if sysValErr != nil {
if _, ok := sysValErr.(*store.ErrNotFound); !ok {
mlog.Error("An error occurred while getting NUMBER_OF_INVITE_EMAILS_SENT from system store", mlog.String("worker", rseworker.name), mlog.Err(sysValErr))
@ -119,21 +138,21 @@ func (rseworker *ResendInvitationEmailWorker) Execute(job *model.Job, elapsedTim
}
func (rseworker *ResendInvitationEmailWorker) setJobSuccess(job *model.Job) {
if err := rseworker.App.Srv().Jobs.SetJobSuccess(job); err != nil {
if err := rseworker.jobServer.SetJobSuccess(job); err != nil {
mlog.Error("Worker: Failed to set success for job", mlog.String("worker", rseworker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
rseworker.setJobError(job, err)
}
}
func (rseworker *ResendInvitationEmailWorker) setJobCancelled(job *model.Job) {
if err := rseworker.App.Srv().Jobs.SetJobCanceled(job); err != nil {
if err := rseworker.jobServer.SetJobCanceled(job); err != nil {
mlog.Error("Worker: Failed to cancel job", mlog.String("worker", rseworker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
rseworker.setJobError(job, err)
}
}
func (rseworker *ResendInvitationEmailWorker) setJobError(job *model.Job, appError *model.AppError) {
if err := rseworker.App.Srv().Jobs.SetJobError(job, appError); err != nil {
if err := rseworker.jobServer.SetJobError(job, appError); err != nil {
mlog.Error("Worker: Failed to set job error", mlog.String("worker", rseworker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
}
}
@ -153,14 +172,14 @@ func (rseworker *ResendInvitationEmailWorker) removeAlreadyJoined(teamID string,
var notJoinedYet []string
for _, email := range emailList {
// check if the user with this email is on the system already
user, appErr := rseworker.App.GetUserByEmail(email)
user, appErr := rseworker.app.GetUserByEmail(email)
if appErr != nil {
notJoinedYet = append(notJoinedYet, email)
continue
}
// now we check if they are part of the team already
userID := []string{user.Id}
members, appErr := rseworker.App.GetTeamMembersByIds(teamID, userID, nil)
members, appErr := rseworker.app.GetTeamMembersByIds(teamID, userID, nil)
if len(members) == 0 || appErr != nil {
notJoinedYet = append(notJoinedYet, email)
}
@ -171,7 +190,7 @@ func (rseworker *ResendInvitationEmailWorker) removeAlreadyJoined(teamID string,
func (rseworker *ResendInvitationEmailWorker) setNumResendEmailSent(job *model.Job, num string) {
sysVar := &model.System{Name: job.Id, Value: num}
if err := rseworker.App.Srv().Store.System().SaveOrUpdate(sysVar); err != nil {
if err := rseworker.store.System().SaveOrUpdate(sysVar); err != nil {
mlog.Error("Unable to save NUMBER_OF_INVITE_EMAIL_SENT", mlog.String("worker", rseworker.name), mlog.Err(err))
}
}
@ -208,7 +227,7 @@ func (rseworker *ResendInvitationEmailWorker) GetDurations(job *model.Job) (int6
}
func (rseworker *ResendInvitationEmailWorker) TearDown(job *model.Job) {
rseworker.App.Srv().Store.System().PermanentDeleteByName(job.Id)
rseworker.store.System().PermanentDeleteByName(job.Id)
rseworker.setJobSuccess(job)
}
@ -225,10 +244,10 @@ func (rseworker *ResendInvitationEmailWorker) ResendEmails(job *model.Job, inter
emailList = rseworker.removeAlreadyJoined(teamID, emailList)
_, appErr := rseworker.App.InviteNewUsersToTeamGracefully(emailList, teamID, job.Data["senderID"], interval)
_, appErr := rseworker.app.InviteNewUsersToTeamGracefully(emailList, teamID, job.Data["senderID"], interval)
if appErr != nil {
mlog.Error("Worker: Failed to send emails", mlog.String("worker", rseworker.name), mlog.String("job_id", job.Id), mlog.String("error", appErr.Error()))
rseworker.setJobError(job, appErr)
}
rseworker.App.Srv().GetTelemetryService().SendTelemetry("track_invite_email_resend", map[string]interface{}{interval: interval})
rseworker.telemetryService.SendTelemetry("track_invite_email_resend", map[string]interface{}{interval: interval})
}

113
jobs/schedulers.go
View File

@ -22,8 +22,8 @@ type Schedulers struct {
isLeader bool
running bool
schedulers []model.Scheduler
nextRunTimes []*time.Time
schedulers map[string]model.Scheduler
nextRunTimes map[string]*time.Time
}
var (
@ -47,66 +47,19 @@ func (srv *JobServer) InitSchedulers() error {
clusterLeaderChanged: make(chan bool, 1),
jobs: srv,
isLeader: true,
schedulers: make(map[string]model.Scheduler),
nextRunTimes: make(map[string]*time.Time),
}
if srv.DataRetentionJob != nil {
schedulers.schedulers = append(schedulers.schedulers, srv.DataRetentionJob.MakeScheduler())
}
if srv.MessageExportJob != nil {
schedulers.schedulers = append(schedulers.schedulers, srv.MessageExportJob.MakeScheduler())
}
if elasticsearchAggregatorInterface := srv.ElasticsearchAggregator; elasticsearchAggregatorInterface != nil {
schedulers.schedulers = append(schedulers.schedulers, elasticsearchAggregatorInterface.MakeScheduler())
}
if ldapSyncInterface := srv.LdapSync; ldapSyncInterface != nil {
schedulers.schedulers = append(schedulers.schedulers, ldapSyncInterface.MakeScheduler())
}
if migrationsInterface := srv.Migrations; migrationsInterface != nil {
schedulers.schedulers = append(schedulers.schedulers, migrationsInterface.MakeScheduler())
}
if pluginsInterface := srv.Plugins; pluginsInterface != nil {
schedulers.schedulers = append(schedulers.schedulers, pluginsInterface.MakeScheduler())
}
if expiryNotifyInterface := srv.ExpiryNotify; expiryNotifyInterface != nil {
schedulers.schedulers = append(schedulers.schedulers, expiryNotifyInterface.MakeScheduler())
}
if activeUsersInterface := srv.ActiveUsers; activeUsersInterface != nil {
schedulers.schedulers = append(schedulers.schedulers, activeUsersInterface.MakeScheduler())
}
if productNoticesInterface := srv.ProductNotices; productNoticesInterface != nil {
schedulers.schedulers = append(schedulers.schedulers, productNoticesInterface.MakeScheduler())
}
if cloudInterface := srv.Cloud; cloudInterface != nil {
schedulers.schedulers = append(schedulers.schedulers, cloudInterface.MakeScheduler())
}
if resendInvitationEmailInterface := srv.ResendInvitationEmails; resendInvitationEmailInterface != nil {
schedulers.schedulers = append(schedulers.schedulers, resendInvitationEmailInterface.MakeScheduler())
}
if importDeleteInterface := srv.ImportDelete; importDeleteInterface != nil {
schedulers.schedulers = append(schedulers.schedulers, importDeleteInterface.MakeScheduler())
}
if exportDeleteInterface := srv.ExportDelete; exportDeleteInterface != nil {
schedulers.schedulers = append(schedulers.schedulers, exportDeleteInterface.MakeScheduler())
}
schedulers.nextRunTimes = make([]*time.Time, len(schedulers.schedulers))
srv.schedulers = schedulers
return nil
}
func (schedulers *Schedulers) AddScheduler(name string, scheduler model.Scheduler) {
schedulers.schedulers[name] = scheduler
}
// Start starts the schedulers. This call is not safe for concurrent use.
// Synchronization should be implemented by the caller.
func (schedulers *Schedulers) Start() {
@ -121,11 +74,11 @@ func (schedulers *Schedulers) Start() {
}()
now := time.Now()
for idx, scheduler := range schedulers.schedulers {
for name, scheduler := range schedulers.schedulers {
if !scheduler.Enabled(schedulers.jobs.Config()) {
schedulers.nextRunTimes[idx] = nil
schedulers.nextRunTimes[name] = nil
} else {
schedulers.setNextRunTime(schedulers.jobs.Config(), idx, now, false)
schedulers.setNextRunTime(schedulers.jobs.Config(), name, now, false)
}
}
@ -139,38 +92,38 @@ func (schedulers *Schedulers) Start() {
case now = <-timer.C:
cfg := schedulers.jobs.Config()
for idx, nextTime := range schedulers.nextRunTimes {
for name, nextTime := range schedulers.nextRunTimes {
if nextTime == nil {
continue
}
if time.Now().After(*nextTime) {
scheduler := schedulers.schedulers[idx]
scheduler := schedulers.schedulers[name]
if scheduler == nil || !schedulers.isLeader || !scheduler.Enabled(cfg) {
continue
}
if _, err := schedulers.scheduleJob(cfg, scheduler); err != nil {
mlog.Error("Failed to schedule job", mlog.String("scheduler", scheduler.Name()), mlog.Err(err))
if _, err := schedulers.scheduleJob(cfg, name, scheduler); err != nil {
mlog.Error("Failed to schedule job", mlog.String("scheduler", name), mlog.Err(err))
continue
}
schedulers.setNextRunTime(cfg, idx, now, true)
schedulers.setNextRunTime(cfg, name, now, true)
}
}
case newCfg := <-schedulers.configChanged:
for idx, scheduler := range schedulers.schedulers {
for name, scheduler := range schedulers.schedulers {
if !schedulers.isLeader || !scheduler.Enabled(newCfg) {
schedulers.nextRunTimes[idx] = nil
schedulers.nextRunTimes[name] = nil
} else {
schedulers.setNextRunTime(newCfg, idx, now, false)
schedulers.setNextRunTime(newCfg, name, now, false)
}
}
case isLeader := <-schedulers.clusterLeaderChanged:
for idx := range schedulers.schedulers {
for name := range schedulers.schedulers {
schedulers.isLeader = isLeader
if !isLeader {
schedulers.nextRunTimes[idx] = nil
schedulers.nextRunTimes[name] = nil
} else {
schedulers.setNextRunTime(schedulers.jobs.Config(), idx, now, false)
schedulers.setNextRunTime(schedulers.jobs.Config(), name, now, false)
}
}
}
@ -192,37 +145,37 @@ func (schedulers *Schedulers) Stop() {
schedulers.running = false
}
func (schedulers *Schedulers) setNextRunTime(cfg *model.Config, idx int, now time.Time, pendingJobs bool) {
scheduler := schedulers.schedulers[idx]
func (schedulers *Schedulers) setNextRunTime(cfg *model.Config, name string, now time.Time, pendingJobs bool) {
scheduler := schedulers.schedulers[name]
if !pendingJobs {
pj, err := schedulers.jobs.CheckForPendingJobsByType(scheduler.JobType())
pj, err := schedulers.jobs.CheckForPendingJobsByType(name)
if err != nil {
mlog.Error("Failed to set next job run time", mlog.Err(err))
schedulers.nextRunTimes[idx] = nil
schedulers.nextRunTimes[name] = nil
return
}
pendingJobs = pj
}
lastSuccessfulJob, err := schedulers.jobs.GetLastSuccessfulJobByType(scheduler.JobType())
lastSuccessfulJob, err := schedulers.jobs.GetLastSuccessfulJobByType(name)
if err != nil {
mlog.Error("Failed to set next job run time", mlog.Err(err))
schedulers.nextRunTimes[idx] = nil
schedulers.nextRunTimes[name] = nil
return
}
schedulers.nextRunTimes[idx] = scheduler.NextScheduleTime(cfg, now, pendingJobs, lastSuccessfulJob)
mlog.Debug("Next run time for scheduler", mlog.String("scheduler_name", scheduler.Name()), mlog.String("next_runtime", fmt.Sprintf("%v", schedulers.nextRunTimes[idx])))
schedulers.nextRunTimes[name] = scheduler.NextScheduleTime(cfg, now, pendingJobs, lastSuccessfulJob)
mlog.Debug("Next run time for scheduler", mlog.String("scheduler_name", name), mlog.String("next_runtime", fmt.Sprintf("%v", schedulers.nextRunTimes[name])))
}
func (schedulers *Schedulers) scheduleJob(cfg *model.Config, scheduler model.Scheduler) (*model.Job, *model.AppError) {
pendingJobs, err := schedulers.jobs.CheckForPendingJobsByType(scheduler.JobType())
func (schedulers *Schedulers) scheduleJob(cfg *model.Config, name string, scheduler model.Scheduler) (*model.Job, *model.AppError) {
pendingJobs, err := schedulers.jobs.CheckForPendingJobsByType(name)
if err != nil {
return nil, err
}
lastSuccessfulJob, err2 := schedulers.jobs.GetLastSuccessfulJobByType(scheduler.JobType())
lastSuccessfulJob, err2 := schedulers.jobs.GetLastSuccessfulJobByType(name)
if err2 != nil {
return nil, err
}

20
jobs/schedulers_test.go
View File

@ -9,7 +9,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/mattermost/mattermost-server/v6/einterfaces/mocks"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/plugin/plugintest/mock"
"github.com/mattermost/mattermost-server/v6/store/storetest"
@ -24,14 +23,6 @@ func (scheduler *MockScheduler) Enabled(cfg *model.Config) bool {
return true
}
func (scheduler *MockScheduler) Name() string {
return "MockScheduler"
}
func (scheduler *MockScheduler) JobType() string {
return model.JobTypeDataRetention
}
func (scheduler *MockScheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
nextTime := time.Now().Add(60 * time.Second)
return &nextTime
@ -70,16 +61,11 @@ func TestScheduler(t *testing.T) {
},
}
jobInterface := new(mocks.DataRetentionJobInterface)
jobInterface.On("MakeScheduler").Return(new(MockScheduler))
jobServer.DataRetentionJob = jobInterface
exportInterface := new(mocks.MessageExportJobInterface)
exportInterface.On("MakeScheduler").Return(new(MockScheduler))
jobServer.MessageExportJob = exportInterface
jobServer.InitSchedulers()
jobServer.RegisterJobType(model.JobTypeDataRetention, nil, new(MockScheduler))
jobServer.RegisterJobType(model.JobTypeMessageExport, nil, new(MockScheduler))
t.Run("Base", func(t *testing.T) {
jobServer.InitSchedulers()
jobServer.StartSchedulers()
time.Sleep(time.Second)

32
jobs/server.go
View File

@ -7,8 +7,6 @@ import (
"sync"
"github.com/mattermost/mattermost-server/v6/einterfaces"
ejobs "github.com/mattermost/mattermost-server/v6/einterfaces/jobs"
tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/services/configservice"
"github.com/mattermost/mattermost-server/v6/store"
@ -19,25 +17,6 @@ type JobServer struct {
Store store.Store
metrics einterfaces.MetricsInterface
DataRetentionJob ejobs.DataRetentionJobInterface
MessageExportJob ejobs.MessageExportJobInterface
ElasticsearchAggregator ejobs.ElasticsearchAggregatorInterface
ElasticsearchIndexer tjobs.IndexerJobInterface
LdapSync ejobs.LdapSyncInterface
Migrations tjobs.MigrationsJobInterface
Plugins tjobs.PluginsJobInterface
BleveIndexer tjobs.IndexerJobInterface
ExpiryNotify tjobs.ExpiryNotifyJobInterface
ProductNotices tjobs.ProductNoticesJobInterface
ActiveUsers tjobs.ActiveUsersJobInterface
ImportProcess tjobs.ImportProcessInterface
ImportDelete tjobs.ImportDeleteInterface
ExportProcess tjobs.ExportProcessInterface
ExportDelete tjobs.ExportDeleteInterface
Cloud ejobs.CloudJobInterface
ResendInvitationEmails ejobs.ResendInvitationEmailJobInterface
ExtractContent tjobs.ExtractContentInterface
// mut is used to protect the following fields from concurrent access.
mut sync.Mutex
workers *Workers
@ -56,6 +35,17 @@ func (srv *JobServer) Config() *model.Config {
return srv.ConfigService.Config()
}
func (srv *JobServer) RegisterJobType(name string, worker model.Worker, scheduler model.Scheduler) {
srv.mut.Lock()
defer srv.mut.Unlock()
if worker != nil {
srv.workers.AddWorker(name, worker)
}
if scheduler != nil {
srv.schedulers.AddScheduler(name, scheduler)
}
}
func (srv *JobServer) StartWorkers() error {
srv.mut.Lock()
defer srv.mut.Unlock()

308
jobs/workers.go
View File

@ -15,24 +15,7 @@ type Workers struct {
ConfigService configservice.ConfigService
Watcher *Watcher
DataRetention model.Worker
MessageExport model.Worker
ElasticsearchIndexing model.Worker
ElasticsearchAggregation model.Worker
LdapSync model.Worker
Migrations model.Worker
Plugins model.Worker
BleveIndexing model.Worker
ExpiryNotify model.Worker
ProductNotices model.Worker
ActiveUsers model.Worker
ImportProcess model.Worker
ImportDelete model.Worker
ExportProcess model.Worker
ExportDelete model.Worker
Cloud model.Worker
ResendInvitationEmail model.Worker
ExtractContent model.Worker
workers map[string]model.Worker
listenerId string
running bool
@ -52,163 +35,39 @@ func (srv *JobServer) InitWorkers() error {
return ErrWorkersRunning
}
workers := &Workers{
ConfigService: srv.ConfigService,
}
workers := NewWorkers(srv.ConfigService)
workers.Watcher = srv.MakeWatcher(workers, DefaultWatcherPollingInterval)
if srv.DataRetentionJob != nil {
workers.DataRetention = srv.DataRetentionJob.MakeWorker()
}
if srv.MessageExportJob != nil {
workers.MessageExport = srv.MessageExportJob.MakeWorker()
}
if elasticsearchIndexerInterface := srv.ElasticsearchIndexer; elasticsearchIndexerInterface != nil {
workers.ElasticsearchIndexing = elasticsearchIndexerInterface.MakeWorker()
}
if elasticsearchAggregatorInterface := srv.ElasticsearchAggregator; elasticsearchAggregatorInterface != nil {
workers.ElasticsearchAggregation = elasticsearchAggregatorInterface.MakeWorker()
}
if ldapSyncInterface := srv.LdapSync; ldapSyncInterface != nil {
workers.LdapSync = ldapSyncInterface.MakeWorker()
}
if migrationsInterface := srv.Migrations; migrationsInterface != nil {
workers.Migrations = migrationsInterface.MakeWorker()
}
if pluginsInterface := srv.Plugins; pluginsInterface != nil {
workers.Plugins = pluginsInterface.MakeWorker()
}
if bleveIndexerInterface := srv.BleveIndexer; bleveIndexerInterface != nil {
workers.BleveIndexing = bleveIndexerInterface.MakeWorker()
}
if expiryNotifyInterface := srv.ExpiryNotify; expiryNotifyInterface != nil {
workers.ExpiryNotify = expiryNotifyInterface.MakeWorker()
}
if activeUsersInterface := srv.ActiveUsers; activeUsersInterface != nil {
workers.ActiveUsers = activeUsersInterface.MakeWorker()
}
if productNoticesInterface := srv.ProductNotices; productNoticesInterface != nil {
workers.ProductNotices = productNoticesInterface.MakeWorker()
}
if importProcessInterface := srv.ImportProcess; importProcessInterface != nil {
workers.ImportProcess = importProcessInterface.MakeWorker()
}
if importDeleteInterface := srv.ImportDelete; importDeleteInterface != nil {
workers.ImportDelete = importDeleteInterface.MakeWorker()
}
if exportProcessInterface := srv.ExportProcess; exportProcessInterface != nil {
workers.ExportProcess = exportProcessInterface.MakeWorker()
}
if exportDeleteInterface := srv.ExportDelete; exportDeleteInterface != nil {
workers.ExportDelete = exportDeleteInterface.MakeWorker()
}
if cloudInterface := srv.Cloud; cloudInterface != nil {
workers.Cloud = cloudInterface.MakeWorker()
}
if resendInvitationEmailInterface := srv.ResendInvitationEmails; resendInvitationEmailInterface != nil {
workers.ResendInvitationEmail = resendInvitationEmailInterface.MakeWorker()
}
if extractContentInterface := srv.ExtractContent; extractContentInterface != nil {
workers.ExtractContent = extractContentInterface.MakeWorker()
}
srv.workers = workers
return nil
}
func NewWorkers(configService configservice.ConfigService) *Workers {
return &Workers{
ConfigService: configService,
workers: make(map[string]model.Worker),
}
}
func (workers *Workers) AddWorker(name string, worker model.Worker) {
workers.workers[name] = worker
}
func (workers *Workers) Get(name string) model.Worker {
return workers.workers[name]
}
// Start starts the workers. This call is not safe for concurrent use.
// Synchronization should be implemented by the caller.
func (workers *Workers) Start() {
mlog.Info("Starting workers")
if workers.DataRetention != nil {
go workers.DataRetention.Run()
}
if workers.MessageExport != nil && *workers.ConfigService.Config().MessageExportSettings.EnableExport {
go workers.MessageExport.Run()
}
if workers.ElasticsearchIndexing != nil && *workers.ConfigService.Config().ElasticsearchSettings.EnableIndexing {
go workers.ElasticsearchIndexing.Run()
}
if workers.ElasticsearchAggregation != nil && *workers.ConfigService.Config().ElasticsearchSettings.EnableIndexing {
go workers.ElasticsearchAggregation.Run()
}
if workers.LdapSync != nil && *workers.ConfigService.Config().LdapSettings.EnableSync {
go workers.LdapSync.Run()
}
if workers.Migrations != nil {
go workers.Migrations.Run()
}
if workers.Plugins != nil {
go workers.Plugins.Run()
}
if workers.BleveIndexing != nil && *workers.ConfigService.Config().BleveSettings.EnableIndexing && *workers.ConfigService.Config().BleveSettings.IndexDir != "" {
go workers.BleveIndexing.Run()
}
if workers.ExpiryNotify != nil {
go workers.ExpiryNotify.Run()
}
if workers.ActiveUsers != nil {
go workers.ActiveUsers.Run()
}
if workers.ProductNotices != nil {
go workers.ProductNotices.Run()
}
if workers.ImportProcess != nil {
go workers.ImportProcess.Run()
}
if workers.ImportDelete != nil {
go workers.ImportDelete.Run()
}
if workers.ExportProcess != nil {
go workers.ExportProcess.Run()
}
if workers.ExportDelete != nil {
go workers.ExportDelete.Run()
}
if workers.Cloud != nil {
go workers.Cloud.Run()
}
if workers.ResendInvitationEmail != nil {
go workers.ResendInvitationEmail.Run()
}
if workers.ExtractContent != nil {
go workers.ExtractContent.Run()
for _, w := range workers.workers {
if w.IsEnabled(workers.ConfigService.Config()) {
go w.Run()
}
}
go workers.Watcher.Start()
@ -220,51 +79,12 @@ func (workers *Workers) Start() {
func (workers *Workers) handleConfigChange(oldConfig *model.Config, newConfig *model.Config) {
mlog.Debug("Workers received config change.")
if workers.DataRetention != nil {
if (!*oldConfig.DataRetentionSettings.EnableMessageDeletion && !*oldConfig.DataRetentionSettings.EnableFileDeletion && !*oldConfig.DataRetentionSettings.EnableBoardsDeletion) && (*newConfig.DataRetentionSettings.EnableMessageDeletion || *newConfig.DataRetentionSettings.EnableFileDeletion || *newConfig.DataRetentionSettings.EnableBoardsDeletion) {
go workers.DataRetention.Run()
} else if (*oldConfig.DataRetentionSettings.EnableMessageDeletion || *oldConfig.DataRetentionSettings.EnableFileDeletion || *oldConfig.DataRetentionSettings.EnableBoardsDeletion) && (!*newConfig.DataRetentionSettings.EnableMessageDeletion && !*newConfig.DataRetentionSettings.EnableFileDeletion && !*newConfig.DataRetentionSettings.EnableBoardsDeletion) {
workers.DataRetention.Stop()
for _, w := range workers.workers {
if w.IsEnabled(oldConfig) && !w.IsEnabled(newConfig) {
w.Stop()
}
}
if workers.MessageExport != nil {
if !*oldConfig.MessageExportSettings.EnableExport && *newConfig.MessageExportSettings.EnableExport {
go workers.MessageExport.Run()
} else if *oldConfig.MessageExportSettings.EnableExport && !*newConfig.MessageExportSettings.EnableExport {
workers.MessageExport.Stop()
}
}
if workers.ElasticsearchIndexing != nil {
if !*oldConfig.ElasticsearchSettings.EnableIndexing && *newConfig.ElasticsearchSettings.EnableIndexing {
go workers.ElasticsearchIndexing.Run()
} else if *oldConfig.ElasticsearchSettings.EnableIndexing && !*newConfig.ElasticsearchSettings.EnableIndexing {
workers.ElasticsearchIndexing.Stop()
}
}
if workers.ElasticsearchAggregation != nil {
if !*oldConfig.ElasticsearchSettings.EnableIndexing && *newConfig.ElasticsearchSettings.EnableIndexing {
go workers.ElasticsearchAggregation.Run()
} else if *oldConfig.ElasticsearchSettings.EnableIndexing && !*newConfig.ElasticsearchSettings.EnableIndexing {
workers.ElasticsearchAggregation.Stop()
}
}
if workers.LdapSync != nil {
if !*oldConfig.LdapSettings.EnableSync && *newConfig.LdapSettings.EnableSync {
go workers.LdapSync.Run()
} else if *oldConfig.LdapSettings.EnableSync && !*newConfig.LdapSettings.EnableSync {
workers.LdapSync.Stop()
}
}
if workers.BleveIndexing != nil {
if !*oldConfig.BleveSettings.EnableIndexing && *newConfig.BleveSettings.EnableIndexing {
go workers.BleveIndexing.Run()
} else if *oldConfig.BleveSettings.EnableIndexing && !*newConfig.BleveSettings.EnableIndexing {
workers.BleveIndexing.Stop()
if !w.IsEnabled(oldConfig) && w.IsEnabled(newConfig) {
go w.Run()
}
}
}
@ -276,76 +96,10 @@ func (workers *Workers) Stop() {
workers.Watcher.Stop()
if workers.DataRetention != nil && (*workers.ConfigService.Config().DataRetentionSettings.EnableMessageDeletion || *workers.ConfigService.Config().DataRetentionSettings.EnableFileDeletion) {
workers.DataRetention.Stop()
}
if workers.MessageExport != nil && *workers.ConfigService.Config().MessageExportSettings.EnableExport {
workers.MessageExport.Stop()
}
if workers.ElasticsearchIndexing != nil && *workers.ConfigService.Config().ElasticsearchSettings.EnableIndexing {
workers.ElasticsearchIndexing.Stop()
}
if workers.ElasticsearchAggregation != nil && *workers.ConfigService.Config().ElasticsearchSettings.EnableIndexing {
workers.ElasticsearchAggregation.Stop()
}
if workers.LdapSync != nil && *workers.ConfigService.Config().LdapSettings.EnableSync {
workers.LdapSync.Stop()
}
if workers.Migrations != nil {
workers.Migrations.Stop()
}
if workers.Plugins != nil {
workers.Plugins.Stop()
}
if workers.BleveIndexing != nil && *workers.ConfigService.Config().BleveSettings.EnableIndexing {
workers.BleveIndexing.Stop()
}
if workers.ExpiryNotify != nil {
workers.ExpiryNotify.Stop()
}
if workers.ActiveUsers != nil {
workers.ActiveUsers.Stop()
}
if workers.ProductNotices != nil {
workers.ProductNotices.Stop()
}
if workers.ImportProcess != nil {
workers.ImportProcess.Stop()
}
if workers.ImportDelete != nil {
workers.ImportDelete.Stop()
}
if workers.ExportProcess != nil {
workers.ExportProcess.Stop()
}
if workers.ExportDelete != nil {
workers.ExportDelete.Stop()
}
if workers.Cloud != nil {
workers.Cloud.Stop()
}
if workers.ResendInvitationEmail != nil {
workers.ResendInvitationEmail.Stop()
}
if workers.ExtractContent != nil {
workers.ExtractContent.Stop()
for _, w := range workers.workers {
if w.IsEnabled(workers.ConfigService.Config()) {
w.Stop()
}
}
workers.running = false

265
migrations/helper_test.go
View File

@ -1,265 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package migrations
import (
"os"
"testing"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/app/request"
"github.com/mattermost/mattermost-server/v6/config"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
"github.com/mattermost/mattermost-server/v6/store/localcachelayer"
)
type TestHelper struct {
App *app.App
Context *request.Context
Server *app.Server
BasicTeam *model.Team
BasicUser *model.User
BasicUser2 *model.User
BasicChannel *model.Channel
BasicPost *model.Post
SystemAdminUser *model.User
tempWorkspace string
TestLogger *mlog.Logger
}
func setupTestHelper(enterprise bool) *TestHelper {
store := mainHelper.GetStore()
store.DropAllTables()
memoryStore := config.NewTestMemoryStore()
newConfig := memoryStore.Get().Clone()
*newConfig.AnnouncementSettings.AdminNoticesEnabled = false
*newConfig.AnnouncementSettings.UserNoticesEnabled = false
memoryStore.Set(newConfig)
var options []app.Option
options = append(options, app.ConfigStore(memoryStore))
options = append(options, app.StoreOverride(mainHelper.Store))
options = append(options, app.SkipPostInitializiation())
testLogger, _ := mlog.NewLogger()
logCfg, _ := config.MloggerConfigFromLoggerConfig(&newConfig.LogSettings, nil, config.GetLogFileLocation)
if errCfg := testLogger.ConfigureTargets(logCfg, nil); errCfg != nil {
panic("failed to configure test logger: " + errCfg.Error())
}
// lock logger config so server init cannot override it during testing.
testLogger.LockConfiguration()
options = append(options, app.SetLogger(testLogger))
s, err := app.NewServer(options...)
if err != nil {
panic(err)
}
// Adds the cache layer to the test store
s.Store, err = localcachelayer.NewLocalCacheLayer(s.Store, s.Metrics, s.Cluster, s.CacheProvider)
if err != nil {
panic(err)
}
th := &TestHelper{
App: app.New(app.ServerConnector(s.Channels())),
Context: &request.Context{},
Server: s,
TestLogger: testLogger,
}
th.App.UpdateConfig(func(cfg *model.Config) { *cfg.TeamSettings.MaxUsersPerTeam = 50 })
th.App.UpdateConfig(func(cfg *model.Config) { *cfg.RateLimitSettings.Enable = false })
prevListenAddress := *th.App.Config().ServiceSettings.ListenAddress
th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = ":0" })
serverErr := th.Server.Start()
if serverErr != nil {
panic(serverErr)
}
th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = prevListenAddress })
th.App.DoAppMigrations()
th.App.Srv().Store.MarkSystemRanUnitTests()
th.App.UpdateConfig(func(cfg *model.Config) { *cfg.TeamSettings.EnableOpenServer = true })
if enterprise {
th.App.Srv().SetLicense(model.NewTestLicense())
th.App.Srv().Jobs.InitWorkers()
th.App.Srv().Jobs.InitSchedulers()
} else {
th.App.Srv().SetLicense(nil)
}
return th
}
func SetupEnterprise(tb testing.TB) *TestHelper {
return setupTestHelper(true)
}
func Setup(tb testing.TB) *TestHelper {
return setupTestHelper(false)
}
func (th *TestHelper) InitBasic() *TestHelper {
th.SystemAdminUser = th.CreateUser()
th.App.UpdateUserRoles(th.SystemAdminUser.Id, model.SystemUserRoleId+" "+model.SystemAdminRoleId, false)
th.SystemAdminUser, _ = th.App.GetUser(th.SystemAdminUser.Id)
th.BasicTeam = th.CreateTeam()
th.BasicUser = th.CreateUser()
th.LinkUserToTeam(th.BasicUser, th.BasicTeam)
th.BasicUser2 = th.CreateUser()
th.LinkUserToTeam(th.BasicUser2, th.BasicTeam)
th.BasicChannel = th.CreateChannel(th.BasicTeam)
th.BasicPost = th.CreatePost(th.BasicChannel)
return th
}
func (*TestHelper) MakeEmail() string {
return "success_" + model.NewId() + "@simulator.amazonses.com"
}
func (th *TestHelper) CreateTeam() *model.Team {
id := model.NewId()
team := &model.Team{
DisplayName: "dn_" + id,
Name: "name" + id,
Email: "success+" + id + "@simulator.amazonses.com",
Type: model.TeamOpen,
}
var err *model.AppError
if team, err = th.App.CreateTeam(th.Context, team); err != nil {
panic(err)
}
return team
}
func (th *TestHelper) CreateUser() *model.User {
id := model.NewId()
user := &model.User{
Email: "success+" + id + "@simulator.amazonses.com",
Username: "un_" + id,
Nickname: "nn_" + id,
Password: "Password1",
EmailVerified: true,
}
var err *model.AppError
if user, err = th.App.CreateUser(th.Context, user); err != nil {
panic(err)
}
return user
}
func (th *TestHelper) CreateChannel(team *model.Team) *model.Channel {
return th.createChannel(team, model.ChannelTypeOpen)
}
func (th *TestHelper) createChannel(team *model.Team, channelType model.ChannelType) *model.Channel {
id := model.NewId()
channel := &model.Channel{
DisplayName: "dn_" + id,
Name: "name_" + id,
Type: channelType,
TeamId: team.Id,
CreatorId: th.BasicUser.Id,
}
var err *model.AppError
if channel, err = th.App.CreateChannel(th.Context, channel, true); err != nil {
panic(err)
}
return channel
}
func (th *TestHelper) CreateDmChannel(user *model.User) *model.Channel {
var err *model.AppError
var channel *model.Channel
if channel, err = th.App.GetOrCreateDirectChannel(th.Context, th.BasicUser.Id, user.Id); err != nil {
panic(err)
}
return channel
}
func (th *TestHelper) CreatePost(channel *model.Channel) *model.Post {
id := model.NewId()
post := &model.Post{
UserId: th.BasicUser.Id,
ChannelId: channel.Id,
Message: "message_" + id,
CreateAt: model.GetMillis() - 10000,
}
var err *model.AppError
if post, err = th.App.CreatePost(th.Context, post, channel, false, true); err != nil {
panic(err)
}
return post
}
func (th *TestHelper) LinkUserToTeam(user *model.User, team *model.Team) {
_, err := th.App.JoinUserToTeam(th.Context, team, user, "")
if err != nil {
panic(err)
}
}
func (th *TestHelper) AddUserToChannel(user *model.User, channel *model.Channel) *model.ChannelMember {
member, err := th.App.AddUserToChannel(user, channel, false)
if err != nil {
panic(err)
}
return member
}
func (th *TestHelper) TearDown() {
// Clean all the caches
th.App.Srv().InvalidateAllCaches()
th.Server.Shutdown()
if th.tempWorkspace != "" {
os.RemoveAll(th.tempWorkspace)
}
}
func (*TestHelper) ResetRoleMigration() {
sqlStore := mainHelper.GetSQLStore()
if _, err := sqlStore.GetMaster().Exec("DELETE from Roles"); err != nil {
panic(err)
}
mainHelper.GetClusterInterface().SendClearRoleCacheMessage()
if _, err := sqlStore.GetMaster().Exec("DELETE from Systems where Name = :Name", map[string]interface{}{"Name": model.AdvancedPermissionsMigrationKey}); err != nil {
panic(err)
}
}
func (th *TestHelper) DeleteAllJobsByTypeAndMigrationKey(jobType string, migrationKey string) {
jobs, err := th.App.Srv().Store.Job().GetAllByType(model.JobTypeMigrations)
if err != nil {
panic(err)
}
for _, job := range jobs {
if key, ok := job.Data[JobDataKeyMigration]; ok && key == migrationKey {
if _, err = th.App.Srv().Store.Job().Delete(job.Id); err != nil {
panic(err)
}
}
}
}

13
model/bulk_export.go Normal file
View File

@ -0,0 +1,13 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package model
// ExportDataDir is the name of the directory were to store additional data
// included with the export (e.g. file attachments).
const ExportDataDir = "data"
type BulkExportOpts struct {
IncludeAttachments bool
CreateArchive bool
}

26
model/job.go
View File

@ -78,29 +78,6 @@ func (j *Job) IsValid() *AppError {
return NewAppError("Job.IsValid", "model.job.is_valid.create_at.app_error", nil, "id="+j.Id, http.StatusBadRequest)
}
switch j.Type {
case JobTypeDataRetention:
case JobTypeElasticsearchPostIndexing:
case JobTypeElasticsearchPostAggregation:
case JobTypeBlevePostIndexing:
case JobTypeLdapSync:
case JobTypeMessageExport:
case JobTypeMigrations:
case JobTypePlugins:
case JobTypeProductNotices:
case JobTypeExpiryNotify:
case JobTypeActiveUsers:
case JobTypeImportProcess:
case JobTypeImportDelete:
case JobTypeExportProcess:
case JobTypeExportDelete:
case JobTypeCloud:
case JobTypeResendInvitationEmail:
case JobTypeExtractContent:
default:
return NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+j.Id, http.StatusBadRequest)
}
switch j.Status {
case JobStatusPending:
case JobStatusInProgress:
@ -119,11 +96,10 @@ type Worker interface {
Run()
Stop()
JobChannel() chan<- Job
IsEnabled(cfg *Config) bool
}
type Scheduler interface {
Name() string
JobType() string
Enabled(cfg *Config) bool
NextScheduleTime(cfg *Config, now time.Time, pendingJobs bool, lastSuccessfulJob *Job) *time.Time
ScheduleJob(cfg *Config, pendingJobs bool, lastSuccessfulJob *Job) (*Job, *AppError)

5
model/upload_session.go
View File

@ -12,8 +12,9 @@ import (
type UploadType string
const (
UploadTypeAttachment UploadType = "attachment"
UploadTypeImport UploadType = "import"
UploadTypeAttachment UploadType = "attachment"
UploadTypeImport UploadType = "import"
IncompleteUploadSuffix = ".tmp"
)
// UploadNoUserID is a "fake" user id used by the API layer when in local mode.

20
plugin/scheduler/plugin.go
View File

@ -1,20 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package scheduler
import (
"github.com/mattermost/mattermost-server/v6/app"
tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces"
)
type PluginsJobInterfaceImpl struct {
App *app.App
}
func init() {
app.RegisterJobsPluginsJobInterface(func(s *app.Server) tjobs.PluginsJobInterface {
a := app.New(app.ServerConnector(s.Channels()))
return &PluginsJobInterfaceImpl{a}
})
}

42
plugin/scheduler/scheduler.go
View File

@ -6,45 +6,15 @@ package scheduler
import (
"time"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/jobs"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
)
const pluginsJobInterval = 24 * 60 * 60 * time.Second
const schedFreq = 24 * time.Hour
type Scheduler struct {
App *app.App
}
func (m *PluginsJobInterfaceImpl) MakeScheduler() model.Scheduler {
return &Scheduler{m.App}
}
func (scheduler *Scheduler) Name() string {
return "PluginsScheduler"
}
func (scheduler *Scheduler) JobType() string {
return model.JobTypePlugins
}
func (scheduler *Scheduler) Enabled(cfg *model.Config) bool {
return true
}
func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
nextTime := time.Now().Add(pluginsJobInterval)
return &nextTime
}
func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
mlog.Debug("Scheduling Job", mlog.String("scheduler", scheduler.Name()))
job, err := scheduler.App.Srv().Jobs.CreateJob(model.JobTypePlugins, nil)
if err != nil {
return nil, err
func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler {
isEnabled := func(cfg *model.Config) bool {
return true
}
return job, nil
return jobs.NewPeriodicScheduler(jobServer, model.JobTypePlugins, schedFreq, isEnabled)
}

21
plugin/scheduler/worker.go
View File

@ -4,29 +4,32 @@
package scheduler
import (
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/jobs"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
)
type AppIface interface {
DeleteAllExpiredPluginKeys() *model.AppError
}
type Worker struct {
name string
stop chan bool
stopped chan bool
jobs chan model.Job
jobServer *jobs.JobServer
app *app.App
app AppIface
}
func (m *PluginsJobInterfaceImpl) MakeWorker() model.Worker {
func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker {
worker := Worker{
name: "Plugins",
stop: make(chan bool, 1),
stopped: make(chan bool, 1),
jobs: make(chan model.Job),
jobServer: m.App.Srv().Jobs,
app: m.App,
jobServer: jobServer,
app: app,
}
return &worker
@ -62,6 +65,10 @@ func (worker *Worker) JobChannel() chan<- model.Job {
return worker.jobs
}
func (worker *Worker) IsEnabled(cfg *model.Config) bool {
return true
}
func (worker *Worker) DoJob(job *model.Job) {
if claimed, err := worker.jobServer.ClaimJob(job); err != nil {
mlog.Info("Worker experienced an error while trying to claim job",
@ -84,14 +91,14 @@ func (worker *Worker) DoJob(job *model.Job) {
}
func (worker *Worker) setJobSuccess(job *model.Job) {
if err := worker.app.Srv().Jobs.SetJobSuccess(job); err != nil {
if err := worker.jobServer.SetJobSuccess(job); err != nil {
mlog.Error("Worker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
worker.setJobError(job, err)
}
}
func (worker *Worker) setJobError(job *model.Job, appError *model.AppError) {
if err := worker.app.Srv().Jobs.SetJobError(job, appError); err != nil {
if err := worker.jobServer.SetJobError(job, appError); err != nil {
mlog.Error("Worker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
}
}

30
services/searchengine/bleveengine/indexer/indexing_job.go
View File

@ -1,7 +1,7 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package ebleveengine
package indexer
import (
"context"
@ -9,9 +9,7 @@ import (
"strconv"
"time"
"github.com/mattermost/mattermost-server/v6/app"
"github.com/mattermost/mattermost-server/v6/jobs"
tjobs "github.com/mattermost/mattermost-server/v6/jobs/interfaces"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/services/searchengine/bleveengine"
"github.com/mattermost/mattermost-server/v6/shared/mlog"
@ -26,28 +24,17 @@ const (
EstimatedUserCount = 10000
)
func init() {
app.RegisterJobsBleveIndexerInterface(func(s *app.Server) tjobs.IndexerJobInterface {
return &BleveIndexerInterfaceImpl{s}
})
}
type BleveIndexerInterfaceImpl struct {
Server *app.Server
}
type BleveIndexerWorker struct {
name string
stop chan bool
stopped chan bool
jobs chan model.Job
jobServer *jobs.JobServer
engine *bleveengine.BleveEngine
engine *bleveengine.BleveEngine
}
func (bi *BleveIndexerInterfaceImpl) MakeWorker() model.Worker {
if bi.Server.SearchEngine.BleveEngine == nil {
func MakeWorker(jobServer *jobs.JobServer, engine *bleveengine.BleveEngine) model.Worker {
if engine == nil {
return nil
}
return &BleveIndexerWorker{
@ -55,9 +42,8 @@ func (bi *BleveIndexerInterfaceImpl) MakeWorker() model.Worker {
stop: make(chan bool, 1),
stopped: make(chan bool, 1),
jobs: make(chan model.Job),
jobServer: bi.Server.Jobs,
engine: bi.Server.SearchEngine.BleveEngine.(*bleveengine.BleveEngine),
jobServer: jobServer,
engine: engine,
}
}
@ -92,6 +78,10 @@ func (worker *BleveIndexerWorker) JobChannel() chan<- model.Job {
return worker.jobs
}
func (worker *BleveIndexerWorker) IsEnabled(cfg *model.Config) bool {
return true
}
func (worker *BleveIndexerWorker) Run() {
mlog.Debug("Worker Started", mlog.String("workername", worker.name))

2
services/searchengine/bleveengine/indexer/indexing_job_test.go
View File

@ -1,7 +1,7 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package ebleveengine
package indexer
import (
"errors"