IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2026-08-26 21:27:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

MM-61407: Remove Bleve (#33430)

Browse Source 
https://mattermost.atlassian.net/browse/MM-61407

```release-note
NONE
```

* webapp i18n

```release-note
NONE
```

* Fix e2e tests

```release-note
NONE
```

* fix roles in e2e tests

```release-note
NONE
```

* some review comments

```release-note
NONE
```

* add back permissions to deprecated list

```release-note
NONE
```
This commit is contained in:
Agniva De Sarker
2025-07-31 09:35:29 +05:30
committed by GitHub
parent 9f92131899
commit 41e88b74ac
62 changed files with 41 additions and 3835 deletions
Download Patch File Download Diff File Expand all files Collapse all files
e2e-tests/cypress/tests/fixtures/console-example-inputs.json
Vendored
+1 -10
View File
@@ -412,14 +412,5 @@
"selector": ""
}
]
},
{
"section": "experimental.bleve",
"disabledInputs": [
{
"path": "/admin_console/experimental/blevesearch",
"selector": "indexDirinput"
}
]
}
]
]
e2e-tests/cypress/tests/fixtures/system-roles-console-access.json
Vendored
+1 -7
View File
@@ -310,11 +310,5 @@
"system_manager": "none",
"system_user_manager": "none",
"system_read_only_admin": "read"
},
{
"section": "experimental.bleve",
"system_manager": "none",
"system_user_manager": "none",
"system_read_only_admin": "read"
}
]
]
e2e-tests/cypress/tests/integration/channels/enterprise/system_console/search_box_spec.js
+2 -2
View File
@@ -46,14 +46,14 @@ describe('System console', () => {
cy.visit('/admin_console');
// * Scroll to the last item of the page and ensure it can be clicked
cy.findByTestId('experimental.bleve').scrollIntoView().click();
cy.findByTestId('experimental.feature_flags').scrollIntoView().click();
});
it('MM-T1634 - Search box should remain visible / in the header as you scroll down the settings list in the left-hand-side', () => {
cy.visit('/admin_console');
// * Scroll to bottom of left hand side
cy.findByTestId('experimental.bleve').scrollIntoView().click();
cy.findByTestId('experimental.feature_flags').scrollIntoView().click();
// * To check if the sidebar is in view, try to click it
cy.get('#adminSidebarFilter').should('be.visible').click();
e2e-tests/cypress/tests/support/api/on_prem_default_config.json
-7
View File
@@ -521,13 +521,6 @@
"Trace": "",
"IgnoredPurgeIndexes": ""
},
"BleveSettings": {
"IndexDir": "",
"EnableIndexing": false,
"EnableSearching": false,
"EnableAutocomplete": false,
"BatchSize": 10000
},
"DataRetentionSettings": {
"EnableMessageDeletion": false,
"EnableFileDeletion": false,
e2e-tests/cypress/tests/support/api/role.js
+2 -2
View File
File diff suppressed because one or more lines are too long
e2e-tests/cypress/tests/utils/admin_console.js
-6
View File
@@ -359,10 +359,4 @@ export const adminConsoleNavigation = [
sidebar: 'Feature Flags',
url: 'admin_console/experimental/feature_flags',
},
{
type: ['team', 'e20'],
header: 'Bleve',
sidebar: 'Bleve',
url: 'admin_console/experimental/blevesearch',
},
];
e2e-tests/playwright/lib/src/server/default_config.ts
-7
View File
@@ -646,13 +646,6 @@ const defaultServerConfig: AdminConfig = {
Trace: '',
IgnoredPurgeIndexes: '',
},
BleveSettings: {
IndexDir: '',
EnableIndexing: false,
EnableSearching: false,
EnableAutocomplete: false,
BatchSize: 10000,
},
DataRetentionSettings: {
EnableMessageDeletion: false,
EnableFileDeletion: false,
server/.golangci.yml
-3
View File
@@ -156,9 +156,6 @@ linters:
platform/services/remotecluster/send_test.go|\
platform/services/remotecluster/sendfile.go|\
platform/services/remotecluster/sendprofileImage_test.go|\
platform/services/searchengine/bleveengine/bleve_test.go|\
platform/services/searchengine/bleveengine/common.go|\
platform/services/searchengine/bleveengine/indexer/indexing_job.go|\
platform/services/sharedchannel/attachment.go|\
platform/services/sharedchannel/channelinvite.go|\
platform/services/sharedchannel/permalink_test.go|\
server/channels/api4/api.go
-4
View File
@@ -93,8 +93,6 @@ type Routes struct {
Elasticsearch *mux.Router // 'api/v4/elasticsearch'
Bleve *mux.Router // 'api/v4/bleve'
DataRetention *mux.Router // 'api/v4/data_retention'
Brand *mux.Router // 'api/v4/brand'
@@ -256,7 +254,6 @@ func Init(srv *app.Server) (*API, error) {
api.BaseRoutes.Reactions = api.BaseRoutes.APIRoot.PathPrefix("/reactions").Subrouter()
api.BaseRoutes.Jobs = api.BaseRoutes.APIRoot.PathPrefix("/jobs").Subrouter()
api.BaseRoutes.Elasticsearch = api.BaseRoutes.APIRoot.PathPrefix("/elasticsearch").Subrouter()
api.BaseRoutes.Bleve = api.BaseRoutes.APIRoot.PathPrefix("/bleve").Subrouter()
api.BaseRoutes.DataRetention = api.BaseRoutes.APIRoot.PathPrefix("/data_retention").Subrouter()
api.BaseRoutes.Emojis = api.BaseRoutes.APIRoot.PathPrefix("/emoji").Subrouter()
@@ -331,7 +328,6 @@ func Init(srv *app.Server) (*API, error) {
api.InitCluster()
api.InitLdap()
api.InitElasticsearch()
api.InitBleve()
api.InitDataRetention()
api.InitBrand()
api.InitJob()
server/channels/api4/apitestlib.go
-4
View File
@@ -174,10 +174,6 @@ func setupTestHelper(tb testing.TB, dbStore store.Store, sqlSettings *model.SqlS
Store: dbStore,
}
if s.Platform().SearchEngine != nil && s.Platform().SearchEngine.BleveEngine != nil && searchEngine != nil {
searchEngine.BleveEngine = s.Platform().SearchEngine.BleveEngine
}
if searchEngine != nil {
th.App.SetSearchEngine(searchEngine)
}
server/channels/api4/bleve.go
-33
View File
@@ -1,33 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package api4
import (
"net/http"
"github.com/mattermost/mattermost/server/public/model"
)
func (api *API) InitBleve() {
api.BaseRoutes.Bleve.Handle("/purge_indexes", api.APISessionRequired(purgeBleveIndexes)).Methods(http.MethodPost)
}
func purgeBleveIndexes(c *Context, w http.ResponseWriter, r *http.Request) {
auditRec := c.MakeAuditRecord(model.AuditEventPurgeBleveIndexes, model.AuditStatusFail)
defer c.LogAuditRec(auditRec)
if !c.App.SessionHasPermissionToAndNotRestrictedAdmin(*c.AppContext.Session(), model.PermissionPurgeBleveIndexes) {
c.SetPermissionError(model.PermissionPurgeBleveIndexes)
return
}
if err := c.App.PurgeBleveIndexes(c.AppContext); err != nil {
c.Err = err
return
}
auditRec.Success()
ReturnStatusOK(w)
}
server/channels/api4/bleve_test.go
-47
View File
@@ -1,47 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package api4
import (
"context"
"testing"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
)
func TestBlevePurgeIndexes(t *testing.T) {
mainHelper.Parallel(t)
th := Setup(t)
defer th.TearDown()
t.Run("as system user", func(t *testing.T) {
resp, err := th.Client.PurgeBleveIndexes(context.Background())
require.Error(t, err)
CheckForbiddenStatus(t, resp)
})
t.Run("as system user with write experimental permission", func(t *testing.T) {
th.AddPermissionToRole(model.PermissionPurgeBleveIndexes.Id, model.SystemUserRoleId)
defer th.RemovePermissionFromRole(model.PermissionSysconsoleWriteExperimental.Id, model.SystemUserRoleId)
resp, err := th.Client.PurgeBleveIndexes(context.Background())
require.NoError(t, err)
CheckOKStatus(t, resp)
})
t.Run("as system admin", func(t *testing.T) {
resp, err := th.SystemAdminClient.PurgeBleveIndexes(context.Background())
require.NoError(t, err)
CheckOKStatus(t, resp)
})
t.Run("as restricted system admin", func(t *testing.T) {
th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ExperimentalSettings.RestrictSystemAdmin = true })
resp, err := th.SystemAdminClient.PurgeBleveIndexes(context.Background())
require.Error(t, err)
CheckForbiddenStatus(t, resp)
})
}
server/channels/app/job.go
-5
View File
@@ -74,8 +74,6 @@ func (a *App) UpdateJobStatus(c request.CTX, job *model.Job, newStatus string) *
func (a *App) SessionHasPermissionToCreateJob(session model.Session, job *model.Job) (bool, *model.Permission) {
switch job.Type {
case model.JobTypeBlevePostIndexing:
return a.SessionHasPermissionTo(session, model.PermissionCreatePostBleveIndexesJob), model.PermissionCreatePostBleveIndexesJob
case model.JobTypeDataRetention:
return a.SessionHasPermissionTo(session, model.PermissionCreateDataRetentionJob), model.PermissionCreateDataRetentionJob
case model.JobTypeMessageExport:
@@ -110,8 +108,6 @@ func (a *App) SessionHasPermissionToManageJob(session model.Session, job *model.
var permission *model.Permission
switch job.Type {
case model.JobTypeBlevePostIndexing:
permission = model.PermissionManagePostBleveIndexesJob
case model.JobTypeDataRetention:
permission = model.PermissionManageDataRetentionJob
case model.JobTypeMessageExport:
@@ -159,7 +155,6 @@ func (a *App) SessionHasPermissionToReadJob(session model.Session, jobType strin
case model.JobTypeLdapSync:
return a.SessionHasPermissionTo(session, model.PermissionReadLdapSyncJob), model.PermissionReadLdapSyncJob
case
model.JobTypeBlevePostIndexing,
model.JobTypeMigrations,
model.JobTypePlugins,
model.JobTypeProductNotices,
server/channels/app/job_test.go
+2 -25
View File
@@ -42,11 +42,6 @@ func TestSessionHasPermissionToCreateJob(t *testing.T) {
defer th.TearDown()
jobs := []model.Job{
{
Id: model.NewId(),
Type: model.JobTypeBlevePostIndexing,
CreateAt: 1000,
},
{
Id: model.NewId(),
Type: model.JobTypeDataRetention,
@@ -65,14 +60,10 @@ func TestSessionHasPermissionToCreateJob(t *testing.T) {
}{
{
Job: jobs[0],
PermissionRequired: model.PermissionCreatePostBleveIndexesJob,
},
{
Job: jobs[1],
PermissionRequired: model.PermissionCreateDataRetentionJob,
},
{
Job: jobs[2],
Job: jobs[1],
PermissionRequired: model.PermissionCreateComplianceExportJob,
},
}
@@ -104,24 +95,10 @@ func TestSessionHasPermissionToCreateJob(t *testing.T) {
ctx := sqlstore.WithMaster(context.Background())
role, _ := th.App.GetRoleByName(ctx, model.SystemReadOnlyAdminRoleId)
role.Permissions = append(role.Permissions, model.PermissionCreatePostBleveIndexesJob.Id)
_, err := th.App.UpdateRole(role)
require.Nil(t, err)
// Now system read only admin should have ability to create a Belve Post Index job but not the others
for _, testCase := range testCases {
hasPermission, permissionRequired := th.App.SessionHasPermissionToCreateJob(session, &testCase.Job)
expectedHasPermission := testCase.Job.Type == model.JobTypeBlevePostIndexing
assert.Equal(t, expectedHasPermission, hasPermission)
require.NotNil(t, permissionRequired)
assert.Equal(t, testCase.PermissionRequired.Id, permissionRequired.Id)
}
role.Permissions = append(role.Permissions, model.PermissionCreateDataRetentionJob.Id)
role.Permissions = append(role.Permissions, model.PermissionCreateComplianceExportJob.Id)
_, err = th.App.UpdateRole(role)
_, err := th.App.UpdateRole(role)
require.Nil(t, err)
// Now system read only admin should have ability to create all jobs
server/channels/app/permissions_migrations.go
+2 -11
View File
@@ -614,17 +614,12 @@ func (a *App) getAddExperimentalSubsectionPermissions() (permissionsMap, error)
// Give the new subsection READ permissions to any user with READ_EXPERIMENTAL
permissionTransformation{
On: permissionExists(model.PermissionSysconsoleReadExperimental.Id),
Add: []string{model.PermissionSysconsoleReadExperimentalBleve.Id, model.PermissionSysconsoleReadExperimentalFeatures.Id, model.PermissionSysconsoleReadExperimentalFeatureFlags.Id},
Add: []string{model.PermissionSysconsoleReadExperimentalFeatures.Id, model.PermissionSysconsoleReadExperimentalFeatureFlags.Id},
},
// Give the new subsection WRITE permissions to any user with WRITE_EXPERIMENTAL
permissionTransformation{
On: permissionExists(model.PermissionSysconsoleWriteExperimental.Id),
Add: []string{model.PermissionSysconsoleWriteExperimentalBleve.Id, model.PermissionSysconsoleWriteExperimentalFeatures.Id, model.PermissionSysconsoleWriteExperimentalFeatureFlags.Id},
},
// Give the ancillary permissions MANAGE_JOBS and PURGE_BLEVE_INDEXES to anyone with WRITE_EXPERIMENTAL_BLEVE
permissionTransformation{
On: permissionExists(model.PermissionSysconsoleWriteExperimentalBleve.Id),
Add: []string{model.PermissionCreatePostBleveIndexesJob.Id, model.PermissionPurgeBleveIndexes.Id},
Add: []string{model.PermissionSysconsoleWriteExperimentalFeatures.Id, model.PermissionSysconsoleWriteExperimentalFeatureFlags.Id},
},
}, nil
}
@@ -1097,10 +1092,6 @@ func (a *App) getAddManageJobAncillaryPermissionsMigration() (permissionsMap, er
On: permissionExists(model.PermissionSysconsoleWriteComplianceDataRetentionPolicy.Id),
Add: []string{model.PermissionManageDataRetentionJob.Id},
},
permissionTransformation{
On: permissionExists(model.PermissionSysconsoleWriteExperimentalBleve.Id),
Add: []string{model.PermissionManagePostBleveIndexesJob.Id},
},
permissionTransformation{
On: permissionExists(model.PermissionSysconsoleWriteComplianceComplianceExport.Id),
Add: []string{model.PermissionManageComplianceExportJob.Id},
server/channels/app/platform/searchengine.go
-5
View File
@@ -86,9 +86,4 @@ func (ps *PlatformService) StopSearchEngine() {
ps.Log().Error("Failed to stop Elasticsearch engine", mlog.Err(err))
}
}
if ps.SearchEngine != nil && ps.SearchEngine.BleveEngine != nil && ps.SearchEngine.BleveEngine.IsActive() {
if err := ps.SearchEngine.BleveEngine.Stop(); err != nil {
ps.Log().Error("Failed to stop Bleve Engine", mlog.Err(err))
}
}
}
server/channels/app/platform/service.go
-6
View File
@@ -30,7 +30,6 @@ import (
"github.com/mattermost/mattermost/server/v8/einterfaces"
"github.com/mattermost/mattermost/server/v8/platform/services/cache"
"github.com/mattermost/mattermost/server/v8/platform/services/searchengine"
"github.com/mattermost/mattermost/server/v8/platform/services/searchengine/bleveengine"
"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
)
@@ -232,11 +231,6 @@ func New(sc ServiceConfig, options ...Option) (*PlatformService, error) {
// Step 3: Search Engine
searchEngine := searchengine.NewBroker(ps.Config())
bleveEngine := bleveengine.NewBleveEngine(ps.Config())
if err := bleveEngine.Start(); err != nil {
return nil, err
}
searchEngine.RegisterBleveEngine(bleveEngine)
ps.SearchEngine = searchEngine
// Step 4: Init Enterprise
server/channels/app/searchengine.go
-12
View File
@@ -53,18 +53,6 @@ func (a *App) PurgeElasticsearchIndexes(c request.CTX, indexes []string) *model.
return appErr
}
func (a *App) PurgeBleveIndexes(c request.CTX) *model.AppError {
engine := a.SearchEngine().BleveEngine
if engine == nil {
err := model.NewAppError("PurgeBleveIndexes", "searchengine.bleve.disabled.error", nil, "", http.StatusNotImplemented)
return err
}
if err := engine.PurgeIndexes(c); err != nil {
return err
}
return nil
}
func (a *App) ActiveSearchBackend() string {
return a.ch.srv.platform.SearchEngine.ActiveEngine()
}
server/channels/app/server.go
-8
View File
@@ -70,8 +70,6 @@ import (
"github.com/mattermost/mattermost/server/v8/platform/services/awsmeter"
"github.com/mattermost/mattermost/server/v8/platform/services/cache"
"github.com/mattermost/mattermost/server/v8/platform/services/remotecluster"
"github.com/mattermost/mattermost/server/v8/platform/services/searchengine/bleveengine"
"github.com/mattermost/mattermost/server/v8/platform/services/searchengine/bleveengine/indexer"
"github.com/mattermost/mattermost/server/v8/platform/services/sharedchannel"
"github.com/mattermost/mattermost/server/v8/platform/services/telemetry"
"github.com/mattermost/mattermost/server/v8/platform/services/upgrader"
@@ -1478,12 +1476,6 @@ func (s *Server) initJobs() {
s.Jobs.RegisterJobType(model.JobTypeAccessControlSync, builder.MakeWorker(), builder.MakeScheduler())
}
s.Jobs.RegisterJobType(
model.JobTypeBlevePostIndexing,
indexer.MakeWorker(s.Jobs, s.platform.SearchEngine.BleveEngine.(*bleveengine.BleveEngine)),
nil,
)
s.Jobs.RegisterJobType(
model.JobTypeMigrations,
migrations.MakeWorker(s.Jobs, s.Store()),
server/channels/app/support_packet.go
-4
View File
@@ -260,10 +260,6 @@ func (a *App) getSupportPacketJobList(rctx request.CTX) (*model.FileData, error)
if err != nil {
rErr = multierror.Append(errors.Wrap(err, "error while getting ES post aggregation jobs"))
}
jobs.BlevePostIndexingJobs, err = a.Srv().Store().Job().GetAllByTypePage(rctx, model.JobTypeBlevePostIndexing, 0, numberOfJobsRuns)
if err != nil {
rErr = multierror.Append(errors.Wrap(err, "error while getting bleve post indexing jobs"))
}
jobs.MigrationJobs, err = a.Srv().Store().Job().GetAllByTypePage(rctx, model.JobTypeMigrations, 0, numberOfJobsRuns)
if err != nil {
rErr = multierror.Append(errors.Wrap(err, "error while getting migration jobs"))
server/channels/app/support_packet_test.go
+1 -7
View File
@@ -502,7 +502,6 @@ func TestGetSupportPacketJobList(t *testing.T) {
assert.Empty(t, jobs.MessageExportJobs)
assert.Empty(t, jobs.ElasticPostIndexingJobs)
assert.Empty(t, jobs.ElasticPostAggregationJobs)
assert.Empty(t, jobs.BlevePostIndexingJobs)
assert.Empty(t, jobs.MigrationJobs)
})
@@ -527,7 +526,6 @@ func TestGetSupportPacketJobList(t *testing.T) {
getJob(model.JobTypeMessageExport),
getJob(model.JobTypeElasticsearchPostIndexing),
getJob(model.JobTypeElasticsearchPostAggregation),
getJob(model.JobTypeBlevePostIndexing),
getJob(model.JobTypeMigrations),
}
@@ -577,13 +575,9 @@ func TestGetSupportPacketJobList(t *testing.T) {
require.Len(t, jobs.ElasticPostAggregationJobs, 1, "Should have 1 elasticsearch post aggregation job")
verifyJob(t, expectedJobs[4], jobs.ElasticPostAggregationJobs[0])
// Verify bleve post indexing jobs
require.Len(t, jobs.BlevePostIndexingJobs, 1, "Should have 1 bleve post indexing job")
verifyJob(t, expectedJobs[5], jobs.BlevePostIndexingJobs[0])
// Verify migration jobs
require.Len(t, jobs.MigrationJobs, 1, "Should have 1 migration job")
verifyJob(t, expectedJobs[6], jobs.MigrationJobs[0])
verifyJob(t, expectedJobs[5], jobs.MigrationJobs[0])
})
}
server/channels/store/searchtest/channel_layer.go
+5 -5
View File
@@ -17,7 +17,7 @@ var searchChannelStoreTests = []searchTest{
{
Name: "Should be able to autocomplete a channel by name",
Fn: testAutocompleteChannelByName,
Tags: []string{EngineElasticSearch, EngineBleve},
Tags: []string{EngineElasticSearch},
},
{
Name: "Should be able to autocomplete a channel by name (Postgres)",
@@ -32,7 +32,7 @@ var searchChannelStoreTests = []searchTest{
{
Name: "Should be able to autocomplete a channel by a part of its name when has parts splitted by - character",
Fn: testAutocompleteChannelByNameSplittedWithDashChar,
Tags: []string{EngineElasticSearch, EngineBleve},
Tags: []string{EngineElasticSearch},
},
{
Name: "Should be able to autocomplete a channel by a part of its name when has parts splitted by - character (Postgres)",
@@ -42,12 +42,12 @@ var searchChannelStoreTests = []searchTest{
{
Name: "Should be able to autocomplete a channel by a part of its name when has parts splitted by _ character",
Fn: testAutocompleteChannelByNameSplittedWithUnderscoreChar,
Tags: []string{EngineElasticSearch, EngineBleve},
Tags: []string{EngineElasticSearch},
},
{
Name: "Should be able to autocomplete a channel by a part of its display name when has parts splitted by whitespace character",
Fn: testAutocompleteChannelByDisplayNameSplittedByWhitespaces,
Tags: []string{EngineElasticSearch, EngineBleve},
Tags: []string{EngineElasticSearch},
},
{
Name: "Should be able to autocomplete retrieving all channels if the term is empty",
@@ -57,7 +57,7 @@ var searchChannelStoreTests = []searchTest{
{
Name: "Should be able to autocomplete channels in a case insensitive manner",
Fn: testSearchChannelsInCaseInsensitiveManner,
Tags: []string{EngineElasticSearch, EngineBleve},
Tags: []string{EngineElasticSearch},
},
{
Name: "Should be able to autocomplete channels in a case insensitive manner (Postgres)",
server/channels/store/searchtest/file_info_layer.go
+1 -1
View File
@@ -23,7 +23,7 @@ var searchFileInfoStoreTests = []searchTest{
{
Name: "Should be able to search posts using pagination",
Fn: testFileInfoSearchFileInfosWithPagination,
Tags: []string{EngineElasticSearch, EngineBleve},
Tags: []string{EngineElasticSearch},
},
{
Name: "Should be able to search for exact phrases in quotes",
server/channels/store/searchtest/post_layer.go
+1 -1
View File
@@ -23,7 +23,7 @@ var searchPostStoreTests = []searchTest{
{
Name: "Should be able to search posts using pagination",
Fn: testSearchPostsWithPagination,
Tags: []string{EngineElasticSearch, EngineBleve},
Tags: []string{EngineElasticSearch},
},
{
Name: "Should return pinned and unpinned posts",
server/channels/store/searchtest/testlib.go
-1
View File
@@ -14,7 +14,6 @@ const (
EngineAll = "all"
EnginePostgres = "postgres"
EngineElasticSearch = "elasticsearch"
EngineBleve = "bleve"
)
type SearchTestEngine struct {
server/channels/store/searchtest/user_layer.go
+4 -4
View File
@@ -27,7 +27,7 @@ var searchUserStoreTests = []searchTest{
{
Name: "Should honor team restrictions when autocompleting users",
Fn: testHonorTeamRestrictionsAutocompletingUsers,
Tags: []string{EngineElasticSearch, EngineBleve},
Tags: []string{EngineElasticSearch},
},
{
Name: "Should return nothing if the user can't access the channels of a given search",
@@ -71,17 +71,17 @@ var searchUserStoreTests = []searchTest{
{
Name: "Should be able to autocomplete a user by part of its username splitted by Dot",
Fn: testAutocompleteUserByUsernameWithDot,
Tags: []string{EngineElasticSearch, EngineBleve},
Tags: []string{EngineElasticSearch},
},
{
Name: "Should be able to autocomplete a user by part of its username splitted by underscore",
Fn: testAutocompleteUserByUsernameWithUnderscore,
Tags: []string{EngineElasticSearch, EngineBleve},
Tags: []string{EngineElasticSearch},
},
{
Name: "Should be able to autocomplete a user by part of its username splitted by hyphen",
Fn: testAutocompleteUserByUsernameWithHyphen,
Tags: []string{EngineElasticSearch, EngineBleve},
Tags: []string{EngineElasticSearch},
},
{
Name: "Should escape the percentage character",
server/channels/testlib/testdata/mysql_migration_warmup.sql
Vendored
+2 -2
View File
File diff suppressed because one or more lines are too long
server/channels/testlib/testdata/postgres_migration_warmup.sql
Vendored
+2 -2
View File
File diff suppressed because one or more lines are too long
server/go.mod
-25
View File
@@ -10,7 +10,6 @@ require (
github.com/aws/aws-sdk-go v1.55.7
github.com/bep/imagemeta v0.12.0
github.com/blang/semver/v4 v4.0.0
github.com/blevesearch/bleve/v2 v2.5.0
github.com/cespare/xxhash/v2 v2.3.0
github.com/dgryski/dgoogauth v0.0.0-20190221195224-5a805980a5f3
github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a
@@ -82,7 +81,6 @@ require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/JalfResi/justext v0.0.0-20221106200834-be571e3e3052 // indirect
github.com/PuerkitoBio/goquery v1.10.3 // indirect
github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect
github.com/STARRY-S/zip v0.2.1 // indirect
github.com/advancedlogic/GoOse v0.0.0-20231203033844-ae6b36caf275 // indirect
github.com/andybalholm/brotli v1.1.2-0.20250424173009-453214e765f3 // indirect
@@ -94,23 +92,6 @@ require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.22.0 // indirect
github.com/bits-and-blooms/bloom/v3 v3.7.0 // indirect
github.com/blevesearch/bleve_index_api v1.2.8 // indirect
github.com/blevesearch/geo v0.2.2 // indirect
github.com/blevesearch/go-faiss v1.0.25 // indirect
github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
github.com/blevesearch/gtreap v0.1.1 // indirect
github.com/blevesearch/mmap-go v1.0.4 // indirect
github.com/blevesearch/scorch_segment_api/v2 v2.3.10 // indirect
github.com/blevesearch/segment v0.9.1 // indirect
github.com/blevesearch/snowballstem v0.9.0 // indirect
github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect
github.com/blevesearch/vellum v1.1.0 // indirect
github.com/blevesearch/zapx/v11 v11.4.2 // indirect
github.com/blevesearch/zapx/v12 v12.4.2 // indirect
github.com/blevesearch/zapx/v13 v13.4.2 // indirect
github.com/blevesearch/zapx/v14 v14.4.2 // indirect
github.com/blevesearch/zapx/v15 v15.4.2 // indirect
github.com/blevesearch/zapx/v16 v16.2.3 // indirect
github.com/bodgit/plumbing v1.3.0 // indirect
github.com/bodgit/sevenzip v1.6.0 // indirect
github.com/bodgit/windows v1.0.1 // indirect
@@ -134,7 +115,6 @@ require (
github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v1.0.0 // indirect
github.com/gomodule/redigo v2.0.0+incompatible // indirect
github.com/google/btree v1.1.3 // indirect
github.com/google/uuid v1.6.0 // indirect
@@ -154,7 +134,6 @@ require (
github.com/isacikgoz/fuzzy v0.2.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jonboulle/clockwork v0.5.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
@@ -169,9 +148,6 @@ require (
github.com/minio/crc64nvme v1.0.2 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/minlz v1.0.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mschoch/smat v0.2.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/ncruces/go-strftime v0.1.9 // indirect
github.com/nwaples/rardecode/v2 v2.1.0 // indirect
@@ -214,7 +190,6 @@ require (
github.com/ulikunitz/xz v0.5.12 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/wiggin77/srslog v1.0.1 // indirect
go.etcd.io/bbolt v1.4.0 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/otel v1.35.0 // indirect
go.opentelemetry.io/otel/metric v1.35.0 // indirect
server/go.sum
-51
View File
@@ -36,8 +36,6 @@ github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lpr
github.com/PuerkitoBio/goquery v1.4.1/go.mod h1:T9ezsOHcCrDCgA8aF1Cqr3sSYbO/xgdy8/R/XiIMAhA=
github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2JW2gggRdg=
github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0=
github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg=
github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4=
github.com/advancedlogic/GoOse v0.0.0-20231203033844-ae6b36caf275 h1:Kuhf+w+ilOGoXaR4O4nZ6Dp+ZS83LdANUjwyMXsPGX4=
@@ -76,49 +74,12 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
github.com/bep/imagemeta v0.12.0 h1:ARf+igs5B7pf079LrqRnwzQ/wEB8Q9v4NSDRZO1/F5k=
github.com/bep/imagemeta v0.12.0/go.mod h1:23AF6O+4fUi9avjiydpKLStUNtJr5hJB4rarG18JpN8=
github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4=
github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/bits-and-blooms/bloom/v3 v3.7.0 h1:VfknkqV4xI+PsaDIsoHueyxVDZrfvMn56jeWUzvzdls=
github.com/bits-and-blooms/bloom/v3 v3.7.0/go.mod h1:VKlUSvp0lFIYqxJjzdnSsZEw4iHb1kOL2tfHTgyJBHg=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
github.com/blevesearch/bleve/v2 v2.5.0 h1:HzYqBy/5/M9Ul9ESEmXzN/3Jl7YpmWBdHM/+zzv/3k4=
github.com/blevesearch/bleve/v2 v2.5.0/go.mod h1:PcJzTPnEynO15dCf9isxOga7YFRa/cMSsbnRwnszXUk=
github.com/blevesearch/bleve_index_api v1.2.8 h1:Y98Pu5/MdlkRyLM0qDHostYo7i+Vv1cDNhqTeR4Sy6Y=
github.com/blevesearch/bleve_index_api v1.2.8/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0=
github.com/blevesearch/geo v0.2.2 h1:nNy+1QNErPyv1Dqpq+MUiykqtsDIU7Fn0A985i+ywIo=
github.com/blevesearch/geo v0.2.2/go.mod h1:K56Q33AzXt2YExVHGObtmRSFYZKYGv0JEN5mdacJJR8=
github.com/blevesearch/go-faiss v1.0.25 h1:lel1rkOUGbT1CJ0YgzKwC7k+XH0XVBHnCVWahdCXk4U=
github.com/blevesearch/go-faiss v1.0.25/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk=
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y=
github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk=
github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
github.com/blevesearch/scorch_segment_api/v2 v2.3.10 h1:Yqk0XD1mE0fDZAJXTjawJ8If/85JxnLd8v5vG/jWE/s=
github.com/blevesearch/scorch_segment_api/v2 v2.3.10/go.mod h1:Z3e6ChN3qyN35yaQpl00MfI5s8AxUJbpTR/DL8QOQ+8=
github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A=
github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ=
github.com/blevesearch/vellum v1.1.0 h1:CinkGyIsgVlYf8Y2LUQHvdelgXr6PYuvoDIajq6yR9w=
github.com/blevesearch/vellum v1.1.0/go.mod h1:QgwWryE8ThtNPxtgWJof5ndPfx0/YMBh+W2weHKPw8Y=
github.com/blevesearch/zapx/v11 v11.4.2 h1:l46SV+b0gFN+Rw3wUI1YdMWdSAVhskYuvxlcgpQFljs=
github.com/blevesearch/zapx/v11 v11.4.2/go.mod h1:4gdeyy9oGa/lLa6D34R9daXNUvfMPZqUYjPwiLmekwc=
github.com/blevesearch/zapx/v12 v12.4.2 h1:fzRbhllQmEMUuAQ7zBuMvKRlcPA5ESTgWlDEoB9uQNE=
github.com/blevesearch/zapx/v12 v12.4.2/go.mod h1:TdFmr7afSz1hFh/SIBCCZvcLfzYvievIH6aEISCte58=
github.com/blevesearch/zapx/v13 v13.4.2 h1:46PIZCO/ZuKZYgxI8Y7lOJqX3Irkc3N8W82QTK3MVks=
github.com/blevesearch/zapx/v13 v13.4.2/go.mod h1:knK8z2NdQHlb5ot/uj8wuvOq5PhDGjNYQQy0QDnopZk=
github.com/blevesearch/zapx/v14 v14.4.2 h1:2SGHakVKd+TrtEqpfeq8X+So5PShQ5nW6GNxT7fWYz0=
github.com/blevesearch/zapx/v14 v14.4.2/go.mod h1:rz0XNb/OZSMjNorufDGSpFpjoFKhXmppH9Hi7a877D8=
github.com/blevesearch/zapx/v15 v15.4.2 h1:sWxpDE0QQOTjyxYbAVjt3+0ieu8NCE0fDRaFxEsp31k=
github.com/blevesearch/zapx/v15 v15.4.2/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw=
github.com/blevesearch/zapx/v16 v16.2.3 h1:7Y0r+a3diEvlazsncexq1qoFOcBd64xwMS7aDm4lo1s=
github.com/blevesearch/zapx/v16 v16.2.3/go.mod h1:wVJ+GtURAaRG9KQAMNYyklq0egV+XJlGcXNCE0OFjjA=
github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU=
github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs=
github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A=
@@ -256,8 +217,6 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -370,8 +329,6 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@@ -465,14 +422,9 @@ github.com/minio/minio-go/v7 v7.0.91/go.mod h1:uvMUcGrpgeSAAI6+sD3818508nUyMULw9
github.com/minio/minlz v1.0.0 h1:Kj7aJZ1//LlTP1DM8Jm7lNKvvJS2m74gyyXXn3+uJWQ=
github.com/minio/minlz v1.0.0/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -709,8 +661,6 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/goldmark v1.7.11 h1:ZCxLyDMtz0nT2HFfsYG8WZ47Trip2+JyLysKcMYE5bo=
github.com/yuin/goldmark v1.7.11/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk=
go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk=
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -1051,7 +1001,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
server/i18n/en.json
-200
View File
@@ -4766,10 +4766,6 @@
"id": "app.channel.get_channels.not_found.app_error",
"translation": "No channels were found."
},
{
"id": "app.channel.get_channels_batch_for_indexing.get.app_error",
"translation": "Unable to get the channels batch for indexing."
},
{
"id": "app.channel.get_channels_by_ids.app_error",
"translation": "Unable to get channels by ids."
@@ -6658,10 +6654,6 @@
"id": "app.post.get_direct_posts.app_error",
"translation": "Unable to get direct posts."
},
{
"id": "app.post.get_files_batch_for_indexing.get.app_error",
"translation": "Unable to get the files batch for indexing."
},
{
"id": "app.post.get_flagged_posts.app_error",
"translation": "Unable to get the flagged posts."
@@ -6682,10 +6674,6 @@
"id": "app.post.get_posts_around.get.app_error",
"translation": "Unable to get the posts for the channel."
},
{
"id": "app.post.get_posts_batch_for_indexing.get.app_error",
"translation": "Unable to get the posts batch for indexing."
},
{
"id": "app.post.get_posts_created_at.app_error",
"translation": "Unable to get the posts for the channel."
@@ -7800,174 +7788,6 @@
"id": "basic_security_check.url.too_long_error",
"translation": "URL is too long"
},
{
"id": "bleveengine.already_started.error",
"translation": "Bleve is already started."
},
{
"id": "bleveengine.create_channel_index.error",
"translation": "Error creating the bleve channel index."
},
{
"id": "bleveengine.create_file_index.error",
"translation": "Error creating the bleve file index."
},
{
"id": "bleveengine.create_post_index.error",
"translation": "Error creating the bleve post index."
},
{
"id": "bleveengine.create_user_index.error",
"translation": "Error creating the bleve user index."
},
{
"id": "bleveengine.delete_channel.error",
"translation": "Failed to delete the channel."
},
{
"id": "bleveengine.delete_channel_posts.error",
"translation": "Failed to delete channel posts"
},
{
"id": "bleveengine.delete_file.error",
"translation": "Failed to delete the file."
},
{
"id": "bleveengine.delete_files_batch.error",
"translation": "Failed to delete the files."
},
{
"id": "bleveengine.delete_post.error",
"translation": "Failed to delete the post."
},
{
"id": "bleveengine.delete_post_files.error",
"translation": "Failed to delete the post files."
},
{
"id": "bleveengine.delete_user.error",
"translation": "Failed to delete the user."
},
{
"id": "bleveengine.delete_user_files.error",
"translation": "Failed to delete the user files."
},
{
"id": "bleveengine.delete_user_posts.error",
"translation": "Failed to delete user posts"
},
{
"id": "bleveengine.index_channel.error",
"translation": "Failed to index the channel."
},
{
"id": "bleveengine.index_file.error",
"translation": "Failed to index the file."
},
{
"id": "bleveengine.index_post.error",
"translation": "Failed to index the post."
},
{
"id": "bleveengine.index_user.error",
"translation": "Failed to index the user."
},
{
"id": "bleveengine.indexer.do_job.bulk_index_channels.batch_error",
"translation": "Failed to index channel batch."
},
{
"id": "bleveengine.indexer.do_job.bulk_index_files.batch_error",
"translation": "Failed to index file batch."
},
{
"id": "bleveengine.indexer.do_job.bulk_index_posts.batch_error",
"translation": "Failed to index post batch."
},
{
"id": "bleveengine.indexer.do_job.bulk_index_users.batch_error",
"translation": "Failed to index user batch."
},
{
"id": "bleveengine.indexer.do_job.engine_inactive",
"translation": "Failed to run Bleve index job: engine is inactive."
},
{
"id": "bleveengine.indexer.do_job.get_oldest_entity.error",
"translation": "The oldest entity (user, channel or post) could not be retrieved from the database."
},
{
"id": "bleveengine.indexer.do_job.parse_end_time.error",
"translation": "Bleve indexing worker failed to parse the end time."
},
{
"id": "bleveengine.indexer.do_job.parse_start_time.error",
"translation": "Bleve indexing worker failed to parse the start time."
},
{
"id": "bleveengine.indexer.index_batch.nothing_left_to_index.error",
"translation": "Trying to index a new batch when all the entities are completed."
},
{
"id": "bleveengine.purge_channel_index.error",
"translation": "Failed to purge channel indexes."
},
{
"id": "bleveengine.purge_file_index.error",
"translation": "Failed to purge file indexes."
},
{
"id": "bleveengine.purge_list.not_implemented",
"translation": "Purge list feature is not available for Bleve."
},
{
"id": "bleveengine.purge_post_index.error",
"translation": "Failed to purge post indexes."
},
{
"id": "bleveengine.purge_user_index.error",
"translation": "Failed to purge user indexes."
},
{
"id": "bleveengine.search_channels.error",
"translation": "Channel search failed to complete."
},
{
"id": "bleveengine.search_files.error",
"translation": "File search failed to complete."
},
{
"id": "bleveengine.search_posts.error",
"translation": "Post search failed to complete."
},
{
"id": "bleveengine.search_users_in_channel.nuchan.error",
"translation": "User search failed to complete."
},
{
"id": "bleveengine.search_users_in_channel.uchan.error",
"translation": "User search failed to complete."
},
{
"id": "bleveengine.search_users_in_team.error",
"translation": "User search failed to complete."
},
{
"id": "bleveengine.stop_channel_index.error",
"translation": "Failed to close channel index."
},
{
"id": "bleveengine.stop_file_index.error",
"translation": "Failed to close file index."
},
{
"id": "bleveengine.stop_post_index.error",
"translation": "Failed to close post index."
},
{
"id": "bleveengine.stop_user_index.error",
"translation": "Failed to close user index."
},
{
"id": "brand.save_brand_image.check_image_limits.app_error",
"translation": "Image limits check failed. Resolution is too high."
@@ -9180,22 +9000,6 @@
"id": "model.config.is_valid.atmos_camo_image_proxy_url.app_error",
"translation": "Invalid RemoteImageProxyURL for atmos/camo. Must be set to your shared key."
},
{
"id": "model.config.is_valid.bleve_search.bulk_indexing_batch_size.app_error",
"translation": "Bleve Bulk Indexing Batch Size must be at least {{.BatchSize}}."
},
{
"id": "model.config.is_valid.bleve_search.enable_autocomplete.app_error",
"translation": "Bleve EnableIndexing setting must be set to true when Bleve EnableAutocomplete is set to true"
},
{
"id": "model.config.is_valid.bleve_search.enable_searching.app_error",
"translation": "Bleve EnableIndexing setting must be set to true when Bleve EnableSearching is set to true"
},
{
"id": "model.config.is_valid.bleve_search.filename.app_error",
"translation": "Bleve IndexingDir setting must be set when Bleve EnableIndexing is set to true"
},
{
"id": "model.config.is_valid.cache_type.app_error",
"translation": "Cache type must be either lru or redis."
@@ -10692,10 +10496,6 @@
"id": "plugin_reattach_request.is_valid.plugin_reattach_config.app_error",
"translation": "Missing plugin reattach config"
},
{
"id": "searchengine.bleve.disabled.error",
"translation": "Error purging Bleve indexes: engine is disabled"
},
{
"id": "sharedchannel.cannot_deliver_post",
"translation": "One or more posts could not be delivered to remote site {{.Remote}} because it is offline. The post(s) will be delivered when the site is online."
server/platform/services/searchengine/bleveengine/bleve.go
-337
View File
@@ -1,337 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package bleveengine
import (
"net/http"
"os"
"path/filepath"
"reflect"
"sync"
"sync/atomic"
"time"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
"github.com/blevesearch/bleve/v2/analysis/analyzer/standard"
"github.com/blevesearch/bleve/v2/mapping"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
)
const (
EngineName = "bleve"
PostIndex = "posts"
FileIndex = "files"
UserIndex = "users"
ChannelIndex = "channels"
)
type BleveEngine struct {
PostIndex bleve.Index
FileIndex bleve.Index
UserIndex bleve.Index
ChannelIndex bleve.Index
Mutex sync.RWMutex
ready int32
cfg *model.Config
indexSync bool
}
var keywordMapping *mapping.FieldMapping
var standardMapping *mapping.FieldMapping
var dateMapping *mapping.FieldMapping
func init() {
keywordMapping = bleve.NewTextFieldMapping()
keywordMapping.Analyzer = keyword.Name
standardMapping = bleve.NewTextFieldMapping()
standardMapping.Analyzer = standard.Name
dateMapping = bleve.NewNumericFieldMapping()
}
func getChannelIndexMapping() *mapping.IndexMappingImpl {
channelMapping := bleve.NewDocumentMapping()
channelMapping.AddFieldMappingsAt("Id", keywordMapping)
channelMapping.AddFieldMappingsAt("Type", keywordMapping)
channelMapping.AddFieldMappingsAt("TeamId", keywordMapping)
channelMapping.AddFieldMappingsAt("NameSuggest", keywordMapping)
channelMapping.AddFieldMappingsAt("UserIDs", keywordMapping)
channelMapping.AddFieldMappingsAt("TeamMemberIDs", keywordMapping)
indexMapping := bleve.NewIndexMapping()
indexMapping.AddDocumentMapping("_default", channelMapping)
return indexMapping
}
func getPostIndexMapping() *mapping.IndexMappingImpl {
postMapping := bleve.NewDocumentMapping()
postMapping.AddFieldMappingsAt("Id", keywordMapping)
postMapping.AddFieldMappingsAt("TeamId", keywordMapping)
postMapping.AddFieldMappingsAt("ChannelId", keywordMapping)
postMapping.AddFieldMappingsAt("UserId", keywordMapping)
postMapping.AddFieldMappingsAt("CreateAt", dateMapping)
postMapping.AddFieldMappingsAt("Message", standardMapping)
postMapping.AddFieldMappingsAt("Type", keywordMapping)
postMapping.AddFieldMappingsAt("Hashtags", standardMapping)
postMapping.AddFieldMappingsAt("Attachments", standardMapping)
indexMapping := bleve.NewIndexMapping()
indexMapping.AddDocumentMapping("_default", postMapping)
return indexMapping
}
func getFileIndexMapping() *mapping.IndexMappingImpl {
fileMapping := bleve.NewDocumentMapping()
fileMapping.AddFieldMappingsAt("Id", keywordMapping)
fileMapping.AddFieldMappingsAt("CreatorId", keywordMapping)
fileMapping.AddFieldMappingsAt("ChannelId", keywordMapping)
fileMapping.AddFieldMappingsAt("CreateAt", dateMapping)
fileMapping.AddFieldMappingsAt("Name", standardMapping)
fileMapping.AddFieldMappingsAt("Content", standardMapping)
fileMapping.AddFieldMappingsAt("Extension", keywordMapping)
fileMapping.AddFieldMappingsAt("Content", standardMapping)
indexMapping := bleve.NewIndexMapping()
indexMapping.AddDocumentMapping("_default", fileMapping)
return indexMapping
}
func getUserIndexMapping() *mapping.IndexMappingImpl {
userMapping := bleve.NewDocumentMapping()
userMapping.AddFieldMappingsAt("Id", keywordMapping)
userMapping.AddFieldMappingsAt("SuggestionsWithFullname", keywordMapping)
userMapping.AddFieldMappingsAt("SuggestionsWithoutFullname", keywordMapping)
userMapping.AddFieldMappingsAt("TeamsIds", keywordMapping)
userMapping.AddFieldMappingsAt("ChannelsIds", keywordMapping)
indexMapping := bleve.NewIndexMapping()
indexMapping.AddDocumentMapping("_default", userMapping)
return indexMapping
}
func NewBleveEngine(cfg *model.Config) *BleveEngine {
return &BleveEngine{
cfg: cfg,
}
}
func (b *BleveEngine) getIndexDir(indexName string) string {
return filepath.Join(*b.cfg.BleveSettings.IndexDir, indexName+".bleve")
}
func (b *BleveEngine) createOrOpenIndex(indexName string, mapping *mapping.IndexMappingImpl) (bleve.Index, error) {
indexPath := b.getIndexDir(indexName)
if index, err := bleve.Open(indexPath); err == nil {
return index, nil
}
index, err := bleve.NewUsing(indexPath, mapping, "scorch", "scorch", map[string]any{
"forceSegmentType": "zap",
"forceSegmentVersion": 15,
})
if err != nil {
return nil, err
}
return index, nil
}
func (b *BleveEngine) openIndexes() *model.AppError {
if atomic.LoadInt32(&b.ready) != 0 {
return model.NewAppError("Bleveengine.Start", "bleveengine.already_started.error", nil, "", http.StatusInternalServerError)
}
var err error
b.PostIndex, err = b.createOrOpenIndex(PostIndex, getPostIndexMapping())
if err != nil {
return model.NewAppError("Bleveengine.Start", "bleveengine.create_post_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
b.FileIndex, err = b.createOrOpenIndex(FileIndex, getFileIndexMapping())
if err != nil {
return model.NewAppError("Bleveengine.Start", "bleveengine.create_file_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
b.UserIndex, err = b.createOrOpenIndex(UserIndex, getUserIndexMapping())
if err != nil {
return model.NewAppError("Bleveengine.Start", "bleveengine.create_user_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
b.ChannelIndex, err = b.createOrOpenIndex(ChannelIndex, getChannelIndexMapping())
if err != nil {
return model.NewAppError("Bleveengine.Start", "bleveengine.create_channel_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
atomic.StoreInt32(&b.ready, 1)
return nil
}
func (b *BleveEngine) Start() *model.AppError {
if !*b.cfg.BleveSettings.EnableIndexing || *b.cfg.BleveSettings.IndexDir == "" {
return nil
}
b.Mutex.Lock()
defer b.Mutex.Unlock()
mlog.Info("EXPERIMENTAL: Starting Bleve")
return b.openIndexes()
}
func (b *BleveEngine) closeIndexes() *model.AppError {
if b.IsActive() {
if err := b.PostIndex.Close(); err != nil {
return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_post_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
if err := b.FileIndex.Close(); err != nil {
return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_file_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
if err := b.UserIndex.Close(); err != nil {
return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_user_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
if err := b.ChannelIndex.Close(); err != nil {
return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_channel_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
}
atomic.StoreInt32(&b.ready, 0)
return nil
}
func (b *BleveEngine) Stop() *model.AppError {
b.Mutex.Lock()
defer b.Mutex.Unlock()
mlog.Info("Stopping Bleve")
return b.closeIndexes()
}
func (b *BleveEngine) IsEnabled() bool {
return b.IsIndexingEnabled()
}
func (b *BleveEngine) IsActive() bool {
return atomic.LoadInt32(&b.ready) == 1
}
func (b *BleveEngine) IsIndexingSync() bool {
return b.indexSync
}
func (b *BleveEngine) RefreshIndexes(_ request.CTX) *model.AppError {
return nil
}
func (b *BleveEngine) GetVersion() int {
return 0
}
func (b *BleveEngine) GetFullVersion() string {
return "0"
}
func (b *BleveEngine) GetPlugins() []string {
return []string{}
}
func (b *BleveEngine) GetName() string {
return EngineName
}
func (b *BleveEngine) TestConfig(rctx request.CTX, cfg *model.Config) *model.AppError {
return nil
}
func (b *BleveEngine) deleteIndexes() *model.AppError {
if err := os.RemoveAll(b.getIndexDir(PostIndex)); err != nil {
return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_post_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
if err := os.RemoveAll(b.getIndexDir(UserIndex)); err != nil {
return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_user_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
if err := os.RemoveAll(b.getIndexDir(ChannelIndex)); err != nil {
return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_channel_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
if err := os.RemoveAll(b.getIndexDir(FileIndex)); err != nil {
return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_file_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
}
func (b *BleveEngine) PurgeIndexes(rctx request.CTX) *model.AppError {
if *b.cfg.BleveSettings.IndexDir == "" {
return nil
}
b.Mutex.Lock()
defer b.Mutex.Unlock()
rctx.Logger().Info("PurgeIndexes Bleve")
if err := b.closeIndexes(); err != nil {
return err
}
if err := b.deleteIndexes(); err != nil {
return err
}
return b.openIndexes()
}
func (b *BleveEngine) PurgeIndexList(rctx request.CTX, indexes []string) *model.AppError {
return model.NewAppError("Bleve.PurgeIndex", "bleveengine.purge_list.not_implemented", nil, "not implemented", http.StatusNotFound)
}
func (b *BleveEngine) DataRetentionDeleteIndexes(rctx request.CTX, cutoff time.Time) *model.AppError {
return nil
}
func (b *BleveEngine) IsAutocompletionEnabled() bool {
return *b.cfg.BleveSettings.EnableAutocomplete
}
func (b *BleveEngine) IsIndexingEnabled() bool {
return *b.cfg.BleveSettings.EnableIndexing
}
func (b *BleveEngine) IsSearchEnabled() bool {
return *b.cfg.BleveSettings.EnableSearching
}
func (b *BleveEngine) UpdateConfig(cfg *model.Config) {
b.Mutex.Lock()
defer b.Mutex.Unlock()
if reflect.DeepEqual(cfg.BleveSettings, b.cfg.BleveSettings) {
return
}
mlog.Info("UpdateConf Bleve")
if *cfg.BleveSettings.EnableIndexing != *b.cfg.BleveSettings.EnableIndexing || *cfg.BleveSettings.IndexDir != *b.cfg.BleveSettings.IndexDir {
if err := b.closeIndexes(); err != nil {
mlog.Error("Error closing Bleve indexes to update the config", mlog.Err(err))
return
}
b.cfg = cfg
if err := b.openIndexes(); err != nil {
mlog.Error("Error opening Bleve indexes after updating the config", mlog.Err(err))
}
return
}
b.cfg = cfg
}
server/platform/services/searchengine/bleveengine/bleve_test.go
-235
View File
@@ -1,235 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package bleveengine
import (
"os"
"testing"
"github.com/blevesearch/bleve/v2"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store/searchlayer"
"github.com/mattermost/mattermost/server/v8/channels/store/searchtest"
"github.com/mattermost/mattermost/server/v8/channels/store/sqlstore"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/testlib"
"github.com/mattermost/mattermost/server/v8/platform/services/searchengine"
)
type BleveEngineTestSuite struct {
suite.Suite
SQLSettings *model.SqlSettings
SQLStore *sqlstore.SqlStore
SearchEngine *searchengine.Broker
Store *searchlayer.SearchStore
BleveEngine *BleveEngine
IndexDir string
Context request.CTX
}
func TestBleveEngineTestSuite(t *testing.T) {
suite.Run(t, &BleveEngineTestSuite{
Context: request.TestContext(t),
})
}
func (s *BleveEngineTestSuite) setupIndexes() {
indexDir, err := os.MkdirTemp("", "mmbleve")
if err != nil {
s.Require().FailNow("Cannot setup bleveengine tests: %s", err.Error())
}
s.IndexDir = indexDir
}
func (s *BleveEngineTestSuite) setupStore() {
driverName := os.Getenv("MM_SQLSETTINGS_DRIVERNAME")
if driverName == "" {
driverName = model.DatabaseDriverPostgres
}
s.SQLSettings = storetest.MakeSqlSettings(driverName)
var err error
s.SQLStore, err = sqlstore.New(*s.SQLSettings, s.Context.Logger(), nil, sqlstore.DisableMorphLogging())
if err != nil {
s.Require().FailNow("Cannot initialize store: %s", err.Error())
}
cfg := &model.Config{}
cfg.SetDefaults()
cfg.BleveSettings.EnableIndexing = model.NewPointer(true)
cfg.BleveSettings.EnableSearching = model.NewPointer(true)
cfg.BleveSettings.EnableAutocomplete = model.NewPointer(true)
cfg.BleveSettings.IndexDir = model.NewPointer(s.IndexDir)
cfg.SqlSettings.DisableDatabaseSearch = model.NewPointer(true)
s.SearchEngine = searchengine.NewBroker(cfg)
s.Store = searchlayer.NewSearchLayer(&testlib.TestStore{Store: s.SQLStore}, s.SearchEngine, cfg)
s.BleveEngine = NewBleveEngine(cfg)
s.BleveEngine.indexSync = true
s.SearchEngine.RegisterBleveEngine(s.BleveEngine)
if err := s.BleveEngine.Start(); err != nil {
s.Require().FailNow("Cannot start bleveengine: %s", err.Error())
}
}
func (s *BleveEngineTestSuite) SetupSuite() {
s.setupIndexes()
s.setupStore()
}
func (s *BleveEngineTestSuite) TearDownSuite() {
os.RemoveAll(s.IndexDir)
s.SQLStore.Close()
storetest.CleanupSqlSettings(s.SQLSettings)
}
func (s *BleveEngineTestSuite) TestBleveSearchStoreTests() {
searchTestEngine := &searchtest.SearchTestEngine{
Driver: searchtest.EngineBleve,
}
s.Run("TestSearchChannelStore", func() {
searchtest.TestSearchChannelStore(s.T(), s.Store, searchTestEngine)
})
s.Run("TestSearchUserStore", func() {
searchtest.TestSearchUserStore(s.T(), s.Store, searchTestEngine)
})
s.Run("TestSearchPostStore", func() {
searchtest.TestSearchPostStore(s.T(), s.Store, searchTestEngine)
})
s.Run("TestSearchFileInfoStore", func() {
searchtest.TestSearchFileInfoStore(s.T(), s.Store, searchTestEngine)
})
}
func (s *BleveEngineTestSuite) TestDeleteChannelPosts() {
s.Run("Should remove all the posts that belongs to a channel", func() {
s.BleveEngine.PurgeIndexes(s.Context)
teamID := model.NewId()
userID := model.NewId()
channelID := model.NewId()
channelToAvoidID := model.NewId()
for range 10 {
post := createPost(userID, channelID)
appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID)
require.Nil(s.T(), appErr)
}
postToAvoid := createPost(userID, channelToAvoidID)
appErr := s.SearchEngine.BleveEngine.IndexPost(postToAvoid, teamID)
require.Nil(s.T(), appErr)
s.SearchEngine.BleveEngine.DeleteChannelPosts(s.Context, channelID)
doc, err := s.BleveEngine.PostIndex.Document(postToAvoid.Id)
require.NoError(s.T(), err)
require.Equal(s.T(), postToAvoid.Id, doc.ID())
numberDocs, err := s.BleveEngine.PostIndex.DocCount()
require.NoError(s.T(), err)
require.Equal(s.T(), 1, int(numberDocs))
})
s.Run("Shouldn't do anything if there is not posts for the selected channel", func() {
s.BleveEngine.PurgeIndexes(s.Context)
teamID := model.NewId()
userID := model.NewId()
channelID := model.NewId()
channelToDeleteID := model.NewId()
post := createPost(userID, channelID)
appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID)
require.Nil(s.T(), appErr)
s.SearchEngine.BleveEngine.DeleteChannelPosts(s.Context, channelToDeleteID)
_, err := s.BleveEngine.PostIndex.Document(post.Id)
require.NoError(s.T(), err)
numberDocs, err := s.BleveEngine.PostIndex.DocCount()
require.NoError(s.T(), err)
require.Equal(s.T(), 1, int(numberDocs))
})
}
func (s *BleveEngineTestSuite) TestDeleteUserPosts() {
s.Run("Should remove all the posts that belongs to a user", func() {
s.BleveEngine.PurgeIndexes(s.Context)
teamID := model.NewId()
userID := model.NewId()
userToAvoidID := model.NewId()
channelID := model.NewId()
for range 10 {
post := createPost(userID, channelID)
appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID)
require.Nil(s.T(), appErr)
}
postToAvoid := createPost(userToAvoidID, channelID)
appErr := s.SearchEngine.BleveEngine.IndexPost(postToAvoid, teamID)
require.Nil(s.T(), appErr)
s.SearchEngine.BleveEngine.DeleteUserPosts(s.Context, userID)
doc, err := s.BleveEngine.PostIndex.Document(postToAvoid.Id)
require.NoError(s.T(), err)
require.Equal(s.T(), postToAvoid.Id, doc.ID())
numberDocs, err := s.BleveEngine.PostIndex.DocCount()
require.NoError(s.T(), err)
require.Equal(s.T(), 1, int(numberDocs))
})
s.Run("Shouldn't do anything if there is not posts for the selected user", func() {
s.BleveEngine.PurgeIndexes(s.Context)
teamID := model.NewId()
userID := model.NewId()
userToDeleteID := model.NewId()
channelID := model.NewId()
post := createPost(userID, channelID)
appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID)
require.Nil(s.T(), appErr)
s.SearchEngine.BleveEngine.DeleteUserPosts(s.Context, userToDeleteID)
_, err := s.BleveEngine.PostIndex.Document(post.Id)
require.NoError(s.T(), err)
numberDocs, err := s.BleveEngine.PostIndex.DocCount()
require.NoError(s.T(), err)
require.Equal(s.T(), 1, int(numberDocs))
})
}
func (s *BleveEngineTestSuite) TestDeletePosts() {
s.BleveEngine.PurgeIndexes(s.Context)
teamID := model.NewId()
userID := model.NewId()
userToAvoidID := model.NewId()
channelID := model.NewId()
for range 10 {
post := createPost(userID, channelID)
appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID)
require.Nil(s.T(), appErr)
}
postToAvoid := createPost(userToAvoidID, channelID)
appErr := s.SearchEngine.BleveEngine.IndexPost(postToAvoid, teamID)
require.Nil(s.T(), appErr)
query := bleve.NewTermQuery(userID)
query.SetField("UserId")
search := bleve.NewSearchRequest(query)
count, err := s.BleveEngine.deletePosts(search, 1)
require.NoError(s.T(), err)
require.Equal(s.T(), 10, int(count))
doc, err := s.BleveEngine.PostIndex.Document(postToAvoid.Id)
require.NoError(s.T(), err)
require.Equal(s.T(), postToAvoid.Id, doc.ID())
numberDocs, err := s.BleveEngine.PostIndex.DocCount()
require.NoError(s.T(), err)
require.Equal(s.T(), 1, int(numberDocs))
}
server/platform/services/searchengine/bleveengine/common.go
-163
View File
@@ -1,163 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package bleveengine
import (
"strings"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/v8/platform/services/searchengine"
)
type BLVChannel struct {
Id string
Type model.ChannelType
UserIDs []string
TeamId []string
TeamMemberIDs []string
NameSuggest []string
}
type BLVUser struct {
Id string
SuggestionsWithFullname []string
SuggestionsWithoutFullname []string
TeamsIds []string
ChannelsIds []string
}
type BLVPost struct {
Id string
TeamId string
ChannelId string
UserId string
CreateAt int64
Message string
Type string
Hashtags []string
Attachments string
}
type BLVFile struct {
Id string
CreatorId string
ChannelId string
CreateAt int64
Name string
Content string
Extension string
}
func BLVChannelFromChannel(channel *model.Channel, userIDs, teamMemberIDs []string) *BLVChannel {
displayNameInputs := searchengine.GetSuggestionInputsSplitBy(channel.DisplayName, " ")
nameInputs := searchengine.GetSuggestionInputsSplitByMultiple(channel.Name, []string{"-", "_"})
return &BLVChannel{
Id: channel.Id,
Type: channel.Type,
TeamId: []string{channel.TeamId},
NameSuggest: append(displayNameInputs, nameInputs...),
UserIDs: userIDs,
TeamMemberIDs: teamMemberIDs,
}
}
func BLVUserFromUserAndTeams(user *model.User, teamsIds, channelsIds []string) *BLVUser {
usernameSuggestions := searchengine.GetSuggestionInputsSplitByMultiple(user.Username, []string{".", "-", "_"})
fullnameStrings := []string{}
if user.FirstName != "" {
fullnameStrings = append(fullnameStrings, user.FirstName)
}
if user.LastName != "" {
fullnameStrings = append(fullnameStrings, user.LastName)
}
fullnameSuggestions := []string{}
if len(fullnameStrings) > 0 {
fullname := strings.Join(fullnameStrings, " ")
fullnameSuggestions = searchengine.GetSuggestionInputsSplitBy(fullname, " ")
}
nicknameSuggestions := []string{}
if user.Nickname != "" {
nicknameSuggestions = searchengine.GetSuggestionInputsSplitBy(user.Nickname, " ")
}
usernameAndNicknameSuggestions := append(usernameSuggestions, nicknameSuggestions...)
return &BLVUser{
Id: user.Id,
SuggestionsWithFullname: append(usernameAndNicknameSuggestions, fullnameSuggestions...),
SuggestionsWithoutFullname: usernameAndNicknameSuggestions,
TeamsIds: teamsIds,
ChannelsIds: channelsIds,
}
}
func BLVUserFromUserForIndexing(userForIndexing *model.UserForIndexing) *BLVUser {
user := &model.User{
Id: userForIndexing.Id,
Username: userForIndexing.Username,
Nickname: userForIndexing.Nickname,
FirstName: userForIndexing.FirstName,
LastName: userForIndexing.LastName,
CreateAt: userForIndexing.CreateAt,
DeleteAt: userForIndexing.DeleteAt,
}
return BLVUserFromUserAndTeams(user, userForIndexing.TeamsIds, userForIndexing.ChannelsIds)
}
func BLVPostFromPost(post *model.Post, teamId string) *BLVPost {
p := &model.PostForIndexing{
TeamId: teamId,
}
post.ShallowCopy(&p.Post)
return BLVPostFromPostForIndexing(p)
}
func BLVPostFromPostForIndexing(post *model.PostForIndexing) *BLVPost {
return &BLVPost{
Id: post.Id,
TeamId: post.TeamId,
ChannelId: post.ChannelId,
UserId: post.UserId,
CreateAt: post.CreateAt,
Message: post.Message,
Type: post.Type,
Hashtags: strings.Fields(post.Hashtags),
}
}
func splitFilenameWords(name string) string {
result := name
result = strings.ReplaceAll(result, "-", " ")
result = strings.ReplaceAll(result, ".", " ")
return result
}
func BLVFileFromFileInfo(fileInfo *model.FileInfo, channelId string) *BLVFile {
return &BLVFile{
Id: fileInfo.Id,
ChannelId: channelId,
CreatorId: fileInfo.CreatorId,
CreateAt: fileInfo.CreateAt,
Content: fileInfo.Content,
Extension: fileInfo.Extension,
Name: fileInfo.Name + " " + splitFilenameWords(fileInfo.Name),
}
}
func BLVFileFromFileForIndexing(file *model.FileForIndexing) *BLVFile {
return &BLVFile{
Id: file.Id,
ChannelId: file.ChannelId,
CreatorId: file.CreatorId,
CreateAt: file.CreateAt,
Content: file.Content,
Extension: file.Extension,
Name: file.Name + " " + splitFilenameWords(file.Name),
}
}
server/platform/services/searchengine/bleveengine/indexer/indexing_job.go
-638
View File
@@ -1,638 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package indexer
import (
"context"
"net/http"
"strconv"
"sync"
"time"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/jobs"
"github.com/mattermost/mattermost/server/v8/platform/services/searchengine/bleveengine"
)
const (
timeBetweenBatches = 100 * time.Millisecond
estimatedPostCount = 10000000
estimatedFilesCount = 100000
estimatedChannelCount = 100000
estimatedUserCount = 10000
)
type BleveIndexerWorker struct {
name string
// stateMut protects stopCh and helps enforce
// ordering in case subsequent Run or Stop calls are made.
stateMut sync.Mutex
stopCh chan struct{}
stoppedCh chan bool
jobs chan model.Job
jobServer *jobs.JobServer
logger mlog.LoggerIFace
engine *bleveengine.BleveEngine
stopped bool
}
func MakeWorker(jobServer *jobs.JobServer, engine *bleveengine.BleveEngine) *BleveIndexerWorker {
if engine == nil {
return nil
}
const workerName = "BleveIndexer"
return &BleveIndexerWorker{
name: workerName,
stoppedCh: make(chan bool, 1),
jobs: make(chan model.Job),
jobServer: jobServer,
logger: jobServer.Logger().With(mlog.String("worker_name", workerName)),
engine: engine,
stopped: true,
}
}
type IndexingProgress struct {
Now time.Time
StartAtTime int64
EndAtTime int64
LastEntityTime int64
TotalPostsCount int64
DonePostsCount int64
DonePosts bool
LastPostID string
TotalFilesCount int64
DoneFilesCount int64
DoneFiles bool
LastFileID string
TotalChannelsCount int64
DoneChannelsCount int64
DoneChannels bool
LastChannelID string
TotalUsersCount int64
DoneUsersCount int64
DoneUsers bool
LastUserID string
}
func (ip *IndexingProgress) CurrentProgress() int64 {
return (ip.DonePostsCount + ip.DoneChannelsCount + ip.DoneUsersCount + ip.DoneFilesCount) * 100 / (ip.TotalPostsCount + ip.TotalChannelsCount + ip.TotalUsersCount + ip.TotalFilesCount)
}
func (ip *IndexingProgress) IsDone() bool {
return ip.DonePosts && ip.DoneChannels && ip.DoneUsers && ip.DoneFiles
}
func (worker *BleveIndexerWorker) JobChannel() chan<- model.Job {
return worker.jobs
}
func (worker *BleveIndexerWorker) IsEnabled(cfg *model.Config) bool {
return true
}
func (worker *BleveIndexerWorker) Run() {
worker.stateMut.Lock()
// We have to re-assign the stop channel again, because
// it might happen that the job was restarted due to a config change.
if worker.stopped {
worker.stopped = false
worker.stopCh = make(chan struct{})
} else {
worker.stateMut.Unlock()
return
}
// Run is called from a separate goroutine and doesn't return.
// So we cannot Unlock in a defer clause.
worker.stateMut.Unlock()
worker.logger.Debug("Worker Started")
defer func() {
worker.logger.Debug("Worker: Finished")
worker.stoppedCh <- true
}()
for {
select {
case <-worker.stopCh:
worker.logger.Debug("Worker: Received stop signal")
return
case job := <-worker.jobs:
worker.DoJob(&job)
}
}
}
func (worker *BleveIndexerWorker) Stop() {
worker.stateMut.Lock()
defer worker.stateMut.Unlock()
// Set to close, and if already closed before, then return.
if worker.stopped {
return
}
worker.stopped = true
worker.logger.Debug("Worker Stopping")
close(worker.stopCh)
<-worker.stoppedCh
}
func (worker *BleveIndexerWorker) DoJob(job *model.Job) {
logger := worker.logger.With(jobs.JobLoggerFields(job)...)
logger.Debug("Worker: Received a new candidate job.")
var appErr *model.AppError
job, appErr = worker.jobServer.ClaimJob(job)
if appErr != nil {
logger.Warn("Worker: Error occurred while trying to claim job", mlog.Err(appErr))
return
} else if job == nil {
return
}
logger.Info("Worker: Indexing job claimed by worker")
if !worker.engine.IsActive() {
appError := model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.do_job.engine_inactive", nil, "", http.StatusInternalServerError)
if err := worker.jobServer.SetJobError(job, appError); err != nil {
logger.Error("Worker: Failed to run job as ")
}
return
}
progress := IndexingProgress{
Now: time.Now(),
DonePosts: false,
DoneChannels: false,
DoneUsers: false,
DoneFiles: false,
StartAtTime: 0,
EndAtTime: model.GetMillis(),
}
// Extract the start and end times, if they are set.
if startString, ok := job.Data["start_time"]; ok {
startInt, err := strconv.ParseInt(startString, 10, 64)
if err != nil {
logger.Error("Worker: Failed to parse start_time for job", mlog.String("start_time", startString), mlog.Err(err))
appError := model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.do_job.parse_start_time.error", nil, "", http.StatusInternalServerError).Wrap(err)
if err := worker.jobServer.SetJobError(job, appError); err != nil {
logger.Error("Worker: Failed to set job error", mlog.Err(err), mlog.NamedErr("set_error", appError))
}
return
}
progress.StartAtTime = startInt
} else {
// Set start time to oldest entity in the database.
// A user or a channel may be created before any post.
oldestEntityCreationTime, err := worker.jobServer.Store.Post().GetOldestEntityCreationTime()
if err != nil {
logger.Error("Worker: Failed to fetch oldest entity for job.", mlog.String("start_time", startString), mlog.Err(err))
appError := model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.do_job.get_oldest_entity.error", nil, "", http.StatusInternalServerError).Wrap(err)
if err := worker.jobServer.SetJobError(job, appError); err != nil {
logger.Error("Worker: Failed to set job error", mlog.Err(err), mlog.NamedErr("set_error", appError))
}
return
}
progress.StartAtTime = oldestEntityCreationTime
}
progress.LastEntityTime = progress.StartAtTime
if endString, ok := job.Data["end_time"]; ok {
endInt, err := strconv.ParseInt(endString, 10, 64)
if err != nil {
logger.Error("Worker: Failed to parse end_time for job", mlog.String("end_time", endString), mlog.Err(err))
appError := model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.do_job.parse_end_time.error", nil, "", http.StatusInternalServerError).Wrap(err)
if err := worker.jobServer.SetJobError(job, appError); err != nil {
logger.Error("Worker: Failed to set job errorv", mlog.Err(err), mlog.NamedErr("set_error", appError))
}
return
}
progress.EndAtTime = endInt
}
if id, ok := job.Data["start_post_id"]; ok {
progress.LastPostID = id
}
if id, ok := job.Data["start_channel_id"]; ok {
progress.LastChannelID = id
}
if id, ok := job.Data["start_user_id"]; ok {
progress.LastUserID = id
}
if id, ok := job.Data["start_file_id"]; ok {
progress.LastFileID = id
}
// Counting all posts may fail or timeout when the posts table is large. If this happens, log a warning, but carry
// on with the indexing job anyway. The only issue is that the progress % reporting will be inaccurate.
if count, err := worker.jobServer.Store.Post().AnalyticsPostCount(&model.PostCountOptions{}); err != nil {
logger.Warn("Worker: Failed to fetch total post count for job. An estimated value will be used for progress reporting.", mlog.Err(err))
progress.TotalPostsCount = estimatedPostCount
} else {
progress.TotalPostsCount = count
}
// Same possible fail as above can happen when counting channels
if count, err := worker.jobServer.Store.Channel().AnalyticsTypeCount("", ""); err != nil {
logger.Warn("Worker: Failed to fetch total channel count for job. An estimated value will be used for progress reporting.", mlog.Err(err))
progress.TotalChannelsCount = estimatedChannelCount
} else {
progress.TotalChannelsCount = count
}
// Same possible fail as above can happen when counting users
if count, err := worker.jobServer.Store.User().Count(model.UserCountOptions{
IncludeBotAccounts: true, // This actually doesn't join with the bots table
// since ExcludeRegularUsers is set to false
}); err != nil {
logger.Warn("Worker: Failed to fetch total user count for job. An estimated value will be used for progress reporting.", mlog.Err(err))
progress.TotalUsersCount = estimatedUserCount
} else {
progress.TotalUsersCount = count
}
// Counting all files may fail or timeout when the file_info table is large. If this happens, log a warning, but carry
// on with the indexing job anyway. The only issue is that the progress % reporting will be inaccurate.
if count, err := worker.jobServer.Store.FileInfo().CountAll(); err != nil {
logger.Warn("Worker: Failed to fetch total file info count for job. An estimated value will be used for progress reporting.", mlog.Err(err))
progress.TotalFilesCount = estimatedFilesCount
} else {
progress.TotalFilesCount = count
}
var cancelContext request.CTX = request.EmptyContext(worker.logger)
cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background())
cancelWatcherChan := make(chan struct{}, 1)
cancelContext = cancelContext.WithContext(cancelCtx)
go worker.jobServer.CancellationWatcher(cancelContext, job.Id, cancelWatcherChan)
defer cancelCancelWatcher()
for {
select {
case <-cancelWatcherChan:
logger.Info("Worker: Indexing job has been canceled via CancellationWatcher")
if err := worker.jobServer.SetJobCanceled(job); err != nil {
logger.Error("Worker: Failed to mark job as cancelled", mlog.Err(err))
}
return
case <-worker.stopCh:
logger.Info("Worker: Indexing has been canceled via Worker Stop")
if err := worker.jobServer.SetJobCanceled(job); err != nil {
logger.Error("Worker: Failed to mark job as canceled", mlog.Err(err))
}
return
case <-time.After(timeBetweenBatches):
var err *model.AppError
if progress, err = worker.IndexBatch(logger, progress); err != nil {
logger.Error("Worker: Failed to index batch for job", mlog.Err(err))
if err2 := worker.jobServer.SetJobError(job, err); err2 != nil {
logger.Error("Worker: Failed to set job error", mlog.Err(err2), mlog.NamedErr("set_error", err))
}
return
}
// Storing the batch progress in metadata.
if job.Data == nil {
job.Data = make(model.StringMap)
}
job.Data["start_time"] = strconv.FormatInt(progress.LastEntityTime, 10)
job.Data["start_post_id"] = progress.LastPostID
job.Data["start_channel_id"] = progress.LastChannelID
job.Data["start_user_id"] = progress.LastUserID
job.Data["start_file_id"] = progress.LastFileID
job.Data["original_start_time"] = strconv.FormatInt(progress.StartAtTime, 10)
job.Data["end_time"] = strconv.FormatInt(progress.EndAtTime, 10)
if err := worker.jobServer.SetJobProgress(job, progress.CurrentProgress()); err != nil {
logger.Error("Worker: Failed to set progress for job", mlog.Err(err))
if err2 := worker.jobServer.SetJobError(job, err); err2 != nil {
logger.Error("Worker: Failed to set error for job", mlog.Err(err2), mlog.NamedErr("set_error", err))
}
return
}
if progress.IsDone() {
if err := worker.jobServer.SetJobSuccess(job); err != nil {
logger.Error("Worker: Failed to set success for job", mlog.Err(err))
if err2 := worker.jobServer.SetJobError(job, err); err2 != nil {
logger.Error("Worker: Failed to set error for job", mlog.Err(err2), mlog.NamedErr("set_error", err))
}
}
logger.Info("Worker: Indexing job finished successfully")
return
}
}
}
}
func (worker *BleveIndexerWorker) IndexBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) {
if !progress.DonePosts {
return worker.IndexPostsBatch(logger, progress)
}
if !progress.DoneChannels {
return worker.IndexChannelsBatch(logger, progress)
}
if !progress.DoneUsers {
return worker.IndexUsersBatch(logger, progress)
}
if !progress.DoneFiles {
return worker.IndexFilesBatch(logger, progress)
}
return progress, model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.index_batch.nothing_left_to_index.error", nil, "", http.StatusInternalServerError)
}
func (worker *BleveIndexerWorker) IndexPostsBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) {
var posts []*model.PostForIndexing
tries := 0
for posts == nil {
var err error
posts, err = worker.jobServer.Store.Post().GetPostsBatchForIndexing(progress.LastEntityTime, progress.LastPostID, *worker.jobServer.Config().BleveSettings.BatchSize)
if err != nil {
if tries >= 10 {
return progress, model.NewAppError("IndexPostsBatch", "app.post.get_posts_batch_for_indexing.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
logger.Warn("Failed to get posts batch for indexing. Retrying.", mlog.Err(err))
// Wait a bit before trying again.
time.Sleep(15 * time.Second)
}
tries++
}
// Handle zero messages.
if len(posts) == 0 {
progress.DonePosts = true
progress.LastEntityTime = progress.StartAtTime
return progress, nil
}
lastPost, err := worker.BulkIndexPosts(posts, progress)
if err != nil {
return progress, err
}
// Our exit condition is when the last post's createAt reaches the initial endAtTime
// set during job creation.
if progress.EndAtTime <= lastPost.CreateAt {
progress.DonePosts = true
progress.LastEntityTime = progress.StartAtTime
} else {
progress.LastEntityTime = lastPost.CreateAt
}
progress.LastPostID = lastPost.Id
progress.DonePostsCount += int64(len(posts))
return progress, nil
}
func (worker *BleveIndexerWorker) BulkIndexPosts(posts []*model.PostForIndexing, progress IndexingProgress) (*model.Post, *model.AppError) {
batch := worker.engine.PostIndex.NewBatch()
for _, post := range posts {
if post.DeleteAt == 0 {
searchPost := bleveengine.BLVPostFromPostForIndexing(post)
batch.Index(searchPost.Id, searchPost)
} else {
batch.Delete(post.Id)
}
}
worker.engine.Mutex.RLock()
defer worker.engine.Mutex.RUnlock()
if err := worker.engine.PostIndex.Batch(batch); err != nil {
return nil, model.NewAppError("BleveIndexerWorker.BulkIndexPosts", "bleveengine.indexer.do_job.bulk_index_posts.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return &posts[len(posts)-1].Post, nil
}
func (worker *BleveIndexerWorker) IndexFilesBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) {
var files []*model.FileForIndexing
tries := 0
for files == nil {
var err error
files, err = worker.jobServer.Store.FileInfo().GetFilesBatchForIndexing(progress.LastEntityTime, progress.LastFileID, true, *worker.jobServer.Config().BleveSettings.BatchSize)
if err != nil {
if tries >= 10 {
return progress, model.NewAppError("IndexFilesBatch", "app.post.get_files_batch_for_indexing.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
logger.Warn("Failed to get files batch for indexing. Retrying.", mlog.Err(err))
// Wait a bit before trying again.
time.Sleep(15 * time.Second)
}
tries++
}
if len(files) == 0 {
progress.DoneFiles = true
progress.LastEntityTime = progress.StartAtTime
return progress, nil
}
lastFile, err := worker.BulkIndexFiles(files, progress)
if err != nil {
return progress, err
}
// Our exit condition is when the last file's createAt reaches the initial endAtTime
// set during job creation.
if progress.EndAtTime <= lastFile.CreateAt {
progress.DoneFiles = true
progress.LastEntityTime = progress.StartAtTime
} else {
progress.LastEntityTime = lastFile.CreateAt
}
progress.LastFileID = lastFile.Id
progress.DoneFilesCount += int64(len(files))
return progress, nil
}
func (worker *BleveIndexerWorker) BulkIndexFiles(files []*model.FileForIndexing, progress IndexingProgress) (*model.FileInfo, *model.AppError) {
batch := worker.engine.FileIndex.NewBatch()
for _, file := range files {
if file.ShouldIndex() {
searchFile := bleveengine.BLVFileFromFileForIndexing(file)
batch.Index(searchFile.Id, searchFile)
} else {
batch.Delete(file.Id)
}
}
worker.engine.Mutex.RLock()
defer worker.engine.Mutex.RUnlock()
if err := worker.engine.FileIndex.Batch(batch); err != nil {
return nil, model.NewAppError("BleveIndexerWorker.BulkIndexPosts", "bleveengine.indexer.do_job.bulk_index_files.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return &files[len(files)-1].FileInfo, nil
}
func (worker *BleveIndexerWorker) IndexChannelsBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) {
var channels []*model.Channel
tries := 0
for channels == nil {
var nErr error
channels, nErr = worker.jobServer.Store.Channel().GetChannelsBatchForIndexing(progress.LastEntityTime, progress.LastChannelID, *worker.jobServer.Config().BleveSettings.BatchSize)
if nErr != nil {
if tries >= 10 {
return progress, model.NewAppError("BleveIndexerWorker.IndexChannelsBatch", "app.channel.get_channels_batch_for_indexing.get.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
logger.Warn("Failed to get channels batch for indexing. Retrying.", mlog.Err(nErr))
// Wait a bit before trying again.
time.Sleep(15 * time.Second)
}
tries++
}
if len(channels) == 0 {
progress.DoneChannels = true
progress.LastEntityTime = progress.StartAtTime
return progress, nil
}
lastChannel, err := worker.BulkIndexChannels(logger, channels, progress)
if err != nil {
return progress, err
}
// Our exit condition is when the last channel's createAt reaches the initial endAtTime
// set during job creation.
if progress.EndAtTime <= lastChannel.CreateAt {
progress.DoneChannels = true
progress.LastEntityTime = progress.StartAtTime
} else {
progress.LastEntityTime = lastChannel.CreateAt
}
progress.LastChannelID = lastChannel.Id
progress.DoneChannelsCount += int64(len(channels))
return progress, nil
}
func (worker *BleveIndexerWorker) BulkIndexChannels(logger mlog.LoggerIFace, channels []*model.Channel, progress IndexingProgress) (*model.Channel, *model.AppError) {
batch := worker.engine.ChannelIndex.NewBatch()
for _, channel := range channels {
if channel.DeleteAt == 0 {
var userIDs []string
var err error
if channel.Type == model.ChannelTypePrivate {
userIDs, err = worker.jobServer.Store.Channel().GetAllChannelMemberIdsByChannelId(channel.Id)
if err != nil {
return nil, model.NewAppError("BleveIndexerWorker.BulkIndexChannels", "bleveengine.indexer.do_job.bulk_index_channels.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
}
// Get teamMember ids from channelid
teamMemberIDs, err := worker.jobServer.Store.Channel().GetTeamMembersForChannel(channel.Id)
if err != nil {
return nil, model.NewAppError("BleveIndexerWorker.BulkIndexChannels", "bleveengine.indexer.do_job.bulk_index_channels.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
searchChannel := bleveengine.BLVChannelFromChannel(channel, userIDs, teamMemberIDs)
batch.Index(searchChannel.Id, searchChannel)
} else {
batch.Delete(channel.Id)
}
}
worker.engine.Mutex.RLock()
defer worker.engine.Mutex.RUnlock()
if err := worker.engine.ChannelIndex.Batch(batch); err != nil {
return nil, model.NewAppError("BleveIndexerWorker.BulkIndexChannels", "bleveengine.indexer.do_job.bulk_index_channels.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return channels[len(channels)-1], nil
}
func (worker *BleveIndexerWorker) IndexUsersBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) {
var users []*model.UserForIndexing
tries := 0
for users == nil {
if usersBatch, err := worker.jobServer.Store.User().GetUsersBatchForIndexing(progress.LastEntityTime, progress.LastUserID, *worker.jobServer.Config().BleveSettings.BatchSize); err != nil {
if tries >= 10 {
return progress, model.NewAppError("IndexUsersBatch", "app.user.get_users_batch_for_indexing.get_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
logger.Warn("Failed to get users batch for indexing. Retrying.", mlog.Err(err))
// Wait a bit before trying again.
time.Sleep(15 * time.Second)
} else {
users = usersBatch
}
tries++
}
if len(users) == 0 {
progress.DoneUsers = true
progress.LastEntityTime = progress.StartAtTime
return progress, nil
}
lastUser, err := worker.BulkIndexUsers(logger, users, progress)
if err != nil {
return progress, err
}
// Our exit condition is when the last user's createAt reaches the initial endAtTime
// set during job creation.
if progress.EndAtTime <= lastUser.CreateAt {
progress.DoneUsers = true
progress.LastEntityTime = progress.StartAtTime
} else {
progress.LastEntityTime = lastUser.CreateAt
}
progress.LastUserID = lastUser.Id
progress.DoneUsersCount += int64(len(users))
return progress, nil
}
func (worker *BleveIndexerWorker) BulkIndexUsers(logger mlog.LoggerIFace, users []*model.UserForIndexing, progress IndexingProgress) (*model.UserForIndexing, *model.AppError) {
batch := worker.engine.UserIndex.NewBatch()
for _, user := range users {
if user.DeleteAt == 0 {
searchUser := bleveengine.BLVUserFromUserForIndexing(user)
batch.Index(searchUser.Id, searchUser)
} else {
batch.Delete(user.Id)
}
}
worker.engine.Mutex.RLock()
defer worker.engine.Mutex.RUnlock()
if err := worker.engine.UserIndex.Batch(batch); err != nil {
return nil, model.NewAppError("BleveIndexerWorker.BulkIndexUsers", "bleveengine.indexer.do_job.bulk_index_users.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return users[len(users)-1], nil
}
server/platform/services/searchengine/bleveengine/indexer/indexing_job_test.go
-75
View File
@@ -1,75 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package indexer
import (
"errors"
"os"
"testing"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/jobs"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
"github.com/mattermost/mattermost/server/v8/channels/utils/testutils"
"github.com/mattermost/mattermost/server/v8/platform/services/searchengine/bleveengine"
)
func TestBleveIndexer(t *testing.T) {
mockStore := &storetest.Store{}
defer mockStore.AssertExpectations(t)
t.Run("Call GetOldestEntityCreationTime for the first indexing call", func(t *testing.T) {
job := &model.Job{
Id: model.NewId(),
CreateAt: model.GetMillis(),
Status: model.JobStatusPending,
Type: model.JobTypeBlevePostIndexing,
}
retJob := *job
retJob.Status = model.JobStatusInProgress
mockStore.JobStore.
On("UpdateStatusOptimistically", job.Id, model.JobStatusPending, model.JobStatusInProgress).
Return(&retJob, nil)
mockStore.JobStore.On("UpdateOptimistically", mock.AnythingOfType("*model.Job"), model.JobStatusInProgress).Return(true, nil)
mockStore.PostStore.On("GetOldestEntityCreationTime").Return(int64(1), errors.New("")) // intentionally return error to return from function
tempDir, err := os.MkdirTemp("", "setupConfigFile")
require.NoError(t, err)
t.Cleanup(func() {
os.RemoveAll(tempDir)
})
cfg := &model.Config{
BleveSettings: model.BleveSettings{
EnableIndexing: model.NewPointer(true),
IndexDir: model.NewPointer(tempDir),
},
}
jobServer := &jobs.JobServer{
Store: mockStore,
ConfigService: &testutils.StaticConfigService{
Cfg: cfg,
},
}
bleveEngine := bleveengine.NewBleveEngine(cfg)
aErr := bleveEngine.Start()
require.Nil(t, aErr)
worker := &BleveIndexerWorker{
jobServer: jobServer,
engine: bleveEngine,
logger: mlog.CreateConsoleTestLogger(t),
}
worker.DoJob(job)
})
}
server/platform/services/searchengine/bleveengine/search.go
-880
View File
@@ -1,880 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package bleveengine
import (
"net/http"
"strings"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/search/query"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
)
const DeletePostsBatchSize = 500
const DeleteFilesBatchSize = 500
func (b *BleveEngine) IndexPost(post *model.Post, teamId string) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
blvPost := BLVPostFromPost(post, teamId)
if err := b.PostIndex.Index(blvPost.Id, blvPost); err != nil {
return model.NewAppError("Bleveengine.IndexPost", "bleveengine.index_post.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
}
func (b *BleveEngine) SearchPosts(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, model.PostSearchMatches, *model.AppError) {
channelQueries := []query.Query{}
for _, channel := range channels {
channelIdQ := bleve.NewTermQuery(channel.Id)
channelIdQ.SetField("ChannelId")
channelQueries = append(channelQueries, channelIdQ)
}
channelDisjunctionQ := bleve.NewDisjunctionQuery(channelQueries...)
var termQueries []query.Query
var notTermQueries []query.Query
var filters []query.Query
var notFilters []query.Query
typeQ := bleve.NewTermQuery("")
typeQ.SetField("Type")
filters = append(filters, typeQ)
for i, params := range searchParams {
var termOperator query.MatchQueryOperator = query.MatchQueryOperatorAnd
if searchParams[0].OrTerms {
termOperator = query.MatchQueryOperatorOr
}
// Date, channels and FromUsers filters come in all
// searchParams iteration, and as they are global to the
// query, we only need to process them once
if i == 0 {
if len(params.InChannels) > 0 {
inChannels := []query.Query{}
for _, channelId := range params.InChannels {
channelQ := bleve.NewTermQuery(channelId)
channelQ.SetField("ChannelId")
inChannels = append(inChannels, channelQ)
}
filters = append(filters, bleve.NewDisjunctionQuery(inChannels...))
}
if len(params.ExcludedChannels) > 0 {
excludedChannels := []query.Query{}
for _, channelId := range params.ExcludedChannels {
channelQ := bleve.NewTermQuery(channelId)
channelQ.SetField("ChannelId")
excludedChannels = append(excludedChannels, channelQ)
}
notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedChannels...))
}
if len(params.FromUsers) > 0 {
fromUsers := []query.Query{}
for _, userId := range params.FromUsers {
userQ := bleve.NewTermQuery(userId)
userQ.SetField("UserId")
fromUsers = append(fromUsers, userQ)
}
filters = append(filters, bleve.NewDisjunctionQuery(fromUsers...))
}
if len(params.ExcludedUsers) > 0 {
excludedUsers := []query.Query{}
for _, userId := range params.ExcludedUsers {
userQ := bleve.NewTermQuery(userId)
userQ.SetField("UserId")
excludedUsers = append(excludedUsers, userQ)
}
notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedUsers...))
}
if params.OnDate != "" {
before, after := params.GetOnDateMillis()
beforeFloat64 := float64(before)
afterFloat64 := float64(after)
onDateQ := bleve.NewNumericRangeQuery(&beforeFloat64, &afterFloat64)
onDateQ.SetField("CreateAt")
filters = append(filters, onDateQ)
} else {
if params.AfterDate != "" || params.BeforeDate != "" {
var rangeMin, rangeMax *float64
if params.AfterDate != "" {
minf := float64(params.GetAfterDateMillis())
rangeMin = &minf
}
if params.BeforeDate != "" {
maxf := float64(params.GetBeforeDateMillis())
rangeMax = &maxf
}
dateQ := bleve.NewNumericRangeQuery(rangeMin, rangeMax)
dateQ.SetField("CreateAt")
filters = append(filters, dateQ)
}
if params.ExcludedAfterDate != "" {
minf := float64(params.GetExcludedAfterDateMillis())
dateQ := bleve.NewNumericRangeQuery(&minf, nil)
dateQ.SetField("CreateAt")
notFilters = append(notFilters, dateQ)
}
if params.ExcludedBeforeDate != "" {
maxf := float64(params.GetExcludedBeforeDateMillis())
dateQ := bleve.NewNumericRangeQuery(nil, &maxf)
dateQ.SetField("CreateAt")
notFilters = append(notFilters, dateQ)
}
if params.ExcludedDate != "" {
before, after := params.GetExcludedDateMillis()
beforef := float64(before)
afterf := float64(after)
onDateQ := bleve.NewNumericRangeQuery(&beforef, &afterf)
onDateQ.SetField("CreateAt")
notFilters = append(notFilters, onDateQ)
}
}
}
if params.IsHashtag {
if params.Terms != "" {
hashtagQ := bleve.NewMatchQuery(params.Terms)
hashtagQ.SetField("Hashtags")
hashtagQ.SetOperator(termOperator)
termQueries = append(termQueries, hashtagQ)
} else if params.ExcludedTerms != "" {
hashtagQ := bleve.NewMatchQuery(params.ExcludedTerms)
hashtagQ.SetField("Hashtags")
hashtagQ.SetOperator(termOperator)
notTermQueries = append(notTermQueries, hashtagQ)
}
} else {
if params.Terms != "" {
terms := []string{}
for term := range strings.SplitSeq(params.Terms, " ") {
if strings.HasSuffix(term, "*") {
messageQ := bleve.NewWildcardQuery(term)
messageQ.SetField("Message")
termQueries = append(termQueries, messageQ)
} else {
terms = append(terms, term)
}
}
if len(terms) > 0 {
messageQ := bleve.NewMatchQuery(strings.Join(terms, " "))
messageQ.SetField("Message")
messageQ.SetOperator(termOperator)
termQueries = append(termQueries, messageQ)
}
}
if params.ExcludedTerms != "" {
messageQ := bleve.NewMatchQuery(params.ExcludedTerms)
messageQ.SetField("Message")
messageQ.SetOperator(termOperator)
notTermQueries = append(notTermQueries, messageQ)
}
}
}
allTermsQ := bleve.NewBooleanQuery()
allTermsQ.AddMustNot(notTermQueries...)
if searchParams[0].OrTerms {
allTermsQ.AddShould(termQueries...)
} else {
allTermsQ.AddMust(termQueries...)
}
query := bleve.NewBooleanQuery()
query.AddMust(channelDisjunctionQ)
if len(termQueries) > 0 || len(notTermQueries) > 0 {
query.AddMust(allTermsQ)
}
if len(filters) > 0 {
query.AddMust(bleve.NewConjunctionQuery(filters...))
}
if len(notFilters) > 0 {
query.AddMustNot(notFilters...)
}
search := bleve.NewSearchRequestOptions(query, perPage, page*perPage, false)
search.SortBy([]string{"-CreateAt"})
results, err := b.PostIndex.Search(search)
if err != nil {
return nil, nil, model.NewAppError("Bleveengine.SearchPosts", "bleveengine.search_posts.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
postIds := []string{}
matches := model.PostSearchMatches{}
for _, r := range results.Hits {
postIds = append(postIds, r.ID)
}
return postIds, matches, nil
}
func (b *BleveEngine) deletePosts(searchRequest *bleve.SearchRequest, batchSize int) (int64, error) {
resultsCount := int64(0)
for {
// As we are deleting the posts after fetching them, we need to keep
// From fixed always to 0
searchRequest.From = 0
searchRequest.Size = batchSize
results, err := b.PostIndex.Search(searchRequest)
if err != nil {
return -1, err
}
batch := b.PostIndex.NewBatch()
for _, post := range results.Hits {
batch.Delete(post.ID)
}
if err := b.PostIndex.Batch(batch); err != nil {
return -1, err
}
resultsCount += int64(results.Hits.Len())
if results.Hits.Len() < batchSize {
break
}
}
return resultsCount, nil
}
func (b *BleveEngine) DeleteChannelPosts(rctx request.CTX, channelID string) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
query := bleve.NewTermQuery(channelID)
query.SetField("ChannelId")
search := bleve.NewSearchRequest(query)
deleted, err := b.deletePosts(search, DeletePostsBatchSize)
if err != nil {
return model.NewAppError("Bleveengine.DeleteChannelPosts",
"bleveengine.delete_channel_posts.error", nil,
err.Error(), http.StatusInternalServerError)
}
rctx.Logger().Info("Posts for channel deleted", mlog.String("channel_id", channelID), mlog.Int("deleted", deleted))
return nil
}
func (b *BleveEngine) DeleteUserPosts(rctx request.CTX, userID string) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
query := bleve.NewTermQuery(userID)
query.SetField("UserId")
search := bleve.NewSearchRequest(query)
deleted, err := b.deletePosts(search, DeletePostsBatchSize)
if err != nil {
return model.NewAppError("Bleveengine.DeleteUserPosts",
"bleveengine.delete_user_posts.error", nil,
err.Error(), http.StatusInternalServerError)
}
rctx.Logger().Info("Posts for user deleted", mlog.String("user_id", userID), mlog.Int("deleted", deleted))
return nil
}
func (b *BleveEngine) DeletePost(post *model.Post) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
if err := b.PostIndex.Delete(post.Id); err != nil {
return model.NewAppError("Bleveengine.DeletePost", "bleveengine.delete_post.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
}
func (b *BleveEngine) IndexChannel(_ request.CTX, channel *model.Channel, userIDs, teamMemberIDs []string) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
blvChannel := BLVChannelFromChannel(channel, userIDs, teamMemberIDs)
if err := b.ChannelIndex.Index(blvChannel.Id, blvChannel); err != nil {
return model.NewAppError("Bleveengine.IndexChannel", "bleveengine.index_channel.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
}
func (b *BleveEngine) SearchChannels(teamId, userID, term string, isGuest, _ bool) ([]string, *model.AppError) {
// This query essentially boils down to (if teamID is passed):
// match teamID == <>
// AND
// match term == <>
// AND
// match (channelType != 'P' || (<> in userIDs && channelType == 'P'))
// (or if teamID is not passed)
// <> in teamMemberIds
// AND
// match term == <>
// AND
// match (channelType != 'P' || (<> in userIDs && channelType == 'P'))
// (or if isGuest is true)
// <> in teamMemberIds
// AND
// match term == <>
// AND
// match (<> in userIDs)
queries := []query.Query{}
if teamId != "" {
teamIdQ := bleve.NewTermQuery(teamId)
teamIdQ.SetField("TeamId")
queries = append(queries, teamIdQ)
} else {
teamMemberQ := bleve.NewTermQuery(userID)
teamMemberQ.SetField("TeamMemberIDs")
queries = append(queries, teamMemberQ)
}
if isGuest {
userQ := bleve.NewBooleanQuery()
userIDQ := bleve.NewTermQuery(userID)
userIDQ.SetField("UserIDs")
userQ.AddMust(userIDQ)
queries = append(queries, userIDQ)
} else {
boolNotPrivate := bleve.NewBooleanQuery()
privateQ := bleve.NewTermQuery(string(model.ChannelTypePrivate))
privateQ.SetField("Type")
boolNotPrivate.AddMustNot(privateQ)
userQ := bleve.NewBooleanQuery()
userIDQ := bleve.NewTermQuery(userID)
userIDQ.SetField("UserIDs")
userQ.AddMust(userIDQ)
userQ.AddMust(privateQ)
channelTypeQ := bleve.NewDisjunctionQuery()
channelTypeQ.AddQuery(boolNotPrivate)
channelTypeQ.AddQuery(userQ) // userID && 'p'
queries = append(queries, channelTypeQ)
}
if term != "" {
nameSuggestQ := bleve.NewPrefixQuery(strings.ToLower(term))
nameSuggestQ.SetField("NameSuggest")
queries = append(queries, nameSuggestQ)
}
query := bleve.NewSearchRequest(bleve.NewConjunctionQuery(queries...))
query.Size = model.ChannelSearchDefaultLimit
results, err := b.ChannelIndex.Search(query)
if err != nil {
return nil, model.NewAppError("Bleveengine.SearchChannels", "bleveengine.search_channels.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
channelIds := []string{}
for _, result := range results.Hits {
channelIds = append(channelIds, result.ID)
}
return channelIds, nil
}
func (b *BleveEngine) DeleteChannel(channel *model.Channel) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
if err := b.ChannelIndex.Delete(channel.Id); err != nil {
return model.NewAppError("Bleveengine.DeleteChannel", "bleveengine.delete_channel.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
}
func (b *BleveEngine) IndexUser(_ request.CTX, user *model.User, teamsIds, channelsIds []string) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
blvUser := BLVUserFromUserAndTeams(user, teamsIds, channelsIds)
if err := b.UserIndex.Index(blvUser.Id, blvUser); err != nil {
return model.NewAppError("Bleveengine.IndexUser", "bleveengine.index_user.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
}
func (b *BleveEngine) SearchUsersInChannel(teamId, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError) {
if restrictedToChannels != nil && len(restrictedToChannels) == 0 {
return []string{}, []string{}, nil
}
// users in channel
var queries []query.Query
if term != "" {
termQ := bleve.NewPrefixQuery(strings.ToLower(term))
if options.AllowFullNames {
termQ.SetField("SuggestionsWithFullname")
} else {
termQ.SetField("SuggestionsWithoutFullname")
}
queries = append(queries, termQ)
}
channelIdQ := bleve.NewTermQuery(channelId)
channelIdQ.SetField("ChannelsIds")
queries = append(queries, channelIdQ)
query := bleve.NewConjunctionQuery(queries...)
uchanSearch := bleve.NewSearchRequest(query)
uchanSearch.Size = options.Limit
uchan, err := b.UserIndex.Search(uchanSearch)
if err != nil {
return nil, nil, model.NewAppError("Bleveengine.SearchUsersInChannel", "bleveengine.search_users_in_channel.uchan.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
// users not in channel
boolQ := bleve.NewBooleanQuery()
if term != "" {
termQ := bleve.NewPrefixQuery(strings.ToLower(term))
if options.AllowFullNames {
termQ.SetField("SuggestionsWithFullname")
} else {
termQ.SetField("SuggestionsWithoutFullname")
}
boolQ.AddMust(termQ)
}
teamIdQ := bleve.NewTermQuery(teamId)
teamIdQ.SetField("TeamsIds")
boolQ.AddMust(teamIdQ)
outsideChannelIdQ := bleve.NewTermQuery(channelId)
outsideChannelIdQ.SetField("ChannelsIds")
boolQ.AddMustNot(outsideChannelIdQ)
if len(restrictedToChannels) > 0 {
restrictedChannelsQ := bleve.NewDisjunctionQuery()
for _, channelId := range restrictedToChannels {
restrictedChannelQ := bleve.NewTermQuery(channelId)
restrictedChannelsQ.AddQuery(restrictedChannelQ)
}
boolQ.AddMust(restrictedChannelsQ)
}
nuchanSearch := bleve.NewSearchRequest(boolQ)
nuchanSearch.Size = options.Limit
nuchan, err := b.UserIndex.Search(nuchanSearch)
if err != nil {
return nil, nil, model.NewAppError("Bleveengine.SearchUsersInChannel", "bleveengine.search_users_in_channel.nuchan.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
uchanIds := []string{}
for _, result := range uchan.Hits {
uchanIds = append(uchanIds, result.ID)
}
nuchanIds := []string{}
for _, result := range nuchan.Hits {
nuchanIds = append(nuchanIds, result.ID)
}
return uchanIds, nuchanIds, nil
}
func (b *BleveEngine) SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError) {
if restrictedToChannels != nil && len(restrictedToChannels) == 0 {
return []string{}, nil
}
var rootQ query.Query
if term == "" && teamId == "" && restrictedToChannels == nil {
rootQ = bleve.NewMatchAllQuery()
} else {
boolQ := bleve.NewBooleanQuery()
if term != "" {
termQ := bleve.NewPrefixQuery(strings.ToLower(term))
if options.AllowFullNames {
termQ.SetField("SuggestionsWithFullname")
} else {
termQ.SetField("SuggestionsWithoutFullname")
}
boolQ.AddMust(termQ)
}
if len(restrictedToChannels) > 0 {
// restricted channels are already filtered by team, so we
// can search only those matches
restrictedChannelsQ := []query.Query{}
for _, channelId := range restrictedToChannels {
channelIdQ := bleve.NewTermQuery(channelId)
channelIdQ.SetField("ChannelsIds")
restrictedChannelsQ = append(restrictedChannelsQ, channelIdQ)
}
boolQ.AddMust(bleve.NewDisjunctionQuery(restrictedChannelsQ...))
} else {
// this means that we only need to restrict by team
if teamId != "" {
teamIdQ := bleve.NewTermQuery(teamId)
teamIdQ.SetField("TeamsIds")
boolQ.AddMust(teamIdQ)
}
}
rootQ = boolQ
}
search := bleve.NewSearchRequest(rootQ)
search.Size = options.Limit
results, err := b.UserIndex.Search(search)
if err != nil {
return nil, model.NewAppError("Bleveengine.SearchUsersInTeam", "bleveengine.search_users_in_team.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
usersIds := []string{}
for _, r := range results.Hits {
usersIds = append(usersIds, r.ID)
}
return usersIds, nil
}
func (b *BleveEngine) DeleteUser(user *model.User) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
if err := b.UserIndex.Delete(user.Id); err != nil {
return model.NewAppError("Bleveengine.DeleteUser", "bleveengine.delete_user.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
}
func (b *BleveEngine) IndexFile(file *model.FileInfo, channelId string) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
blvFile := BLVFileFromFileInfo(file, channelId)
if err := b.FileIndex.Index(blvFile.Id, blvFile); err != nil {
return model.NewAppError("Bleveengine.IndexFile", "bleveengine.index_file.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
}
func (b *BleveEngine) SearchFiles(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, *model.AppError) {
channelQueries := []query.Query{}
for _, channel := range channels {
channelIdQ := bleve.NewTermQuery(channel.Id)
channelIdQ.SetField("ChannelId")
channelQueries = append(channelQueries, channelIdQ)
}
channelDisjunctionQ := bleve.NewDisjunctionQuery(channelQueries...)
var termQueries []query.Query
var notTermQueries []query.Query
var filters []query.Query
var notFilters []query.Query
for i, params := range searchParams {
var termOperator query.MatchQueryOperator = query.MatchQueryOperatorAnd
if searchParams[0].OrTerms {
termOperator = query.MatchQueryOperatorOr
}
// Date, channels and FromUsers filters come in all
// searchParams iteration, and as they are global to the
// query, we only need to process them once
if i == 0 {
if len(params.InChannels) > 0 {
inChannels := []query.Query{}
for _, channelId := range params.InChannels {
channelQ := bleve.NewTermQuery(channelId)
channelQ.SetField("ChannelId")
inChannels = append(inChannels, channelQ)
}
filters = append(filters, bleve.NewDisjunctionQuery(inChannels...))
}
if len(params.ExcludedChannels) > 0 {
excludedChannels := []query.Query{}
for _, channelId := range params.ExcludedChannels {
channelQ := bleve.NewTermQuery(channelId)
channelQ.SetField("ChannelId")
excludedChannels = append(excludedChannels, channelQ)
}
notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedChannels...))
}
if len(params.FromUsers) > 0 {
fromUsers := []query.Query{}
for _, userId := range params.FromUsers {
userQ := bleve.NewTermQuery(userId)
userQ.SetField("CreatorId")
fromUsers = append(fromUsers, userQ)
}
filters = append(filters, bleve.NewDisjunctionQuery(fromUsers...))
}
if len(params.ExcludedUsers) > 0 {
excludedUsers := []query.Query{}
for _, userId := range params.ExcludedUsers {
userQ := bleve.NewTermQuery(userId)
userQ.SetField("CreatorId")
excludedUsers = append(excludedUsers, userQ)
}
notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedUsers...))
}
if len(params.Extensions) > 0 {
extensions := []query.Query{}
for _, extension := range params.Extensions {
extensionQ := bleve.NewTermQuery(extension)
extensionQ.SetField("Extension")
extensions = append(extensions, extensionQ)
}
filters = append(filters, bleve.NewDisjunctionQuery(extensions...))
}
if len(params.ExcludedExtensions) > 0 {
excludedExtensions := []query.Query{}
for _, extension := range params.ExcludedExtensions {
extensionQ := bleve.NewTermQuery(extension)
extensionQ.SetField("Extension")
excludedExtensions = append(excludedExtensions, extensionQ)
}
notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedExtensions...))
}
if params.OnDate != "" {
before, after := params.GetOnDateMillis()
beforeFloat64 := float64(before)
afterFloat64 := float64(after)
onDateQ := bleve.NewNumericRangeQuery(&beforeFloat64, &afterFloat64)
onDateQ.SetField("CreateAt")
filters = append(filters, onDateQ)
} else {
if params.AfterDate != "" || params.BeforeDate != "" {
var rangeMin, rangeMax *float64
if params.AfterDate != "" {
minf := float64(params.GetAfterDateMillis())
rangeMin = &minf
}
if params.BeforeDate != "" {
maxf := float64(params.GetBeforeDateMillis())
rangeMax = &maxf
}
dateQ := bleve.NewNumericRangeQuery(rangeMin, rangeMax)
dateQ.SetField("CreateAt")
filters = append(filters, dateQ)
}
if params.ExcludedAfterDate != "" {
minf := float64(params.GetExcludedAfterDateMillis())
dateQ := bleve.NewNumericRangeQuery(&minf, nil)
dateQ.SetField("CreateAt")
notFilters = append(notFilters, dateQ)
}
if params.ExcludedBeforeDate != "" {
maxf := float64(params.GetExcludedBeforeDateMillis())
dateQ := bleve.NewNumericRangeQuery(nil, &maxf)
dateQ.SetField("CreateAt")
notFilters = append(notFilters, dateQ)
}
if params.ExcludedDate != "" {
before, after := params.GetExcludedDateMillis()
beforef := float64(before)
afterf := float64(after)
onDateQ := bleve.NewNumericRangeQuery(&beforef, &afterf)
onDateQ.SetField("CreateAt")
notFilters = append(notFilters, onDateQ)
}
}
}
if params.Terms != "" {
terms := []string{}
for term := range strings.SplitSeq(params.Terms, " ") {
if strings.HasSuffix(term, "*") {
nameQ := bleve.NewWildcardQuery(term)
nameQ.SetField("Name")
contentQ := bleve.NewWildcardQuery(term)
contentQ.SetField("Content")
termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
} else {
terms = append(terms, term)
}
}
if len(terms) > 0 {
nameQ := bleve.NewMatchQuery(strings.Join(terms, " "))
nameQ.SetField("Name")
nameQ.SetOperator(termOperator)
contentQ := bleve.NewMatchQuery(strings.Join(terms, " "))
contentQ.SetField("Content")
contentQ.SetOperator(termOperator)
termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
}
}
if params.ExcludedTerms != "" {
nameQ := bleve.NewMatchQuery(params.ExcludedTerms)
nameQ.SetField("Name")
nameQ.SetOperator(termOperator)
contentQ := bleve.NewMatchQuery(params.ExcludedTerms)
contentQ.SetField("Content")
contentQ.SetOperator(termOperator)
notTermQueries = append(notTermQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
}
}
allTermsQ := bleve.NewBooleanQuery()
allTermsQ.AddMustNot(notTermQueries...)
if searchParams[0].OrTerms {
allTermsQ.AddShould(termQueries...)
} else {
allTermsQ.AddMust(termQueries...)
}
query := bleve.NewBooleanQuery()
query.AddMust(channelDisjunctionQ)
if len(termQueries) > 0 || len(notTermQueries) > 0 {
query.AddMust(allTermsQ)
}
if len(filters) > 0 {
query.AddMust(bleve.NewConjunctionQuery(filters...))
}
if len(notFilters) > 0 {
query.AddMustNot(notFilters...)
}
search := bleve.NewSearchRequestOptions(query, perPage, page*perPage, false)
search.SortBy([]string{"-CreateAt"})
results, err := b.FileIndex.Search(search)
if err != nil {
return nil, model.NewAppError("Bleveengine.SearchFiles", "bleveengine.search_files.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
fileIds := []string{}
for _, r := range results.Hits {
fileIds = append(fileIds, r.ID)
}
return fileIds, nil
}
func (b *BleveEngine) DeleteFile(fileID string) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
if err := b.FileIndex.Delete(fileID); err != nil {
return model.NewAppError("Bleveengine.DeleteFile", "bleveengine.delete_file.error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
}
func (b *BleveEngine) deleteFiles(searchRequest *bleve.SearchRequest, batchSize int) (int64, error) {
resultsCount := int64(0)
for {
// As we are deleting the files after fetching them, we need to keep
// From fixed always to 0
searchRequest.From = 0
searchRequest.Size = batchSize
results, err := b.FileIndex.Search(searchRequest)
if err != nil {
return -1, err
}
batch := b.FileIndex.NewBatch()
for _, file := range results.Hits {
batch.Delete(file.ID)
}
if err := b.FileIndex.Batch(batch); err != nil {
return -1, err
}
resultsCount += int64(results.Hits.Len())
if results.Hits.Len() < batchSize {
break
}
}
return resultsCount, nil
}
func (b *BleveEngine) DeleteUserFiles(rctx request.CTX, userID string) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
query := bleve.NewTermQuery(userID)
query.SetField("CreatorId")
search := bleve.NewSearchRequest(query)
deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
if err != nil {
return model.NewAppError("Bleveengine.DeleteUserFiles",
"bleveengine.delete_user_files.error", nil,
err.Error(), http.StatusInternalServerError)
}
rctx.Logger().Info("Files for user deleted", mlog.String("user_id", userID), mlog.Int("deleted", deleted))
return nil
}
func (b *BleveEngine) DeletePostFiles(rctx request.CTX, postID string) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
query := bleve.NewTermQuery(postID)
query.SetField("PostId")
search := bleve.NewSearchRequest(query)
deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
if err != nil {
return model.NewAppError("Bleveengine.DeletePostFiles",
"bleveengine.delete_post_files.error", nil,
err.Error(), http.StatusInternalServerError)
}
rctx.Logger().Info("Files for post deleted", mlog.String("post_id", postID), mlog.Int("deleted", deleted))
return nil
}
func (b *BleveEngine) DeleteFilesBatch(rctx request.CTX, endTime, limit int64) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
endTimeFloat := float64(endTime)
query := bleve.NewNumericRangeQuery(nil, &endTimeFloat)
query.SetField("CreateAt")
search := bleve.NewSearchRequestOptions(query, int(limit), 0, false)
search.SortBy([]string{"-CreateAt"})
deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
if err != nil {
return model.NewAppError("Bleveengine.DeleteFilesBatch",
"bleveengine.delete_files_batch.error", nil,
err.Error(), http.StatusInternalServerError)
}
rctx.Logger().Info("Files in batch deleted", mlog.Int("endTime", endTime), mlog.Int("limit", limit), mlog.Int("deleted", deleted))
return nil
}
server/platform/services/searchengine/bleveengine/testlib.go
-23
View File
@@ -1,23 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package bleveengine
import (
"strconv"
"github.com/mattermost/mattermost/server/public/model"
)
func createPost(userId string, channelId string) *model.Post {
post := &model.Post{
Message: model.NewRandomString(15),
ChannelId: channelId,
PendingPostId: model.NewId() + ":" + strconv.FormatInt(model.GetMillis(), 10),
UserId: userId,
CreateAt: 1000000,
}
post.PreSave()
return post
}
server/platform/services/searchengine/searchengine.go
-12
View File
@@ -17,14 +17,9 @@ func (seb *Broker) RegisterElasticsearchEngine(es SearchEngineInterface) {
seb.ElasticsearchEngine = es
}
func (seb *Broker) RegisterBleveEngine(be SearchEngineInterface) {
seb.BleveEngine = be
}
type Broker struct {
cfg *model.Config
ElasticsearchEngine SearchEngineInterface
BleveEngine SearchEngineInterface
}
func (seb *Broker) UpdateConfig(cfg *model.Config) *model.AppError {
@@ -33,10 +28,6 @@ func (seb *Broker) UpdateConfig(cfg *model.Config) *model.AppError {
seb.ElasticsearchEngine.UpdateConfig(cfg)
}
if seb.BleveEngine != nil {
seb.BleveEngine.UpdateConfig(cfg)
}
return nil
}
@@ -45,9 +36,6 @@ func (seb *Broker) GetActiveEngines() []SearchEngineInterface {
if seb.ElasticsearchEngine != nil && seb.ElasticsearchEngine.IsActive() {
engines = append(engines, seb.ElasticsearchEngine)
}
if seb.BleveEngine != nil && seb.BleveEngine.IsActive() && seb.BleveEngine.IsIndexingEnabled() {
engines = append(engines, seb.BleveEngine)
}
return engines
}
server/platform/services/searchengine/searchengine_test.go
-9
View File
@@ -21,21 +21,12 @@ func TestActiveEngine(t *testing.T) {
esMock.On("IsActive").Return(true)
esMock.On("GetName").Return("elasticsearch")
bleveMock := &mocks.SearchEngineInterface{}
bleveMock.On("IsActive").Return(true)
bleveMock.On("IsIndexingEnabled").Return(true)
bleveMock.On("GetName").Return("bleve")
assert.Equal(t, "database", b.ActiveEngine())
b.ElasticsearchEngine = esMock
assert.Equal(t, "elasticsearch", b.ActiveEngine())
b.ElasticsearchEngine = nil
b.BleveEngine = bleveMock
assert.Equal(t, "bleve", b.ActiveEngine())
b.BleveEngine = nil
*b.cfg.SqlSettings.DisableDatabaseSearch = true
assert.Equal(t, "none", b.ActiveEngine())
server/platform/services/telemetry/telemetry.go
-8
View File
@@ -74,7 +74,6 @@ const (
TrackConfigDisplay = "config_display"
TrackConfigGuestAccounts = "config_guest_accounts"
TrackConfigImageProxy = "config_image_proxy"
TrackConfigBleve = "config_bleve"
TrackConfigExport = "config_export"
TrackConfigWrangler = "config_wrangler"
TrackConfigConnectedWorkspaces = "config_connected_workspaces"
@@ -945,13 +944,6 @@ func (ts *TelemetryService) trackConfig() {
"isdefault_remote_image_proxy_options": isDefault(*cfg.ImageProxySettings.RemoteImageProxyOptions, ""),
}
configs[TrackConfigBleve] = map[string]any{
"enable_indexing": *cfg.BleveSettings.EnableIndexing,
"enable_searching": *cfg.BleveSettings.EnableSearching,
"enable_autocomplete": *cfg.BleveSettings.EnableAutocomplete,
"bulk_indexing_batch_size": *cfg.BleveSettings.BatchSize,
}
configs[TrackConfigExport] = map[string]any{
"retention_days": *cfg.ExportSettings.RetentionDays,
}
server/public/model/client4.go
-16
View File
@@ -442,10 +442,6 @@ func (c *Client4) elasticsearchRoute() string {
return "/elasticsearch"
}
func (c *Client4) bleveRoute() string {
return "/bleve"
}
func (c *Client4) commandsRoute() string {
return "/commands"
}
@@ -6500,18 +6496,6 @@ func (c *Client4) PurgeElasticsearchIndexes(ctx context.Context) (*Response, err
return BuildResponse(r), nil
}
// Bleve Section
// PurgeBleveIndexes immediately deletes all Bleve indexes.
func (c *Client4) PurgeBleveIndexes(ctx context.Context) (*Response, error) {
r, err := c.DoAPIPost(ctx, c.bleveRoute()+"/purge_indexes", "")
if err != nil {
return BuildResponse(r), err
}
defer closeBody(r)
return BuildResponse(r), nil
}
// Data Retention Section
// GetDataRetentionPolicy will get the current global data retention policy details.
server/public/model/config.go
-61
View File
@@ -224,9 +224,6 @@ const (
ElasticsearchSettingsESBackend = "elasticsearch"
ElasticsearchSettingsOSBackend = "opensearch"
BleveSettingsDefaultIndexDir = ""
BleveSettingsDefaultBatchSize = 10000
DataRetentionSettingsDefaultMessageRetentionDays = 365
DataRetentionSettingsDefaultMessageRetentionHours = 0
DataRetentionSettingsDefaultFileRetentionDays = 365
@@ -3146,37 +3143,6 @@ func (s *ElasticsearchSettings) SetDefaults() {
}
}
type BleveSettings struct {
IndexDir *string `access:"experimental_bleve"` // telemetry: none
EnableIndexing *bool `access:"experimental_bleve"`
EnableSearching *bool `access:"experimental_bleve"`
EnableAutocomplete *bool `access:"experimental_bleve"`
BulkIndexingTimeWindowSeconds *int `json:",omitempty"` // telemetry: none
BatchSize *int `access:"experimental_bleve"`
}
func (bs *BleveSettings) SetDefaults() {
if bs.IndexDir == nil {
bs.IndexDir = NewPointer(BleveSettingsDefaultIndexDir)
}
if bs.EnableIndexing == nil {
bs.EnableIndexing = NewPointer(false)
}
if bs.EnableSearching == nil {
bs.EnableSearching = NewPointer(false)
}
if bs.EnableAutocomplete == nil {
bs.EnableAutocomplete = NewPointer(false)
}
if bs.BatchSize == nil {
bs.BatchSize = NewPointer(BleveSettingsDefaultBatchSize)
}
}
type DataRetentionSettings struct {
EnableMessageDeletion *bool `access:"compliance_data_retention_policy"`
EnableFileDeletion *bool `access:"compliance_data_retention_policy"`
@@ -3883,7 +3849,6 @@ type Config struct {
ExperimentalSettings ExperimentalSettings
AnalyticsSettings AnalyticsSettings
ElasticsearchSettings ElasticsearchSettings
BleveSettings BleveSettings
DataRetentionSettings DataRetentionSettings
MessageExportSettings MessageExportSettings
JobSettings JobSettings
@@ -3995,7 +3960,6 @@ func (o *Config) SetDefaults() {
o.ComplianceSettings.SetDefaults()
o.LocalizationSettings.SetDefaults()
o.ElasticsearchSettings.SetDefaults()
o.BleveSettings.SetDefaults()
o.NativeAppSettings.SetDefaults()
o.DataRetentionSettings.SetDefaults()
o.RateLimitSettings.SetDefaults()
@@ -4085,10 +4049,6 @@ func (o *Config) IsValid() *AppError {
return appErr
}
if appErr := o.BleveSettings.isValid(); appErr != nil {
return appErr
}
if appErr := o.DataRetentionSettings.isValid(); appErr != nil {
return appErr
}
@@ -4647,27 +4607,6 @@ func (s *ElasticsearchSettings) isValid() *AppError {
return nil
}
func (bs *BleveSettings) isValid() *AppError {
if *bs.EnableIndexing {
if *bs.IndexDir == "" {
return NewAppError("Config.IsValid", "model.config.is_valid.bleve_search.filename.app_error", nil, "", http.StatusBadRequest)
}
} else {
if *bs.EnableSearching {
return NewAppError("Config.IsValid", "model.config.is_valid.bleve_search.enable_searching.app_error", nil, "", http.StatusBadRequest)
}
if *bs.EnableAutocomplete {
return NewAppError("Config.IsValid", "model.config.is_valid.bleve_search.enable_autocomplete.app_error", nil, "", http.StatusBadRequest)
}
}
minBatchSize := 1
if *bs.BatchSize < minBatchSize {
return NewAppError("Config.IsValid", "model.config.is_valid.bleve_search.bulk_indexing_batch_size.app_error", map[string]any{"BatchSize": minBatchSize}, "", http.StatusBadRequest)
}
return nil
}
func (s *DataRetentionSettings) isValid() *AppError {
if s.MessageRetentionDays == nil || *s.MessageRetentionDays < 0 {
return NewAppError("Config.IsValid", "model.config.is_valid.data_retention.message_retention_days_too_low.app_error", nil, "", http.StatusBadRequest)
server/public/model/config_test.go
+1 -2
View File
@@ -34,8 +34,7 @@ func TestConfigDefaults(t *testing.T) {
if v.Type().Kind() == reflect.Ptr {
// Ignoring these 2 settings.
// TODO: remove them completely in v8.0.
if name == "config.BleveSettings.BulkIndexingTimeWindowSeconds" ||
name == "config.ElasticsearchSettings.BulkIndexingTimeWindowSeconds" ||
if name == "config.ElasticsearchSettings.BulkIndexingTimeWindowSeconds" ||
name == "config.ClusterSettings.EnableExperimentalGossipEncryption" {
return
}
server/public/model/job.go
-2
View File
@@ -15,7 +15,6 @@ const (
JobTypeCLIMessageExport = "cli_message_export"
JobTypeElasticsearchPostIndexing = "elasticsearch_post_indexing"
JobTypeElasticsearchPostAggregation = "elasticsearch_post_aggregation"
JobTypeBlevePostIndexing = "bleve_post_indexing"
JobTypeLdapSync = "ldap_sync"
JobTypeMigrations = "migrations"
JobTypePlugins = "plugins"
@@ -60,7 +59,6 @@ var AllJobTypes = [...]string{
JobTypeMessageExport,
JobTypeElasticsearchPostIndexing,
JobTypeElasticsearchPostAggregation,
JobTypeBlevePostIndexing,
JobTypeLdapSync,
JobTypeMigrations,
JobTypePlugins,
server/public/model/permission.go
+11 -5
View File
@@ -841,6 +841,7 @@ func initializePermissions() {
PermissionScopeSystem,
}
// DEPRECATED
PermissionPurgeBleveIndexes = &Permission{
"purge_bleve_indexes",
"",
@@ -848,6 +849,7 @@ func initializePermissions() {
PermissionScopeSystem,
}
// DEPRECATED
PermissionCreatePostBleveIndexesJob = &Permission{
"create_post_bleve_indexes_job",
"",
@@ -855,6 +857,7 @@ func initializePermissions() {
PermissionScopeSystem,
}
// DEPRECATED
PermissionManagePostBleveIndexesJob = &Permission{
"manage_post_bleve_indexes_job",
"",
@@ -2095,12 +2098,15 @@ func initializePermissions() {
"",
PermissionScopeSystem,
}
// DEPRECATED
PermissionSysconsoleReadExperimentalBleve = &Permission{
"sysconsole_read_experimental_bleve",
"",
"",
PermissionScopeSystem,
}
// DEPRECATED
PermissionSysconsoleWriteExperimentalBleve = &Permission{
"sysconsole_write_experimental_bleve",
"",
@@ -2331,7 +2337,6 @@ func initializePermissions() {
PermissionSysconsoleReadComplianceCustomTermsOfService,
PermissionSysconsoleReadExperimentalFeatures,
PermissionSysconsoleReadExperimentalFeatureFlags,
PermissionSysconsoleReadExperimentalBleve,
PermissionSysconsoleReadProductsBoards,
PermissionSysconsoleReadIPFilters,
}
@@ -2391,7 +2396,6 @@ func initializePermissions() {
PermissionSysconsoleWriteComplianceCustomTermsOfService,
PermissionSysconsoleWriteExperimentalFeatures,
PermissionSysconsoleWriteExperimentalFeatureFlags,
PermissionSysconsoleWriteExperimentalBleve,
PermissionSysconsoleWriteProductsBoards,
PermissionSysconsoleWriteIPFilters,
}
@@ -2451,9 +2455,6 @@ func initializePermissions() {
PermissionManageElasticsearchPostAggregationJob,
PermissionReadElasticsearchPostIndexingJob,
PermissionReadElasticsearchPostAggregationJob,
PermissionPurgeBleveIndexes,
PermissionCreatePostBleveIndexesJob,
PermissionManagePostBleveIndexesJob,
PermissionCreateLdapSyncJob,
PermissionManageLdapSyncJob,
PermissionReadLdapSyncJob,
@@ -2578,6 +2579,11 @@ func initializePermissions() {
PermissionSysconsoleWriteIntegrations,
PermissionSysconsoleReadCompliance,
PermissionSysconsoleWriteCompliance,
PermissionPurgeBleveIndexes,
PermissionCreatePostBleveIndexesJob,
PermissionManagePostBleveIndexesJob,
PermissionSysconsoleReadExperimentalBleve,
PermissionSysconsoleWriteExperimentalBleve,
}
PlaybookScopedPermissions := []*Permission{
server/public/model/role.go
-6
View File
@@ -163,11 +163,6 @@ func init() {
PermissionSysconsoleReadComplianceComplianceMonitoring.Id: {
PermissionReadAudits,
},
PermissionSysconsoleWriteExperimentalBleve.Id: {
PermissionCreatePostBleveIndexesJob,
PermissionPurgeBleveIndexes,
PermissionManagePostBleveIndexesJob,
},
PermissionSysconsoleWriteAuthenticationLdap.Id: {
PermissionCreateLdapSyncJob,
PermissionManageLdapSyncJob,
@@ -265,7 +260,6 @@ func init() {
PermissionSysconsoleReadComplianceCustomTermsOfService.Id,
PermissionSysconsoleReadExperimentalFeatures.Id,
PermissionSysconsoleReadExperimentalFeatureFlags.Id,
PermissionSysconsoleReadExperimentalBleve.Id,
PermissionSysconsoleReadProductsBoards.Id,
}
server/public/model/support_packet.go
-1
View File
@@ -98,7 +98,6 @@ type SupportPacketJobList struct {
MessageExportJobs []*Job `yaml:"message_export_jobs"`
ElasticPostIndexingJobs []*Job `yaml:"elastic_post_indexing_jobs"`
ElasticPostAggregationJobs []*Job `yaml:"elastic_post_aggregation_jobs"`
BlevePostIndexingJobs []*Job `yaml:"bleve_post_indexin_jobs"`
MigrationJobs []*Job `yaml:"migration_jobs"`
}
server/public/model/utils_test.go
+1 -2
View File
@@ -1036,8 +1036,7 @@ func checkNowhereNil(t *testing.T, name string, value any) bool {
case reflect.Ptr:
// Ignoring these 2 settings.
// TODO: remove them completely in v8.0.
if name == "config.BleveSettings.BulkIndexingTimeWindowSeconds" ||
name == "config.ElasticsearchSettings.BulkIndexingTimeWindowSeconds" ||
if name == "config.ElasticsearchSettings.BulkIndexingTimeWindowSeconds" ||
name == "config.ClusterSettings.EnableExperimentalGossipEncryption" {
return true
}
webapp/channels/src/actions/admin_actions.jsx
-14
View File
@@ -405,20 +405,6 @@ export async function rebuildChannelsIndex(success, error) {
success();
}
export async function blevePurgeIndexes(success, error) {
const purgeBleveIndexes = bindClientFunc({
clientFunc: Client4.purgeBleveIndexes,
params: [],
});
const {data, error: err} = await dispatch(purgeBleveIndexes);
if (data && success) {
success(data);
} else if (err && error) {
error({id: err.server_error_id, ...err});
}
}
export function setNavigationBlocked(blocked) {
return {
type: ActionTypes.SET_NAVIGATION_BLOCKED,
webapp/channels/src/components/admin_console/__snapshots__/bleve_settings.test.tsx.snap
-387
View File
@@ -1,387 +0,0 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`components/BleveSettings should match snapshot, disabled 1`] = `
<form
className="form-horizontal"
onSubmit={[Function]}
role="form"
>
<div
className="wrapper--fixed"
>
<AdminHeader>
<MemoizedFormattedMessage
defaultMessage="Bleve"
id="admin.bleve.title"
/>
</AdminHeader>
<Memo(SettingsGroup)>
<Memo(BooleanSetting)
helpText={
<Memo(MemoizedFormattedMessage)
defaultMessage="When true, indexing of new posts occurs automatically. Search queries will use database search until \\"Enable Bleve for search queries\\" is enabled. <link>Learn more about Bleve in our documentation.</link>"
id="admin.bleve.enableIndexingDescription"
values={
Object {
"link": [Function],
}
}
/>
}
id="enableIndexing"
label={
<Memo(MemoizedFormattedMessage)
defaultMessage="Enable Bleve Indexing:"
id="admin.bleve.enableIndexingTitle"
/>
}
onChange={[Function]}
setByEnv={false}
value={false}
/>
<AdminTextSetting
helpText={
<Memo(MemoizedFormattedMessage)
defaultMessage="Directory path to use for store bleve indexes."
id="admin.bleve.indexDirDescription"
/>
}
id="indexDir"
label={
<Memo(MemoizedFormattedMessage)
defaultMessage="Index Directory:"
id="admin.bleve.indexDirTitle"
/>
}
onChange={[Function]}
setByEnv={false}
value=""
/>
<SettingSet
label={
<Memo(MemoizedFormattedMessage)
defaultMessage="Bulk Indexing:"
id="admin.bleve.bulkIndexingTitle"
/>
}
>
<div
className="job-table-setting"
>
<Connect(JobTable)
createJobButtonText={
<Memo(MemoizedFormattedMessage)
defaultMessage="Index Now"
id="admin.bleve.createJob.title"
/>
}
createJobHelpText={
<Memo(MemoizedFormattedMessage)
defaultMessage="All users, channels and posts in the database will be indexed from oldest to newest. Bleve is available during indexing but search results may be incomplete until the indexing job is complete."
id="admin.bleve.createJob.help"
/>
}
disabled={true}
getExtraInfoText={[Function]}
jobType="bleve_post_indexing"
/>
</div>
</SettingSet>
<RequestButton
buttonText={
<Memo(MemoizedFormattedMessage)
defaultMessage="Purge Index"
id="admin.bleve.purgeIndexesButton"
/>
}
disabled={true}
errorMessage={
Object {
"defaultMessage": "Failed to purge indexes: {error}",
"id": "admin.bleve.purgeIndexesButton.error",
}
}
helpText={
<Memo(MemoizedFormattedMessage)
defaultMessage="Purging will entirely remove the content of the Bleve index directory. Search results may be incomplete until a bulk index of the existing database is rebuilt."
id="admin.bleve.purgeIndexesHelpText"
/>
}
id="purgeIndexesSection"
includeDetailedError={false}
label={
<Memo(MemoizedFormattedMessage)
defaultMessage="Purge Indexes:"
id="admin.bleve.purgeIndexesButton.label"
/>
}
requestAction={[MockFunction]}
saveNeeded={false}
showSuccessMessage={true}
successMessage={
Object {
"defaultMessage": "Indexes purged successfully.",
"id": "admin.bleve.purgeIndexesButton.success",
}
}
/>
<Memo(BooleanSetting)
disabled={true}
helpText={
<Memo(MemoizedFormattedMessage)
defaultMessage="When true, Bleve will be used for all search queries using the latest index. Search results may be incomplete until a bulk index of the existing post database is finished. When false, database search is used."
id="admin.bleve.enableSearchingDescription"
/>
}
id="enableSearching"
label={
<Memo(MemoizedFormattedMessage)
defaultMessage="Enable Bleve for search queries:"
id="admin.bleve.enableSearchingTitle"
/>
}
onChange={[Function]}
setByEnv={false}
value={false}
/>
<Memo(BooleanSetting)
disabled={true}
helpText={
<Memo(MemoizedFormattedMessage)
defaultMessage="When true, Bleve will be used for all autocompletion queries on users and channels using the latest index. Autocompletion results may be incomplete until a bulk index of the existing users and channels database is finished. When false, database autocomplete is used."
id="admin.bleve.enableAutocompleteDescription"
/>
}
id="enableAutocomplete"
label={
<Memo(MemoizedFormattedMessage)
defaultMessage="Enable Bleve for autocomplete queries:"
id="admin.bleve.enableAutocompleteTitle"
/>
}
onChange={[Function]}
setByEnv={false}
value={false}
/>
</Memo(SettingsGroup)>
<div
className="admin-console-save"
>
<SaveButton
disabled={true}
onClick={[Function]}
saving={false}
savingMessage={
<Memo(MemoizedFormattedMessage)
defaultMessage="Saving Config..."
id="admin.saving"
/>
}
/>
<WithTooltip
title=""
>
<div
className="error-message"
>
<Memo(FormError)
error={null}
/>
</div>
</WithTooltip>
</div>
</div>
</form>
`;
exports[`components/BleveSettings should match snapshot, enabled 1`] = `
<form
className="form-horizontal"
onSubmit={[Function]}
role="form"
>
<div
className="wrapper--fixed"
>
<AdminHeader>
<MemoizedFormattedMessage
defaultMessage="Bleve"
id="admin.bleve.title"
/>
</AdminHeader>
<Memo(SettingsGroup)>
<Memo(BooleanSetting)
helpText={
<Memo(MemoizedFormattedMessage)
defaultMessage="When true, indexing of new posts occurs automatically. Search queries will use database search until \\"Enable Bleve for search queries\\" is enabled. <link>Learn more about Bleve in our documentation.</link>"
id="admin.bleve.enableIndexingDescription"
values={
Object {
"link": [Function],
}
}
/>
}
id="enableIndexing"
label={
<Memo(MemoizedFormattedMessage)
defaultMessage="Enable Bleve Indexing:"
id="admin.bleve.enableIndexingTitle"
/>
}
onChange={[Function]}
setByEnv={false}
value={true}
/>
<AdminTextSetting
helpText={
<Memo(MemoizedFormattedMessage)
defaultMessage="Directory path to use for store bleve indexes."
id="admin.bleve.indexDirDescription"
/>
}
id="indexDir"
label={
<Memo(MemoizedFormattedMessage)
defaultMessage="Index Directory:"
id="admin.bleve.indexDirTitle"
/>
}
onChange={[Function]}
setByEnv={false}
value="bleve.idx"
/>
<SettingSet
label={
<Memo(MemoizedFormattedMessage)
defaultMessage="Bulk Indexing:"
id="admin.bleve.bulkIndexingTitle"
/>
}
>
<div
className="job-table-setting"
>
<Connect(JobTable)
createJobButtonText={
<Memo(MemoizedFormattedMessage)
defaultMessage="Index Now"
id="admin.bleve.createJob.title"
/>
}
createJobHelpText={
<Memo(MemoizedFormattedMessage)
defaultMessage="All users, channels and posts in the database will be indexed from oldest to newest. Bleve is available during indexing but search results may be incomplete until the indexing job is complete."
id="admin.bleve.createJob.help"
/>
}
disabled={false}
getExtraInfoText={[Function]}
jobType="bleve_post_indexing"
/>
</div>
</SettingSet>
<RequestButton
buttonText={
<Memo(MemoizedFormattedMessage)
defaultMessage="Purge Index"
id="admin.bleve.purgeIndexesButton"
/>
}
disabled={false}
errorMessage={
Object {
"defaultMessage": "Failed to purge indexes: {error}",
"id": "admin.bleve.purgeIndexesButton.error",
}
}
helpText={
<Memo(MemoizedFormattedMessage)
defaultMessage="Purging will entirely remove the content of the Bleve index directory. Search results may be incomplete until a bulk index of the existing database is rebuilt."
id="admin.bleve.purgeIndexesHelpText"
/>
}
id="purgeIndexesSection"
includeDetailedError={false}
label={
<Memo(MemoizedFormattedMessage)
defaultMessage="Purge Indexes:"
id="admin.bleve.purgeIndexesButton.label"
/>
}
requestAction={[MockFunction]}
saveNeeded={false}
showSuccessMessage={true}
successMessage={
Object {
"defaultMessage": "Indexes purged successfully.",
"id": "admin.bleve.purgeIndexesButton.success",
}
}
/>
<Memo(BooleanSetting)
helpText={
<Memo(MemoizedFormattedMessage)
defaultMessage="When true, Bleve will be used for all search queries using the latest index. Search results may be incomplete until a bulk index of the existing post database is finished. When false, database search is used."
id="admin.bleve.enableSearchingDescription"
/>
}
id="enableSearching"
label={
<Memo(MemoizedFormattedMessage)
defaultMessage="Enable Bleve for search queries:"
id="admin.bleve.enableSearchingTitle"
/>
}
onChange={[Function]}
setByEnv={false}
value={false}
/>
<Memo(BooleanSetting)
helpText={
<Memo(MemoizedFormattedMessage)
defaultMessage="When true, Bleve will be used for all autocompletion queries on users and channels using the latest index. Autocompletion results may be incomplete until a bulk index of the existing users and channels database is finished. When false, database autocomplete is used."
id="admin.bleve.enableAutocompleteDescription"
/>
}
id="enableAutocomplete"
label={
<Memo(MemoizedFormattedMessage)
defaultMessage="Enable Bleve for autocomplete queries:"
id="admin.bleve.enableAutocompleteTitle"
/>
}
onChange={[Function]}
setByEnv={false}
value={false}
/>
</Memo(SettingsGroup)>
<div
className="admin-console-save"
>
<SaveButton
disabled={true}
onClick={[Function]}
saving={false}
savingMessage={
<Memo(MemoizedFormattedMessage)
defaultMessage="Saving Config..."
id="admin.saving"
/>
}
/>
<WithTooltip
title=""
>
<div
className="error-message"
>
<Memo(FormError)
error={null}
/>
</div>
</WithTooltip>
</div>
</div>
</form>
`;
webapp/channels/src/components/admin_console/__snapshots__/database_settings.test.tsx.snap
+1 -1
View File
@@ -387,7 +387,7 @@ exports[`components/DatabaseSettings should match snapshot 1`] = `
className="help-text"
>
<MemoizedFormattedMessage
defaultMessage="Shows the currently active backend used for search. Values can be none, database, elasticsearch, bleve etc."
defaultMessage="Shows the currently active backend used for search. Values can be none, database, elasticsearch."
id="admin.database.search_backend.help_text"
/>
</div>
webapp/channels/src/components/admin_console/admin_definition.tsx
-15
View File
@@ -57,7 +57,6 @@ import BillingHistory, {searchableStrings as billingHistorySearchableStrings} fr
import BillingSubscriptions, {searchableStrings as billingSubscriptionSearchableStrings} from './billing/billing_subscriptions';
import CompanyInfo, {searchableStrings as billingCompanyInfoSearchableStrings} from './billing/company_info';
import CompanyInfoEdit from './billing/company_info_edit';
import BleveSettings, {searchableStrings as bleveSearchableStrings} from './bleve_settings';
import BrandImageSetting from './brand_image_setting/brand_image_setting';
import ClientSideUserIdsSetting from './client_side_userids_setting';
import ClusterSettings, {searchableStrings as clusterSearchableStrings} from './cluster_settings';
@@ -6276,20 +6275,6 @@ const AdminDefinition: AdminDefinitionType = {
component: FeatureFlags,
},
},
bleve: {
url: 'experimental/blevesearch',
title: defineMessage({id: 'admin.sidebar.blevesearch', defaultMessage: 'Bleve'}),
isHidden: it.any(
it.configIsTrue('ExperimentalSettings', 'RestrictSystemAdmin'),
it.not(it.userHasReadPermissionOnResource(RESOURCE_KEYS.EXPERIMENTAL.BLEVE)),
),
isDisabled: it.not(it.userHasWritePermissionOnResource(RESOURCE_KEYS.EXPERIMENTAL.BLEVE)),
searchableStrings: bleveSearchableStrings,
schema: {
id: 'BleveSettings',
component: BleveSettings,
},
},
},
},
};
webapp/channels/src/components/admin_console/admin_sidebar/__snapshots__/admin_sidebar.test.tsx.snap
-99
View File
@@ -857,17 +857,6 @@ exports[`components/AdminSidebar should match snapshot 1`] = `
/>
}
/>
<AdminSidebarSection
definitionKey="experimental.bleve"
key="experimental.bleve"
name="experimental/blevesearch"
title={
<Memo(MemoizedFormattedMessage)
defaultMessage="Bleve"
id="admin.sidebar.blevesearch"
/>
}
/>
</AdminSidebarCategory>
</ul>
</SearchKeywordMarking>
@@ -1947,17 +1936,6 @@ exports[`components/AdminSidebar should match snapshot with license with enterpr
/>
}
/>
<AdminSidebarSection
definitionKey="experimental.bleve"
key="experimental.bleve"
name="experimental/blevesearch"
title={
<Memo(MemoizedFormattedMessage)
defaultMessage="Bleve"
id="admin.sidebar.blevesearch"
/>
}
/>
</AdminSidebarCategory>
</ul>
</SearchKeywordMarking>
@@ -3024,17 +3002,6 @@ exports[`components/AdminSidebar should match snapshot with license with enterpr
/>
}
/>
<AdminSidebarSection
definitionKey="experimental.bleve"
key="experimental.bleve"
name="experimental/blevesearch"
title={
<Memo(MemoizedFormattedMessage)
defaultMessage="Bleve"
id="admin.sidebar.blevesearch"
/>
}
/>
</AdminSidebarCategory>
</ul>
</SearchKeywordMarking>
@@ -4012,17 +3979,6 @@ exports[`components/AdminSidebar should match snapshot with license with profess
/>
}
/>
<AdminSidebarSection
definitionKey="experimental.bleve"
key="experimental.bleve"
name="experimental/blevesearch"
title={
<Memo(MemoizedFormattedMessage)
defaultMessage="Bleve"
id="admin.sidebar.blevesearch"
/>
}
/>
</AdminSidebarCategory>
</ul>
</SearchKeywordMarking>
@@ -4717,17 +4673,6 @@ exports[`components/AdminSidebar should match snapshot with workspace optimizati
/>
}
/>
<AdminSidebarSection
definitionKey="experimental.bleve"
key="experimental.bleve"
name="experimental/blevesearch"
title={
<Memo(MemoizedFormattedMessage)
defaultMessage="Bleve"
id="admin.sidebar.blevesearch"
/>
}
/>
</AdminSidebarCategory>
</ul>
</SearchKeywordMarking>
@@ -5477,17 +5422,6 @@ exports[`components/AdminSidebar should match snapshot, not prevent the console
/>
}
/>
<AdminSidebarSection
definitionKey="experimental.bleve"
key="experimental.bleve"
name="experimental/blevesearch"
title={
<Memo(MemoizedFormattedMessage)
defaultMessage="Bleve"
id="admin.sidebar.blevesearch"
/>
}
/>
</AdminSidebarCategory>
</ul>
</SearchKeywordMarking>
@@ -6182,17 +6116,6 @@ exports[`components/AdminSidebar should match snapshot, render plugins without a
/>
}
/>
<AdminSidebarSection
definitionKey="experimental.bleve"
key="experimental.bleve"
name="experimental/blevesearch"
title={
<Memo(MemoizedFormattedMessage)
defaultMessage="Bleve"
id="admin.sidebar.blevesearch"
/>
}
/>
</AdminSidebarCategory>
</ul>
</SearchKeywordMarking>
@@ -7084,17 +7007,6 @@ exports[`components/AdminSidebar should match snapshot, with license (with all f
/>
}
/>
<AdminSidebarSection
definitionKey="experimental.bleve"
key="experimental.bleve"
name="experimental/blevesearch"
title={
<Memo(MemoizedFormattedMessage)
defaultMessage="Bleve"
id="admin.sidebar.blevesearch"
/>
}
/>
</AdminSidebarCategory>
</ul>
</SearchKeywordMarking>
@@ -8072,17 +7984,6 @@ exports[`components/AdminSidebar should match snapshot, with license (without an
/>
}
/>
<AdminSidebarSection
definitionKey="experimental.bleve"
key="experimental.bleve"
name="experimental/blevesearch"
title={
<Memo(MemoizedFormattedMessage)
defaultMessage="Bleve"
id="admin.sidebar.blevesearch"
/>
}
/>
</AdminSidebarCategory>
</ul>
</SearchKeywordMarking>
webapp/channels/src/components/admin_console/bleve_settings.test.tsx
-51
View File
@@ -1,51 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
import {shallow} from 'enzyme';
import React from 'react';
import type {AdminConfig} from '@mattermost/types/config';
import BleveSettings from 'components/admin_console/bleve_settings';
jest.mock('actions/admin_actions.jsx', () => {
return {
blevePurgeIndexes: jest.fn(),
};
});
describe('components/BleveSettings', () => {
test('should match snapshot, disabled', () => {
const config = {
BleveSettings: {
IndexDir: '',
EnableIndexing: false,
EnableSearching: false,
EnableAutocomplete: false,
},
} as AdminConfig;
const wrapper = shallow(
<BleveSettings
config={config}
/>,
);
expect(wrapper).toMatchSnapshot();
});
test('should match snapshot, enabled', () => {
const config = {
BleveSettings: {
IndexDir: 'bleve.idx',
EnableIndexing: true,
EnableSearching: false,
EnableAutocomplete: false,
},
} as AdminConfig;
const wrapper = shallow(
<BleveSettings
config={config}
/>,
);
expect(wrapper).toMatchSnapshot();
});
});
webapp/channels/src/components/admin_console/bleve_settings.tsx
-243
View File
@@ -1,243 +0,0 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
import React from 'react';
import {FormattedMessage, defineMessage, defineMessages} from 'react-intl';
import type {AdminConfig} from '@mattermost/types/config';
import type {Job} from '@mattermost/types/jobs';
import {blevePurgeIndexes} from 'actions/admin_actions.jsx';
import ExternalLink from 'components/external_link';
import {JobStatuses, JobTypes} from 'utils/constants';
import BooleanSetting from './boolean_setting';
import JobsTable from './jobs';
import OLDAdminSettings from './old_admin_settings';
import type {BaseProps, BaseState} from './old_admin_settings';
import RequestButton from './request_button/request_button';
import SettingSet from './setting_set';
import SettingsGroup from './settings_group';
import TextSetting from './text_setting';
type Props = BaseProps & {
config: AdminConfig;
};
type State = BaseState & {
indexDir: string;
enableIndexing: boolean;
enableSearching: boolean;
enableAutocomplete: boolean;
canSave: boolean;
canPurgeAndIndex: boolean;
};
const messages = defineMessages({
title: {id: 'admin.bleve.title', defaultMessage: 'Bleve'},
enableIndexingTitle: {id: 'admin.bleve.enableIndexingTitle', defaultMessage: 'Enable Bleve Indexing:'},
enableIndexingDescription: {id: 'admin.bleve.enableIndexingDescription', defaultMessage: 'When true, indexing of new posts occurs automatically. Search queries will use database search until "Enable Bleve for search queries" is enabled. <link>Learn more about Bleve in our documentation.</link>'},
bulkIndexingTitle: {id: 'admin.bleve.bulkIndexingTitle', defaultMessage: 'Bulk Indexing:'},
createJob_help: {id: 'admin.bleve.createJob.help', defaultMessage: 'All users, channels and posts in the database will be indexed from oldest to newest. Bleve is available during indexing but search results may be incomplete until the indexing job is complete.'},
purgeIndexesHelpText: {id: 'admin.bleve.purgeIndexesHelpText', defaultMessage: 'Purging will entirely remove the content of the Bleve index directory. Search results may be incomplete until a bulk index of the existing database is rebuilt.'},
purgeIndexesButton: {id: 'admin.bleve.purgeIndexesButton', defaultMessage: 'Purge Index'},
purgeIndexesButton_label: {id: 'admin.bleve.purgeIndexesButton.label', defaultMessage: 'Purge Indexes:'},
enableSearchingTitle: {id: 'admin.bleve.enableSearchingTitle', defaultMessage: 'Enable Bleve for search queries:'},
enableSearchingDescription: {id: 'admin.bleve.enableSearchingDescription', defaultMessage: 'When true, Bleve will be used for all search queries using the latest index. Search results may be incomplete until a bulk index of the existing post database is finished. When false, database search is used.'},
});
export const searchableStrings = [
messages.title,
messages.enableIndexingTitle,
messages.enableIndexingDescription,
messages.bulkIndexingTitle,
messages.createJob_help,
messages.purgeIndexesHelpText,
messages.purgeIndexesButton,
messages.purgeIndexesButton_label,
messages.enableSearchingTitle,
messages.enableSearchingDescription,
];
export default class BleveSettings extends OLDAdminSettings<Props, State> {
getConfigFromState = (config: Props['config']) => {
if (config && config.BleveSettings) {
config.BleveSettings.IndexDir = this.state.indexDir;
config.BleveSettings.EnableIndexing = this.state.enableIndexing;
config.BleveSettings.EnableSearching = this.state.enableSearching;
config.BleveSettings.EnableAutocomplete = this.state.enableAutocomplete;
}
return config;
};
getStateFromConfig(config: Props['config']) {
return {
enableIndexing: config.BleveSettings.EnableIndexing,
indexDir: config.BleveSettings.IndexDir,
enableSearching: config.BleveSettings.EnableSearching,
enableAutocomplete: config.BleveSettings.EnableAutocomplete,
canSave: true,
canPurgeAndIndex: config.BleveSettings.EnableIndexing,
};
}
handleSettingChanged = (id: string, value: boolean) => {
if (id === 'enableIndexing') {
if (value === false) {
this.setState({
enableSearching: false,
enableAutocomplete: false,
});
}
}
if (id !== 'enableSearching' && id !== 'enableAutocomplete') {
this.setState({
canPurgeAndIndex: false,
});
}
this.handleChange(id, value);
};
handleSaved = () => {
this.setState({
canPurgeAndIndex: this.state.enableIndexing && this.state.indexDir !== '',
});
};
canSave = () => {
return this.state.canSave;
};
getExtraInfo(job: Job) {
if (job.status === JobStatuses.IN_PROGRESS) {
return (
<FormattedMessage
id='admin.bleve.percentComplete'
defaultMessage='{percent}% Complete'
values={{percent: Number(job.progress)}}
/>
);
}
return <></>;
}
renderTitle() {
return (<FormattedMessage {...messages.title}/>);
}
renderSettings = () => {
return (
<SettingsGroup>
<BooleanSetting
id='enableIndexing'
label={<FormattedMessage {...messages.enableIndexingTitle}/>}
helpText={
<FormattedMessage
{...messages.enableIndexingDescription}
values={{
link: (chunks) => (
<ExternalLink
href='https://docs.mattermost.com/deploy/bleve-search.html'
location='bleve_settings'
>
{chunks}
</ExternalLink>
),
}}
/>
}
value={this.state.enableIndexing}
onChange={this.handleSettingChanged}
setByEnv={this.isSetByEnv('BleveSettings.EnableIndexing')}
disabled={this.props.isDisabled}
/>
<TextSetting
id='indexDir'
label={
<FormattedMessage
id='admin.bleve.indexDirTitle'
defaultMessage='Index Directory:'
/>
}
helpText={
<FormattedMessage
id='admin.bleve.indexDirDescription'
defaultMessage='Directory path to use for store bleve indexes.'
/>
}
value={this.state.indexDir}
onChange={this.handleSettingChanged}
setByEnv={this.isSetByEnv('BleveSettings.IndexDir')}
disabled={this.props.isDisabled}
/>
<SettingSet
label={<FormattedMessage {...messages.bulkIndexingTitle}/>}
>
<div className='job-table-setting'>
<JobsTable
jobType={JobTypes.BLEVE_POST_INDEXING}
disabled={!this.state.canPurgeAndIndex || Boolean(this.props.isDisabled)}
createJobButtonText={
<FormattedMessage
id='admin.bleve.createJob.title'
defaultMessage='Index Now'
/>
}
createJobHelpText={<FormattedMessage {...messages.createJob_help}/>}
getExtraInfoText={this.getExtraInfo}
/>
</div>
</SettingSet>
<RequestButton
id='purgeIndexesSection'
requestAction={blevePurgeIndexes}
helpText={<FormattedMessage {...messages.purgeIndexesHelpText}/>}
buttonText={<FormattedMessage {...messages.purgeIndexesButton}/>}
successMessage={defineMessage({
id: 'admin.bleve.purgeIndexesButton.success',
defaultMessage: 'Indexes purged successfully.',
})}
errorMessage={defineMessage({
id: 'admin.bleve.purgeIndexesButton.error',
defaultMessage: 'Failed to purge indexes: {error}',
})}
disabled={!this.state.canPurgeAndIndex || this.props.isDisabled}
label={<FormattedMessage {...messages.purgeIndexesButton_label}/>}
/>
<BooleanSetting
id='enableSearching'
label={<FormattedMessage {...messages.enableSearchingTitle}/>}
helpText={<FormattedMessage {...messages.enableSearchingDescription}/>}
value={this.state.enableSearching}
disabled={!this.state.enableIndexing || this.props.isDisabled}
onChange={this.handleSettingChanged}
setByEnv={this.isSetByEnv('BleveSettings.EnableSearching')}
/>
<BooleanSetting
id='enableAutocomplete'
label={
<FormattedMessage
id='admin.bleve.enableAutocompleteTitle'
defaultMessage='Enable Bleve for autocomplete queries:'
/>
}
helpText={
<FormattedMessage
id='admin.bleve.enableAutocompleteDescription'
defaultMessage='When true, Bleve will be used for all autocompletion queries on users and channels using the latest index. Autocompletion results may be incomplete until a bulk index of the existing users and channels database is finished. When false, database autocomplete is used.'
/>
}
value={this.state.enableAutocomplete}
disabled={!this.state.enableIndexing || this.props.isDisabled}
onChange={this.handleSettingChanged}
setByEnv={this.isSetByEnv('BleveSettings.EnableAutocomplete')}
/>
</SettingsGroup>
);
};
}
webapp/channels/src/components/admin_console/database_settings.tsx
+1 -1
View File
@@ -419,7 +419,7 @@ export default class DatabaseSettings extends OLDAdminSettings<Props, State> {
<div className='help-text'>
<FormattedMessage
id='admin.database.search_backend.help_text'
defaultMessage='Shows the currently active backend used for search. Values can be none, database, elasticsearch, bleve etc.'
defaultMessage='Shows the currently active backend used for search. Values can be none, database, elasticsearch.'
/>
</div>
</div>
webapp/channels/src/components/admin_console/system_roles/system_role/__snapshots__/system_role_permissions.test.tsx.snap
-6
View File
@@ -390,9 +390,6 @@ exports[`admin_console/system_role_permissions should match snapshot 1`] = `
Object {
"name": "experimental_feature_flags",
},
Object {
"name": "experimental_bleve",
},
],
}
}
@@ -705,9 +702,6 @@ exports[`admin_console/system_role_permissions should match snapshot with isLice
Object {
"name": "experimental_feature_flags",
},
Object {
"name": "experimental_bleve",
},
],
}
}
webapp/channels/src/components/admin_console/system_roles/system_role/strings.tsx
-6
View File
@@ -434,10 +434,4 @@ export const sectionStrings: Record<string, Record<string, MessageDescriptor>> =
defaultMessage: 'Feature Flags',
},
}),
experimental_bleve: defineMessages({
name: {
id: 'admin.permissions.sysconsole_section_experimental_bleve.name',
defaultMessage: 'Bleve',
},
}),
};
webapp/channels/src/components/admin_console/system_roles/system_role/system_role_permissions.tsx
-1
View File
@@ -146,7 +146,6 @@ const sectionsList: SystemSection[] = [
subsections: [
{name: 'experimental_features'},
{name: 'experimental_feature_flags'},
{name: 'experimental_bleve'},
],
},
];
webapp/channels/src/i18n/en.json
-20
View File
@@ -519,24 +519,6 @@
"admin.billing.subscriptions.billing_summary.noBillingHistory.link": "See how billing works",
"admin.billing.subscriptions.billing_summary.noBillingHistory.title": "No billing history yet",
"admin.billing.subscriptions.billing_summary.upcomingInvoice.has_more_line_items": "And {count} more items",
"admin.bleve.bulkIndexingTitle": "Bulk Indexing:",
"admin.bleve.createJob.help": "All users, channels and posts in the database will be indexed from oldest to newest. Bleve is available during indexing but search results may be incomplete until the indexing job is complete.",
"admin.bleve.createJob.title": "Index Now",
"admin.bleve.enableAutocompleteDescription": "When true, Bleve will be used for all autocompletion queries on users and channels using the latest index. Autocompletion results may be incomplete until a bulk index of the existing users and channels database is finished. When false, database autocomplete is used.",
"admin.bleve.enableAutocompleteTitle": "Enable Bleve for autocomplete queries:",
"admin.bleve.enableIndexingDescription": "When true, indexing of new posts occurs automatically. Search queries will use database search until \"Enable Bleve for search queries\" is enabled. {documentationLink}",
"admin.bleve.enableIndexingTitle": "Enable Bleve Indexing:",
"admin.bleve.enableSearchingDescription": "When true, Bleve will be used for all search queries using the latest index. Search results may be incomplete until a bulk index of the existing post database is finished. When false, database search is used.",
"admin.bleve.enableSearchingTitle": "Enable Bleve for search queries:",
"admin.bleve.indexDirDescription": "Directory path to use for store bleve indexes.",
"admin.bleve.indexDirTitle": "Index Directory:",
"admin.bleve.percentComplete": "{percent}% Complete",
"admin.bleve.purgeIndexesButton": "Purge Index",
"admin.bleve.purgeIndexesButton.error": "Failed to purge indexes: {error}",
"admin.bleve.purgeIndexesButton.label": "Purge Indexes:",
"admin.bleve.purgeIndexesButton.success": "Indexes purged successfully.",
"admin.bleve.purgeIndexesHelpText": "Purging will entirely remove the content of the Bleve index directory. Search results may be incomplete until a bulk index of the existing database is rebuilt.",
"admin.bleve.title": "Bleve",
"admin.cacheSettings.cacheType.lru": "LRU",
"admin.cacheSettings.cacheType.redis": "Redis",
"admin.cacheSettings.cacheTypeDesc": "The type of the cache backend. E.g.: \"redis\" or \"lru\"",
@@ -2104,7 +2086,6 @@
"admin.permissions.sysconsole_section_environment_web_server.name": "Web Server",
"admin.permissions.sysconsole_section_environment.description": "Review server environment configuration such as URLs, database and performance.",
"admin.permissions.sysconsole_section_environment.name": "Environment",
"admin.permissions.sysconsole_section_experimental_bleve.name": "Bleve",
"admin.permissions.sysconsole_section_experimental_feature_flags.name": "Feature Flags",
"admin.permissions.sysconsole_section_experimental_features.name": "Features",
"admin.permissions.sysconsole_section_experimental.description": "Review the settings of experimental features",
@@ -2683,7 +2664,6 @@
"admin.sidebar.authentication": "Authentication",
"admin.sidebar.billing": "Billing & Account",
"admin.sidebar.billing_history": "Billing History",
"admin.sidebar.blevesearch": "Bleve",
"admin.sidebar.channels": "Channels",
"admin.sidebar.company_info": "Company Information",
"admin.sidebar.compliance": "Compliance",
webapp/platform/types/src/config.ts
-9
View File
@@ -882,14 +882,6 @@ export type ElasticsearchSettings = {
IgnoredPurgeIndexes: string;
};
export type BleveSettings = {
IndexDir: string;
EnableIndexing: boolean;
EnableSearching: boolean;
EnableAutocomplete: boolean;
BatchSize: number;
};
export type DataRetentionSettings = {
EnableMessageDeletion: boolean;
EnableFileDeletion: boolean;
@@ -1059,7 +1051,6 @@ export type AdminConfig = {
AnalyticsSettings: AnalyticsSettings;
CacheSettings: CacheSettings;
ElasticsearchSettings: ElasticsearchSettings;
BleveSettings: BleveSettings;
DataRetentionSettings: DataRetentionSettings;
MessageExportSettings: MessageExportSettings;
JobSettings: JobSettings;