IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2025-02-25 18:55:24 -06:00
Code Issues Packages Projects Releases Wiki Activity

MM-11359: support excluding results from search (#11196)

Browse Source
This commit is contained in:
Siyuan Liu
2019-08-12 05:03:42 -07:00
committed by Hanzei
parent a4091cd363
commit e4bb8cd887
5 changed files with 1340 additions and 401 deletions
Download Patch File Download Diff File Expand all files Collapse all files

311
store/sqlstore/post_store.go
View File

@@ -799,33 +799,136 @@ var specialSearchChar = []string{
":",
}
func (s *SqlPostStore) buildCreateDateFilterClause(params *model.SearchParams, queryParams map[string]interface{}) (string, map[string]interface{}) {
searchQuery := ""
// handle after: before: on: filters
if len(params.OnDate) > 0 {
onDateStart, onDateEnd := params.GetOnDateMillis()
queryParams["OnDateStart"] = strconv.FormatInt(onDateStart, 10)
queryParams["OnDateEnd"] = strconv.FormatInt(onDateEnd, 10)
// between `on date` start of day and end of day
searchQuery += "AND CreateAt BETWEEN :OnDateStart AND :OnDateEnd "
} else {
if len(params.ExcludedDate) > 0 {
excludedDateStart, excludedDateEnd := params.GetExcludedDateMillis()
queryParams["ExcludedDateStart"] = strconv.FormatInt(excludedDateStart, 10)
queryParams["ExcludedDateEnd"] = strconv.FormatInt(excludedDateEnd, 10)
searchQuery += "AND CreateAt NOT BETWEEN :ExcludedDateStart AND :ExcludedDateEnd "
}
if len(params.AfterDate) > 0 {
afterDate := params.GetAfterDateMillis()
queryParams["AfterDate"] = strconv.FormatInt(afterDate, 10)
// greater than `after date`
searchQuery += "AND CreateAt >= :AfterDate "
}
if len(params.BeforeDate) > 0 {
beforeDate := params.GetBeforeDateMillis()
queryParams["BeforeDate"] = strconv.FormatInt(beforeDate, 10)
// less than `before date`
searchQuery += "AND CreateAt <= :BeforeDate "
}
if len(params.ExcludedAfterDate) > 0 {
afterDate := params.GetExcludedAfterDateMillis()
queryParams["ExcludedAfterDate"] = strconv.FormatInt(afterDate, 10)
searchQuery += "AND CreateAt < :ExcludedAfterDate "
}
if len(params.ExcludedBeforeDate) > 0 {
beforeDate := params.GetExcludedBeforeDateMillis()
queryParams["ExcludedBeforeDate"] = strconv.FormatInt(beforeDate, 10)
searchQuery += "AND CreateAt > :ExcludedBeforeDate "
}
}
return searchQuery, queryParams
}
func (s *SqlPostStore) buildSearchChannelFilterClause(channels []string, paramPrefix string, exclusion bool, queryParams map[string]interface{}) (string, map[string]interface{}) {
if len(channels) == 0 {
return "", queryParams
}
clauseSlice := []string{}
for i, channel := range channels {
paramName := paramPrefix + strconv.FormatInt(int64(i), 10)
clauseSlice = append(clauseSlice, ":"+paramName)
queryParams[paramName] = channel
}
clause := strings.Join(clauseSlice, ", ")
if exclusion {
return "AND Name NOT IN (" + clause + ")", queryParams
}
return "AND Name IN (" + clause + ")", queryParams
}
func (s *SqlPostStore) buildSearchUserFilterClause(users []string, paramPrefix string, exclusion bool, queryParams map[string]interface{}) (string, map[string]interface{}) {
if len(users) == 0 {
return "", queryParams
}
clauseSlice := []string{}
for i, user := range users {
paramName := paramPrefix + strconv.FormatInt(int64(i), 10)
clauseSlice = append(clauseSlice, ":"+paramName)
queryParams[paramName] = user
}
clause := strings.Join(clauseSlice, ", ")
if exclusion {
return "AND Username NOT IN (" + clause + ")", queryParams
}
return "AND Username IN (" + clause + ")", queryParams
}
func (s *SqlPostStore) buildSearchPostFilterClause(fromUsers []string, excludedUsers []string, queryParams map[string]interface{}) (string, map[string]interface{}) {
if len(fromUsers) == 0 && len(excludedUsers) == 0 {
return "", queryParams
}
filterQuery := `
AND UserId IN (
SELECT
Id
FROM
Users,
TeamMembers
WHERE
TeamMembers.TeamId = :TeamId
AND Users.Id = TeamMembers.UserId
FROM_USER_FILTER
EXCLUDED_USER_FILTER)`
fromUserClause, queryParams := s.buildSearchUserFilterClause(fromUsers, "FromUser", false, queryParams)
filterQuery = strings.Replace(filterQuery, "FROM_USER_FILTER", fromUserClause, 1)
excludedUserClause, queryParams := s.buildSearchUserFilterClause(excludedUsers, "ExcludedUser", true, queryParams)
filterQuery = strings.Replace(filterQuery, "EXCLUDED_USER_FILTER", excludedUserClause, 1)
return filterQuery, queryParams
}
func (s *SqlPostStore) Search(teamId string, userId string, params *model.SearchParams) (*model.PostList, *model.AppError) {
queryParams := map[string]interface{}{
"TeamId": teamId,
"UserId": userId,
}
termMap := map[string]bool{}
terms := params.Terms
list := model.NewPostList()
if terms == "" && len(params.InChannels) == 0 && len(params.FromUsers) == 0 && len(params.OnDate) == 0 && len(params.AfterDate) == 0 && len(params.BeforeDate) == 0 {
if params.Terms == "" && params.ExcludedTerms == "" &&
len(params.InChannels) == 0 && len(params.ExcludedChannels) == 0 &&
len(params.FromUsers) == 0 && len(params.ExcludedUsers) == 0 &&
len(params.OnDate) == 0 && len(params.AfterDate) == 0 && len(params.BeforeDate) == 0 {
return list, nil
}
searchType := "Message"
if params.IsHashtag {
searchType = "Hashtags"
for _, term := range strings.Split(terms, " ") {
termMap[strings.ToUpper(term)] = true
}
}
// these chars have special meaning and can be treated as spaces
for _, c := range specialSearchChar {
terms = strings.Replace(terms, c, " ", -1)
}
var posts []*model.Post
deletedQueryPart := "AND DeleteAt = 0"
@@ -858,116 +961,62 @@ func (s *SqlPostStore) Search(teamId string, userId string, params *model.Search
AND (TeamId = :TeamId OR TeamId = '')
` + userIdPart + `
` + deletedQueryPart + `
CHANNEL_FILTER)
IN_CHANNEL_FILTER
EXCLUDED_CHANNEL_FILTER)
CREATEDATE_CLAUSE
SEARCH_CLAUSE
ORDER BY CreateAt DESC
LIMIT 100`
if len(params.InChannels) > 1 {
inClause := ":InChannel0"
queryParams["InChannel0"] = params.InChannels[0]
inChannelClause, queryParams := s.buildSearchChannelFilterClause(params.InChannels, "InChannel", false, queryParams)
searchQuery = strings.Replace(searchQuery, "IN_CHANNEL_FILTER", inChannelClause, 1)
for i := 1; i < len(params.InChannels); i++ {
paramName := "InChannel" + strconv.FormatInt(int64(i), 10)
inClause += ", :" + paramName
queryParams[paramName] = params.InChannels[i]
excludedChannelClause, queryParams := s.buildSearchChannelFilterClause(params.ExcludedChannels, "ExcludedChannel", true, queryParams)
searchQuery = strings.Replace(searchQuery, "EXCLUDED_CHANNEL_FILTER", excludedChannelClause, 1)
postFilterClause, queryParams := s.buildSearchPostFilterClause(params.FromUsers, params.ExcludedUsers, queryParams)
searchQuery = strings.Replace(searchQuery, "POST_FILTER", postFilterClause, 1)
createDateFilterClause, queryParams := s.buildCreateDateFilterClause(params, queryParams)
searchQuery = strings.Replace(searchQuery, "CREATEDATE_CLAUSE", createDateFilterClause, 1)
termMap := map[string]bool{}
terms := params.Terms
excludedTerms := params.ExcludedTerms
searchType := "Message"
if params.IsHashtag {
searchType = "Hashtags"
for _, term := range strings.Split(terms, " ") {
termMap[strings.ToUpper(term)] = true
}
searchQuery = strings.Replace(searchQuery, "CHANNEL_FILTER", "AND Name IN ("+inClause+")", 1)
} else if len(params.InChannels) == 1 {
queryParams["InChannel"] = params.InChannels[0]
searchQuery = strings.Replace(searchQuery, "CHANNEL_FILTER", "AND Name = :InChannel", 1)
} else {
searchQuery = strings.Replace(searchQuery, "CHANNEL_FILTER", "", 1)
}
if len(params.FromUsers) > 1 {
inClause := ":FromUser0"
queryParams["FromUser0"] = params.FromUsers[0]
for i := 1; i < len(params.FromUsers); i++ {
paramName := "FromUser" + strconv.FormatInt(int64(i), 10)
inClause += ", :" + paramName
queryParams[paramName] = params.FromUsers[i]
}
searchQuery = strings.Replace(searchQuery, "POST_FILTER", `
AND UserId IN (
SELECT
Id
FROM
Users,
TeamMembers
WHERE
TeamMembers.TeamId = :TeamId
AND Users.Id = TeamMembers.UserId
AND Username IN (`+inClause+`))`, 1)
} else if len(params.FromUsers) == 1 {
queryParams["FromUser"] = params.FromUsers[0]
searchQuery = strings.Replace(searchQuery, "POST_FILTER", `
AND UserId IN (
SELECT
Id
FROM
Users,
TeamMembers
WHERE
TeamMembers.TeamId = :TeamId
AND Users.Id = TeamMembers.UserId
AND Username = :FromUser)`, 1)
} else {
searchQuery = strings.Replace(searchQuery, "POST_FILTER", "", 1)
// these chars have special meaning and can be treated as spaces
for _, c := range specialSearchChar {
terms = strings.Replace(terms, c, " ", -1)
excludedTerms = strings.Replace(excludedTerms, c, " ", -1)
}
// handle after: before: on: filters
if len(params.AfterDate) > 1 || len(params.BeforeDate) > 1 || len(params.OnDate) > 1 {
if len(params.OnDate) > 1 {
onDateStart, onDateEnd := params.GetOnDateMillis()
queryParams["OnDateStart"] = strconv.FormatInt(onDateStart, 10)
queryParams["OnDateEnd"] = strconv.FormatInt(onDateEnd, 10)
// between `on date` start of day and end of day
searchQuery = strings.Replace(searchQuery, "CREATEDATE_CLAUSE", "AND CreateAt BETWEEN :OnDateStart AND :OnDateEnd ", 1)
} else if len(params.AfterDate) > 1 && len(params.BeforeDate) > 1 {
afterDate := params.GetAfterDateMillis()
beforeDate := params.GetBeforeDateMillis()
queryParams["OnDateStart"] = strconv.FormatInt(afterDate, 10)
queryParams["OnDateEnd"] = strconv.FormatInt(beforeDate, 10)
// between clause
searchQuery = strings.Replace(searchQuery, "CREATEDATE_CLAUSE", "AND CreateAt BETWEEN :OnDateStart AND :OnDateEnd ", 1)
} else if len(params.AfterDate) > 1 {
afterDate := params.GetAfterDateMillis()
queryParams["AfterDate"] = strconv.FormatInt(afterDate, 10)
// greater than `after date`
searchQuery = strings.Replace(searchQuery, "CREATEDATE_CLAUSE", "AND CreateAt >= :AfterDate ", 1)
} else if len(params.BeforeDate) > 1 {
beforeDate := params.GetBeforeDateMillis()
queryParams["BeforeDate"] = strconv.FormatInt(beforeDate, 10)
// less than `before date`
searchQuery = strings.Replace(searchQuery, "CREATEDATE_CLAUSE", "AND CreateAt <= :BeforeDate ", 1)
}
} else {
// no create date filters set
searchQuery = strings.Replace(searchQuery, "CREATEDATE_CLAUSE", "", 1)
}
if terms == "" {
if terms == "" && excludedTerms == "" {
// we've already confirmed that we have a channel or user to search for
searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", "", 1)
} else if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
// Parse text for wildcards
if wildcard, err := regexp.Compile(`\*($| )`); err == nil {
terms = wildcard.ReplaceAllLiteralString(terms, ":* ")
excludedTerms = wildcard.ReplaceAllLiteralString(excludedTerms, ":* ")
}
excludeClause := ""
if excludedTerms != "" {
excludeClause = " & !(" + strings.Join(strings.Fields(excludedTerms), " | ") + ")"
}
if params.OrTerms {
terms = strings.Join(strings.Fields(terms), " | ")
queryParams["Terms"] = "(" + strings.Join(strings.Fields(terms), " | ") + ")" + excludeClause
} else {
terms = strings.Join(strings.Fields(terms), " & ")
queryParams["Terms"] = "(" + strings.Join(strings.Fields(terms), " & ") + ")" + excludeClause
}
searchClause := fmt.Sprintf("AND to_tsvector('english', %s) @@ to_tsquery(:Terms)", searchType)
@@ -976,43 +1025,45 @@ func (s *SqlPostStore) Search(teamId string, userId string, params *model.Search
searchClause := fmt.Sprintf("AND MATCH (%s) AGAINST (:Terms IN BOOLEAN MODE)", searchType)
searchQuery = strings.Replace(searchQuery, "SEARCH_CLAUSE", searchClause, 1)
if !params.OrTerms {
splitTerms := strings.Fields(terms)
for i, t := range strings.Fields(terms) {
splitTerms[i] = "+" + t
}
excludeClause := ""
if excludedTerms != "" {
excludeClause = " -(" + excludedTerms + ")"
}
terms = strings.Join(splitTerms, " ")
if params.OrTerms {
queryParams["Terms"] = terms + excludeClause
} else {
splitTerms := []string{}
for _, t := range strings.Fields(terms) {
splitTerms = append(splitTerms, "+"+t)
}
queryParams["Terms"] = strings.Join(splitTerms, " ") + excludeClause
}
}
queryParams["Terms"] = terms
_, err := s.GetSearchReplica().Select(&posts, searchQuery, queryParams)
if err != nil {
mlog.Warn(fmt.Sprintf("Query error searching posts: %v", err.Error()))
// Don't return the error to the caller as it is of no use to the user. Instead return an empty set of search results.
return list, nil
}
for _, p := range posts {
if searchType == "Hashtags" {
exactMatch := false
for _, tag := range strings.Split(p.Hashtags, " ") {
if termMap[strings.ToUpper(tag)] {
exactMatch = true
} else {
for _, p := range posts {
if searchType == "Hashtags" {
exactMatch := false
for _, tag := range strings.Split(p.Hashtags, " ") {
if termMap[strings.ToUpper(tag)] {
exactMatch = true
break
}
}
if !exactMatch {
continue
}
}
if !exactMatch {
continue
}
list.AddPost(p)
list.AddOrder(p.Id)
}
list.AddPost(p)
list.AddOrder(p.Id)
}
list.MakeNonNil()
return list, nil
}

158
store/storetest/post_store.go
View File

@@ -1170,10 +1170,46 @@ func testPostStoreSearch(t *testing.T, ss store.Store) {
teamId := model.NewId()
userId := model.NewId()
u1 := &model.User{}
u1.Username = "usera1"
u1.Email = MakeEmail()
u1, err := ss.User().Save(u1)
require.Nil(t, err)
t1 := &model.TeamMember{}
t1.TeamId = teamId
t1.UserId = u1.Id
_, err = ss.Team().SaveMember(t1, 1000)
require.Nil(t, err)
u2 := &model.User{}
u2.Username = "userb2"
u2.Email = MakeEmail()
u2, err = ss.User().Save(u2)
require.Nil(t, err)
t2 := &model.TeamMember{}
t2.TeamId = teamId
t2.UserId = u2.Id
_, err = ss.Team().SaveMember(t2, 1000)
require.Nil(t, err)
u3 := &model.User{}
u3.Username = "userc3"
u3.Email = MakeEmail()
u3, err = ss.User().Save(u3)
require.Nil(t, err)
t3 := &model.TeamMember{}
t3.TeamId = teamId
t3.UserId = u3.Id
_, err = ss.Team().SaveMember(t3, 1000)
require.Nil(t, err)
c1 := &model.Channel{}
c1.TeamId = teamId
c1.DisplayName = "Channel1"
c1.Name = "zz" + model.NewId() + "b"
c1.Name = "channel-x"
c1.Type = model.CHANNEL_OPEN
c1, _ = ss.Channel().Save(c1, -1)
@@ -1181,20 +1217,20 @@ func testPostStoreSearch(t *testing.T, ss store.Store) {
m1.ChannelId = c1.Id
m1.UserId = userId
m1.NotifyProps = model.GetDefaultChannelNotifyProps()
_, err := ss.Channel().SaveMember(&m1)
_, err = ss.Channel().SaveMember(&m1)
require.Nil(t, err)
c2 := &model.Channel{}
c2.TeamId = teamId
c2.DisplayName = "Channel1"
c2.Name = "zz" + model.NewId() + "b"
c2.DisplayName = "Channel2"
c2.Name = "channel-y"
c2.Type = model.CHANNEL_OPEN
c2, _ = ss.Channel().Save(c2, -1)
c3 := &model.Channel{}
c3.TeamId = teamId
c3.DisplayName = "Channel1"
c3.Name = "zz" + model.NewId() + "b"
c3.DisplayName = "Channel3"
c3.Name = "channel-z"
c3.Type = model.CHANNEL_OPEN
c3, _ = ss.Channel().Save(c3, -1)
@@ -1209,37 +1245,37 @@ func testPostStoreSearch(t *testing.T, ss store.Store) {
o1 := &model.Post{}
o1.ChannelId = c1.Id
o1.UserId = model.NewId()
o1.Message = "corey mattermost new york"
o1.UserId = u1.Id
o1.Message = "corey mattermost new york United States"
o1, err = ss.Post().Save(o1)
require.Nil(t, err)
o1a := &model.Post{}
o1a.ChannelId = c1.Id
o1a.UserId = model.NewId()
o1a.Message = "corey mattermost new york"
o1a.Message = "corey mattermost new york United States"
o1a.Type = model.POST_JOIN_CHANNEL
_, err = ss.Post().Save(o1a)
require.Nil(t, err)
o2 := &model.Post{}
o2.ChannelId = c1.Id
o2.UserId = model.NewId()
o2.Message = "New Jersey is where John is from"
o2.UserId = u2.Id
o2.Message = "New Jersey United States is where John is from"
o2, err = ss.Post().Save(o2)
require.Nil(t, err)
o3 := &model.Post{}
o3.ChannelId = c2.Id
o3.UserId = model.NewId()
o3.Message = "New Jersey is where John is from corey new york"
o3.Message = "New Jersey United States is where John is from corey new york"
_, err = ss.Post().Save(o3)
require.Nil(t, err)
o4 := &model.Post{}
o4.ChannelId = c1.Id
o4.UserId = model.NewId()
o4.Hashtags = "#hashtag"
o4.Hashtags = "#hashtag #tagme"
o4.Message = "(message)blargh"
o4, err = ss.Post().Save(o4)
require.Nil(t, err)
@@ -1247,7 +1283,7 @@ func testPostStoreSearch(t *testing.T, ss store.Store) {
o5 := &model.Post{}
o5.ChannelId = c1.Id
o5.UserId = model.NewId()
o5.Hashtags = "#secret #howdy"
o5.Hashtags = "#secret #howdy #tagme"
o5, err = ss.Post().Save(o5)
require.Nil(t, err)
@@ -1260,8 +1296,8 @@ func testPostStoreSearch(t *testing.T, ss store.Store) {
o7 := &model.Post{}
o7.ChannelId = c3.Id
o7.UserId = model.NewId()
o7.Message = "New Jersey is where John is from corey new york"
o7.UserId = u3.Id
o7.Message = "New Jersey United States is where John is from corey new york"
o7, err = ss.Post().Save(o7)
require.Nil(t, err)
@@ -1314,6 +1350,12 @@ func testPostStoreSearch(t *testing.T, ss store.Store) {
1,
[]string{o5.Id},
},
{
"hashtag-search-with-exclusion",
&model.SearchParams{Terms: "#tagme", ExcludedTerms: "#hashtag", IsHashtag: true},
1,
[]string{o5.Id},
},
{
"no-match-mention",
&model.SearchParams{Terms: "@thisshouldmatchnothing", IsHashtag: true},
@@ -1326,18 +1368,48 @@ func testPostStoreSearch(t *testing.T, ss store.Store) {
0,
[]string{},
},
{
"exclude-search",
&model.SearchParams{Terms: "united", ExcludedTerms: "jersey"},
1,
[]string{o1.Id},
},
{
"multiple-words-search",
&model.SearchParams{Terms: "corey new york"},
1,
[]string{o1.Id},
},
{
"multiple-words-with-exclusion-search",
&model.SearchParams{Terms: "united states", ExcludedTerms: "jersey"},
1,
[]string{o1.Id},
},
{
"multiple-excluded-words-search",
&model.SearchParams{Terms: "united", ExcludedTerms: "corey john"},
0,
[]string{},
},
{
"multiple-wildcard-search",
&model.SearchParams{Terms: "matter* jer*"},
0,
[]string{},
},
{
"multiple-wildcard-with-exclusion-search",
&model.SearchParams{Terms: "unite* state*", ExcludedTerms: "jers*"},
1,
[]string{o1.Id},
},
{
"multiple-wildcard-excluded-words-search",
&model.SearchParams{Terms: "united states", ExcludedTerms: "jers* yor*"},
0,
[]string{},
},
{
"search-with-work-next-to-a-symbol",
&model.SearchParams{Terms: "message blargh"},
@@ -1350,6 +1422,60 @@ func testPostStoreSearch(t *testing.T, ss store.Store) {
2,
[]string{o1.Id, o2.Id},
},
{
"exclude-search-with-or",
&model.SearchParams{Terms: "york jersey", ExcludedTerms: "john", OrTerms: true},
1,
[]string{o1.Id},
},
{
"search-with-from-user",
&model.SearchParams{Terms: "united states", FromUsers: []string{"usera1"}, IncludeDeletedChannels: true},
1,
[]string{o1.Id},
},
{
"search-with-multiple-from-user",
&model.SearchParams{Terms: "united states", FromUsers: []string{"usera1", "userc3"}, IncludeDeletedChannels: true},
2,
[]string{o1.Id, o7.Id},
},
{
"search-with-excluded-user",
&model.SearchParams{Terms: "united states", ExcludedUsers: []string{"usera1"}, IncludeDeletedChannels: true},
2,
[]string{o2.Id, o7.Id},
},
{
"search-with-multiple-excluded-user",
&model.SearchParams{Terms: "united states", ExcludedUsers: []string{"usera1", "userb2"}, IncludeDeletedChannels: true},
1,
[]string{o7.Id},
},
{
"search-with-deleted-and-channel-filter",
&model.SearchParams{Terms: "Jersey corey", InChannels: []string{"channel-x"}, IncludeDeletedChannels: true, OrTerms: true},
2,
[]string{o1.Id, o2.Id},
},
{
"search-with-deleted-and-multiple-channel-filter",
&model.SearchParams{Terms: "Jersey corey", InChannels: []string{"channel-x", "channel-z"}, IncludeDeletedChannels: true, OrTerms: true},
3,
[]string{o1.Id, o2.Id, o7.Id},
},
{
"search-with-deleted-and-excluded-channel-filter",
&model.SearchParams{Terms: "Jersey corey", ExcludedChannels: []string{"channel-x"}, IncludeDeletedChannels: true, OrTerms: true},
1,
[]string{o7.Id},
},
{
"search-with-deleted-and-multiple-excluded-channel-filter",
&model.SearchParams{Terms: "Jersey corey", ExcludedChannels: []string{"channel-x", "channel-z"}, IncludeDeletedChannels: true, OrTerms: true},
0,
[]string{},
},
{
"search-with-or-and-deleted",
&model.SearchParams{Terms: "Jersey corey", OrTerms: true, IncludeDeletedChannels: true},