Precompute multibyte mention keywords once per post (#38038)

server/channels/app/mention_parser_standard.go

@@ -15,12 +15,25 @@ var _ MentionParser = &StandardMentionParser{}
 type StandardMentionParser struct {
 	keywords MentionKeywords
 
+	// multibyteKeywords holds the subset of keywords containing a multibyte character. It's computed
+	// once here because isKeywordMultibyte needs it for every word of every post, and recomputing it
+	// there makes parsing cost the product of the number of keywords and the number of words.
+	multibyteKeywords []string
+
 	results *MentionResults
 }
 
 func makeStandardMentionParser(keywords MentionKeywords) *StandardMentionParser {
+	var multibyteKeywords []string
+	for keyword := range keywords {
+		if len(keyword) != utf8.RuneCountInString(keyword) {
+			multibyteKeywords = append(multibyteKeywords, keyword)
+		}
+	}
+
 	return &StandardMentionParser{
-		keywords: keywords,
+		keywords:          keywords,
+		multibyteKeywords: multibyteKeywords,
 
 		results: &MentionResults{},
 	}
@@ -86,7 +99,7 @@ func (p *StandardMentionParser) ProcessText(text string) {
 			}
 		}
 
-		if ids, match := isKeywordMultibyte(p.keywords, word); match {
+		if ids, match := p.isKeywordMultibyte(word); match {
 			p.addMentions(ids, KeywordMention)
 		}
 	}
@@ -139,22 +152,19 @@ func (p *StandardMentionParser) addMentions(ids []MentionableID, mentionType Men
 }
 
 // isKeywordMultibyte checks if a word containing a multibyte character contains a multibyte keyword
-func isKeywordMultibyte(keywords MentionKeywords, word string) ([]MentionableID, bool) {
+func (p *StandardMentionParser) isKeywordMultibyte(word string) ([]MentionableID, bool) {
 	ids := []MentionableID{}
 	match := false
-	var multibyteKeywords []string
-	for keyword := range keywords {
-		if len(keyword) != utf8.RuneCountInString(keyword) {
-			multibyteKeywords = append(multibyteKeywords, keyword)
+
+	if len(p.multibyteKeywords) == 0 || len(word) == utf8.RuneCountInString(word) {
+		return ids, match
+	}
+
+	for _, key := range p.multibyteKeywords {
+		if strings.Contains(word, key) {
+			ids, match = p.keywords[key]
 		}
 	}
 
-	if len(word) != utf8.RuneCountInString(word) {
-		for _, key := range multibyteKeywords {
-			if strings.Contains(word, key) {
-				ids, match = keywords[key]
-			}
-		}
-	}
 	return ids, match
 }

server/channels/app/mention_parser_standard_test.go

@@ -4,15 +4,60 @@
 package app
 
 import (
+	"fmt"
+	"maps"
+	"strings"
 	"testing"
 
 	"github.com/mattermost/mattermost/server/public/model"
 	"github.com/stretchr/testify/assert"
 )
 
+// withFillerKeywords returns the given keywords along with count additional single-byte keywords
+// that are not expected to match anything.
+func withFillerKeywords(keywords map[string][]string, count int) map[string][]string {
+	result := make(map[string][]string, len(keywords)+count)
+	maps.Copy(result, keywords)
+
+	filler := model.NewId()
+	for i := range count {
+		result[fmt.Sprintf("keyword%d", i)] = []string{filler}
+	}
+
+	return result
+}
+
+func TestMakeStandardMentionParser(t *testing.T) {
+	mainHelper.Parallel(t)
+
+	t.Run("should precompute only the multibyte keywords", func(t *testing.T) {
+		id := model.NewId()
+		p := makeStandardMentionParser(mapsToMentionKeywords(map[string][]string{
+			"apple":  {id},
+			"banana": {id},
+			"番茄":     {id},
+			"世界":     {id},
+			"café":   {id},
+		}, nil))
+
+		assert.ElementsMatch(t, []string{"番茄", "世界", "café"}, p.multibyteKeywords)
+	})
+
+	t.Run("should precompute nothing when no keyword is multibyte", func(t *testing.T) {
+		id := model.NewId()
+		p := makeStandardMentionParser(mapsToMentionKeywords(map[string][]string{
+			"apple":  {id},
+			"banana": {id},
+		}, nil))
+
+		assert.Empty(t, p.multibyteKeywords)
+	})
+}
+
 func TestIsKeywordMultibyte(t *testing.T) {
 	mainHelper.Parallel(t)
 	id1 := model.NewId()
+	id2 := model.NewId()
 
 	for name, tc := range map[string]struct {
 		Message     string
@@ -101,6 +146,38 @@ func TestIsKeywordMultibyte(t *testing.T) {
 				Mentions: nil,
 			},
 		},
+		"MultibyteCharacterAlongsideManySingleByteKeywords": {
+			Message:  "我爱吃番茄炒饭",
+			Keywords: withFillerKeywords(map[string][]string{"番茄": {id1}}, 500),
+			Expected: &MentionResults{
+				Mentions: map[string]MentionType{
+					id1: KeywordMention,
+				},
+			},
+		},
+		"MultibyteCharacterWithOnlySingleByteKeywords": {
+			Message:  "我爱吃番茄炒饭",
+			Keywords: map[string][]string{"tomato": {id1}},
+			Expected: &MentionResults{
+				Mentions: nil,
+			},
+		},
+		"SingleByteWordWithOnlyMultibyteKeywords": {
+			Message:  "the quick brown fox",
+			Keywords: map[string][]string{"番茄": {id1}},
+			Expected: &MentionResults{
+				Mentions: nil,
+			},
+		},
+		"MultipleMultibyteKeywordsWhereOnlyOneMatches": {
+			Message:  "我爱吃番茄炒饭",
+			Keywords: map[string][]string{"番茄": {id1}, "世界": {id2}},
+			Expected: &MentionResults{
+				Mentions: map[string]MentionType{
+					id1: KeywordMention,
+				},
+			},
+		},
 	} {
 		t.Run(name, func(t *testing.T) {
 			post := &model.Post{
@@ -435,3 +512,48 @@ func TestProcessText(t *testing.T) {
 		})
 	}
 }
+
+// makeBenchmarkKeywords returns count single-byte keywords, optionally alongside one multibyte
+// keyword, all belonging to a single user.
+func makeBenchmarkKeywords(count int, multibyteKeyword string) MentionKeywords {
+	keywords := make(MentionKeywords, count+1)
+
+	id := mentionableUserID(model.NewId())
+	for i := range count {
+		keywords[fmt.Sprintf("keyword%d", i)] = []MentionableID{id}
+	}
+
+	if multibyteKeyword != "" {
+		keywords[multibyteKeyword] = []MentionableID{id}
+	}
+
+	return keywords
+}
+
+func BenchmarkGetExplicitMentions(b *testing.B) {
+	// Roughly 1800 words, matching the scale of a long message in a busy channel.
+	asciiMessage := strings.Repeat("the quick brown fox jumps over the lazy dog ", 200)
+	multibyteMessage := strings.Repeat("こんにちは、世界 the quick brown fox ", 200)
+
+	for _, tc := range []struct {
+		name             string
+		message          string
+		multibyteKeyword string
+	}{
+		{name: "ascii post, no multibyte keywords", message: asciiMessage},
+		{name: "ascii post, one multibyte keyword", message: asciiMessage, multibyteKeyword: "世界"},
+		{name: "multibyte post, one multibyte keyword", message: multibyteMessage, multibyteKeyword: "世界"},
+	} {
+		for _, numKeywords := range []int{10, 1000, 10000, 55000} {
+			b.Run(fmt.Sprintf("%s/keywords=%d", tc.name, numKeywords), func(b *testing.B) {
+				keywords := makeBenchmarkKeywords(numKeywords, tc.multibyteKeyword)
+				post := &model.Post{Message: tc.message}
+
+				b.ReportAllocs()
+				for b.Loop() {
+					getExplicitMentions(post, keywords, true)
+				}
+			})
+		}
+	}
+}