mirror of
https://github.com/discourse/discourse.git
synced 2025-02-25 18:55:32 -06:00
FIX: do not remove stop words when using English locale
PG already handles English stop words, the list in cppjieba is bigger than the list PG uses, which in turn causes confusion cause words such as "volume" are stripped using cppijieba stop word list We will follow up with another commit here to apply the Chinese word stopwords, but for now to eliminate the confusion we are skipping applying the stopword list when the dictionary in PG is in English.
This commit is contained in:
@@ -4,6 +4,20 @@ require 'rails_helper'
|
||||
|
||||
describe Search do
|
||||
|
||||
context "#prepare_data" do
|
||||
it "does not remove English stop words in mixed mode" do
|
||||
SiteSetting.search_tokenize_chinese_japanese_korean = true
|
||||
|
||||
tokenized = Search.prepare_data("monkey 吃香蕉 in a loud volume")
|
||||
expect(tokenized).to eq("monkey 吃 香蕉 in a loud volume")
|
||||
|
||||
SiteSetting.default_locale = 'zh_CN'
|
||||
|
||||
tokenized = Search.prepare_data("monkey 吃香蕉 in a loud volume")
|
||||
expect(tokenized).to eq("monkey 吃 香蕉 loud")
|
||||
end
|
||||
end
|
||||
|
||||
context "#ts_config" do
|
||||
it "maps locales to correct Postgres dictionaries" do
|
||||
expect(Search.ts_config).to eq("english")
|
||||
|
||||
Reference in New Issue
Block a user