mirror of
https://github.com/discourse/discourse.git
synced 2025-02-25 18:55:32 -06:00
FIX: search within topic not working correctly in CJK
We were splitting the term prior to search causing everything to miss
This commit is contained in:
@@ -55,20 +55,26 @@ class Search
|
||||
end
|
||||
|
||||
def self.prepare_data(search_data, purpose = :query)
|
||||
data = search_data.squish
|
||||
# TODO cppjieba_rb is designed for chinese, we need something else for Japanese
|
||||
# Korean appears to be safe cause words are already space seperated
|
||||
# For Japanese we should investigate using kakasi
|
||||
if ['zh_TW', 'zh_CN', 'ja'].include?(SiteSetting.default_locale) || SiteSetting.search_tokenize_chinese_japanese_korean
|
||||
require 'cppjieba_rb' unless defined? CppjiebaRb
|
||||
mode = (purpose == :query ? :query : :mix)
|
||||
data = CppjiebaRb.segment(search_data, mode: mode)
|
||||
data = CppjiebaRb.filter_stop_word(data).join(' ')
|
||||
end
|
||||
purpose ||= :query
|
||||
|
||||
data = search_data.dup
|
||||
data.force_encoding("UTF-8")
|
||||
if SiteSetting.search_ignore_accents
|
||||
data = strip_diacritics(data)
|
||||
if purpose != :topic
|
||||
# TODO cppjieba_rb is designed for chinese, we need something else for Japanese
|
||||
# Korean appears to be safe cause words are already space seperated
|
||||
# For Japanese we should investigate using kakasi
|
||||
if ['zh_TW', 'zh_CN', 'ja'].include?(SiteSetting.default_locale) || SiteSetting.search_tokenize_chinese_japanese_korean
|
||||
require 'cppjieba_rb' unless defined? CppjiebaRb
|
||||
mode = (purpose == :query ? :query : :mix)
|
||||
data = CppjiebaRb.segment(search_data, mode: mode)
|
||||
data = CppjiebaRb.filter_stop_word(data).join(' ')
|
||||
else
|
||||
data.squish!
|
||||
end
|
||||
|
||||
if SiteSetting.search_ignore_accents
|
||||
data = strip_diacritics(data)
|
||||
end
|
||||
end
|
||||
data
|
||||
end
|
||||
@@ -155,7 +161,7 @@ class Search
|
||||
term = process_advanced_search!(term)
|
||||
|
||||
if term.present?
|
||||
@term = Search.prepare_data(term)
|
||||
@term = Search.prepare_data(term, Topic === @search_context ? :topic : nil)
|
||||
@original_term = PG::Connection.escape_string(@term)
|
||||
end
|
||||
|
||||
|
||||
Reference in New Issue
Block a user